diff --git a/benchmarks/multi_node/agentic_srt.sh b/benchmarks/multi_node/agentic_srt.sh deleted file mode 100644 index 79a36da52..000000000 --- a/benchmarks/multi_node/agentic_srt.sh +++ /dev/null @@ -1,121 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -set -x - -# Client-only agentic trace replay for srt-slurm multinode jobs. -# srt-slurm owns server startup; this script runs as benchmark.type=custom -# against the already-ready frontend on the head node. - -INFMAX_CONTAINER_WORKSPACE="${INFMAX_CONTAINER_WORKSPACE:-/infmax-workspace}" -source "$INFMAX_CONTAINER_WORKSPACE/benchmarks/benchmark_lib.sh" - -check_env_vars MODEL MODEL_PREFIX FRAMEWORK PRECISION CONC RESULT_FILENAME DURATION - -BASE_RESULT_DIR="${RESULT_DIR:-/logs/agentic}" -BASE_RESULT_FILENAME="$RESULT_FILENAME" -read -r -a CONCURRENCIES <<< "${CONC_LIST:-$CONC}" - -if [ "${#CONCURRENCIES[@]}" -eq 0 ]; then - echo "ERROR: CONC_LIST must contain at least one concurrency" >&2 - exit 1 -fi -for concurrency in "${CONCURRENCIES[@]}"; do - if ! [[ "$concurrency" =~ ^[1-9][0-9]*$ ]]; then - echo "ERROR: invalid agentic concurrency: $concurrency" >&2 - exit 1 - fi -done - -resolve_trace_source -install_agentic_deps - -wait_for_agentic_servers_idle() { - local timeout_seconds="${AIPERF_DRAIN_TIMEOUT_SECONDS:-1800}" - local poll_seconds="${AIPERF_DRAIN_POLL_SECONDS:-10}" - local frontend_metrics_url="http://localhost:${PORT}/metrics" - - "$AIPERF_PYTHON" - \ - "$timeout_seconds" \ - "$poll_seconds" \ - "$frontend_metrics_url" \ - "${AIPERF_SERVER_METRICS_URLS:-}" <<'PY' -import sys -import time -import urllib.request - -timeout_seconds = int(sys.argv[1]) -poll_seconds = int(sys.argv[2]) -frontend_url = sys.argv[3] -worker_urls = [url for url in sys.argv[4].split(",") if url] -deadline = time.monotonic() + timeout_seconds -idle_polls = 0 - - -def fetch_metrics(url: str) -> str: - with urllib.request.urlopen(url, timeout=10) as response: - return response.read().decode("utf-8") - - -def metric_sum(metrics: str, name: str) -> float: - total = 0.0 - for line in metrics.splitlines(): - if not line or line.startswith("#"): - continue - fields = line.split() - if len(fields) < 2 or fields[0].split("{", 1)[0] != name: - continue - total += float(fields[1]) - return total - - -while time.monotonic() < deadline: - try: - frontend_metrics = fetch_metrics(frontend_url) - frontend_active = metric_sum(frontend_metrics, "dynamo_frontend_active_requests") - worker_active = 0.0 - for worker_url in worker_urls: - worker_metrics = fetch_metrics(worker_url) - worker_active += metric_sum(worker_metrics, "vllm:num_requests_running") - worker_active += metric_sum(worker_metrics, "vllm:num_requests_waiting") - print( - f"Agentic drain status: frontend_active={frontend_active:g} " - f"worker_running_or_waiting={worker_active:g}", - flush=True, - ) - if frontend_active == 0 and worker_active == 0: - idle_polls += 1 - if idle_polls >= 3: - print("Agentic servers remained idle for three polls", flush=True) - raise SystemExit(0) - else: - idle_polls = 0 - except Exception as error: - idle_polls = 0 - print(f"Agentic drain metrics query failed: {error}", file=sys.stderr, flush=True) - time.sleep(poll_seconds) - -raise SystemExit(f"Agentic servers did not drain within {timeout_seconds} seconds") -PY -} - -# The AgentX scenario's first-turn cache-bust marker includes AIPerf's unique -# per-invocation benchmark ID. Each point therefore gets a disjoint KV keyspace -# while its own warmup and profile phases share markers. This makes sequential -# points comparable without restarting the engines or inheriting warmed trace -# prefixes from an earlier concurrency. -for index in "${!CONCURRENCIES[@]}"; do - concurrency="${CONCURRENCIES[$index]}" - export CONC="$concurrency" - export RESULT_FILENAME="${BASE_RESULT_FILENAME}_conc${concurrency}" - RESULT_DIR="${BASE_RESULT_DIR}/conc_${concurrency}" - - mkdir -p "$RESULT_DIR" - - echo "Running agentic concurrency $concurrency of: ${CONCURRENCIES[*]}" - build_replay_cmd "$RESULT_DIR" - run_agentic_replay_and_write_outputs "$RESULT_DIR" - - if [ "$index" -lt "$(( ${#CONCURRENCIES[@]} - 1 ))" ]; then - wait_for_agentic_servers_idle - fi -done diff --git a/benchmarks/multi_node/deprecated/minimaxm2.5_fp8_mi355x_vllm-disagg.sh b/benchmarks/multi_node/deprecated/minimaxm2.5_fp8_mi355x_vllm-disagg.sh deleted file mode 100644 index a9a28d889..000000000 --- a/benchmarks/multi_node/deprecated/minimaxm2.5_fp8_mi355x_vllm-disagg.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then - export PREFILL_ENABLE_EP=false -else - export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then - export PREFILL_ENABLE_DP=true -else - export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then - export DECODE_ENABLE_EP=false -else - export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then - export DECODE_ENABLE_DP=true -else - export DECODE_ENABLE_DP=false -fi - -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO} \ - "${NODELIST:-}") - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/dsr1_fp4_mi355x_sglang-disagg.sh b/benchmarks/multi_node/dsr1_fp4_mi355x_sglang-disagg.sh deleted file mode 100644 index d17d1a323..000000000 --- a/benchmarks/multi_node/dsr1_fp4_mi355x_sglang-disagg.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -# Use upstreamed multi_node scripts (no external clone needed) -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -# Set up SGL launch script-specific environment variables -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -# Launch jobs based on ISL/OSL -# Replace ' ' in CONC_LIST with 'x' such that the concurrency list is represented -# by a list of numbers delimited by 'x'. This is because of how the underlying launch script -# expects the concurrencies. -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO}) - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/dsr1_fp8_mi355x_sglang-disagg.sh b/benchmarks/multi_node/dsr1_fp8_mi355x_sglang-disagg.sh deleted file mode 100644 index a8c0d2743..000000000 --- a/benchmarks/multi_node/dsr1_fp8_mi355x_sglang-disagg.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -# Use upstreamed multi_node scripts (no external clone needed) -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -# Set up SGL launch script-specific environment variables -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -# Launch jobs based on ISL/OSL -# Replace ' ' in CONC_LIST with 'x' such that the concurrency list is represented -# by a list of numbers delimited by 'x'. This is because of how the underlying launch script -# expects the concurrencies. -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO}) - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" \ No newline at end of file diff --git a/benchmarks/multi_node/dsv4_fp4_mi355x_atom-disagg.sh b/benchmarks/multi_node/dsv4_fp4_mi355x_atom-disagg.sh deleted file mode 100644 index d17d1a323..000000000 --- a/benchmarks/multi_node/dsv4_fp4_mi355x_atom-disagg.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -# Use upstreamed multi_node scripts (no external clone needed) -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -# Set up SGL launch script-specific environment variables -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -# Launch jobs based on ISL/OSL -# Replace ' ' in CONC_LIST with 'x' such that the concurrency list is represented -# by a list of numbers delimited by 'x'. This is because of how the underlying launch script -# expects the concurrencies. -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO}) - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/glm5_fp8_mi355x_sglang-disagg.sh b/benchmarks/multi_node/glm5_fp8_mi355x_sglang-disagg.sh deleted file mode 100755 index 7cbc6afe7..000000000 --- a/benchmarks/multi_node/glm5_fp8_mi355x_sglang-disagg.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -# Use upstreamed multi_node scripts (no external clone needed) -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -# Set up SGL launch script-specific environment variables -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -# Launch jobs based on ISL/OSL -# Replace ' ' in CONC_LIST with 'x' such that the concurrency list is represented -# by a list of numbers delimited by 'x'. This is because of how the underlying launch script -# expects the concurrencies. -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO} \ - "${NODELIST:-}") - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/gptoss_fp4_gb200_dynamo-trt.sh b/benchmarks/multi_node/gptoss_fp4_gb200_dynamo-trt.sh deleted file mode 100644 index a08e6f4b0..000000000 --- a/benchmarks/multi_node/gptoss_fp4_gb200_dynamo-trt.sh +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/bash - -set -x - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_MAX_NUM_TOKENS \ - PREFILL_MAX_BATCH_SIZE \ - DECODE_MAX_NUM_TOKENS \ - DECODE_MAX_BATCH_SIZE \ - DECODE_GPU_MEM_FRACTION \ - MODEL_PATH \ - SERVED_MODEL_NAME \ - RUNNER_NAME - -if [[ "$SPEC_DECODING" == "mtp" ]]; then - check_env_vars DECODE_MTP_SIZE -else - DECODE_MTP_SIZE="0" -fi - -PERFORMANCE_SWEEPS_PATH="components/backends/trtllm/performance_sweeps" - -echo "Cloning Dynamo repository..." -git clone https://github.com/ai-dynamo/dynamo.git -cd dynamo -git checkout release/0.5.1-rc0.20260105 -git submodule update --init --recursive - -cd "$PERFORMANCE_SWEEPS_PATH" - -# Set up environment variables based on ISL/OSL -if [ "$ISL" = "1024" ] && [ "$OSL" = "1024" ]; then - export CACHE_TRANSCEIVER_MAX_NUM_TOKENS=1024 -elif [ "$ISL" = "8192" ] && [ "$OSL" = "1024" ]; then - export CACHE_TRANSCEIVER_MAX_NUM_TOKENS=8448 -else - echo "Unsupported ISL/OSL combination: $ISL/$OSL" - exit 1 -fi - -kind=dynamo_disagg -additional_slurm_args="--time=04:00:00" -ntasks_per_node=4 - -gen_nodes=$(((DECODE_TP + 3)/4 * DECODE_NUM_WORKERS)) -total_nodes=$((PREFILL_NUM_WORKERS + gen_nodes)) -total_tasks=$((total_nodes * ntasks_per_node)) - -decode_eplb_num_slots=0 - -sbatch --nodes=${total_nodes} \ - --ntasks=${total_tasks} \ - --ntasks-per-node=${ntasks_per_node} \ - --job-name="${RUNNER_NAME}" \ - --segment=${total_nodes} ${additional_slurm_args} \ - benchmark_disagg.slurm \ - ${PREFILL_NUM_WORKERS} ${PREFILL_TP} \ - ${PREFILL_MAX_BATCH_SIZE} ${PREFILL_MAX_NUM_TOKENS} \ - ${PREFILL_DP_ATTN} ${DECODE_NUM_WORKERS} \ - ${DECODE_TP} ${DECODE_EP} ${DECODE_MAX_BATCH_SIZE} \ - ${DECODE_MAX_NUM_TOKENS} ${DECODE_DP_ATTN} \ - ${DECODE_GPU_MEM_FRACTION} ${decode_eplb_num_slots} \ - ${DECODE_MTP_SIZE} "${CONC_LIST}" \ - ${gen_nodes} ${kind} \ - ${MODEL_PATH} ${SERVED_MODEL_NAME} \ - ${IMAGE} ${ISL} ${OSL} diff --git a/benchmarks/multi_node/minimaxm3_fp4_mi355x_atom-disagg.sh b/benchmarks/multi_node/minimaxm3_fp4_mi355x_atom-disagg.sh deleted file mode 100644 index 505f74319..000000000 --- a/benchmarks/multi_node/minimaxm3_fp4_mi355x_atom-disagg.sh +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -# Use upstreamed multi_node scripts (no external clone needed) -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -# Set up SGL launch script-specific environment variables -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -# No MTP for MiniMax-M3 -export SPEC_DECODING="none" -export DECODE_MTP_SIZE=0 - -# Block size 128 -export KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-auto}" -export BLOCK_SIZE="${BLOCK_SIZE:-128}" -export MEM_FRAC_STATIC="${MEM_FRAC_STATIC:-0.8}" -export MAX_MODEL_LEN=32768 -export MAX_NUM_SEQS="${MAX_NUM_SEQS:-128}" -export MAX_NUM_BATCHED_TOKENS="${MAX_NUM_BATCHED_TOKENS:-32768}" - -# Launch jobs based on ISL/OSL -# Replace ' ' in CONC_LIST with 'x' such that the concurrency list is represented -# by a list of numbers delimited by 'x'. This is because of how the underlying launch script -# expects the concurrencies. -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO}) - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/minimaxm3_fp4_mi355x_vllm-disagg.sh b/benchmarks/multi_node/minimaxm3_fp4_mi355x_vllm-disagg.sh deleted file mode 100755 index 2658b8615..000000000 --- a/benchmarks/multi_node/minimaxm3_fp4_mi355x_vllm-disagg.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then - export PREFILL_ENABLE_EP=false -else - export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then - export PREFILL_ENABLE_DP=true -else - export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then - export DECODE_ENABLE_EP=false -else - export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then - export DECODE_ENABLE_DP=true -else - export DECODE_ENABLE_DP=false -fi - -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO} \ - "${NODE_LIST:-}") - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/minimaxm3_fp8_mi355x_atom-disagg.sh b/benchmarks/multi_node/minimaxm3_fp8_mi355x_atom-disagg.sh deleted file mode 100644 index 505f74319..000000000 --- a/benchmarks/multi_node/minimaxm3_fp8_mi355x_atom-disagg.sh +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -# Use upstreamed multi_node scripts (no external clone needed) -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -# Set up SGL launch script-specific environment variables -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -# No MTP for MiniMax-M3 -export SPEC_DECODING="none" -export DECODE_MTP_SIZE=0 - -# Block size 128 -export KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-auto}" -export BLOCK_SIZE="${BLOCK_SIZE:-128}" -export MEM_FRAC_STATIC="${MEM_FRAC_STATIC:-0.8}" -export MAX_MODEL_LEN=32768 -export MAX_NUM_SEQS="${MAX_NUM_SEQS:-128}" -export MAX_NUM_BATCHED_TOKENS="${MAX_NUM_BATCHED_TOKENS:-32768}" - -# Launch jobs based on ISL/OSL -# Replace ' ' in CONC_LIST with 'x' such that the concurrency list is represented -# by a list of numbers delimited by 'x'. This is because of how the underlying launch script -# expects the concurrencies. -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO}) - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/minimaxm3_fp8_mi355x_vllm-disagg.sh b/benchmarks/multi_node/minimaxm3_fp8_mi355x_vllm-disagg.sh deleted file mode 100644 index f54940e29..000000000 --- a/benchmarks/multi_node/minimaxm3_fp8_mi355x_vllm-disagg.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -export TIME_LIMIT="08:00:00" -# MiniMax-M3 MXFP8 (~414 GB) is pre-staged in this cluster's shared HF cache -# (/it-share/hf-hub-cache/models--MiniMaxAI--MiniMax-M3-MXFP8), not the default -# /it-share/data the launcher sets. Point the disagg model dir there for M3 only; -# submit.sh exports MODEL_DIR=$MODEL_PATH and job.slurm resolves the snapshot under -# it and bind-mounts MODEL_DIR into the prefill/decode serving containers. -export MODEL_PATH=/it-share/hf-hub-cache -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then - export PREFILL_ENABLE_EP=false -else - export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then - export PREFILL_ENABLE_DP=true -else - export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then - export DECODE_ENABLE_EP=false -else - export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then - export DECODE_ENABLE_DP=true -else - export DECODE_ENABLE_DP=false -fi - -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO} \ - "${NODELIST:-}") - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/qwen3.5_fp4_mi355x_sglang-disagg.sh b/benchmarks/multi_node/qwen3.5_fp4_mi355x_sglang-disagg.sh deleted file mode 100755 index 1494b1d1c..000000000 --- a/benchmarks/multi_node/qwen3.5_fp4_mi355x_sglang-disagg.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -# Use upstreamed multi_node scripts (no external clone needed) -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -# Set up SGL launch script-specific environment variables -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -# Launch jobs based on ISL/OSL -# Replace ' ' in CONC_LIST with 'x' such that the concurrency list is represented -# by a list of numbers delimited by 'x'. This is because of how the underlying launch script -# expects the concurrencies. -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO} \ - ${NODE_LIST:-}) - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/qwen3.5_fp8_mi355x_sglang-disagg.sh b/benchmarks/multi_node/qwen3.5_fp8_mi355x_sglang-disagg.sh deleted file mode 100755 index 1494b1d1c..000000000 --- a/benchmarks/multi_node/qwen3.5_fp8_mi355x_sglang-disagg.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -# Use upstreamed multi_node scripts (no external clone needed) -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -# Set up SGL launch script-specific environment variables -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -# Launch jobs based on ISL/OSL -# Replace ' ' in CONC_LIST with 'x' such that the concurrency list is represented -# by a list of numbers delimited by 'x'. This is because of how the underlying launch script -# expects the concurrencies. -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO} \ - ${NODE_LIST:-}) - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/minimax-m3-gb200-vllm-fixes.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/minimax-m3-gb200-vllm-fixes.sh deleted file mode 100755 index c0eed0a51..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/configs/minimax-m3-gb200-vllm-fixes.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -python3 - <<'PYEOF' -from importlib.util import find_spec -from pathlib import Path - -spec = find_spec("vllm") -if not spec or not spec.origin: - raise RuntimeError("vllm is not installed") -root = Path(spec.origin).parent -patches = { - root / "distributed/device_communicators/flashinfer_all_reduce.py": [ - ( - " comm_backend=comm_backend,\n" - " group=group,\n", - " comm_backend=comm_backend,\n" - ' force_oneshot_support=backend == "mnnvl",\n' - " group=group,\n", - ), - ], - root / "models/minimax_m3/nvidia/sparse_attention_msa.py": [ - ( - " prefill_topk = topk[:, nd:num_tokens, :]\n", - " prefill_topk = topk[:, nd:num_tokens, :].contiguous()\n", - ), - ], -} -for path, edits in patches.items(): - source = path.read_text() - for old, new in edits: - if new in source: - continue - if source.count(old) != 1: - raise RuntimeError(f"missing or ambiguous patch anchor in {path}") - source = source.replace(old, new, 1) - path.write_text(source) -PYEOF diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/minimax-m3-gb300-vllm-fixes.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/minimax-m3-gb300-vllm-fixes.sh deleted file mode 100755 index c0eed0a51..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/configs/minimax-m3-gb300-vllm-fixes.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -python3 - <<'PYEOF' -from importlib.util import find_spec -from pathlib import Path - -spec = find_spec("vllm") -if not spec or not spec.origin: - raise RuntimeError("vllm is not installed") -root = Path(spec.origin).parent -patches = { - root / "distributed/device_communicators/flashinfer_all_reduce.py": [ - ( - " comm_backend=comm_backend,\n" - " group=group,\n", - " comm_backend=comm_backend,\n" - ' force_oneshot_support=backend == "mnnvl",\n' - " group=group,\n", - ), - ], - root / "models/minimax_m3/nvidia/sparse_attention_msa.py": [ - ( - " prefill_topk = topk[:, nd:num_tokens, :]\n", - " prefill_topk = topk[:, nd:num_tokens, :].contiguous()\n", - ), - ], -} -for path, edits in patches.items(): - source = path.read_text() - for old, new in edits: - if new in source: - continue - if source.count(old) != 1: - raise RuntimeError(f"missing or ambiguous patch anchor in {path}") - source = source.replace(old, new, 1) - path.write_text(source) -PYEOF diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/minimax-m3-vllm-fixes.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/minimax-m3-vllm-fixes.sh deleted file mode 100755 index 02862bba3..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/configs/minimax-m3-vllm-fixes.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -python3 - <<'PYEOF' -from importlib.util import find_spec -from pathlib import Path - -spec = find_spec("vllm") -if not spec or not spec.origin: - raise RuntimeError("vllm is not installed") -root = Path(spec.origin).parent -patches = { - root / "models/minimax_m3/nvidia/sparse_attention_msa.py": [ - ( - " prefill_topk = topk[:, nd:num_tokens, :]\n", - " prefill_topk = topk[:, nd:num_tokens, :].contiguous()\n", - ), - ], - root / "distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py": [ - ( - " for i, local_len in enumerate(self.block_len_per_layer):\n", - " total_kv_heads = self.transfer_topo.total_num_kv_heads\n" - " local_heads = self.transfer_topo.local_physical_heads\n" - " remote_heads = max(1, total_kv_heads // remote_tp_size)\n" - " for i, local_len in enumerate(self.block_len_per_layer):\n", - ), - ( - "remote_len == (local_len * tp_ratio) // block_size_ratio,", - "remote_len == (local_len * remote_heads // local_heads) " - "// block_size_ratio,", - ), - ( - "remote_len == local_len // (-tp_ratio),", - "remote_len == local_len * remote_heads // local_heads,", - ), - ], -} -for path, edits in patches.items(): - source = path.read_text() - for old, new in edits: - if new in source: - continue - if source.count(old) != 1: - raise RuntimeError(f"missing or ambiguous patch anchor in {path}") - source = source.replace(old, new, 1) - path.write_text(source) -PYEOF diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p1d-dep8-dep16-6-c512.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p1d-dep8-dep16-6-c512.yaml deleted file mode 100644 index f46e40782..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p1d-dep8-dep16-6-c512.yaml +++ /dev/null @@ -1,156 +0,0 @@ -name: "disagg-gb200-1p1d-dep8-dep16-6-c512" - - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85" - precision: "fp4" - -dynamo: - hash: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" - install: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 2 - prefill_workers: 1 - gpus_per_prefill: 8 - decode_nodes: 4 - decode_workers: 1 - gpus_per_decode: 16 - -frontend: - type: dynamo - enable_multiple_frontends: false - env: - DYN_ROUTER_LOAD_BLOCK_SIZE: "1" - args: - router-mode: "kv" - router-kv-overlap-score-weight: 0 - router-queue-threshold: 64 - router-temperature: 0.5 - no-kv-events: true - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_ENABLE_THINKING: "1" - SGLANG_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_FIX_HASH_MEGA_MOE: "1" - SGLANG_OPT_USE_FAST_MASK_EP: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - SGLANG_OPT_FP8_WO_A_GEMM: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_ENABLE_THINKING: "1" - SGLANG_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_FIX_HASH_MEGA_MOE: "1" - SGLANG_OPT_USE_FAST_MASK_EP: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":88,"num_max_nvl_chunked_send_tokens":28,"num_max_nvl_chunked_recv_tokens":512},"normal_combine": {"num_sms":88,"num_max_nvl_chunked_send_tokens":16,"num_max_nvl_chunked_recv_tokens":512}}' - moe-dense-tp-size: 1 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - mem-fraction-static: 0.80 - max-running-requests: 1024 - chunked-prefill-size: 65536 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - load-balance-method: "total_requests" - moe-a2a-backend: "megamoe" - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - disaggregation-decode-polling-interval: 8 - - mem-fraction-static: 0.94 - swa-full-tokens-ratio: 0.056 - context-length: 9216 - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - max-running-requests: 21504 - cuda-graph-max-bs: 1280 - - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "512" - req_rate: "inf" - use_chat_template: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p1d-tp8-tp8-4-c1.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p1d-tp8-tp8-4-c1.yaml deleted file mode 100644 index 6305f653a..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p1d-tp8-tp8-4-c1.yaml +++ /dev/null @@ -1,117 +0,0 @@ -name: "disagg-gb200-1p1d-tp8-tp8-4-c1" - - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85" - precision: "fp4" - -dynamo: - hash: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" - install: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 2 - prefill_workers: 1 - gpus_per_prefill: 8 - decode_nodes: 2 - decode_workers: 1 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_ENABLE_THINKING: "1" - SGLANG_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_ENABLE_THINKING: "1" - SGLANG_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - disable-radix-cache: true - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 1 - - moe-runner-backend: "flashinfer_mxfp4" - disable-flashinfer-autotune: true - - mem-fraction-static: 0.90 - max-running-requests: 1024 - cuda-graph-max-bs: 512 - chunked-prefill-size: 65536 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - disable-radix-cache: true - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 1 - - moe-runner-backend: "flashinfer_mxfp4" - disable-flashinfer-autotune: true - - mem-fraction-static: 0.9 - max-running-requests: 1024 - cuda-graph-max-bs: 512 - swa-full-tokens-ratio: 0.1 - context-length: 16384 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "1" - req_rate: "inf" - use_chat_template: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p2d-dep8-dep16-10-c256.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p2d-dep8-dep16-10-c256.yaml deleted file mode 100644 index ff0bb705a..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p2d-dep8-dep16-10-c256.yaml +++ /dev/null @@ -1,156 +0,0 @@ -name: "disagg-gb200-1p2d-dep8-dep16-10-c256" - - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85" - precision: "fp4" - -dynamo: - hash: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" - install: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 2 - prefill_workers: 1 - gpus_per_prefill: 8 - decode_nodes: 8 - decode_workers: 2 - gpus_per_decode: 16 - -frontend: - type: dynamo - enable_multiple_frontends: false - env: - DYN_ROUTER_LOAD_BLOCK_SIZE: "1" - args: - router-mode: "kv" - router-kv-overlap-score-weight: 0 - router-queue-threshold: 64 - router-temperature: 0.5 - no-kv-events: true - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_ENABLE_THINKING: "1" - SGLANG_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_FIX_HASH_MEGA_MOE: "1" - SGLANG_OPT_USE_FAST_MASK_EP: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - SGLANG_OPT_FP8_WO_A_GEMM: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_ENABLE_THINKING: "1" - SGLANG_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_FIX_HASH_MEGA_MOE: "1" - SGLANG_OPT_USE_FAST_MASK_EP: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":88,"num_max_nvl_chunked_send_tokens":28,"num_max_nvl_chunked_recv_tokens":512},"normal_combine": {"num_sms":88,"num_max_nvl_chunked_send_tokens":16,"num_max_nvl_chunked_recv_tokens":512}}' - moe-dense-tp-size: 1 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - mem-fraction-static: 0.80 - max-running-requests: 1024 - chunked-prefill-size: 65536 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - load-balance-method: "total_requests" - moe-a2a-backend: "megamoe" - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - disaggregation-decode-polling-interval: 8 - - mem-fraction-static: 0.94 - swa-full-tokens-ratio: 0.056 - context-length: 9216 - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - max-running-requests: 21504 - cuda-graph-max-bs: 1280 - - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "256" - req_rate: "inf" - use_chat_template: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p4d-dep8-tp8-10-c64.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p4d-dep8-tp8-10-c64.yaml deleted file mode 100644 index 84349c277..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p4d-dep8-tp8-10-c64.yaml +++ /dev/null @@ -1,142 +0,0 @@ -name: "disagg-gb200-1p4d-dep8-tp8-10-c64" - - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85" - precision: "fp4" - -dynamo: - hash: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" - install: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 2 - prefill_workers: 1 - gpus_per_prefill: 8 - decode_nodes: 8 - decode_workers: 4 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: false - env: - DYN_ROUTER_LOAD_BLOCK_SIZE: "1" - args: - router-mode: "kv" - router-kv-overlap-score-weight: 0 - router-queue-threshold: 64 - router-temperature: 0.5 - no-kv-events: true - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_ENABLE_THINKING: "1" - SGLANG_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_FIX_HASH_MEGA_MOE: "1" - SGLANG_OPT_USE_FAST_MASK_EP: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - SGLANG_OPT_FP8_WO_A_GEMM: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_ENABLE_THINKING: "1" - SGLANG_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_USE_JIT_NORM: "1" - SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" - SGLANG_OPT_USE_TOPK_V2: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":88,"num_max_nvl_chunked_send_tokens":28,"num_max_nvl_chunked_recv_tokens":512},"normal_combine": {"num_sms":88,"num_max_nvl_chunked_send_tokens":16,"num_max_nvl_chunked_recv_tokens":512}}' - moe-dense-tp-size: 1 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - mem-fraction-static: 0.80 - max-running-requests: 1024 - chunked-prefill-size: 65536 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - disable-radix-cache: true - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 1 - - moe-runner-backend: "flashinfer_mxfp4" - disable-flashinfer-autotune: true - - mem-fraction-static: 0.9 - max-running-requests: 1024 - cuda-graph-max-bs: 512 - swa-full-tokens-ratio: 0.1 - context-length: 16384 - - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "64" - req_rate: "inf" - use_chat_template: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-2p1d-dep8-dep16-8-c1536.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-2p1d-dep8-dep16-8-c1536.yaml deleted file mode 100644 index 478c91b04..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-2p1d-dep8-dep16-8-c1536.yaml +++ /dev/null @@ -1,156 +0,0 @@ -name: "disagg-gb200-2p1d-dep8-dep16-8-c1536" - - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85" - precision: "fp4" - -dynamo: - hash: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" - install: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 4 - prefill_workers: 2 - gpus_per_prefill: 8 - decode_nodes: 4 - decode_workers: 1 - gpus_per_decode: 16 - -frontend: - type: dynamo - enable_multiple_frontends: false - env: - DYN_ROUTER_LOAD_BLOCK_SIZE: "1" - args: - router-mode: "kv" - router-kv-overlap-score-weight: 0 - router-queue-threshold: 64 - router-temperature: 0.5 - no-kv-events: true - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_ENABLE_THINKING: "1" - SGLANG_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_FIX_HASH_MEGA_MOE: "1" - SGLANG_OPT_USE_FAST_MASK_EP: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - SGLANG_OPT_FP8_WO_A_GEMM: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_ENABLE_THINKING: "1" - SGLANG_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_FIX_HASH_MEGA_MOE: "1" - SGLANG_OPT_USE_FAST_MASK_EP: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":88,"num_max_nvl_chunked_send_tokens":28,"num_max_nvl_chunked_recv_tokens":512},"normal_combine": {"num_sms":88,"num_max_nvl_chunked_send_tokens":16,"num_max_nvl_chunked_recv_tokens":512}}' - moe-dense-tp-size: 1 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - mem-fraction-static: 0.80 - max-running-requests: 1024 - chunked-prefill-size: 65536 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - load-balance-method: "total_requests" - moe-a2a-backend: "megamoe" - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - disaggregation-decode-polling-interval: 8 - - mem-fraction-static: 0.94 - swa-full-tokens-ratio: 0.056 - context-length: 9216 - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - max-running-requests: 21504 - cuda-graph-max-bs: 1280 - - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "1536" - req_rate: "inf" - use_chat_template: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-4p1d-dep8-dep16-12-c4096.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-4p1d-dep8-dep16-12-c4096.yaml deleted file mode 100644 index 11434ca53..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-4p1d-dep8-dep16-12-c4096.yaml +++ /dev/null @@ -1,156 +0,0 @@ -name: "disagg-gb200-4p1d-dep8-dep16-12-c4096" - - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85" - precision: "fp4" - -dynamo: - hash: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" - install: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 8 - prefill_workers: 4 - gpus_per_prefill: 8 - decode_nodes: 4 - decode_workers: 1 - gpus_per_decode: 16 - -frontend: - type: dynamo - enable_multiple_frontends: false - env: - DYN_ROUTER_LOAD_BLOCK_SIZE: "1" - args: - router-mode: "kv" - router-kv-overlap-score-weight: 0 - router-queue-threshold: 64 - router-temperature: 0.5 - no-kv-events: true - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_ENABLE_THINKING: "1" - SGLANG_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_FIX_HASH_MEGA_MOE: "1" - SGLANG_OPT_USE_FAST_MASK_EP: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - SGLANG_OPT_FP8_WO_A_GEMM: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_ENABLE_THINKING: "1" - SGLANG_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_FIX_HASH_MEGA_MOE: "1" - SGLANG_OPT_USE_FAST_MASK_EP: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":88,"num_max_nvl_chunked_send_tokens":28,"num_max_nvl_chunked_recv_tokens":512},"normal_combine": {"num_sms":88,"num_max_nvl_chunked_send_tokens":16,"num_max_nvl_chunked_recv_tokens":512}}' - moe-dense-tp-size: 1 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - mem-fraction-static: 0.80 - max-running-requests: 1024 - chunked-prefill-size: 65536 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - load-balance-method: "total_requests" - moe-a2a-backend: "megamoe" - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - disaggregation-decode-polling-interval: 8 - - mem-fraction-static: 0.94 - swa-full-tokens-ratio: 0.056 - context-length: 9216 - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - max-running-requests: 21504 - cuda-graph-max-bs: 1280 - - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "4096" - req_rate: "inf" - use_chat_template: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-5p1d-dep8-dep16-14-c8192.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-5p1d-dep8-dep16-14-c8192.yaml deleted file mode 100644 index 9962318e8..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-5p1d-dep8-dep16-14-c8192.yaml +++ /dev/null @@ -1,156 +0,0 @@ -name: "disagg-gb200-5p1d-dep8-dep16-14-c8192" - - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85" - precision: "fp4" - -dynamo: - hash: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" - install: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 10 - prefill_workers: 5 - gpus_per_prefill: 8 - decode_nodes: 4 - decode_workers: 1 - gpus_per_decode: 16 - -frontend: - type: dynamo - enable_multiple_frontends: false - env: - DYN_ROUTER_LOAD_BLOCK_SIZE: "1" - args: - router-mode: "kv" - router-kv-overlap-score-weight: 0 - router-queue-threshold: 64 - router-temperature: 0.5 - no-kv-events: true - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_ENABLE_THINKING: "1" - SGLANG_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_FIX_HASH_MEGA_MOE: "1" - SGLANG_OPT_USE_FAST_MASK_EP: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - SGLANG_OPT_FP8_WO_A_GEMM: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_ENABLE_THINKING: "1" - SGLANG_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_FIX_HASH_MEGA_MOE: "1" - SGLANG_OPT_USE_FAST_MASK_EP: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":88,"num_max_nvl_chunked_send_tokens":28,"num_max_nvl_chunked_recv_tokens":512},"normal_combine": {"num_sms":88,"num_max_nvl_chunked_send_tokens":16,"num_max_nvl_chunked_recv_tokens":512}}' - moe-dense-tp-size: 1 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - mem-fraction-static: 0.80 - max-running-requests: 1024 - chunked-prefill-size: 65536 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - load-balance-method: "total_requests" - moe-a2a-backend: "megamoe" - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - disaggregation-decode-polling-interval: 8 - - mem-fraction-static: 0.94 - swa-full-tokens-ratio: 0.056 - context-length: 9216 - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - max-running-requests: 21504 - cuda-graph-max-bs: 1280 - - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "8192" - req_rate: "inf" - use_chat_template: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-6p1d-dep8-dep12-15-c8192.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-6p1d-dep8-dep12-15-c8192.yaml deleted file mode 100644 index e88d4b7d5..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-6p1d-dep8-dep12-15-c8192.yaml +++ /dev/null @@ -1,156 +0,0 @@ -name: "disagg-gb200-6p1d-dep8-dep12-15-c8192" - - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85" - precision: "fp4" - -dynamo: - hash: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" - install: true - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 12 - prefill_workers: 6 - gpus_per_prefill: 8 - decode_nodes: 3 - decode_workers: 1 - gpus_per_decode: 12 - -frontend: - type: dynamo - enable_multiple_frontends: false - env: - DYN_ROUTER_LOAD_BLOCK_SIZE: "1" - args: - router-mode: "kv" - router-kv-overlap-score-weight: 0 - router-queue-threshold: 64 - router-temperature: 0.5 - no-kv-events: true - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_ENABLE_THINKING: "1" - SGLANG_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_FIX_HASH_MEGA_MOE: "1" - SGLANG_OPT_USE_FAST_MASK_EP: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - SGLANG_OPT_FP8_WO_A_GEMM: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_ENABLE_THINKING: "1" - SGLANG_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_FIX_HASH_MEGA_MOE: "1" - SGLANG_OPT_USE_FAST_MASK_EP: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_NEXTN_MEGA_MOE: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":88,"num_max_nvl_chunked_send_tokens":28,"num_max_nvl_chunked_recv_tokens":512},"normal_combine": {"num_sms":88,"num_max_nvl_chunked_send_tokens":16,"num_max_nvl_chunked_recv_tokens":512}}' - moe-dense-tp-size: 1 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - mem-fraction-static: 0.80 - max-running-requests: 1024 - chunked-prefill-size: 65536 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - load-balance-method: "total_requests" - moe-a2a-backend: "megamoe" - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - disaggregation-decode-polling-interval: 8 - - mem-fraction-static: 0.94 - swa-full-tokens-ratio: 0.056 - context-length: 9216 - tensor-parallel-size: 12 - data-parallel-size: 12 - expert-parallel-size: 12 - enable-dp-attention: true - enable-dp-lm-head: true - max-running-requests: 21504 - cuda-graph-max-bs: 1280 - - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "8192" - req_rate: "inf" - use_chat_template: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-low-latency-1p1d-tp8-tp8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-low-latency-1p1d-tp8-tp8-mtp.yaml deleted file mode 100644 index 0b2423a8e..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-low-latency-1p1d-tp8-tp8-mtp.yaml +++ /dev/null @@ -1,124 +0,0 @@ -name: "dsv4-pro-gb200-disagg-8k1k-low-latency-1p1d-tp8-tp8-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85" - precision: "fp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 2 - prefill_workers: 1 - gpus_per_prefill: 8 - decode_nodes: 2 - decode_workers: 1 - gpus_per_decode: 8 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 1 - - moe-runner-backend: "flashinfer_mxfp4" - disable-flashinfer-autotune: true - - mem-fraction-static: 0.9 - max-running-requests: 16 - cuda-graph-max-bs: 8 - chunked-prefill-size: 65536 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 1 - - moe-runner-backend: "flashinfer_mxfp4" - disable-flashinfer-autotune: true - - speculative-algo: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - mem-fraction-static: 0.9 - max-running-requests: 8 - cuda-graph-max-bs: 8 - swa-full-tokens-ratio: 0.1 - context-length: 16384 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "1" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-low-latency-1p6d-dep8-tp8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-low-latency-1p6d-dep8-tp8-mtp.yaml deleted file mode 100644 index 79c9a46bd..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-low-latency-1p6d-dep8-tp8-mtp.yaml +++ /dev/null @@ -1,131 +0,0 @@ -name: "dsv4-pro-gb200-disagg-8k1k-low-latency-1p6d-dep8-tp8-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85" - precision: "fp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 2 - prefill_workers: 1 - gpus_per_prefill: 8 - decode_nodes: 12 - decode_workers: 6 - gpus_per_decode: 8 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' - - mem-fraction-static: 0.9 - max-running-requests: 256 - cuda-graph-max-bs: 128 - chunked-prefill-size: 65536 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 8 - data-parallel-size: 1 - expert-parallel-size: 1 - - moe-runner-backend: "flashinfer_mxfp4" - disable-flashinfer-autotune: true - - speculative-algo: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - mem-fraction-static: 0.9 - max-running-requests: 128 - cuda-graph-max-bs: 128 - swa-full-tokens-ratio: 0.1 - context-length: 16384 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "32x64x128" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-1p1d-dep8-dep16-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-1p1d-dep8-dep16-mtp.yaml deleted file mode 100644 index 1bf4f0e85..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-1p1d-dep8-dep16-mtp.yaml +++ /dev/null @@ -1,143 +0,0 @@ -name: "dsv4-pro-gb200-disagg-8k1k-mid-curve-1p1d-dep8-dep16-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85" - precision: "fp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 2 - prefill_workers: 1 - gpus_per_prefill: 8 - decode_nodes: 4 - decode_workers: 1 - gpus_per_decode: 16 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "2048" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' - - mem-fraction-static: 0.8 - max-running-requests: 512 - cuda-graph-max-bs: 512 - chunked-prefill-size: 65536 - stream-interval: 60 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' - - speculative-algo: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - mem-fraction-static: 0.85 - max-running-requests: 1024 - cuda-graph-max-bs: 1024 - swa-full-tokens-ratio: 0.15 - context-length: 16384 - stream-interval: 60 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "1024" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-2p1d-dep8-dep16-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-2p1d-dep8-dep16-mtp.yaml deleted file mode 100644 index 82519e378..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-2p1d-dep8-dep16-mtp.yaml +++ /dev/null @@ -1,143 +0,0 @@ -name: "dsv4-pro-gb200-disagg-8k1k-mid-curve-2p1d-dep8-dep16-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85" - precision: "fp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 4 - prefill_workers: 2 - gpus_per_prefill: 8 - decode_nodes: 4 - decode_workers: 1 - gpus_per_decode: 16 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' - - mem-fraction-static: 0.8 - max-running-requests: 1024 - cuda-graph-max-bs: 1024 - chunked-prefill-size: 65536 - stream-interval: 60 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' - - speculative-algo: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - mem-fraction-static: 0.85 - max-running-requests: 2048 - cuda-graph-max-bs: 1024 - swa-full-tokens-ratio: 0.15 - context-length: 16384 - stream-interval: 60 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "2048" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-3p1d-dep8-dep16-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-3p1d-dep8-dep16-mtp.yaml deleted file mode 100644 index e69c5e604..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-3p1d-dep8-dep16-mtp.yaml +++ /dev/null @@ -1,143 +0,0 @@ -name: "dsv4-pro-gb200-disagg-8k1k-mid-curve-3p1d-dep8-dep16-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85" - precision: "fp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 6 - prefill_workers: 3 - gpus_per_prefill: 8 - decode_nodes: 4 - decode_workers: 1 - gpus_per_decode: 16 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' - - mem-fraction-static: 0.8 - max-running-requests: 1024 - cuda-graph-max-bs: 1024 - chunked-prefill-size: 65536 - stream-interval: 60 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' - - speculative-algo: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - mem-fraction-static: 0.85 - max-running-requests: 4096 - cuda-graph-max-bs: 1024 - swa-full-tokens-ratio: 0.15 - context-length: 16384 - stream-interval: 60 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "3072" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-4p1d-dep8-dep16-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-4p1d-dep8-dep16-mtp.yaml deleted file mode 100644 index 73bcecaec..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-4p1d-dep8-dep16-mtp.yaml +++ /dev/null @@ -1,143 +0,0 @@ -name: "dsv4-pro-gb200-disagg-8k1k-mid-curve-4p1d-dep8-dep16-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85" - precision: "fp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 8 - prefill_workers: 4 - gpus_per_prefill: 8 - decode_nodes: 4 - decode_workers: 1 - gpus_per_decode: 16 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' - - mem-fraction-static: 0.8 - max-running-requests: 1024 - cuda-graph-max-bs: 1024 - chunked-prefill-size: 65536 - stream-interval: 60 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' - - speculative-algo: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - mem-fraction-static: 0.85 - max-running-requests: 6144 - cuda-graph-max-bs: 1024 - swa-full-tokens-ratio: 0.15 - context-length: 16384 - stream-interval: 60 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "6144" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-5p1d-dep8-dep16-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-5p1d-dep8-dep16-mtp.yaml deleted file mode 100644 index 66829c404..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-5p1d-dep8-dep16-mtp.yaml +++ /dev/null @@ -1,143 +0,0 @@ -name: "dsv4-pro-gb200-disagg-8k1k-mid-curve-5p1d-dep8-dep16-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85" - precision: "fp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 10 - prefill_workers: 5 - gpus_per_prefill: 8 - decode_nodes: 4 - decode_workers: 1 - gpus_per_decode: 16 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' - - mem-fraction-static: 0.8 - max-running-requests: 1024 - cuda-graph-max-bs: 1024 - chunked-prefill-size: 65536 - stream-interval: 60 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' - - speculative-algo: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - mem-fraction-static: 0.85 - max-running-requests: 16384 - cuda-graph-max-bs: 1024 - swa-full-tokens-ratio: 0.15 - context-length: 16384 - stream-interval: 60 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "8192" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-6p1d-dep8-dep16-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-6p1d-dep8-dep16-mtp.yaml deleted file mode 100644 index 34b71a918..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-6p1d-dep8-dep16-mtp.yaml +++ /dev/null @@ -1,143 +0,0 @@ -name: "dsv4-pro-gb200-disagg-8k1k-mid-curve-6p1d-dep8-dep16-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "92f5b3b8d7dd5ab9179d4b1034bd2c1c0803693e" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85" - precision: "fp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 12 - prefill_workers: 6 - gpus_per_prefill: 8 - decode_nodes: 4 - decode_workers: 1 - gpus_per_decode: 16 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_FIX_MEGA_MOE_MEMORY: "1" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' - - mem-fraction-static: 0.8 - max-running-requests: 1024 - cuda-graph-max-bs: 1024 - chunked-prefill-size: 65536 - stream-interval: 60 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' - - speculative-algo: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - mem-fraction-static: 0.85 - max-running-requests: 21504 - cuda-graph-max-bs: 1024 - swa-full-tokens-ratio: 0.15 - context-length: 16384 - stream-interval: 60 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "16384" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" - diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-10p1d-dep4-dep32-18-c2500.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-10p1d-dep4-dep32-18-c2500.yaml deleted file mode 100644 index 528aa5721..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-10p1d-dep4-dep32-18-c2500.yaml +++ /dev/null @@ -1,158 +0,0 @@ -name: "disagg-gb300-10p1d-dep4-dep32-18-c2500" - -# Weiliang wide-EP sweep point: EP=32, 10P+8D = 18 nodes, conc=2500. -# Matches srt-slurm PR#173 zip_override EP=32 topology. -# Env vars and sglang_config from InferenceX main (not Weiliang's 0510 image). - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260520-425dffbd" - precision: "fp4" - -dynamo: - hash: "81d0555ee23519cea80a42b4fe824e30368b7300" - install: true - -slurm: - time_limit: "03:00:00" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 10 - prefill_workers: 10 - gpus_per_prefill: 4 - decode_nodes: 8 - decode_workers: 1 - gpus_per_decode: 32 - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - env: - DYN_ROUTER_LOAD_BLOCK_SIZE: "1" - args: - router-mode: "kv" - router-kv-overlap-score-weight: 0 - router-queue-threshold: 64 - router-temperature: 0.5 - no-kv-events: true - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - - enable-dp-attention: true - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":88,"num_max_nvl_chunked_send_tokens":28,"num_max_nvl_chunked_recv_tokens":512},"normal_combine": {"num_sms":88,"num_max_nvl_chunked_send_tokens":16,"num_max_nvl_chunked_recv_tokens":512}}' - moe-dense-tp-size: 1 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - enable-dp-lm-head: true - - mem-fraction-static: 0.90 - max-running-requests: 512 - cuda-graph-max-bs: 512 - chunked-prefill-size: 32768 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - moe-a2a-backend: "megamoe" - - moe-dense-tp-size: 1 - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - disaggregation-decode-polling-interval: 8 - - mem-fraction-static: 0.94 - swa-full-tokens-ratio: 0.20 - context-length: 9216 - tensor-parallel-size: 32 - data-parallel-size: 32 - expert-parallel-size: 32 - enable-dp-attention: true - enable-dp-lm-head: true - max-running-requests: 18432 - cuda-graph-max-bs: 1280 - - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "2500" - req_rate: "inf" - use_chat_template: false - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-12p1d-dep4-dep24-18-c3000.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-12p1d-dep4-dep24-18-c3000.yaml deleted file mode 100644 index 32a5124c2..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-12p1d-dep4-dep24-18-c3000.yaml +++ /dev/null @@ -1,158 +0,0 @@ -name: "disagg-gb300-12p1d-dep4-dep24-18-c3000" - -# Weiliang wide-EP sweep point: EP=24, 12P+6D = 18 nodes, conc=3000. -# Matches srt-slurm PR#173 zip_override EP=24 topology. -# Env vars and sglang_config from InferenceX main (not Weiliang's 0510 image). - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260520-425dffbd" - precision: "fp4" - -dynamo: - hash: "81d0555ee23519cea80a42b4fe824e30368b7300" - install: true - -slurm: - time_limit: "03:00:00" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 12 - prefill_workers: 12 - gpus_per_prefill: 4 - decode_nodes: 6 - decode_workers: 1 - gpus_per_decode: 24 - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - env: - DYN_ROUTER_LOAD_BLOCK_SIZE: "1" - args: - router-mode: "kv" - router-kv-overlap-score-weight: 0 - router-queue-threshold: 64 - router-temperature: 0.5 - no-kv-events: true - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - - enable-dp-attention: true - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":88,"num_max_nvl_chunked_send_tokens":28,"num_max_nvl_chunked_recv_tokens":512},"normal_combine": {"num_sms":88,"num_max_nvl_chunked_send_tokens":16,"num_max_nvl_chunked_recv_tokens":512}}' - moe-dense-tp-size: 1 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - enable-dp-lm-head: true - - mem-fraction-static: 0.90 - max-running-requests: 512 - cuda-graph-max-bs: 512 - chunked-prefill-size: 32768 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - moe-a2a-backend: "megamoe" - - moe-dense-tp-size: 1 - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - disaggregation-decode-polling-interval: 8 - - mem-fraction-static: 0.94 - swa-full-tokens-ratio: 0.20 - context-length: 9216 - tensor-parallel-size: 24 - data-parallel-size: 24 - expert-parallel-size: 24 - enable-dp-attention: true - enable-dp-lm-head: true - max-running-requests: 18432 - cuda-graph-max-bs: 1280 - - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "3000" - req_rate: "inf" - use_chat_template: false - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-14p1d-dep4-dep16-18-c8192.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-14p1d-dep4-dep16-18-c8192.yaml deleted file mode 100644 index adc6b1550..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-14p1d-dep4-dep16-18-c8192.yaml +++ /dev/null @@ -1,158 +0,0 @@ -name: "disagg-gb300-14p1d-dep4-dep16-18-c8192" - -# Weiliang wide-EP sweep point: EP=16, 14P+4D = 18 nodes, conc=8192. -# Matches srt-slurm PR#173 zip_override EP=16 topology. -# Env vars and sglang_config from InferenceX main (not Weiliang's 0510 image). - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260520-425dffbd" - precision: "fp4" - -dynamo: - hash: "81d0555ee23519cea80a42b4fe824e30368b7300" - install: true - -slurm: - time_limit: "03:00:00" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 14 - prefill_workers: 14 - gpus_per_prefill: 4 - decode_nodes: 4 - decode_workers: 1 - gpus_per_decode: 16 - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - env: - DYN_ROUTER_LOAD_BLOCK_SIZE: "1" - args: - router-mode: "kv" - router-kv-overlap-score-weight: 0 - router-queue-threshold: 64 - router-temperature: 0.5 - no-kv-events: true - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - - enable-dp-attention: true - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":88,"num_max_nvl_chunked_send_tokens":28,"num_max_nvl_chunked_recv_tokens":512},"normal_combine": {"num_sms":88,"num_max_nvl_chunked_send_tokens":16,"num_max_nvl_chunked_recv_tokens":512}}' - moe-dense-tp-size: 1 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - enable-dp-lm-head: true - - mem-fraction-static: 0.90 - max-running-requests: 512 - cuda-graph-max-bs: 512 - chunked-prefill-size: 32768 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - moe-a2a-backend: "megamoe" - - moe-dense-tp-size: 1 - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - disaggregation-decode-polling-interval: 8 - - mem-fraction-static: 0.94 - swa-full-tokens-ratio: 0.20 - context-length: 9216 - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - max-running-requests: 18432 - cuda-graph-max-bs: 1280 - - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "8192" - req_rate: "inf" - use_chat_template: false - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-15p1d-dep4-dep12-18-c12000.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-15p1d-dep4-dep12-18-c12000.yaml deleted file mode 100644 index 73ad15750..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-15p1d-dep4-dep12-18-c12000.yaml +++ /dev/null @@ -1,158 +0,0 @@ -name: "disagg-gb300-15p1d-dep4-dep12-18-c12000" - -# Weiliang wide-EP sweep point: EP=12, 15P+3D = 18 nodes, conc=12000. -# Matches srt-slurm PR#173 zip_override EP=12 topology. -# Env vars and sglang_config from InferenceX main (not Weiliang's 0510 image). - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260520-425dffbd" - precision: "fp4" - -dynamo: - hash: "81d0555ee23519cea80a42b4fe824e30368b7300" - install: true - -slurm: - time_limit: "03:00:00" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 15 - prefill_workers: 15 - gpus_per_prefill: 4 - decode_nodes: 3 - decode_workers: 1 - gpus_per_decode: 12 - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - env: - DYN_ROUTER_LOAD_BLOCK_SIZE: "1" - args: - router-mode: "kv" - router-kv-overlap-score-weight: 0 - router-queue-threshold: 64 - router-temperature: 0.5 - no-kv-events: true - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - - enable-dp-attention: true - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":88,"num_max_nvl_chunked_send_tokens":28,"num_max_nvl_chunked_recv_tokens":512},"normal_combine": {"num_sms":88,"num_max_nvl_chunked_send_tokens":16,"num_max_nvl_chunked_recv_tokens":512}}' - moe-dense-tp-size: 1 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - enable-dp-lm-head: true - - mem-fraction-static: 0.90 - max-running-requests: 512 - cuda-graph-max-bs: 512 - chunked-prefill-size: 32768 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - moe-a2a-backend: "megamoe" - - moe-dense-tp-size: 1 - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - disaggregation-decode-polling-interval: 8 - - mem-fraction-static: 0.94 - swa-full-tokens-ratio: 0.20 - context-length: 9216 - tensor-parallel-size: 12 - data-parallel-size: 12 - expert-parallel-size: 12 - enable-dp-attention: true - enable-dp-lm-head: true - max-running-requests: 18432 - cuda-graph-max-bs: 1280 - - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "12000" - req_rate: "inf" - use_chat_template: false - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-1p1d-dep4-dep16-5-c1024.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-1p1d-dep4-dep16-5-c1024.yaml deleted file mode 100644 index 7cdb779c7..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-1p1d-dep4-dep16-5-c1024.yaml +++ /dev/null @@ -1,182 +0,0 @@ -name: "disagg-gb300-1p1d-dep4-dep16-5-c1024" - -# 8k/1k high-throughput topology for the wideep DSV4-Pro setup. -# -# Schema/values come from PR #1213 (513cbef) — that PR introduced the -# `dsv4-pro-gb300-fp4` upstream-style recipe with two `zip_override` -# variants (wideep [0] / narrow_ep [1]) and `backend.benchmark`. Our -# pinned srtctl (NVIDIA/srt-slurm @ sa-submission-q2-2026) doesn't -# support either: `zip_override_*_hightpt` rejects with `Unknown field` -# and `benchmark` only validates at top level. So this file inlines the -# wideep [0] override and lifts `benchmark` back out — same operational -# values, schema the pinned srtctl will accept. -# -# Other adjustments back to the InferenceX cluster shape: container & -# model.path restored to the aliases mapped in launch_gb300.sh's -# srtslurm.yaml (`lmsysorg/sglang:deepseek-v4-grace-blackwell` and -# `deepseek-v4-pro`); `dynamo.install: true` added so the container -# (which has no dynamo baked in) installs from the pinned hash. -# -# Cluster-specific items NOT inlined (require InferenceX-side equivalents): -# - slurm.partition (the source cluster uses `hpc-mid`) -# - frontend.nginx_container (yangminl's `nginx-1.27.4.sqsh` path) -# - extra_mount: yangminl/sglang-patched/sglang. Earlier diff analysis -# showed only `expert_location_dispatch.py` topk_ids int32 cast is an -# active runtime diff vs container sglang; other patched files are -# env-gated dead code under the same SGLANG_OPT_* flags this yaml -# already sets. -# -# DG-related env intentionally diverged (DG cache path is host-specific): -# - SGLANG_DG_CACHE_DIR=/configs/deepgemm_cache (yangminl host) -# - SGLANG_JIT_DEEPGEMM_PRECOMPILE=0 (yangminl uses prebuilt cache) -# This yaml uses SGLANG_JIT_DEEPGEMM_FAST_WARMUP=1 instead. - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260520-425dffbd" - precision: "fp4" - -dynamo: - hash: "81d0555ee23519cea80a42b4fe824e30368b7300" - install: true - -slurm: - time_limit: "03:00:00" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 4 - decode_workers: 1 - gpus_per_decode: 16 - -frontend: - type: dynamo - enable_multiple_frontends: false - env: - DYN_ROUTER_LOAD_BLOCK_SIZE: "1" - args: - router-mode: "kv" - router-kv-overlap-score-weight: 0 - router-queue-threshold: 64 - router-temperature: 0.5 - no-kv-events: true - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - # is single-node only and corrupts results in 2-node decode setups. - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - - enable-dp-attention: true - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":88,"num_max_nvl_chunked_send_tokens":28,"num_max_nvl_chunked_recv_tokens":512},"normal_combine": {"num_sms":88,"num_max_nvl_chunked_send_tokens":16,"num_max_nvl_chunked_recv_tokens":512}}' - moe-dense-tp-size: 1 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - mem-fraction-static: 0.90 - max-running-requests: 512 - chunked-prefill-size: 32768 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - load-balance-method: "total_requests" - moe-a2a-backend: "megamoe" - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - disaggregation-decode-polling-interval: 8 - - mem-fraction-static: 0.94 - swa-full-tokens-ratio: 0.056 - context-length: 9216 - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - max-running-requests: 21504 - cuda-graph-max-bs: 1280 - - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "1024" - req_rate: "inf" - use_chat_template: false - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-1p1d-tp4-tp4-2-c1.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-1p1d-tp4-tp4-2-c1.yaml deleted file mode 100644 index 9382dd6dd..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-1p1d-tp4-tp4-2-c1.yaml +++ /dev/null @@ -1,165 +0,0 @@ -name: "disagg-gb300-1p1d-tp4-tp4-2-c1" - -# 8k/1k high-throughput topology for the wideep DSV4-Pro setup. -# -# Schema/values come from PR #1213 (513cbef) — that PR introduced the -# `dsv4-pro-gb300-fp4` upstream-style recipe with two `zip_override` -# variants (wideep [0] / narrow_ep [1]) and `backend.benchmark`. Our -# pinned srtctl (NVIDIA/srt-slurm @ sa-submission-q2-2026) doesn't -# support either: `zip_override_*_hightpt` rejects with `Unknown field` -# and `benchmark` only validates at top level. So this file inlines the -# wideep [0] override and lifts `benchmark` back out — same operational -# values, schema the pinned srtctl will accept. -# -# Other adjustments back to the InferenceX cluster shape: container & -# model.path restored to the aliases mapped in launch_gb300.sh's -# srtslurm.yaml (`lmsysorg/sglang:deepseek-v4-grace-blackwell` and -# `deepseek-v4-pro`); `dynamo.install: true` added so the container -# (which has no dynamo baked in) installs from the pinned hash. -# -# Cluster-specific items NOT inlined (require InferenceX-side equivalents): -# - slurm.partition (the source cluster uses `hpc-mid`) -# - frontend.nginx_container (yangminl's `nginx-1.27.4.sqsh` path) -# - extra_mount: yangminl/sglang-patched/sglang. Earlier diff analysis -# showed only `expert_location_dispatch.py` topk_ids int32 cast is an -# active runtime diff vs container sglang; other patched files are -# env-gated dead code under the same SGLANG_OPT_* flags this yaml -# already sets. -# -# DG-related env intentionally diverged (DG cache path is host-specific): -# - SGLANG_DG_CACHE_DIR=/configs/deepgemm_cache (yangminl host) -# - SGLANG_JIT_DEEPGEMM_PRECOMPILE=0 (yangminl uses prebuilt cache) -# This yaml uses SGLANG_JIT_DEEPGEMM_FAST_WARMUP=1 instead. - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260520-425dffbd" - precision: "fp4" - -# See ../1k1k/disagg-gb200-1p1d-dep8-tep8.yaml for the dynamo pin -# rationale. Hash bumped from PR #1213 to track the dynamo-sglang dsv4 -# dev branch. -dynamo: - hash: "81d0555ee23519cea80a42b4fe824e30368b7300" - install: true - -slurm: - time_limit: "03:00:00" - -# Match yangminl's working all-dynamo.yaml on the source cluster: -# cpus-per-task=144 — without this slurm hands out 1 CPU/task, which -# turns the dynamo `hash:` cold source build (~500 rust crates, -# ravif/exr/zip/pyo3 stack) into a 30+ min serial compile. With 144 -# cargo finishes in ~5 min. -# mem=0 — slurm's "give the whole node's memory"; needed -# for sglang loading 671B FP4 weights + dynamo build at the same -# time without OOM. -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -# Topology: 7 prefill (TP=4 / DP=4 / EP=4 / 1 node each) + 1 decode -# (TP=8 / DP=8 / EP=8 / 2 nodes). 9 nodes total. -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 1 - decode_workers: 1 - gpus_per_decode: 4 - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - # is single-node only and corrupts results in 2-node decode setups. - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - disable-radix-cache: true - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - - moe-runner-backend: "flashinfer_mxfp4" - disable-flashinfer-autotune: true - - mem-fraction-static: 0.90 - max-running-requests: 512 - cuda-graph-max-bs: 512 - chunked-prefill-size: 32768 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - disable-radix-cache: true - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - - moe-runner-backend: "flashinfer_mxfp4" - disable-flashinfer-autotune: true - - mem-fraction-static: 0.9 - max-running-requests: 1024 - cuda-graph-max-bs: 512 - swa-full-tokens-ratio: 0.1 - context-length: 16384 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "1" - req_rate: "inf" - use_chat_template: false - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-8p1d-dep4-dep40-18-c2048.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-8p1d-dep4-dep40-18-c2048.yaml deleted file mode 100644 index 269b92e12..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-gb300-8p1d-dep4-dep40-18-c2048.yaml +++ /dev/null @@ -1,159 +0,0 @@ -name: "disagg-gb300-8p1d-dep4-dep40-18-c2048" - -# Weiliang wide-EP sweep point: EP=40, 8P+10D = 18 nodes, conc=2048. -# Matches srt-slurm PR#173 zip_override EP=40 topology. -# Env vars and sglang_config from InferenceX main (not Weiliang's 0510 image). - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-cu13-20260520-425dffbd" - precision: "fp4" - -dynamo: - hash: "81d0555ee23519cea80a42b4fe824e30368b7300" - install: true - -slurm: - time_limit: "03:00:00" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 8 - prefill_workers: 8 - gpus_per_prefill: 4 - decode_nodes: 10 - decode_workers: 1 - gpus_per_decode: 40 - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - env: - DYN_ROUTER_LOAD_BLOCK_SIZE: "1" - args: - router-mode: "kv" - router-kv-overlap-score-weight: 0 - router-queue-threshold: 64 - router-temperature: 0.5 - no-kv-events: true - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_FORCE_MISS: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "1280" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_USE_ONLINE_COMPRESS: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_LOG_FORWARD_ITERS: "1" - SGLANG_LOG_MS: "1" - SGLANG_REQUEST_STATE_WAIT_TIMEOUT: "60" - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - - enable-dp-attention: true - moe-a2a-backend: "megamoe" - deepep-config: '{"normal_dispatch":{"num_sms":88,"num_max_nvl_chunked_send_tokens":28,"num_max_nvl_chunked_recv_tokens":512},"normal_combine": {"num_sms":88,"num_max_nvl_chunked_send_tokens":16,"num_max_nvl_chunked_recv_tokens":512}}' - moe-dense-tp-size: 1 - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - enable-dp-lm-head: true - - mem-fraction-static: 0.90 - max-running-requests: 512 - cuda-graph-max-bs: 512 - chunked-prefill-size: 32768 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - trust-remote-code: true - watchdog-timeout: 86400 - skip-tokenizer-init: true - stream-interval: 60 - - moe-a2a-backend: "megamoe" - - moe-dense-tp-size: 1 - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - disaggregation-decode-polling-interval: 8 - - mem-fraction-static: 0.94 - swa-full-tokens-ratio: 0.20 - context-length: 9216 - tensor-parallel-size: 40 - data-parallel-size: 40 - expert-parallel-size: 40 - ep-num-redundant-experts: 16 - enable-dp-attention: true - enable-dp-lm-head: true - max-running-requests: 18400 - cuda-graph-max-bs: 1280 - - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "2048" - req_rate: "inf" - use_chat_template: false - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-high-conc-6p1d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-high-conc-6p1d-dep4-dep8-mtp.yaml deleted file mode 100644 index 39e11b719..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-high-conc-6p1d-dep4-dep8-mtp.yaml +++ /dev/null @@ -1,141 +0,0 @@ -name: "dsv4-pro-gb300-disagg-8k1k-high-conc-6p1d-dep4-dep8-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "81d0555ee23519cea80a42b4fe824e30368b7300" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-20260527-14f81a67" - precision: "mxfp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 6 - prefill_workers: 6 - gpus_per_prefill: 4 - decode_nodes: 2 - decode_workers: 1 - gpus_per_decode: 8 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_DISABLE_REUSE: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_DISABLE_REUSE: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" # CAR_V2 is single-node only. - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 # gates dsv4 chat-encoding spec. - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - - mem-fraction-static: 0.9 - max-running-requests: 1024 - cuda-graph-max-bs: 1024 - chunked-prefill-size: 32768 - stream-interval: 60 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 # gates dsv4 chat-encoding spec. - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - - speculative-algo: "EAGLE" - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 - - mem-fraction-static: 0.9 - max-running-requests: 4096 - cuda-graph-max-bs: 1024 - swa-full-tokens-ratio: 0.15 - context-length: 9216 - stream-interval: 60 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "4096" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-high-conc-8p1d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-high-conc-8p1d-dep4-dep8-mtp.yaml deleted file mode 100644 index a4cf477ed..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-high-conc-8p1d-dep4-dep8-mtp.yaml +++ /dev/null @@ -1,141 +0,0 @@ -name: "dsv4-pro-gb300-disagg-8k1k-high-conc-8p1d-dep4-dep8-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "81d0555ee23519cea80a42b4fe824e30368b7300" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-20260527-14f81a67" - precision: "mxfp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 8 - prefill_workers: 8 - gpus_per_prefill: 4 - decode_nodes: 2 - decode_workers: 1 - gpus_per_decode: 8 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_DISABLE_REUSE: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_DISABLE_REUSE: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" # CAR_V2 is single-node only. - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 # gates dsv4 chat-encoding spec. - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - - mem-fraction-static: 0.9 - max-running-requests: 1024 - cuda-graph-max-bs: 1024 - chunked-prefill-size: 32768 - stream-interval: 60 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 # gates dsv4 chat-encoding spec. - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - - speculative-algo: "EAGLE" - speculative-num-steps: 1 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 2 - - mem-fraction-static: 0.85 - max-running-requests: 8192 - cuda-graph-max-bs: 1280 - swa-full-tokens-ratio: 0.1 - context-length: 9216 - stream-interval: 60 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "8192" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml deleted file mode 100644 index be5c4cf38..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml +++ /dev/null @@ -1,121 +0,0 @@ -name: "dsv4-pro-gb300-disagg-8k1k-low-latency-1p1d-tp4-tp4-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "81d0555ee23519cea80a42b4fe824e30368b7300" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-20260527-14f81a67" - precision: "mxfp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - decode_nodes: 1 - decode_workers: 1 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_DISABLE_REUSE: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_DISABLE_REUSE: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - # SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2 intentionally NOT set: CAR_V2 - # is single-node only and corrupts results in 2-node decode setups. - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 # gates dsv4 chat-encoding spec. - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - - moe-runner-backend: "flashinfer_mxfp4" - disable-flashinfer-autotune: true - - mem-fraction-static: 0.9 - max-running-requests: 8 - cuda-graph-max-bs: 8 - chunked-prefill-size: 32768 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 # gates dsv4 chat-encoding spec. - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - - moe-runner-backend: "flashinfer_mxfp4" - disable-flashinfer-autotune: true - - speculative-algo: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - mem-fraction-static: 0.9 - max-running-requests: 8 - cuda-graph-max-bs: 8 - swa-full-tokens-ratio: 0.1 - context-length: 16384 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "1" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml deleted file mode 100644 index 5657ad84d..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml +++ /dev/null @@ -1,130 +0,0 @@ -name: "dsv4-pro-gb300-disagg-8k1k-low-latency-1p6d-dep4-tp4-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "81d0555ee23519cea80a42b4fe824e30368b7300" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-20260527-14f81a67" - precision: "mxfp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - decode_nodes: 6 - decode_workers: 6 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_DISABLE_REUSE: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_DISABLE_REUSE: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - # SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2 intentionally NOT set: CAR_V2 - # is single-node only and corrupts results in 2-node decode setups. - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 # gates dsv4 chat-encoding spec. - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - - mem-fraction-static: 0.9 - max-running-requests: 128 - cuda-graph-max-bs: 128 - chunked-prefill-size: 32768 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 # gates dsv4 chat-encoding spec. - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - - moe-runner-backend: "flashinfer_mxfp4" - disable-flashinfer-autotune: true - - speculative-algo: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - mem-fraction-static: 0.9 - max-running-requests: 128 - cuda-graph-max-bs: 128 - swa-full-tokens-ratio: 0.1 - context-length: 16384 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "8x32x64" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep16-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep16-mtp.yaml deleted file mode 100644 index f4ae3076c..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep16-mtp.yaml +++ /dev/null @@ -1,141 +0,0 @@ -name: "dsv4-pro-gb300-disagg-8k1k-mid-curve-1p1d-dep4-dep16-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "81d0555ee23519cea80a42b4fe824e30368b7300" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-20260527-14f81a67" - precision: "mxfp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 4 - decode_workers: 1 - gpus_per_decode: 16 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_DISABLE_REUSE: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_DISABLE_REUSE: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "2048" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" # CAR_V2 is single-node only. - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 # gates dsv4 chat-encoding spec. - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - - mem-fraction-static: 0.9 - max-running-requests: 256 - cuda-graph-max-bs: 256 - chunked-prefill-size: 32768 - stream-interval: 60 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 # gates dsv4 chat-encoding spec. - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - - speculative-algo: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - mem-fraction-static: 0.94 - max-running-requests: 3072 - cuda-graph-max-bs: 256 - swa-full-tokens-ratio: 0.15 - context-length: 16384 - stream-interval: 60 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "256" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep8-mtp.yaml deleted file mode 100644 index 4f9f90276..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep8-mtp.yaml +++ /dev/null @@ -1,141 +0,0 @@ -name: "dsv4-pro-gb300-disagg-8k1k-mid-curve-1p1d-dep4-dep8-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "81d0555ee23519cea80a42b4fe824e30368b7300" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-20260527-14f81a67" - precision: "mxfp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 2 - decode_workers: 1 - gpus_per_decode: 8 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_DISABLE_REUSE: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_DISABLE_REUSE: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "2048" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" # CAR_V2 is single-node only. - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 # gates dsv4 chat-encoding spec. - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - - mem-fraction-static: 0.9 - max-running-requests: 256 - cuda-graph-max-bs: 256 - chunked-prefill-size: 32768 - stream-interval: 60 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 # gates dsv4 chat-encoding spec. - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - - speculative-algo: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - mem-fraction-static: 0.94 - max-running-requests: 3072 - cuda-graph-max-bs: 256 - swa-full-tokens-ratio: 0.15 - context-length: 16384 - stream-interval: 60 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "256" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-2p1d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-2p1d-dep4-dep8-mtp.yaml deleted file mode 100644 index 17018a57e..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-2p1d-dep4-dep8-mtp.yaml +++ /dev/null @@ -1,141 +0,0 @@ -name: "dsv4-pro-gb300-disagg-8k1k-mid-curve-2p1d-dep4-dep8-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "81d0555ee23519cea80a42b4fe824e30368b7300" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-20260527-14f81a67" - precision: "mxfp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 2 - prefill_workers: 2 - gpus_per_prefill: 4 - decode_nodes: 2 - decode_workers: 1 - gpus_per_decode: 8 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_DISABLE_REUSE: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_DISABLE_REUSE: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" # CAR_V2 is single-node only. - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 # gates dsv4 chat-encoding spec. - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - - mem-fraction-static: 0.9 - max-running-requests: 512 - cuda-graph-max-bs: 512 - chunked-prefill-size: 32768 - stream-interval: 60 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 # gates dsv4 chat-encoding spec. - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - - speculative-algo: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - mem-fraction-static: 0.9 - max-running-requests: 3072 - cuda-graph-max-bs: 512 - swa-full-tokens-ratio: 0.15 - context-length: 16384 - stream-interval: 60 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "512" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-4p1d-dep4-dep8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-4p1d-dep4-dep8-mtp.yaml deleted file mode 100644 index 15578537a..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-4p1d-dep4-dep8-mtp.yaml +++ /dev/null @@ -1,141 +0,0 @@ -name: "dsv4-pro-gb300-disagg-8k1k-mid-curve-4p1d-dep4-dep8-mtp" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 8 - -dynamo: - hash: "81d0555ee23519cea80a42b4fe824e30368b7300" - install: true - -model: - path: "deepseek-v4-pro" - container: "lmsysorg/sglang:nightly-dev-20260527-14f81a67" - precision: "mxfp4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 4 - prefill_workers: 4 - gpus_per_prefill: 4 - decode_nodes: 2 - decode_workers: 1 - gpus_per_decode: 8 - -backend: - type: sglang - - prefill_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_DISABLE_REUSE: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - - decode_environment: - PYTHONUNBUFFERED: "1" - SGLANG_RADIX_DISABLE_REUSE: "1" - SGLANG_JIT_DEEPGEMM_FAST_WARMUP: "1" - SGLANG_DEFAULT_THINKING: "1" - SGLANG_DSV4_REASONING_EFFORT: "max" - SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" - - SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "4096" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS: "1" - SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND: "1" - - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" - SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "0" # CAR_V2 is single-node only. - - sglang_config: - prefill: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 # gates dsv4 chat-encoding spec. - - disaggregation-mode: "prefill" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - - mem-fraction-static: 0.9 - max-running-requests: 1024 - cuda-graph-max-bs: 1024 - chunked-prefill-size: 32768 - stream-interval: 60 - - decode: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - model-path: "/model/" - trust-remote-code: true - tool-call-parser: deepseekv4 # gates dsv4 chat-encoding spec. - - disaggregation-mode: "decode" - disaggregation-transfer-backend: mooncake - - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - - enable-dp-attention: true - enable-dp-lm-head: true - - moe-a2a-backend: "megamoe" - - speculative-algo: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - mem-fraction-static: 0.9 - max-running-requests: 3072 - cuda-graph-max-bs: 1024 - swa-full-tokens-ratio: 0.15 - context-length: 16384 - stream-interval: 60 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - random_range_ratio: 0.8 - concurrencies: "1024" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.sglang_deepseek_v4.SGLangDeepseekV4Tokenizer" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp2_throughput_2p1d.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp2_throughput_2p1d.yaml deleted file mode 100644 index 37225d44e..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp2_throughput_2p1d.yaml +++ /dev/null @@ -1,143 +0,0 @@ -name: "b200-fp4-mtp2-throughput-2p-dep4-1d-dep8" - -# Derived from the srt-slurm b200-fp4 8k1k recipe (recipes/b200-fp4/8k1k.yaml -# base + override_mtp2_throughput_2p1d, DEP4 prefill / DEP8 decode, MTP2). -# One flat YAML per concrete topology, matching the InferenceX glm5 disagg -# layout (sglang//-//disagg//...). - -dynamo: - hash: "5b4bc1dd70965017a737c71b19db5a0aeaa88727" - install: true - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 3 - nginx_container: nginx-sqsh - -model: - path: "dsr1" - container: "lmsysorg/sglang:v0.5.12.post1" - precision: "fp4" - -resources: - gpu_type: "b200" - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 2 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_node: 8 - -backend: - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_ENABLE_JIT_DEEPGEMM: "false" - SGLANG_ENABLE_SPEC_V2: "1" - UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_ENABLE_JIT_DEEPGEMM: "false" - SGLANG_ENABLE_SPEC_V2: "1" - UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" - - sglang_config: - prefill: - disaggregation-mode: "prefill" - served-model-name: "deepseek-ai/DeepSeek-R1" - trust-remote-code: true - disable-radix-cache: true - kv-cache-dtype: "fp8_e4m3" - attention-backend: "trtllm_mla" - quantization: "modelopt_fp4" - moe-runner-backend: "flashinfer_trtllm" - stream-interval: 50 - watchdog-timeout: 1000000 - context-length: 9600 - mem-fraction-static: 0.95 - max-total-tokens: 32768 - chunked-prefill-size: 24576 - cuda-graph-max-bs: 256 - max-running-requests: 2048 - scheduler-recv-interval: 1 - load-balance-method: "round_robin" - disaggregation-bootstrap-port: null - data-parallel-size: 4 - tensor-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - fp4-gemm-backend: "flashinfer_trtllm" - disaggregation-transfer-backend: nixl - speculative-algorithm: "EAGLE" - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 - - decode: - disaggregation-mode: "decode" - served-model-name: "deepseek-ai/DeepSeek-R1" - prefill-round-robin-balance: true - trust-remote-code: true - disable-radix-cache: true - kv-cache-dtype: "fp8_e4m3" - attention-backend: "trtllm_mla" - quantization: "modelopt_fp4" - moe-runner-backend: "flashinfer_trtllm" - disaggregation-bootstrap-port: 30001 - stream-interval: 50 - watchdog-timeout: 1000000 - context-length: 9600 - mem-fraction-static: 0.85 - chunked-prefill-size: 8192 - cuda-graph-max-bs: 1024 - max-running-requests: 2048 - scheduler-recv-interval: 1 - data-parallel-size: 8 - tensor-parallel-size: 8 - expert-parallel-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - fp4-gemm-backend: "flashinfer_trtllm" - disaggregation-transfer-backend: nixl - speculative-algorithm: "EAGLE" - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - req_rate: 300 - concurrencies: "768" - use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp2_throughput_3p1d.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp2_throughput_3p1d.yaml deleted file mode 100644 index d8e401620..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp2_throughput_3p1d.yaml +++ /dev/null @@ -1,143 +0,0 @@ -name: "b200-fp4-mtp2-throughput-3p-dep4-1d-dep8" - -# Derived from the srt-slurm b200-fp4 8k1k recipe (recipes/b200-fp4/8k1k.yaml -# base + override_mtp2_throughput_3p1d, DEP4 prefill / DEP8 decode, MTP2). -# One flat YAML per concrete topology, matching the InferenceX glm5 disagg -# layout (sglang//-//disagg//...). - -dynamo: - hash: "5b4bc1dd70965017a737c71b19db5a0aeaa88727" - install: true - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 3 - nginx_container: nginx-sqsh - -model: - path: "dsr1" - container: "lmsysorg/sglang:v0.5.12.post1" - precision: "fp4" - -resources: - gpu_type: "b200" - prefill_nodes: 2 - decode_nodes: 1 - prefill_workers: 3 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_node: 8 - -backend: - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_ENABLE_JIT_DEEPGEMM: "false" - SGLANG_ENABLE_SPEC_V2: "1" - UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_ENABLE_JIT_DEEPGEMM: "false" - SGLANG_ENABLE_SPEC_V2: "1" - UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" - - sglang_config: - prefill: - disaggregation-mode: "prefill" - served-model-name: "deepseek-ai/DeepSeek-R1" - trust-remote-code: true - disable-radix-cache: true - kv-cache-dtype: "fp8_e4m3" - attention-backend: "trtllm_mla" - quantization: "modelopt_fp4" - moe-runner-backend: "flashinfer_trtllm" - stream-interval: 50 - watchdog-timeout: 1000000 - context-length: 9600 - mem-fraction-static: 0.95 - max-total-tokens: 32768 - chunked-prefill-size: 24576 - cuda-graph-max-bs: 256 - max-running-requests: 2048 - scheduler-recv-interval: 1 - load-balance-method: "round_robin" - disaggregation-bootstrap-port: null - data-parallel-size: 4 - tensor-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - fp4-gemm-backend: "flashinfer_trtllm" - disaggregation-transfer-backend: nixl - speculative-algorithm: "EAGLE" - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 - - decode: - disaggregation-mode: "decode" - served-model-name: "deepseek-ai/DeepSeek-R1" - prefill-round-robin-balance: true - trust-remote-code: true - disable-radix-cache: true - kv-cache-dtype: "fp8_e4m3" - attention-backend: "trtllm_mla" - quantization: "modelopt_fp4" - moe-runner-backend: "flashinfer_trtllm" - disaggregation-bootstrap-port: 30001 - stream-interval: 50 - watchdog-timeout: 1000000 - context-length: 9600 - mem-fraction-static: 0.85 - chunked-prefill-size: 8192 - cuda-graph-max-bs: 1024 - max-running-requests: 2048 - scheduler-recv-interval: 1 - data-parallel-size: 8 - tensor-parallel-size: 8 - expert-parallel-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - fp4-gemm-backend: "flashinfer_trtllm" - disaggregation-transfer-backend: nixl - speculative-algorithm: "EAGLE" - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - req_rate: 300 - concurrencies: "1024" - use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp2_throughput_5p1d.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp2_throughput_5p1d.yaml deleted file mode 100644 index bcbedcb68..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp2_throughput_5p1d.yaml +++ /dev/null @@ -1,143 +0,0 @@ -name: "b200-fp4-mtp2-throughput-5p-dep4-1d-dep8" - -# Derived from the srt-slurm b200-fp4 8k1k recipe (recipes/b200-fp4/8k1k.yaml -# base + override_mtp2_throughput_5p1d, DEP4 prefill / DEP8 decode, MTP2). -# One flat YAML per concrete topology, matching the InferenceX glm5 disagg -# layout (sglang//-//disagg//...). - -dynamo: - hash: "5b4bc1dd70965017a737c71b19db5a0aeaa88727" - install: true - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 3 - nginx_container: nginx-sqsh - -model: - path: "dsr1" - container: "lmsysorg/sglang:v0.5.12.post1" - precision: "fp4" - -resources: - gpu_type: "b200" - prefill_nodes: 3 - decode_nodes: 1 - prefill_workers: 5 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_node: 8 - -backend: - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_ENABLE_JIT_DEEPGEMM: "false" - SGLANG_ENABLE_SPEC_V2: "1" - UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_ENABLE_JIT_DEEPGEMM: "false" - SGLANG_ENABLE_SPEC_V2: "1" - UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" - - sglang_config: - prefill: - disaggregation-mode: "prefill" - served-model-name: "deepseek-ai/DeepSeek-R1" - trust-remote-code: true - disable-radix-cache: true - kv-cache-dtype: "fp8_e4m3" - attention-backend: "trtllm_mla" - quantization: "modelopt_fp4" - moe-runner-backend: "flashinfer_trtllm" - stream-interval: 50 - watchdog-timeout: 1000000 - context-length: 9600 - mem-fraction-static: 0.95 - max-total-tokens: 32768 - chunked-prefill-size: 24576 - cuda-graph-max-bs: 256 - max-running-requests: 2048 - scheduler-recv-interval: 1 - load-balance-method: "round_robin" - disaggregation-bootstrap-port: null - data-parallel-size: 4 - tensor-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - fp4-gemm-backend: "flashinfer_trtllm" - disaggregation-transfer-backend: nixl - speculative-algorithm: "EAGLE" - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 - - decode: - disaggregation-mode: "decode" - served-model-name: "deepseek-ai/DeepSeek-R1" - prefill-round-robin-balance: true - trust-remote-code: true - disable-radix-cache: true - kv-cache-dtype: "fp8_e4m3" - attention-backend: "trtllm_mla" - quantization: "modelopt_fp4" - moe-runner-backend: "flashinfer_trtllm" - disaggregation-bootstrap-port: 30001 - stream-interval: 50 - watchdog-timeout: 1000000 - context-length: 9600 - mem-fraction-static: 0.85 - chunked-prefill-size: 8192 - cuda-graph-max-bs: 1024 - max-running-requests: 2048 - scheduler-recv-interval: 1 - data-parallel-size: 8 - tensor-parallel-size: 8 - expert-parallel-size: 8 - enable-dp-attention: true - enable-dp-lm-head: true - fp4-gemm-backend: "flashinfer_trtllm" - disaggregation-transfer-backend: nixl - speculative-algorithm: "EAGLE" - speculative-num-steps: 2 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 3 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - req_rate: 300 - concurrencies: "2048" - use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp_lowlat_0.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp_lowlat_0.yaml deleted file mode 100644 index 160854ebc..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp_lowlat_0.yaml +++ /dev/null @@ -1,140 +0,0 @@ -name: "b200-fp4-mtp-low-latency-1p-tp4-5d-tp8" - -# Derived from the srt-slurm b200-fp4 8k1k recipe (recipes/b200-fp4/8k1k.yaml -# base + zip_override_mtp_lowlat[0], 1p-tp4 prefill / 5d-tp8 decode). -# One flat YAML per concrete topology, matching the InferenceX glm5 disagg -# layout (sglang//-//disagg//...). - -dynamo: - hash: "5b4bc1dd70965017a737c71b19db5a0aeaa88727" - install: true - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 3 - nginx_container: nginx-sqsh - -model: - path: "dsr1" - container: "lmsysorg/sglang:v0.5.12.post1" - precision: "fp4" - -resources: - gpu_type: "b200" - prefill_nodes: 1 - decode_nodes: 5 - prefill_workers: 1 - decode_workers: 5 - gpus_per_prefill: 4 - gpus_per_node: 8 - -backend: - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_ENABLE_JIT_DEEPGEMM: "false" - SGLANG_ENABLE_SPEC_V2: "1" - UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_ENABLE_JIT_DEEPGEMM: "false" - SGLANG_ENABLE_SPEC_V2: "1" - UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" - - sglang_config: - prefill: - disaggregation-mode: "prefill" - served-model-name: "deepseek-ai/DeepSeek-R1" - trust-remote-code: true - disable-radix-cache: true - kv-cache-dtype: "fp8_e4m3" - attention-backend: "trtllm_mla" - quantization: "modelopt_fp4" - moe-runner-backend: "flashinfer_trtllm" - stream-interval: 50 - watchdog-timeout: 1000000 - context-length: 9600 - mem-fraction-static: 0.95 - max-total-tokens: 32768 - chunked-prefill-size: 24576 - cuda-graph-max-bs: 256 - max-running-requests: 512 - scheduler-recv-interval: 10 - load-balance-method: "round_robin" - disaggregation-bootstrap-port: 30001 - data-parallel-size: 1 - tensor-parallel-size: 4 - expert-parallel-size: 1 - enable-dp-attention: false - fp4-gemm-backend: "flashinfer_trtllm" - disaggregation-transfer-backend: nixl - speculative-algorithm: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - decode: - disaggregation-mode: "decode" - served-model-name: "deepseek-ai/DeepSeek-R1" - prefill-round-robin-balance: true - trust-remote-code: true - disable-radix-cache: true - kv-cache-dtype: "fp8_e4m3" - attention-backend: "trtllm_mla" - quantization: "modelopt_fp4" - moe-runner-backend: "flashinfer_trtllm" - disaggregation-bootstrap-port: 30001 - stream-interval: 50 - watchdog-timeout: 1000000 - context-length: 9600 - mem-fraction-static: 0.85 - chunked-prefill-size: 8192 - cuda-graph-max-bs: 128 - max-running-requests: 512 - scheduler-recv-interval: 10 - tensor-parallel-size: 8 - expert-parallel-size: 1 - enable-dp-attention: false - fp4-gemm-backend: "flashinfer_trtllm" - disaggregation-transfer-backend: nixl - speculative-algorithm: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - req_rate: 300 - concurrencies: "4x8x16x32x64" - use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp_lowlat_1.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp_lowlat_1.yaml deleted file mode 100644 index fd1d4a4f5..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp_lowlat_1.yaml +++ /dev/null @@ -1,140 +0,0 @@ -name: "b200-fp4-mtp-low-latency-1p-tp4-3d-tp8" - -# Derived from the srt-slurm b200-fp4 8k1k recipe (recipes/b200-fp4/8k1k.yaml -# base + zip_override_mtp_lowlat[1], 1p-tp4 prefill / 3d-tp8 decode). -# One flat YAML per concrete topology, matching the InferenceX glm5 disagg -# layout (sglang//-//disagg//...). - -dynamo: - hash: "5b4bc1dd70965017a737c71b19db5a0aeaa88727" - install: true - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 3 - nginx_container: nginx-sqsh - -model: - path: "dsr1" - container: "lmsysorg/sglang:v0.5.12.post1" - precision: "fp4" - -resources: - gpu_type: "b200" - prefill_nodes: 1 - decode_nodes: 3 - prefill_workers: 1 - decode_workers: 3 - gpus_per_prefill: 4 - gpus_per_node: 8 - -backend: - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_ENABLE_JIT_DEEPGEMM: "false" - SGLANG_ENABLE_SPEC_V2: "1" - UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_ENABLE_JIT_DEEPGEMM: "false" - SGLANG_ENABLE_SPEC_V2: "1" - UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" - - sglang_config: - prefill: - disaggregation-mode: "prefill" - served-model-name: "deepseek-ai/DeepSeek-R1" - trust-remote-code: true - disable-radix-cache: true - kv-cache-dtype: "fp8_e4m3" - attention-backend: "trtllm_mla" - quantization: "modelopt_fp4" - moe-runner-backend: "flashinfer_trtllm" - stream-interval: 50 - watchdog-timeout: 1000000 - context-length: 9600 - mem-fraction-static: 0.95 - max-total-tokens: 32768 - chunked-prefill-size: 24576 - cuda-graph-max-bs: 256 - max-running-requests: 512 - scheduler-recv-interval: 10 - load-balance-method: "round_robin" - disaggregation-bootstrap-port: 30001 - data-parallel-size: 1 - tensor-parallel-size: 4 - expert-parallel-size: 1 - enable-dp-attention: false - fp4-gemm-backend: "flashinfer_trtllm" - disaggregation-transfer-backend: nixl - speculative-algorithm: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - decode: - disaggregation-mode: "decode" - served-model-name: "deepseek-ai/DeepSeek-R1" - prefill-round-robin-balance: true - trust-remote-code: true - disable-radix-cache: true - kv-cache-dtype: "fp8_e4m3" - attention-backend: "trtllm_mla" - quantization: "modelopt_fp4" - moe-runner-backend: "flashinfer_trtllm" - disaggregation-bootstrap-port: 30001 - stream-interval: 50 - watchdog-timeout: 1000000 - context-length: 9600 - mem-fraction-static: 0.85 - chunked-prefill-size: 8192 - cuda-graph-max-bs: 128 - max-running-requests: 512 - scheduler-recv-interval: 10 - tensor-parallel-size: 8 - expert-parallel-size: 1 - enable-dp-attention: false - fp4-gemm-backend: "flashinfer_trtllm" - disaggregation-transfer-backend: nixl - speculative-algorithm: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - req_rate: 300 - concurrencies: "32x64x128x256x512" - use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp_lowlat_2.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp_lowlat_2.yaml deleted file mode 100644 index bcceaf872..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp_lowlat_2.yaml +++ /dev/null @@ -1,140 +0,0 @@ -name: "b200-fp4-mtp-low-latency-1p-tp4-1d-tp8" - -# Derived from the srt-slurm b200-fp4 8k1k recipe (recipes/b200-fp4/8k1k.yaml -# base + zip_override_mtp_lowlat[2], 1p-tp4 prefill / 1d-tp8 decode). -# One flat YAML per concrete topology, matching the InferenceX glm5 disagg -# layout (sglang//-//disagg//...). - -dynamo: - hash: "5b4bc1dd70965017a737c71b19db5a0aeaa88727" - install: true - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 3 - nginx_container: nginx-sqsh - -model: - path: "dsr1" - container: "lmsysorg/sglang:v0.5.12.post1" - precision: "fp4" - -resources: - gpu_type: "b200" - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_node: 8 - -backend: - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_ENABLE_JIT_DEEPGEMM: "false" - SGLANG_ENABLE_SPEC_V2: "1" - UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_ENABLE_JIT_DEEPGEMM: "false" - SGLANG_ENABLE_SPEC_V2: "1" - UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" - - sglang_config: - prefill: - disaggregation-mode: "prefill" - served-model-name: "deepseek-ai/DeepSeek-R1" - trust-remote-code: true - disable-radix-cache: true - kv-cache-dtype: "fp8_e4m3" - attention-backend: "trtllm_mla" - quantization: "modelopt_fp4" - moe-runner-backend: "flashinfer_trtllm" - stream-interval: 50 - watchdog-timeout: 1000000 - context-length: 9600 - mem-fraction-static: 0.95 - max-total-tokens: 32768 - chunked-prefill-size: 24576 - cuda-graph-max-bs: 256 - max-running-requests: 512 - scheduler-recv-interval: 10 - load-balance-method: "round_robin" - disaggregation-bootstrap-port: 30001 - data-parallel-size: 1 - tensor-parallel-size: 4 - expert-parallel-size: 1 - enable-dp-attention: false - fp4-gemm-backend: "flashinfer_trtllm" - disaggregation-transfer-backend: nixl - speculative-algorithm: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - - decode: - disaggregation-mode: "decode" - served-model-name: "deepseek-ai/DeepSeek-R1" - prefill-round-robin-balance: true - trust-remote-code: true - disable-radix-cache: true - kv-cache-dtype: "fp8_e4m3" - attention-backend: "trtllm_mla" - quantization: "modelopt_fp4" - moe-runner-backend: "flashinfer_trtllm" - disaggregation-bootstrap-port: 30001 - stream-interval: 50 - watchdog-timeout: 1000000 - context-length: 9600 - mem-fraction-static: 0.85 - chunked-prefill-size: 8192 - cuda-graph-max-bs: 128 - max-running-requests: 512 - scheduler-recv-interval: 10 - tensor-parallel-size: 8 - expert-parallel-size: 1 - enable-dp-attention: false - fp4-gemm-backend: "flashinfer_trtllm" - disaggregation-transfer-backend: nixl - speculative-algorithm: "EAGLE" - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - req_rate: 300 - concurrencies: "32x64x128x256x512" - use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml deleted file mode 100644 index 31ac6edf8..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml +++ /dev/null @@ -1,173 +0,0 @@ -name: "gb300-fp4-glm5_8k1k_lowlat_0" - -# Ported from upstream srt-slurm recipes/gb300-fp4/glm5.yaml (PR #152). -# Upstream uses a single combined file with `zip_override_*` arrays -# expanded by srtctl across zip indices. We split into one flat yaml -# per concrete topology to match the InferenceX dsv4 sglang convention -# (see ../deepseek-v4/8k1k/*.yaml). All shared base envs and the -# prefill sglang_config are inlined here verbatim from the upstream -# `base:` block; the decode block is the upstream base plus the -# topology-specific override from this zip index. - -model: - path: "glm-5-fp4" - container: "lmsysorg/sglang:v0.5.11-cu130" - precision: "fp4" - -# Released dynamo wheel; unlike hash-based sources, this recipe does not -# require a persistent /configs/dynamo-wheels build cache. -dynamo: - version: "1.1.0" - -slurm: - time_limit: "03:00:00" - -# Mirror dsv4 sglang recipes: cpus-per-task=144 avoids the 1-CPU -# default that turns dynamo install + sglang weight load into a serial -# crawl; mem=0 grants whole-node memory. -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 3 - decode_workers: 3 - gpus_per_decode: 4 - -frontend: - type: dynamo - -backend: - type: sglang - - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" - SGLANG_MOE_NVFP4_DISPATCH: "1" - - sglang_config: - prefill: - # Model configuration - served-model-name: "GLM-5-FP4" - trust-remote-code: true - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - - # Disaggregation mode - disaggregation-mode: "prefill" - disaggregation-transfer-backend: "nixl" - - # Size limits - max-running-requests: 256 - cuda-graph-max-bs: 256 - mem-fraction-static: 0.7 - context-length: 9600 - chunked-prefill-size: 32768 - max-prefill-tokens: 8192 - - # Parallelism - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: "total_tokens" - - # Backend - nsa-decode-backend: "trtllm" - nsa-prefill-backend: "trtllm" - moe-runner-backend: "flashinfer_trtllm" - fp4-gemm-backend: "flashinfer_cutlass" - - # Other flags - # disable-shared-experts-fusion: true - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - - decode: - # Model configuration - served-model-name: "GLM-5-FP4" - trust-remote-code: true - - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - - # Disaggregation mode - disaggregation-mode: "decode" - disaggregation-transfer-backend: "nixl" - - # Memory and token limits - mem-fraction-static: 0.8 - context-length: 9600 - - # Backend - nsa-decode-backend: "trtllm" - nsa-prefill-backend: "trtllm" - moe-runner-backend: "flashinfer_cutedsl" - fp4-gemm-backend: "flashinfer_cutlass" - - # Detokenizer - skip-tokenizer-init: true - stream-interval: 30 - - # Other flags - # disable-shared-experts-fusion: true - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - # Parallelism (override from upstream zip_override_*_lowlat) - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - enable-flashinfer-allreduce-fusion: true - - moe-runner-backend: "flashinfer_trtllm" - max-running-requests: 128 - cuda-graph-max-bs: 128 - - - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "128" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_1.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_1.yaml deleted file mode 100644 index 95ffd216b..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_1.yaml +++ /dev/null @@ -1,173 +0,0 @@ -name: "gb300-fp4-glm5_8k1k_lowlat_1" - -# Ported from upstream srt-slurm recipes/gb300-fp4/glm5.yaml (PR #152). -# Upstream uses a single combined file with `zip_override_*` arrays -# expanded by srtctl across zip indices. We split into one flat yaml -# per concrete topology to match the InferenceX dsv4 sglang convention -# (see ../deepseek-v4/8k1k/*.yaml). All shared base envs and the -# prefill sglang_config are inlined here verbatim from the upstream -# `base:` block; the decode block is the upstream base plus the -# topology-specific override from this zip index. - -model: - path: "glm-5-fp4" - container: "lmsysorg/sglang:v0.5.11-cu130" - precision: "fp4" - -# Released dynamo wheel; unlike hash-based sources, this recipe does not -# require a persistent /configs/dynamo-wheels build cache. -dynamo: - version: "1.1.0" - -slurm: - time_limit: "03:00:00" - -# Mirror dsv4 sglang recipes: cpus-per-task=144 avoids the 1-CPU -# default that turns dynamo install + sglang weight load into a serial -# crawl; mem=0 grants whole-node memory. -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 5 - decode_workers: 5 - gpus_per_decode: 4 - -frontend: - type: dynamo - -backend: - type: sglang - - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" - SGLANG_MOE_NVFP4_DISPATCH: "1" - - sglang_config: - prefill: - # Model configuration - served-model-name: "GLM-5-FP4" - trust-remote-code: true - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - - # Disaggregation mode - disaggregation-mode: "prefill" - disaggregation-transfer-backend: "nixl" - - # Size limits - max-running-requests: 256 - cuda-graph-max-bs: 256 - mem-fraction-static: 0.7 - context-length: 9600 - chunked-prefill-size: 32768 - max-prefill-tokens: 8192 - - # Parallelism - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: "total_tokens" - - # Backend - nsa-decode-backend: "trtllm" - nsa-prefill-backend: "trtllm" - moe-runner-backend: "flashinfer_trtllm" - fp4-gemm-backend: "flashinfer_cutlass" - - # Other flags - # disable-shared-experts-fusion: true - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - - decode: - # Model configuration - served-model-name: "GLM-5-FP4" - trust-remote-code: true - - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - - # Disaggregation mode - disaggregation-mode: "decode" - disaggregation-transfer-backend: "nixl" - - # Memory and token limits - mem-fraction-static: 0.8 - context-length: 9600 - - # Backend - nsa-decode-backend: "trtllm" - nsa-prefill-backend: "trtllm" - moe-runner-backend: "flashinfer_cutedsl" - fp4-gemm-backend: "flashinfer_cutlass" - - # Detokenizer - skip-tokenizer-init: true - stream-interval: 30 - - # Other flags - # disable-shared-experts-fusion: true - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - # Parallelism (override from upstream zip_override_*_lowlat) - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - enable-flashinfer-allreduce-fusion: true - - moe-runner-backend: "flashinfer_trtllm" - max-running-requests: 64 - cuda-graph-max-bs: 64 - - - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "64" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_2.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_2.yaml deleted file mode 100644 index 506dede54..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_2.yaml +++ /dev/null @@ -1,173 +0,0 @@ -name: "gb300-fp4-glm5_8k1k_lowlat_2" - -# Ported from upstream srt-slurm recipes/gb300-fp4/glm5.yaml (PR #152). -# Upstream uses a single combined file with `zip_override_*` arrays -# expanded by srtctl across zip indices. We split into one flat yaml -# per concrete topology to match the InferenceX dsv4 sglang convention -# (see ../deepseek-v4/8k1k/*.yaml). All shared base envs and the -# prefill sglang_config are inlined here verbatim from the upstream -# `base:` block; the decode block is the upstream base plus the -# topology-specific override from this zip index. - -model: - path: "glm-5-fp4" - container: "lmsysorg/sglang:v0.5.11-cu130" - precision: "fp4" - -# Released dynamo wheel; unlike hash-based sources, this recipe does not -# require a persistent /configs/dynamo-wheels build cache. -dynamo: - version: "1.1.0" - -slurm: - time_limit: "03:00:00" - -# Mirror dsv4 sglang recipes: cpus-per-task=144 avoids the 1-CPU -# default that turns dynamo install + sglang weight load into a serial -# crawl; mem=0 grants whole-node memory. -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 9 - decode_workers: 9 - gpus_per_decode: 4 - -frontend: - type: dynamo - -backend: - type: sglang - - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" - SGLANG_MOE_NVFP4_DISPATCH: "1" - - sglang_config: - prefill: - # Model configuration - served-model-name: "GLM-5-FP4" - trust-remote-code: true - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - - # Disaggregation mode - disaggregation-mode: "prefill" - disaggregation-transfer-backend: "nixl" - - # Size limits - max-running-requests: 256 - cuda-graph-max-bs: 256 - mem-fraction-static: 0.7 - context-length: 9600 - chunked-prefill-size: 32768 - max-prefill-tokens: 8192 - - # Parallelism - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: "total_tokens" - - # Backend - nsa-decode-backend: "trtllm" - nsa-prefill-backend: "trtllm" - moe-runner-backend: "flashinfer_trtllm" - fp4-gemm-backend: "flashinfer_cutlass" - - # Other flags - # disable-shared-experts-fusion: true - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - - decode: - # Model configuration - served-model-name: "GLM-5-FP4" - trust-remote-code: true - - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - - # Disaggregation mode - disaggregation-mode: "decode" - disaggregation-transfer-backend: "nixl" - - # Memory and token limits - mem-fraction-static: 0.8 - context-length: 9600 - - # Backend - nsa-decode-backend: "trtllm" - nsa-prefill-backend: "trtllm" - moe-runner-backend: "flashinfer_cutedsl" - fp4-gemm-backend: "flashinfer_cutlass" - - # Detokenizer - skip-tokenizer-init: true - stream-interval: 30 - - # Other flags - # disable-shared-experts-fusion: true - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - # Parallelism (override from upstream zip_override_*_lowlat) - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - enable-flashinfer-allreduce-fusion: true - - moe-runner-backend: "flashinfer_trtllm" - max-running-requests: 32 - cuda-graph-max-bs: 32 - - - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "32" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_3.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_3.yaml deleted file mode 100644 index 43e96c0a3..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_3.yaml +++ /dev/null @@ -1,173 +0,0 @@ -name: "gb300-fp4-glm5_8k1k_lowlat_3" - -# Ported from upstream srt-slurm recipes/gb300-fp4/glm5.yaml (PR #152). -# Upstream uses a single combined file with `zip_override_*` arrays -# expanded by srtctl across zip indices. We split into one flat yaml -# per concrete topology to match the InferenceX dsv4 sglang convention -# (see ../deepseek-v4/8k1k/*.yaml). All shared base envs and the -# prefill sglang_config are inlined here verbatim from the upstream -# `base:` block; the decode block is the upstream base plus the -# topology-specific override from this zip index. - -model: - path: "glm-5-fp4" - container: "lmsysorg/sglang:v0.5.11-cu130" - precision: "fp4" - -# Released dynamo wheel; unlike hash-based sources, this recipe does not -# require a persistent /configs/dynamo-wheels build cache. -dynamo: - version: "1.1.0" - -slurm: - time_limit: "03:00:00" - -# Mirror dsv4 sglang recipes: cpus-per-task=144 avoids the 1-CPU -# default that turns dynamo install + sglang weight load into a serial -# crawl; mem=0 grants whole-node memory. -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 15 - decode_workers: 15 - gpus_per_decode: 4 - -frontend: - type: dynamo - -backend: - type: sglang - - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" - SGLANG_MOE_NVFP4_DISPATCH: "1" - - sglang_config: - prefill: - # Model configuration - served-model-name: "GLM-5-FP4" - trust-remote-code: true - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - - # Disaggregation mode - disaggregation-mode: "prefill" - disaggregation-transfer-backend: "nixl" - - # Size limits - max-running-requests: 256 - cuda-graph-max-bs: 256 - mem-fraction-static: 0.7 - context-length: 9600 - chunked-prefill-size: 32768 - max-prefill-tokens: 8192 - - # Parallelism - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: "total_tokens" - - # Backend - nsa-decode-backend: "trtllm" - nsa-prefill-backend: "trtllm" - moe-runner-backend: "flashinfer_trtllm" - fp4-gemm-backend: "flashinfer_cutlass" - - # Other flags - # disable-shared-experts-fusion: true - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - - decode: - # Model configuration - served-model-name: "GLM-5-FP4" - trust-remote-code: true - - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - - # Disaggregation mode - disaggregation-mode: "decode" - disaggregation-transfer-backend: "nixl" - - # Memory and token limits - mem-fraction-static: 0.8 - context-length: 9600 - - # Backend - nsa-decode-backend: "trtllm" - nsa-prefill-backend: "trtllm" - moe-runner-backend: "flashinfer_cutedsl" - fp4-gemm-backend: "flashinfer_cutlass" - - # Detokenizer - skip-tokenizer-init: true - stream-interval: 30 - - # Other flags - # disable-shared-experts-fusion: true - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - # Parallelism (override from upstream zip_override_*_lowlat) - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - enable-flashinfer-allreduce-fusion: true - - moe-runner-backend: "flashinfer_trtllm" - max-running-requests: 16 - cuda-graph-max-bs: 16 - - - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "16" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_4.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_4.yaml deleted file mode 100644 index 2bcc483e9..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_4.yaml +++ /dev/null @@ -1,173 +0,0 @@ -name: "gb300-fp4-glm5_8k1k_lowlat_4" - -# Ported from upstream srt-slurm recipes/gb300-fp4/glm5.yaml (PR #152). -# Upstream uses a single combined file with `zip_override_*` arrays -# expanded by srtctl across zip indices. We split into one flat yaml -# per concrete topology to match the InferenceX dsv4 sglang convention -# (see ../deepseek-v4/8k1k/*.yaml). All shared base envs and the -# prefill sglang_config are inlined here verbatim from the upstream -# `base:` block; the decode block is the upstream base plus the -# topology-specific override from this zip index. - -model: - path: "glm-5-fp4" - container: "lmsysorg/sglang:v0.5.11-cu130" - precision: "fp4" - -# Released dynamo wheel; unlike hash-based sources, this recipe does not -# require a persistent /configs/dynamo-wheels build cache. -dynamo: - version: "1.1.0" - -slurm: - time_limit: "03:00:00" - -# Mirror dsv4 sglang recipes: cpus-per-task=144 avoids the 1-CPU -# default that turns dynamo install + sglang weight load into a serial -# crawl; mem=0 grants whole-node memory. -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - prefill_workers: 1 - gpus_per_prefill: 4 - decode_nodes: 17 - decode_workers: 17 - gpus_per_decode: 4 - -frontend: - type: dynamo - -backend: - type: sglang - - prefill_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - - decode_environment: - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - PYTHONUNBUFFERED: "1" - DYN_SKIP_SGLANG_LOG_FORMATTING: "1" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - MC_TE_METRIC: "true" - MC_FORCE_MNNVL: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" - SGLANG_MOE_NVFP4_DISPATCH: "1" - - sglang_config: - prefill: - # Model configuration - served-model-name: "GLM-5-FP4" - trust-remote-code: true - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - - # Disaggregation mode - disaggregation-mode: "prefill" - disaggregation-transfer-backend: "nixl" - - # Size limits - max-running-requests: 256 - cuda-graph-max-bs: 256 - mem-fraction-static: 0.7 - context-length: 9600 - chunked-prefill-size: 32768 - max-prefill-tokens: 8192 - - # Parallelism - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - load-balance-method: "total_tokens" - - # Backend - nsa-decode-backend: "trtllm" - nsa-prefill-backend: "trtllm" - moe-runner-backend: "flashinfer_trtllm" - fp4-gemm-backend: "flashinfer_cutlass" - - # Other flags - # disable-shared-experts-fusion: true - enable-flashinfer-allreduce-fusion: true - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - - decode: - # Model configuration - served-model-name: "GLM-5-FP4" - trust-remote-code: true - - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - - # Disaggregation mode - disaggregation-mode: "decode" - disaggregation-transfer-backend: "nixl" - - # Memory and token limits - mem-fraction-static: 0.8 - context-length: 9600 - - # Backend - nsa-decode-backend: "trtllm" - nsa-prefill-backend: "trtllm" - moe-runner-backend: "flashinfer_cutedsl" - fp4-gemm-backend: "flashinfer_cutlass" - - # Detokenizer - skip-tokenizer-init: true - stream-interval: 30 - - # Other flags - # disable-shared-experts-fusion: true - disable-radix-cache: true - weight-loader-prefetch-checkpoints: true - model-loader-extra-config: '{"enable_multithread_load": true}' - # Parallelism (override from upstream zip_override_*_lowlat) - tensor-parallel-size: 4 - expert-parallel-size: 1 - data-parallel-size: 1 - enable-flashinfer-allreduce-fusion: true - - moe-runner-backend: "flashinfer_trtllm" - max-running-requests: 1 - cuda-graph-max-bs: 1 - - - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "12" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/1k1k/1p1d-dep4-dep16.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/1k1k/1p1d-dep4-dep16.yaml deleted file mode 100644 index c110e1599..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/1k1k/1p1d-dep4-dep16.yaml +++ /dev/null @@ -1,155 +0,0 @@ -name: "qwen3.5-1p1d-dep4-dep16" - -setup_script: rebuild-deepep.sh - -dynamo: - hash: 46520ca59afe992fb5ef61b3197b2316f8df9b2b - install: true - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 1 - nginx_container: nginx - -model: - path: "qwen3.5-fp8" - container: "lmsysorg/sglang:nightly-dev-cu13-20260608-303757cc" - precision: "fp8" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 4 - prefill_workers: 1 - decode_workers: 1 - -backend: - type: sglang - - prefill_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - decode_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - MC_TE_METRIC: "true" - SGLANG_DG_CACHE_DIR: "/tmp/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" - SGLANG_HEALTH_CHECK_TIMEOUT: "1800" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - sglang_config: - prefill: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - trust-remote-code: true - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 2048 - mamba-ssm-dtype: "bfloat16" - disaggregation-mode: "prefill" - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - mem-fraction-static: 0.8 - chunked-prefill-size: 65536 - load-balance-method: "round_robin" - watchdog-timeout: 1000000 - disable-cuda-graph: true - log-level: "info" - page-size: 64 - attention-backend: "trtllm_mha" - moe-runner-backend: "flashinfer_trtllm" - - decode: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - trust-remote-code: true - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - prefill-round-robin-balance: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 128 - mamba-ssm-dtype: "bfloat16" - - disaggregation-mode: "decode" - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - - chunked-prefill-size: 4096 - context-length: 4096 - mem-fraction-static: 0.80 - max-mamba-cache-size: 2048 - max-running-requests: 2048 - cuda-graph-max-bs: 128 - watchdog-timeout: 1000000 - - page-size: 64 - attention-backend: "trtllm_mha" - moe-runner-backend: "deep_gemm" - moe-a2a-backend: "deepep" - deepep-mode: "low_latency" - ep-dispatch-algorithm: "static" - eplb-algorithm: "deepseek" - - decode-log-interval: 1 - stream-interval: 50 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - req_rate: "inf" - random_range_ratio: 0.8 - concurrencies: "512x1024x2048" \ No newline at end of file diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/1k1k/1p1d-tp4-tp4.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/1k1k/1p1d-tp4-tp4.yaml deleted file mode 100644 index 143f339d0..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/1k1k/1p1d-tp4-tp4.yaml +++ /dev/null @@ -1,123 +0,0 @@ -name: "qwen3.5-1p1d-tp4-tp4" - -sbatch_directives: - mem: "0" - -dynamo: - hash: 46520ca59afe992fb5ef61b3197b2316f8df9b2b - install: true - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 1 - nginx_container: nginx - -model: - path: "qwen3.5-fp8" - container: "lmsysorg/sglang:nightly-dev-cu13-20260608-303757cc" - precision: "fp8" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - -backend: - type: sglang - - prefill_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - - decode_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_HEALTH_CHECK_TIMEOUT: "3600" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - MC_FORCE_MNNVL: "1" - MC_TE_METRIC: "true" - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" - - sglang_config: - prefill: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - trust-remote-code: true - attention-backend: "trtllm_mha" - tensor-parallel-size: 4 - mamba-ssm-dtype: "bfloat16" - moe-runner-backend: "flashinfer_trtllm" - disable-radix-cache: true - max-running-requests: 1024 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - context-length: 4096 - cuda-graph-max-bs: 1024 - decode-log-interval: 1 - stream-interval: 50 - disaggregation-mode: "prefill" - - decode: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - trust-remote-code: true - attention-backend: "trtllm_mha" - tensor-parallel-size: 4 - mamba-ssm-dtype: "bfloat16" - moe-runner-backend: "flashinfer_trtllm" - disable-radix-cache: true - max-running-requests: 1024 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - context-length: 4096 - cuda-graph-max-bs: 1024 - decode-log-interval: 1 - stream-interval: 50 - disaggregation-mode: "decode" - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - req_rate: "inf" - num_prompts_mult: 10 - num_warmup_mult: 1 - random_range_ratio: 0.8 - concurrencies: "1x2x4x8x16x32x64" \ No newline at end of file diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/1k1k/2p1d-dep4-dep16.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/1k1k/2p1d-dep4-dep16.yaml deleted file mode 100644 index 7b4ae6a03..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/1k1k/2p1d-dep4-dep16.yaml +++ /dev/null @@ -1,155 +0,0 @@ -name: "qwen3.5-2p1d-dep4-dep16" - -setup_script: rebuild-deepep.sh - -dynamo: - hash: 46520ca59afe992fb5ef61b3197b2316f8df9b2b - install: true - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 2 - nginx_container: nginx - -model: - path: "qwen3.5-fp8" - container: "lmsysorg/sglang:nightly-dev-cu13-20260608-303757cc" - precision: "fp8" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 2 - decode_nodes: 4 - prefill_workers: 2 - decode_workers: 1 - -backend: - type: sglang - - prefill_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - decode_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - MC_TE_METRIC: "true" - SGLANG_DG_CACHE_DIR: "/tmp/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" - SGLANG_HEALTH_CHECK_TIMEOUT: "1800" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - sglang_config: - prefill: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - trust-remote-code: true - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 2048 - mamba-ssm-dtype: "bfloat16" - disaggregation-mode: "prefill" - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - mem-fraction-static: 0.8 - chunked-prefill-size: 65536 - load-balance-method: "round_robin" - watchdog-timeout: 1000000 - disable-cuda-graph: true - log-level: "info" - page-size: 64 - attention-backend: "trtllm_mha" - moe-runner-backend: "flashinfer_trtllm" - - decode: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - trust-remote-code: true - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - prefill-round-robin-balance: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 128 - mamba-ssm-dtype: "bfloat16" - - disaggregation-mode: "decode" - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - - chunked-prefill-size: 4096 - context-length: 8192 - mem-fraction-static: 0.75 - max-mamba-cache-size: 4096 - max-running-requests: 4096 - cuda-graph-max-bs: 256 - watchdog-timeout: 1000000 - - page-size: 64 - attention-backend: "trtllm_mha" - moe-runner-backend: "deep_gemm" - moe-a2a-backend: "deepep" - deepep-mode: "low_latency" - ep-dispatch-algorithm: "static" - eplb-algorithm: "deepseek" - - decode-log-interval: 1 - stream-interval: 50 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - req_rate: "inf" - random_range_ratio: 0.8 - concurrencies: "4096" \ No newline at end of file diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/1p1d-tp4-tp4.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/1p1d-tp4-tp4.yaml deleted file mode 100644 index d869b247a..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/1p1d-tp4-tp4.yaml +++ /dev/null @@ -1,121 +0,0 @@ -name: "qwen3.5-1p1d-tp4-tp4" - -sbatch_directives: - mem: "0" - -dynamo: - hash: 46520ca59afe992fb5ef61b3197b2316f8df9b2b - install: true - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 1 - nginx_container: nginx - -model: - path: "qwen3.5-fp8" - container: "lmsysorg/sglang:nightly-dev-cu13-20260608-303757cc" - precision: "fp8" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - -backend: - type: sglang - - prefill_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - MC_FORCE_MNNVL: "1" - - decode_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_HEALTH_CHECK_TIMEOUT: "3600" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - MC_FORCE_MNNVL: "1" - MC_TE_METRIC: "true" - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" - - sglang_config: - prefill: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - trust-remote-code: true - attention-backend: "trtllm_mha" - tensor-parallel-size: 4 - mamba-ssm-dtype: "bfloat16" - moe-runner-backend: "flashinfer_trtllm" - disable-radix-cache: true - max-running-requests: 1024 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - context-length: 16384 - cuda-graph-max-bs: 1024 - decode-log-interval: 1 - stream-interval: 50 - disaggregation-mode: "prefill" - - decode: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - trust-remote-code: true - attention-backend: "trtllm_mha" - tensor-parallel-size: 4 - mamba-ssm-dtype: "bfloat16" - moe-runner-backend: "flashinfer_trtllm" - disable-radix-cache: true - max-running-requests: 1024 - mem-fraction-static: 0.8 - chunked-prefill-size: 16384 - max-prefill-tokens: 16384 - context-length: 16384 - cuda-graph-max-bs: 1024 - decode-log-interval: 1 - stream-interval: 50 - disaggregation-mode: "decode" - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - req_rate: "inf" - random_range_ratio: 0.8 - concurrencies: "1x2x4x8x16x32x64x128" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/4p1d-dep4-dep16.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/4p1d-dep4-dep16.yaml deleted file mode 100644 index 0eb6c5881..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/4p1d-dep4-dep16.yaml +++ /dev/null @@ -1,160 +0,0 @@ -name: "qwen3.5-4p1d-dep4-dep16" - -setup_script: rebuild-deepep.sh - -sbatch_directives: - mem: "0" - -dynamo: - hash: 46520ca59afe992fb5ef61b3197b2316f8df9b2b - install: true - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 3 - nginx_container: nginx - -model: - path: "qwen3.5-fp8" - container: "lmsysorg/sglang:nightly-dev-cu13-20260608-303757cc" - precision: "fp8" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 4 - decode_nodes: 4 - prefill_workers: 4 - decode_workers: 1 - -backend: - type: sglang - - prefill_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - decode_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - MC_TE_METRIC: "true" - SGLANG_DG_CACHE_DIR: "/tmp/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" - SGLANG_HEALTH_CHECK_TIMEOUT: "1800" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - sglang_config: - prefill: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - trust-remote-code: true - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 2048 - mamba-ssm-dtype: "bfloat16" - disaggregation-mode: "prefill" - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - mem-fraction-static: 0.8 - chunked-prefill-size: 65536 - load-balance-method: "round_robin" - watchdog-timeout: 1000000 - disable-cuda-graph: true - log-level: "info" - page-size: 64 - attention-backend: "trtllm_mha" - moe-runner-backend: "flashinfer_trtllm" - - decode: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - trust-remote-code: true - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - prefill-round-robin-balance: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 128 - mamba-ssm-dtype: "bfloat16" - - disaggregation-mode: "decode" - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - - chunked-prefill-size: 4096 - context-length: 16384 - mem-fraction-static: 0.80 - max-mamba-cache-size: 2048 - max-running-requests: 2048 - cuda-graph-max-bs: 128 - watchdog-timeout: 1000000 - - page-size: 64 - attention-backend: "trtllm_mha" - moe-runner-backend: "deep_gemm" - moe-a2a-backend: "deepep" - deepep-mode: "low_latency" - ep-dispatch-algorithm: "static" - eplb-algorithm: "deepseek" - - decode-log-interval: 1 - stream-interval: 50 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - req_rate: "inf" - num_prompts_mult: 20 - num_warmup_mult: 2 - random_range_ratio: 0.8 - concurrencies: "1024" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/8p1d-dep4-dep16.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/8p1d-dep4-dep16.yaml deleted file mode 100644 index 58f2604a7..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp8/8k1k/8p1d-dep4-dep16.yaml +++ /dev/null @@ -1,161 +0,0 @@ -name: "qwen3.5-8p1d-dep4-dep16" - -setup_script: rebuild-deepep.sh - -sbatch_directives: - mem: "0" - -infra: - etcd_nats_dedicated_node: true - -dynamo: - hash: 46520ca59afe992fb5ef61b3197b2316f8df9b2b - install: true - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 4 - nginx_container: nginx - -model: - path: "qwen3.5-fp8" - container: "lmsysorg/sglang:nightly-dev-cu13-20260608-303757cc" - precision: "fp8" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 8 - decode_nodes: 4 - prefill_workers: 8 - decode_workers: 1 - -backend: - type: sglang - - prefill_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - decode_environment: - SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "3600" - TORCH_NCCL_WATCHDOG_TIMEOUT_SEC: "3600" - TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: "3600" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - MC_FORCE_MNNVL: "1" - MC_TE_METRIC: "true" - SGLANG_DG_CACHE_DIR: "/tmp/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "512" - SGLANG_HEALTH_CHECK_TIMEOUT: "1800" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - sglang_config: - prefill: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - trust-remote-code: true - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 2048 - mamba-ssm-dtype: "bfloat16" - disaggregation-mode: "prefill" - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - mem-fraction-static: 0.8 - chunked-prefill-size: 65536 - load-balance-method: "round_robin" - watchdog-timeout: 1000000 - disable-cuda-graph: true - log-level: "info" - page-size: 64 - attention-backend: "trtllm_mha" - moe-runner-backend: "flashinfer_trtllm" - - decode: - served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" - model-path: "/model/" - trust-remote-code: true - quantization: "fp8" - kv-cache-dtype: "fp8_e4m3" - - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - prefill-round-robin-balance: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 128 - mamba-ssm-dtype: "bfloat16" - - disaggregation-mode: "decode" - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - - chunked-prefill-size: 4096 - context-length: 16384 - mem-fraction-static: 0.80 - max-mamba-cache-size: 2048 - max-running-requests: 2048 - cuda-graph-max-bs: 128 - watchdog-timeout: 1000000 - - page-size: 64 - attention-backend: "trtllm_mha" - moe-runner-backend: "deep_gemm" - moe-a2a-backend: "deepep" - deepep-mode: "low_latency" - ep-dispatch-algorithm: "static" - eplb-algorithm: "deepseek" - - decode-log-interval: 1 - stream-interval: 50 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - req_rate: "inf" - random_range_ratio: 0.8 - concurrencies: "2048x4096" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml deleted file mode 100644 index 71a0ffb9f..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml +++ /dev/null @@ -1,175 +0,0 @@ -# Qwen3.5-397B-A17B-NVFP4 Disaggregated 1P1D: TP4 Prefill + TP4 Decode -# Pure tensor parallel, no expert parallel (STP) -# 8k1k sa-bench concurrency sweep on GB300 -# -# Values taken from ni_experiment_config of the -# sa-qwen-3.5-8k1k-fp4-baseline-low-latency study, row -# qwen3.5-1p_tp4x1d_tp4-aligned-ccsweep (CSV pareto export 2026-06-05). - -name: "gb300-fp4-qwen3.5_8k1k_lowlat_0" - -model: - path: "qwen3.5-fp4" - container: "dynamo-sglang" - precision: "fp4" - -dynamo: - version: "1.1.0" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 2 - nginx_container: nginx - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - -backend: - type: sglang - - prefill_environment: - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - SGLANG_ENABLE_SPEC_V2: "1" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_ENABLE_JIT_DEEPGEMM: "true" - SGLANG_ENABLE_FLASHINFER_GEMM: "true" - FLASHINFER_DISABLE_VERSION_CHECK: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - - decode_environment: - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - SGLANG_ENABLE_SPEC_V2: "1" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - MC_FORCE_MNNVL: "1" - MC_TE_METRIC: "true" - SGLANG_ENABLE_JIT_DEEPGEMM: "true" - SGLANG_ENABLE_FLASHINFER_GEMM: "true" - FLASHINFER_DISABLE_VERSION_CHECK: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_HEALTH_CHECK_TIMEOUT: "1800" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - sglang_config: - prefill: - served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4" - model-path: "/model/" - trust-remote-code: true - - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - - reasoning-parser: "qwen3" - tool-call-parser: "qwen3_coder" - - quantization: "modelopt_fp4" - fp4-gemm-backend: "flashinfer_cutlass" - kv-cache-dtype: "fp8_e4m3" - - mamba-scheduler-strategy: "no_buffer" - mamba-ssm-dtype: "bfloat16" - mamba-track-interval: 2048 - - attention-backend: "trtllm_mha" - mm-attention-backend: "triton_attn" - moe-runner-backend: "flashinfer_trtllm" - linear-attn-decode-backend: "flashinfer" - - disaggregation-mode: "prefill" - disable-radix-cache: true - - mem-fraction-static: 0.8 - context-length: 9236 - max-total-tokens: 128000 - max-running-requests: 128 - cuda-graph-max-bs: 4 - chunked-prefill-size: 32768 - max-prefill-tokens: 32768 - scheduler-recv-interval: 10 - stream-interval: 30 - load-balance-method: "round_robin" - page-size: 64 - watchdog-timeout: 1000000 - log-level: "info" - - decode: - served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4" - model-path: "/model/" - trust-remote-code: true - - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - - reasoning-parser: "qwen3" - tool-call-parser: "qwen3_coder" - - quantization: "modelopt_fp4" - fp4-gemm-backend: "flashinfer_cutlass" - kv-cache-dtype: "fp8_e4m3" - - mamba-scheduler-strategy: "no_buffer" - mamba-ssm-dtype: "bfloat16" - mamba-track-interval: 128 - - attention-backend: "trtllm_mha" - mm-attention-backend: "triton_attn" - moe-runner-backend: "flashinfer_trtllm" - linear-attn-decode-backend: "flashinfer" - - disaggregation-mode: "decode" - disable-radix-cache: true - - mem-fraction-static: 0.8 - context-length: 9236 - max-total-tokens: 1500000 - max-mamba-cache-size: 256 - max-running-requests: 128 - cuda-graph-max-bs: 256 - chunked-prefill-size: 32768 - max-prefill-tokens: 32768 - scheduler-recv-interval: 10 - stream-interval: 30 - page-size: 64 - watchdog-timeout: 1000000 - decode-log-interval: 50 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "1x4x8x16x32x64x256" - req_rate: "inf" - random_range_ratio: 0.8 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_0.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_0.yaml deleted file mode 100644 index 00e576439..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_0.yaml +++ /dev/null @@ -1,177 +0,0 @@ -# Qwen3.5-397B-A17B-NVFP4 Disaggregated 5P1D wide-EP -# Prefill: 5 workers @ TP4/DP4/EP4 with DP-attn (per-node, DEP4) -# Decode: 1 worker @ TP16/DP16/EP16 with DP-attn + TBO (DEP16, 4 nodes) -# Total: 36 GB300 GPUs (5*4 + 4*4); 8k1k concurrency 1024/2048/3072. -# -# Values taken from ni_experiment_config of pareto row -# qwen3.5-dep16-fia2a-tbo-cc1024x2048x3072-dynamo-tot-nixl -# (sa-qwen-3.5-8k1k-fp4-baseline-mid-pareto study). - -name: "gb300-fp4-qwen3.5_8k1k_maxtpt_0" - -model: - path: "qwen3.5-fp4" - container: "dynamo-sglang" - precision: "fp4" - -dynamo: - version: "1.1.0" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 2 - nginx_container: nginx - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 5 - prefill_workers: 5 - decode_nodes: 4 - decode_workers: 1 - -backend: - type: sglang - - prefill_environment: - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - SGLANG_ENABLE_SPEC_V2: "1" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_ENABLE_JIT_DEEPGEMM: "true" - SGLANG_ENABLE_FLASHINFER_GEMM: "true" - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: "cutlass" - FLASHINFER_DISABLE_VERSION_CHECK: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_ENABLE_NIXL: "1" - - decode_environment: - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - SGLANG_ENABLE_SPEC_V2: "1" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - MC_FORCE_MNNVL: "1" - MC_TE_METRIC: "true" - SGLANG_ENABLE_JIT_DEEPGEMM: "true" - SGLANG_ENABLE_FLASHINFER_GEMM: "true" - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: "cutlass" - SGLANG_MOE_NVFP4_DISPATCH: "1" - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: "1" - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: "1" - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: "1" - FLASHINFER_DISABLE_VERSION_CHECK: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "1024" - SGLANG_HEALTH_CHECK_TIMEOUT: "1800" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - SGLANG_ENABLE_NIXL: "1" - - sglang_config: - prefill: - served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4" - model-path: "/model/" - trust-remote-code: true - - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 2048 - mamba-ssm-dtype: "bfloat16" - - disaggregation-mode: "prefill" - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - disaggregation-transfer-backend: "nixl" - - mem-fraction-static: 0.8 - max-total-tokens: 128000 - chunked-prefill-size: 65536 - load-balance-method: "round_robin" - watchdog-timeout: 1000000 - log-level: "info" - page-size: 64 - - attention-backend: "trtllm_mha" - moe-runner-backend: "flashinfer_trtllm" - linear-attn-decode-backend: "flashinfer" - - decode: - served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4" - model-path: "/model/" - trust-remote-code: true - - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - enable-two-batch-overlap: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 128 - mamba-ssm-dtype: "bfloat16" - - disaggregation-mode: "decode" - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - disaggregation-transfer-backend: "nixl" - - chunked-prefill-size: 4096 - max-mamba-cache-size: 4096 - max-total-tokens: 2200000 - max-running-requests: 4096 - mem-fraction-static: 0.8 - watchdog-timeout: 1000000 - page-size: 64 - - attention-backend: "trtllm_mha" - moe-runner-backend: "flashinfer_cutedsl" - moe-a2a-backend: "flashinfer" - disable-shared-experts-fusion: true - linear-attn-decode-backend: "flashinfer" - - decode-log-interval: 50 - stream-interval: 50 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "2048" - req_rate: "inf" - random_range_ratio: 0.8 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_1.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_1.yaml deleted file mode 100644 index ecab28509..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_1.yaml +++ /dev/null @@ -1,175 +0,0 @@ -# Qwen3.5-397B-A17B-NVFP4 Disaggregated 6P1D wide-EP -# Prefill: 6 workers @ TP4/DP4/EP4 with DP-attn (per-node, DEP4) -# Decode: 1 worker @ TP16/DP16/EP16 with DP-attn + TBO (DEP16, 4 nodes) -# Total: 40 GB300 GPUs (6*4 + 4*4); 8k1k concurrency 5120. -# -# Values taken from ni_experiment_config of pareto row -# qwen3.5-6p_dep4x1d_dep16-fia2a-tbo-cc5120-dynamo-tot-mooncake -# (sa-qwen-3.5-8k1k-fp4-baseline-mid-pareto study). - -name: "gb300-fp4-qwen3.5_8k1k_maxtpt_1" - -model: - path: "qwen3.5-fp4" - container: "dynamo-sglang" - precision: "fp4" - -dynamo: - version: "1.1.0" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 2 - nginx_container: nginx - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 6 - prefill_workers: 6 - decode_nodes: 4 - decode_workers: 1 - -backend: - type: sglang - - prefill_environment: - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - SGLANG_ENABLE_SPEC_V2: "1" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_ENABLE_JIT_DEEPGEMM: "true" - SGLANG_ENABLE_FLASHINFER_GEMM: "true" - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: "cutlass" - FLASHINFER_DISABLE_VERSION_CHECK: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - - decode_environment: - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - SGLANG_ENABLE_SPEC_V2: "1" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - MC_FORCE_MNNVL: "1" - MC_TE_METRIC: "true" - SGLANG_ENABLE_JIT_DEEPGEMM: "true" - SGLANG_ENABLE_FLASHINFER_GEMM: "true" - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: "cutlass" - SGLANG_MOE_NVFP4_DISPATCH: "1" - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: "1" - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: "1" - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: "1" - FLASHINFER_DISABLE_VERSION_CHECK: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "1024" - SGLANG_HEALTH_CHECK_TIMEOUT: "1800" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - sglang_config: - prefill: - served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4" - model-path: "/model/" - trust-remote-code: true - - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 2048 - mamba-ssm-dtype: "bfloat16" - - disaggregation-mode: "prefill" - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - disaggregation-transfer-backend: "mooncake" - - mem-fraction-static: 0.8 - max-total-tokens: 128000 - chunked-prefill-size: 65536 - load-balance-method: "round_robin" - watchdog-timeout: 1000000 - log-level: "info" - page-size: 64 - - attention-backend: "trtllm_mha" - moe-runner-backend: "flashinfer_trtllm" - linear-attn-decode-backend: "flashinfer" - - decode: - served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4" - model-path: "/model/" - trust-remote-code: true - - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - enable-two-batch-overlap: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 128 - mamba-ssm-dtype: "bfloat16" - - disaggregation-mode: "decode" - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - disaggregation-transfer-backend: "mooncake" - - chunked-prefill-size: 5120 - max-mamba-cache-size: 5120 - max-total-tokens: 3200000 - max-running-requests: 5120 - mem-fraction-static: 0.8 - watchdog-timeout: 1000000 - page-size: 64 - - attention-backend: "trtllm_mha" - moe-runner-backend: "flashinfer_cutedsl" - moe-a2a-backend: "flashinfer" - disable-shared-experts-fusion: true - linear-attn-decode-backend: "flashinfer" - - decode-log-interval: 50 - stream-interval: 50 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "5120" - req_rate: "inf" - random_range_ratio: 0.8 diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_2.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_2.yaml deleted file mode 100644 index d35f44469..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_2.yaml +++ /dev/null @@ -1,175 +0,0 @@ -# Qwen3.5-397B-A17B-NVFP4 Disaggregated 7P1D wide-EP -# Prefill: 7 workers @ TP4/DP4/EP4 with DP-attn (per-node, DEP4) -# Decode: 1 worker @ TP16/DP16/EP16 with DP-attn + TBO (DEP16, 4 nodes) -# Total: 44 GB300 GPUs (7*4 + 4*4); 8k1k concurrency 5120. -# -# Values taken from ni_experiment_config of pareto row -# qwen3.5-7p_dep4x1d_dep16-fia2a-tbo-cc5120-dynamo-tot-mooncake -# (sa-qwen-3.5-8k1k-fp4-baseline-mid-pareto study). - -name: "gb300-fp4-qwen3.5_8k1k_maxtpt_2" - -model: - path: "qwen3.5-fp4" - container: "dynamo-sglang" - precision: "fp4" - -dynamo: - version: "1.1.0" - -frontend: - type: dynamo - enable_multiple_frontends: true - num_additional_frontends: 2 - nginx_container: nginx - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 7 - prefill_workers: 7 - decode_nodes: 4 - decode_workers: 1 - -backend: - type: sglang - - prefill_environment: - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - SGLANG_ENABLE_SPEC_V2: "1" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - MC_FORCE_MNNVL: "1" - SGLANG_ENABLE_JIT_DEEPGEMM: "true" - SGLANG_ENABLE_FLASHINFER_GEMM: "true" - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: "cutlass" - FLASHINFER_DISABLE_VERSION_CHECK: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - - decode_environment: - NO_COLOR: "1" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - SGLANG_ENABLE_SPEC_V2: "1" - PYTHONUNBUFFERED: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - MC_FORCE_MNNVL: "1" - MC_TE_METRIC: "true" - SGLANG_ENABLE_JIT_DEEPGEMM: "true" - SGLANG_ENABLE_FLASHINFER_GEMM: "true" - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: "cutlass" - SGLANG_MOE_NVFP4_DISPATCH: "1" - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: "1" - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: "1" - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: "1" - FLASHINFER_DISABLE_VERSION_CHECK: "1" - SGLANG_DG_CACHE_DIR: "/configs/deepgemm-cache" - FLASHINFER_WORKSPACE_BASE: "/configs/flashinfer-cache" - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: "1" - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: "1024" - SGLANG_HEALTH_CHECK_TIMEOUT: "1800" - SGLANG_HEALTH_STARTING_OK: "1" - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: "0" - - sglang_config: - prefill: - served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4" - model-path: "/model/" - trust-remote-code: true - - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 2048 - mamba-ssm-dtype: "bfloat16" - - disaggregation-mode: "prefill" - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - disaggregation-transfer-backend: "mooncake" - - mem-fraction-static: 0.8 - max-total-tokens: 128000 - chunked-prefill-size: 65536 - load-balance-method: "round_robin" - watchdog-timeout: 1000000 - log-level: "info" - page-size: 64 - - attention-backend: "trtllm_mha" - moe-runner-backend: "flashinfer_trtllm" - linear-attn-decode-backend: "flashinfer" - - decode: - served-model-name: "nvidia/Qwen3.5-397B-A17B-NVFP4" - model-path: "/model/" - trust-remote-code: true - - quantization: "modelopt_fp4" - kv-cache-dtype: "fp8_e4m3" - - tensor-parallel-size: 16 - data-parallel-size: 16 - expert-parallel-size: 16 - enable-dp-attention: true - enable-dp-lm-head: true - enable-two-batch-overlap: true - - mamba-scheduler-strategy: "no_buffer" - mamba-track-interval: 128 - mamba-ssm-dtype: "bfloat16" - - disaggregation-mode: "decode" - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - disaggregation-transfer-backend: "mooncake" - - chunked-prefill-size: 5120 - max-mamba-cache-size: 5120 - max-total-tokens: 3200000 - max-running-requests: 5120 - mem-fraction-static: 0.8 - watchdog-timeout: 1000000 - page-size: 64 - - attention-backend: "trtllm_mha" - moe-runner-backend: "flashinfer_cutedsl" - moe-a2a-backend: "flashinfer" - disable-shared-experts-fusion: true - linear-attn-decode-backend: "flashinfer" - - decode-log-interval: 50 - stream-interval: 50 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "5120" - req_rate: "inf" - random_range_ratio: 0.8 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/agg-gb200-low-latency-mtp2.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/agg-gb200-low-latency-mtp2.yaml deleted file mode 100644 index ffc8e4ea1..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/agg-gb200-low-latency-mtp2.yaml +++ /dev/null @@ -1,86 +0,0 @@ -name: "svf-vllm-agg-gb200-low-latency-mtp2" - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.20.1-ubuntu2404" - precision: "fp4" - -dynamo: - install: true - wheel: "1.2.0.dev20260426" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - agg_nodes: 2 - agg_workers: 1 - gpus_per_agg: 8 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - aggregated_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - TORCH_SYMMMEM: "NVSHMEM" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_TLS: "cuda_copy,cuda_ipc,tcp" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_P2P_LEVEL: NVL - vllm_config: - aggregated: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 8 - pipeline-parallel-size: 1 - speculative-config: '{"method":"mtp","num_speculative_tokens":2}' - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' - attention-config: '{"use_fp4_indexer_cache":true}' - tokenizer-mode: deepseek_v4 - max-model-len: 9280 - max-num-seqs: 4 - max-num-batched-tokens: 8192 - max-cudagraph-capture-size: 4 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - block-size: 256 - gpu-memory-utilization: 0.9 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "1" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - revision: "0366e4e064385807ea86b088a5c6c878ff23343b" - container: - image: "vllm/vllm-openai:v0.20.1-ubuntu2404" - frameworks: - dynamo: "1.2.0.dev20260426" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-latency-c1.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-latency-c1.yaml deleted file mode 100644 index e81ef8631..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-latency-c1.yaml +++ /dev/null @@ -1,113 +0,0 @@ -name: "svf-vllm-disagg-b200-low-latency-c1" -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.23.0" - precision: "fp4" -dynamo: - install: true - wheel: "1.2.0.dev20260426" -setup_script: vllm-container-deps.sh -slurm: - time_limit: "8:00:00" -health_check: - interval_seconds: 10 - max_attempts: 1440 -sbatch_directives: - segment: "1" -resources: - decode_nodes: 1 - decode_workers: 1 - gpu_type: b200 - gpus_per_decode: 8 - gpus_per_node: 8 - gpus_per_prefill: 8 - prefill_nodes: 1 - prefill_workers: 1 -infra: - etcd_nats_dedicated_node: true -frontend: - enable_multiple_frontends: false - type: dynamo -backend: - connector: null - decode_environment: - NCCL_CUMEM_ENABLE: '1' - TILELANG_CLEANUP_TEMP_FILES: '1' - UCX_MEMTYPE_CACHE: n - UCX_MEMTYPE_REG_WHOLE: n - UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 - VLLM_SERVER_DEV_MODE: '1' - prefill_environment: - NCCL_CUMEM_ENABLE: '1' - TILELANG_CLEANUP_TEMP_FILES: '1' - UCX_MEMTYPE_CACHE: n - UCX_MEMTYPE_REG_WHOLE: n - UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 - VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: '2048' - VLLM_SERVER_DEV_MODE: '1' - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: '1024' - type: vllm - vllm_config: - decode: - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - enable-sleep-mode: true - gpu-memory-utilization: 0.9 - kv-cache-dtype: fp8 - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - max-cudagraph-capture-size: 256 - max-model-len: 9280 - max-num-batched-tokens: 256 - max-num-seqs: 256 - no-disable-hybrid-kv-cache-manager: true - no-enable-flashinfer-autotune: true - no-enable-prefix-caching: true - pipeline-parallel-size: 1 - served-model-name: deepseek-ai/DeepSeek-V4-Pro - stream-interval: 50 - tensor-parallel-size: 8 - tokenizer-mode: deepseek_v4 - trust-remote-code: true - prefill: - block-size: 256 - data-parallel-rpc-port: 13345 - data-parallel-size: 8 - enable-expert-parallel: true - enable-sleep-mode: true - enforce-eager: true - gpu-memory-utilization: 0.8 - kv-cache-dtype: fp8 - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - max-model-len: 9280 - max-num-batched-tokens: 32768 - max-num-seqs: 16 - no-async-scheduling: true - no-disable-hybrid-kv-cache-manager: true - no-enable-flashinfer-autotune: true - no-enable-prefix-caching: true - numa-bind: true - offload-group-size: 3 - offload-num-in-group: 1 - offload-prefetch-step: 2 - pipeline-parallel-size: 1 - served-model-name: deepseek-ai/DeepSeek-V4-Pro - tensor-parallel-size: 1 - tokenizer-mode: deepseek_v4 - trust-remote-code: true -benchmark: - concurrencies: "1" - custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" - isl: 8192 - osl: 1024 - req_rate: "inf" - type: "sa-bench" - use_chat_template: true -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - revision: "0366e4e064385807ea86b088a5c6c878ff23343b" - container: - image: "vllm/vllm-openai:v0.23.0" - frameworks: - dynamo: "1.2.0.dev20260426" - vllm: "0.23.0" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-latency-c32-c128.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-latency-c32-c128.yaml deleted file mode 100644 index b29644015..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-latency-c32-c128.yaml +++ /dev/null @@ -1,113 +0,0 @@ -name: "svf-vllm-disagg-b200-low-latency-c32-c128" -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.23.0" - precision: "fp4" -dynamo: - install: true - wheel: "1.2.0.dev20260426" -setup_script: vllm-container-deps.sh -slurm: - time_limit: "8:00:00" -health_check: - interval_seconds: 10 - max_attempts: 1440 -sbatch_directives: - segment: "1" -resources: - decode_nodes: 1 - decode_workers: 1 - gpu_type: b200 - gpus_per_decode: 8 - gpus_per_node: 8 - gpus_per_prefill: 8 - prefill_nodes: 1 - prefill_workers: 1 -infra: - etcd_nats_dedicated_node: false -frontend: - enable_multiple_frontends: false - type: dynamo -backend: - connector: null - decode_environment: - NCCL_CUMEM_ENABLE: '1' - TILELANG_CLEANUP_TEMP_FILES: '1' - UCX_MEMTYPE_CACHE: n - UCX_MEMTYPE_REG_WHOLE: n - UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 - VLLM_SERVER_DEV_MODE: '1' - prefill_environment: - NCCL_CUMEM_ENABLE: '1' - TILELANG_CLEANUP_TEMP_FILES: '1' - UCX_MEMTYPE_CACHE: n - UCX_MEMTYPE_REG_WHOLE: n - UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 - VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: '2048' - VLLM_SERVER_DEV_MODE: '1' - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: '1024' - type: vllm - vllm_config: - decode: - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - enable-sleep-mode: true - gpu-memory-utilization: 0.9 - kv-cache-dtype: fp8 - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - max-cudagraph-capture-size: 128 - max-model-len: 9280 - max-num-batched-tokens: 128 - max-num-seqs: 128 - no-disable-hybrid-kv-cache-manager: true - no-enable-flashinfer-autotune: true - no-enable-prefix-caching: true - pipeline-parallel-size: 1 - served-model-name: deepseek-ai/DeepSeek-V4-Pro - stream-interval: 50 - tensor-parallel-size: 8 - tokenizer-mode: deepseek_v4 - trust-remote-code: true - prefill: - block-size: 256 - data-parallel-rpc-port: 13345 - data-parallel-size: 8 - enable-expert-parallel: true - enable-sleep-mode: true - enforce-eager: true - gpu-memory-utilization: 0.8 - kv-cache-dtype: fp8 - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - max-model-len: 9280 - max-num-batched-tokens: 32768 - max-num-seqs: 16 - no-async-scheduling: true - no-disable-hybrid-kv-cache-manager: true - no-enable-flashinfer-autotune: true - no-enable-prefix-caching: true - numa-bind: true - offload-group-size: 3 - offload-num-in-group: 1 - offload-prefetch-step: 2 - pipeline-parallel-size: 1 - served-model-name: deepseek-ai/DeepSeek-V4-Pro - tensor-parallel-size: 1 - tokenizer-mode: deepseek_v4 - trust-remote-code: true -benchmark: - concurrencies: "32x128" - custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" - isl: 8192 - osl: 1024 - req_rate: "inf" - type: "sa-bench" - use_chat_template: true -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - revision: "0366e4e064385807ea86b088a5c6c878ff23343b" - container: - image: "vllm/vllm-openai:v0.23.0" - frameworks: - dynamo: "1.2.0.dev20260426" - vllm: "0.23.0" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-latency-c64.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-latency-c64.yaml deleted file mode 100644 index b566c05fa..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-latency-c64.yaml +++ /dev/null @@ -1,113 +0,0 @@ -name: "svf-vllm-disagg-b200-low-latency-c64" -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.23.0" - precision: "fp4" -dynamo: - install: true - wheel: "1.2.0.dev20260426" -setup_script: vllm-container-deps.sh -slurm: - time_limit: "8:00:00" -health_check: - interval_seconds: 10 - max_attempts: 1440 -sbatch_directives: - segment: "1" -resources: - decode_nodes: 1 - decode_workers: 1 - gpu_type: b200 - gpus_per_decode: 8 - gpus_per_node: 8 - gpus_per_prefill: 8 - prefill_nodes: 1 - prefill_workers: 1 -infra: - etcd_nats_dedicated_node: false -frontend: - enable_multiple_frontends: false - type: dynamo -backend: - connector: null - decode_environment: - NCCL_CUMEM_ENABLE: '1' - TILELANG_CLEANUP_TEMP_FILES: '1' - UCX_MEMTYPE_CACHE: n - UCX_MEMTYPE_REG_WHOLE: n - UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 - VLLM_SERVER_DEV_MODE: '1' - prefill_environment: - NCCL_CUMEM_ENABLE: '1' - TILELANG_CLEANUP_TEMP_FILES: '1' - UCX_MEMTYPE_CACHE: n - UCX_MEMTYPE_REG_WHOLE: n - UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 - VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: '2048' - VLLM_SERVER_DEV_MODE: '1' - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: '1024' - type: vllm - vllm_config: - decode: - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - enable-sleep-mode: true - gpu-memory-utilization: 0.9 - kv-cache-dtype: fp8 - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - max-cudagraph-capture-size: 256 - max-model-len: 9280 - max-num-batched-tokens: 256 - max-num-seqs: 256 - no-disable-hybrid-kv-cache-manager: true - no-enable-flashinfer-autotune: true - no-enable-prefix-caching: true - pipeline-parallel-size: 1 - served-model-name: deepseek-ai/DeepSeek-V4-Pro - stream-interval: 50 - tensor-parallel-size: 8 - tokenizer-mode: deepseek_v4 - trust-remote-code: true - prefill: - block-size: 256 - data-parallel-rpc-port: 13345 - data-parallel-size: 8 - enable-expert-parallel: true - enable-sleep-mode: true - enforce-eager: true - gpu-memory-utilization: 0.8 - kv-cache-dtype: fp8 - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - max-model-len: 9280 - max-num-batched-tokens: 32768 - max-num-seqs: 16 - no-async-scheduling: true - no-disable-hybrid-kv-cache-manager: true - no-enable-flashinfer-autotune: true - no-enable-prefix-caching: true - numa-bind: true - offload-group-size: 3 - offload-num-in-group: 1 - offload-prefetch-step: 2 - pipeline-parallel-size: 1 - served-model-name: deepseek-ai/DeepSeek-V4-Pro - tensor-parallel-size: 1 - tokenizer-mode: deepseek_v4 - trust-remote-code: true -benchmark: - concurrencies: "64" - custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" - isl: 8192 - osl: 1024 - req_rate: "inf" - type: "sa-bench" - use_chat_template: true -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - revision: "0366e4e064385807ea86b088a5c6c878ff23343b" - container: - image: "vllm/vllm-openai:v0.23.0" - frameworks: - dynamo: "1.2.0.dev20260426" - vllm: "0.23.0" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-middle-c256.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-middle-c256.yaml deleted file mode 100644 index 43190836a..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-middle-c256.yaml +++ /dev/null @@ -1,117 +0,0 @@ -name: "svf-vllm-disagg-b200-low-middle-c256" -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.23.0" - precision: "fp4" -dynamo: - install: true - wheel: "1.2.0.dev20260426" -setup_script: vllm-container-deps.sh -slurm: - time_limit: "8:00:00" -health_check: - interval_seconds: 10 - max_attempts: 1440 -sbatch_directives: - segment: "1" -resources: - decode_nodes: 1 - decode_workers: 1 - gpu_type: b200 - gpus_per_decode: 8 - gpus_per_node: 8 - gpus_per_prefill: 8 - prefill_nodes: 1 - prefill_workers: 1 -infra: - etcd_nats_dedicated_node: false -frontend: - enable_multiple_frontends: false - type: dynamo -backend: - connector: null - decode_environment: - NCCL_CUMEM_ENABLE: '1' - TILELANG_CLEANUP_TEMP_FILES: '1' - UCX_MEMTYPE_CACHE: n - UCX_MEMTYPE_REG_WHOLE: n - UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 - VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: '2048' - VLLM_SERVER_DEV_MODE: '1' - prefill_environment: - NCCL_CUMEM_ENABLE: '1' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - TILELANG_CLEANUP_TEMP_FILES: '1' - UCX_MEMTYPE_CACHE: n - UCX_MEMTYPE_REG_WHOLE: n - UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 - VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: '2048' - VLLM_SERVER_DEV_MODE: '1' - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: '1024' - type: vllm - vllm_config: - decode: - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - data-parallel-rpc-port: 13345 - data-parallel-size: 8 - enable-ep-weight-filter: true - enable-expert-parallel: true - enable-sleep-mode: true - gpu-memory-utilization: 0.9 - kv-cache-dtype: fp8 - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - max-cudagraph-capture-size: 256 - max-model-len: 9280 - max-num-batched-tokens: 256 - max-num-seqs: 256 - no-disable-hybrid-kv-cache-manager: true - no-enable-flashinfer-autotune: true - no-enable-prefix-caching: true - pipeline-parallel-size: 1 - served-model-name: deepseek-ai/DeepSeek-V4-Pro - stream-interval: 50 - tensor-parallel-size: 1 - tokenizer-mode: deepseek_v4 - trust-remote-code: true - prefill: - block-size: 256 - data-parallel-rpc-port: 13345 - data-parallel-size: 8 - enable-ep-weight-filter: true - enable-expert-parallel: true - enable-sleep-mode: true - enforce-eager: true - gpu-memory-utilization: 0.95 - kv-cache-dtype: fp8 - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - max-model-len: 9280 - max-num-batched-tokens: 32768 - max-num-seqs: 16 - no-async-scheduling: true - no-disable-hybrid-kv-cache-manager: true - no-enable-flashinfer-autotune: true - no-enable-prefix-caching: true - numa-bind: true - pipeline-parallel-size: 1 - served-model-name: deepseek-ai/DeepSeek-V4-Pro - tensor-parallel-size: 1 - tokenizer-mode: deepseek_v4 - trust-remote-code: true -benchmark: - concurrencies: "256" - custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" - isl: 8192 - osl: 1024 - req_rate: "inf" - type: "sa-bench" - use_chat_template: true -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - revision: "0366e4e064385807ea86b088a5c6c878ff23343b" - container: - image: "vllm/vllm-openai:v0.23.0" - frameworks: - dynamo: "1.2.0.dev20260426" - vllm: "0.23.0" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-middle-c512.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-middle-c512.yaml deleted file mode 100644 index 283bb839f..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-middle-c512.yaml +++ /dev/null @@ -1,118 +0,0 @@ -name: "svf-vllm-disagg-b200-low-middle-c512" -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.23.0" - precision: "fp4" -dynamo: - install: true - wheel: "1.2.0.dev20260426" -setup_script: vllm-container-deps.sh -slurm: - time_limit: "8:00:00" -health_check: - max_attempts: 1440 - interval_seconds: 10 -sbatch_directives: - segment: "1" -resources: - gpu_type: b200 - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 8 -infra: - etcd_nats_dedicated_node: false -frontend: - type: dynamo - enable_multiple_frontends: false -backend: - type: vllm - connector: null - prefill_environment: - TILELANG_CLEANUP_TEMP_FILES: '1' - NCCL_CUMEM_ENABLE: '1' - VLLM_SERVER_DEV_MODE: '1' - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: '1024' - VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: '2048' - PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True - UCX_MEMTYPE_CACHE: n - UCX_MEMTYPE_REG_WHOLE: n - UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 - decode_environment: - TILELANG_CLEANUP_TEMP_FILES: '1' - NCCL_CUMEM_ENABLE: '1' - VLLM_SERVER_DEV_MODE: '1' - UCX_MEMTYPE_CACHE: n - UCX_MEMTYPE_REG_WHOLE: n - UCX_NET_DEVICES: mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1 - VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: '2048' - VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: '0' - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: deepseek-ai/DeepSeek-V4-Pro - kv-cache-dtype: fp8 - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enable-ep-weight-filter: true - enforce-eager: true - max-model-len: 9280 - max-num-seqs: 16 - max-num-batched-tokens: 32768 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - no-async-scheduling: true - block-size: 256 - gpu-memory-utilization: 0.95 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - numa-bind: true - tokenizer-mode: deepseek_v4 - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: deepseek-ai/DeepSeek-V4-Pro - kv-cache-dtype: fp8 - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 9280 - max-num-seqs: 512 - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 512 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - tokenizer-mode: deepseek_v4 -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "512" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - revision: "0366e4e064385807ea86b088a5c6c878ff23343b" - container: - image: "vllm/vllm-openai:v0.23.0" - frameworks: - dynamo: "1.2.0.dev20260426" - vllm: "0.23.0" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b300-high-tpt-megamoe.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b300-high-tpt-megamoe.yaml deleted file mode 100644 index 9247a31da..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b300-high-tpt-megamoe.yaml +++ /dev/null @@ -1,130 +0,0 @@ -name: "svf-vllm-disagg-b300-high-tpt-megamoe" - -# B300 adaptation of the DSV4 GB200/B200 vLLM disagg recipe. Each worker uses -# one full 8-GPU B300 node, plus a dedicated NATS/etcd infra node. -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.23.0" - precision: "fp4" - -dynamo: - install: true - wheel: "1.2.0.dev20260426" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 2 - decode_nodes: 1 - prefill_workers: 2 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: true - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - prefill_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - NCCL_CUMEM_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - decode_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - NCCL_CUMEM_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enable-ep-weight-filter: true - attention-config: '{"use_fp4_indexer_cache": true}' - enforce-eager: true - max-model-len: 9280 - max-num-seqs: 16 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 32768 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - no-async-scheduling: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' - gpu-memory-utilization: 0.85 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - numa-bind: true - tokenizer-mode: deepseek_v4 - reasoning-parser: deepseek_v4 - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enable-ep-weight-filter: true - attention-config: '{"use_fp4_indexer_cache": true}' - max-model-len: 9280 - max-num-seqs: 512 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 512 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' - gpu-memory-utilization: 0.85 - stream-interval: 50 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - tokenizer-mode: deepseek_v4 - reasoning-parser: deepseek_v4 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "4096" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - revision: "0366e4e064385807ea86b088a5c6c878ff23343b" - container: - image: "vllm/vllm-openai:v0.23.0" - frameworks: - dynamo: "1.2.0.dev20260426" - vllm: "0.23.0" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b300-low-latency.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b300-low-latency.yaml deleted file mode 100644 index 260e16fc7..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b300-low-latency.yaml +++ /dev/null @@ -1,124 +0,0 @@ -name: "svf-vllm-disagg-b300-low-latency" - -# B300 adaptation of the DSV4 GB200/B200 vLLM disagg recipe. Each worker uses -# one full 8-GPU B300 node, plus a dedicated NATS/etcd infra node. -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.23.0" - precision: "fp4" - -dynamo: - install: true - wheel: "1.2.0.dev20260426" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: true - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - prefill_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - NCCL_CUMEM_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - decode_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - NCCL_CUMEM_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - attention-config: '{"use_fp4_indexer_cache": true}' - enforce-eager: true - max-model-len: 16384 - max-num-seqs: 16 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 32768 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - no-async-scheduling: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' - gpu-memory-utilization: 0.8 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - numa-bind: true - offload-group-size: 3 - offload-num-in-group: 1 - offload-prefetch-step: 2 - tokenizer-mode: deepseek_v4 - reasoning-parser: deepseek_v4 - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 8 - pipeline-parallel-size: 1 - max-model-len: 16384 - max-num-seqs: 256 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 256 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - tokenizer-mode: deepseek_v4 - reasoning-parser: deepseek_v4 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "1x32x64x128" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" - -identity: - container: - image: "vllm/vllm-openai:v0.23.0" - frameworks: - dynamo: "1.2.0.dev20260426" - vllm: "0.23.0" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b300-mid-curve-megamoe.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b300-mid-curve-megamoe.yaml deleted file mode 100644 index 98b701790..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-b300-mid-curve-megamoe.yaml +++ /dev/null @@ -1,130 +0,0 @@ -name: "svf-vllm-disagg-b300-mid-curve-megamoe" - -# B300 adaptation of the DSV4 GB200/B200 vLLM disagg recipe. Each worker uses -# one full 8-GPU B300 node, plus a dedicated NATS/etcd infra node. -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.23.0" - precision: "fp4" - -dynamo: - install: true - wheel: "1.2.0.dev20260426" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: true - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - prefill_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - NCCL_CUMEM_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - decode_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - NCCL_CUMEM_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enable-ep-weight-filter: true - attention-config: '{"use_fp4_indexer_cache": true}' - enforce-eager: true - max-model-len: 9280 - max-num-seqs: 16 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 32768 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - no-async-scheduling: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' - gpu-memory-utilization: 0.85 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - numa-bind: true - tokenizer-mode: deepseek_v4 - reasoning-parser: deepseek_v4 - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enable-ep-weight-filter: true - attention-config: '{"use_fp4_indexer_cache": true}' - max-model-len: 9280 - max-num-seqs: 512 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 512 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' - gpu-memory-utilization: 0.85 - stream-interval: 50 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - tokenizer-mode: deepseek_v4 - reasoning-parser: deepseek_v4 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "256x1024" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - revision: "0366e4e064385807ea86b088a5c6c878ff23343b" - container: - image: "vllm/vllm-openai:v0.23.0" - frameworks: - dynamo: "1.2.0.dev20260426" - vllm: "0.23.0" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-high-tpt-megamoe-mtp2.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-high-tpt-megamoe-mtp2.yaml deleted file mode 100644 index 24b134c1b..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-high-tpt-megamoe-mtp2.yaml +++ /dev/null @@ -1,145 +0,0 @@ -name: "svf-vllm-disagg-gb200-high-tpt-megamoe-mtp2" - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.20.1-ubuntu2404" - precision: "fp4" - -dynamo: - install: true - wheel: "1.2.0.dev20260426" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 4 - decode_nodes: 2 - prefill_workers: 2 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: true - -frontend: - type: dynamo - enable_multiple_frontends: false -backend: - type: vllm - connector: null - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - TORCH_SYMMMEM: "NVSHMEM" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" - # VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - # VLLM_MOE_ROUTING_SIMULATION_STRATEGY: "uniform_random" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_TLS: "cuda_copy,cuda_ipc,tcp" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_P2P_LEVEL: NVL - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - TORCH_SYMMMEM: "NVSHMEM" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - # VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - # VLLM_MOE_ROUTING_SIMULATION_STRATEGY: "uniform_random" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_TLS: "cuda_copy,cuda_ipc,tcp" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_P2P_LEVEL: NVL - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enable-ep-weight-filter: true - moe-backend: deep_gemm_mega_moe - enforce-eager: true - speculative-config: '{"method":"mtp","num_speculative_tokens":2}' - attention-config: '{"use_fp4_indexer_cache":true}' - max-model-len: 9280 - max-num-seqs: 16 - max-num-batched-tokens: 32768 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - no-async-scheduling: true - block-size: 256 - gpu-memory-utilization: 0.94 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - numa-bind: true - tokenizer-mode: deepseek_v4 - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enable-ep-weight-filter: true - moe-backend: deep_gemm_mega_moe - speculative-config: '{"method":"mtp","num_speculative_tokens":2}' - attention-config: '{"use_fp4_indexer_cache":true}' - max-model-len: 9280 - max-num-seqs: 512 - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 1024 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - tokenizer-mode: deepseek_v4 -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "1024" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - revision: "0366e4e064385807ea86b088a5c6c878ff23343b" - container: - image: "vllm/vllm-openai:v0.20.1-ubuntu2404" - frameworks: - dynamo: "1.2.0.dev20260426" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-high-tpt-megamoe.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-high-tpt-megamoe.yaml deleted file mode 100644 index a78b83ce2..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-high-tpt-megamoe.yaml +++ /dev/null @@ -1,155 +0,0 @@ -name: "svf-vllm-disagg-gb200-high-tpt-megamoe" - -# Mirrored from NVIDIA/srt-slurm aflowers/vllm-gb200-v0.20.0 branch: -# recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-high-tpt-megamoe.yaml -# -# Topology: 2 prefill (DEP=8 each) + 1 decode (DEP=8). 7 nodes total with a -# dedicated NATS/etcd infra node. MegaMOE high-throughput point at concurrency -# 4096 with no CPU/NVMe offload. -# -# Local deltas vs upstream: -# * model.path alias renamed deepseekv4-fp4 -> deepseek-v4-pro to match -# SRT_SLURM_MODEL_PREFIX in runners/launch_gb200-nv.sh. -# * model.container set to vllm/vllm-openai:v0.20.0-ubuntu2404 to -# match nvidia-master.yaml image (which the launch script registers as -# the alias key in srtslurm.yaml). Upstream variants ship either the -# non-dynamo floating tag or a sha256 pin. -# * slurm.time_limit + health_check set to 8h / 1440 attempts to -# absorb cold-cache /mnt/numa1 model loads. -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.20.0-ubuntu2404" - precision: "fp4" - -dynamo: - install: true - wheel: "1.2.0.dev20260426" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 4 - decode_nodes: 2 - prefill_workers: 2 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: true - -frontend: - type: dynamo - enable_multiple_frontends: false -backend: - type: vllm - connector: null - prefill_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - TORCH_SYMMMEM: "NVSHMEM" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" - # VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - # VLLM_MOE_ROUTING_SIMULATION_STRATEGY: "uniform_random" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_TLS: "cuda_copy,cuda_ipc,tcp" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_P2P_LEVEL: NVL - decode_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - TORCH_SYMMMEM: "NVSHMEM" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - # VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - # VLLM_MOE_ROUTING_SIMULATION_STRATEGY: "uniform_random" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_TLS: "cuda_copy,cuda_ipc,tcp" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_P2P_LEVEL: NVL - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enable-ep-weight-filter: true - moe-backend: deep_gemm_mega_moe - enforce-eager: true - max-model-len: 9280 - max-num-seqs: 16 - max-num-batched-tokens: 32768 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - no-async-scheduling: true - block-size: 256 - gpu-memory-utilization: 0.95 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - numa-bind: true - tokenizer-mode: deepseek_v4 - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enable-ep-weight-filter: true - moe-backend: deep_gemm_mega_moe - max-model-len: 9280 - max-num-seqs: 512 - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 512 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - tokenizer-mode: deepseek_v4 -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "4096" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - revision: "0366e4e064385807ea86b088a5c6c878ff23343b" - container: - image: "vllm/vllm-openai:v0.20.0-ubuntu2404" - frameworks: - dynamo: "1.2.0.dev20260426" - vllm: "0.20.0" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-low-latency-mtp2.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-low-latency-mtp2.yaml deleted file mode 100644 index 7e0d09a0e..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-low-latency-mtp2.yaml +++ /dev/null @@ -1,131 +0,0 @@ -name: svf-vllm-disagg-gb200-low-latency-mtp2 - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.20.1-ubuntu2404" - precision: "fp4" - -dynamo: - install: true - wheel: "1.2.0.dev20260426" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: gb200 - gpus_per_node: 4 - prefill_nodes: 2 - decode_nodes: 8 - prefill_workers: 1 - decode_workers: 4 - gpus_per_prefill: 8 - gpus_per_decode: 8 -infra: - etcd_nats_dedicated_node: true -frontend: - type: dynamo - enable_multiple_frontends: false -backend: - type: vllm - connector: null - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: '3600' - TILELANG_CLEANUP_TEMP_FILES: '1' - VLLM_USE_NCCL_SYMM_MEM: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - VLLM_SERVER_DEV_MODE: '1' - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: '1024' - VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: '2048' - UCX_MEMTYPE_CACHE: n - UCX_MEMTYPE_REG_WHOLE: n - UCX_TLS: cuda_copy,cuda_ipc,tcp - UCX_CUDA_IPC_ENABLE_MNNVL: y - NCCL_P2P_LEVEL: NVL - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: '3600' - TILELANG_CLEANUP_TEMP_FILES: '1' - VLLM_USE_NCCL_SYMM_MEM: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - VLLM_SERVER_DEV_MODE: '1' - UCX_MEMTYPE_CACHE: n - UCX_MEMTYPE_REG_WHOLE: n - UCX_TLS: cuda_copy,cuda_ipc,tcp - UCX_CUDA_IPC_ENABLE_MNNVL: y - NCCL_P2P_LEVEL: NVL - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: deepseek-ai/DeepSeek-V4-Pro - kv-cache-dtype: fp8 - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-hybrid-lb: true - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enforce-eager: true - speculative-config: '{"method":"mtp","num_speculative_tokens":2}' - attention-config: '{"use_fp4_indexer_cache":true}' - max-model-len: 9280 - max-num-seqs: 8 - max-num-batched-tokens: 16384 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - no-async-scheduling: true - block-size: 256 - gpu-memory-utilization: 0.85 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - numa-bind: true - tokenizer-mode: deepseek_v4 - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: deepseek-ai/DeepSeek-V4-Pro - kv-cache-dtype: fp8 - tensor-parallel-size: 8 - pipeline-parallel-size: 1 - speculative-config: '{"method":"mtp","num_speculative_tokens":2}' - attention-config: '{"use_fp4_indexer_cache":true}' - max-model-len: 9280 - max-num-seqs: 256 - max-cudagraph-capture-size: 256 - max-num-batched-tokens: 256 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - tokenizer-mode: deepseek_v4 -benchmark: - type: sa-bench - isl: 8192 - osl: 1024 - concurrencies: 16x32x64 - req_rate: inf - use_chat_template: true - custom_tokenizer: sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - revision: "0366e4e064385807ea86b088a5c6c878ff23343b" - container: - image: "vllm/vllm-openai:v0.20.1-ubuntu2404" - frameworks: - dynamo: "1.2.0.dev20260426" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-low-latency.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-low-latency.yaml deleted file mode 100644 index 6c5dec41d..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-low-latency.yaml +++ /dev/null @@ -1,149 +0,0 @@ -name: "svf-vllm-disagg-gb200-low-latency" - -# Mirrored from NVIDIA/srt-slurm aflowers/vllm-gb200-v0.20.0 branch: -# recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-low-latency.yaml -# -# Topology: 1 prefill (DEP=8) + 1 decode (TP=8). 5 nodes total with a -# dedicated NATS/etcd infra node. Single-concurrency point for low latency. -# -# Local deltas vs upstream: -# * model.path alias renamed deepseekv4-fp4 -> deepseek-v4-pro to match -# SRT_SLURM_MODEL_PREFIX in runners/launch_gb200-nv.sh. -# * model.container set to vllm/vllm-openai:v0.20.0-ubuntu2404 to -# match nvidia-master.yaml image (which the launch script registers as -# the alias key in srtslurm.yaml). Upstream variants ship either the -# non-dynamo floating tag or a sha256 pin. -# * slurm.time_limit + health_check set to 8h / 1440 attempts to -# absorb cold-cache /mnt/numa1 model loads. -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.20.0-ubuntu2404" - precision: "fp4" - -dynamo: - install: true - wheel: "1.2.0.dev20260426" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 2 - decode_nodes: 2 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: true - -frontend: - type: dynamo - enable_multiple_frontends: false -backend: - type: vllm - connector: null - prefill_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" - # VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - # VLLM_MOE_ROUTING_SIMULATION_STRATEGY: "uniform_random" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_TLS: "cuda_copy,cuda_ipc,tcp" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_P2P_LEVEL: NVL - decode_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - # VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - # VLLM_MOE_ROUTING_SIMULATION_STRATEGY: "uniform_random" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_TLS: "cuda_copy,cuda_ipc,tcp" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_P2P_LEVEL: NVL - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enforce-eager: true - max-model-len: 16384 - max-num-seqs: 16 - max-num-batched-tokens: 32768 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - no-async-scheduling: true - block-size: 256 - gpu-memory-utilization: 0.8 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - numa-bind: true - offload-group-size: 3 - offload-num-in-group: 1 - offload-prefetch-step: 2 - # offload-params: "w13_weight w2_weight w13_weight_scale w2_weight_scale wq_b wo_a wo_b shared_experts" - tokenizer-mode: deepseek_v4 - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 8 - pipeline-parallel-size: 1 -# data-parallel-size: 8 -# data-parallel-rpc-port: 13345 -# enable-expert-parallel: true - max-model-len: 16384 - max-num-seqs: 256 - max-cudagraph-capture-size: 256 - max-num-batched-tokens: 256 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - tokenizer-mode: deepseek_v4 -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "1" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" - -identity: - container: - image: "vllm/vllm-openai:v0.20.0-ubuntu2404" - frameworks: - dynamo: "1.2.0.dev20260426" - vllm: "0.20.0" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-low-middle-curve.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-low-middle-curve.yaml deleted file mode 100644 index 98ec9acda..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-low-middle-curve.yaml +++ /dev/null @@ -1,151 +0,0 @@ -name: "svf-vllm-disagg-gb200-low-middle-curve" - -# Mirrored from NVIDIA/srt-slurm aflowers/vllm-gb200-v0.20.0 branch: -# recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-low-middle-curve.yaml -# -# Topology: 1 prefill (DEP=8) + 4 decode (TP=8). 11 nodes total with a -# dedicated NATS/etcd infra node. Low-middle curve points at concurrencies -# 256 and 512. -# -# Local deltas vs upstream: -# * model.path alias renamed deepseekv4-fp4 -> deepseek-v4-pro to match -# SRT_SLURM_MODEL_PREFIX in runners/launch_gb200-nv.sh. -# * model.container set to vllm/vllm-openai:v0.20.0-ubuntu2404 to -# match nvidia-master.yaml image (which the launch script registers as -# the alias key in srtslurm.yaml). Upstream variants ship either the -# non-dynamo floating tag or a sha256 pin. -# * slurm.time_limit + health_check set to 8h / 1440 attempts to -# absorb cold-cache /mnt/numa1 model loads. -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.20.0-ubuntu2404" - precision: "fp4" - -dynamo: - install: true - wheel: "1.2.0.dev20260426" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 2 - decode_nodes: 8 - prefill_workers: 1 - decode_workers: 4 - gpus_per_prefill: 8 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: true - -frontend: - type: dynamo - enable_multiple_frontends: false -backend: - type: vllm - connector: null - prefill_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" - # VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - # VLLM_MOE_ROUTING_SIMULATION_STRATEGY: "uniform_random" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_TLS: "cuda_copy,cuda_ipc,tcp" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_P2P_LEVEL: NVL - decode_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - # VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - # VLLM_MOE_ROUTING_SIMULATION_STRATEGY: "uniform_random" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_TLS: "cuda_copy,cuda_ipc,tcp" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_P2P_LEVEL: NVL - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enable-ep-weight-filter: true - enforce-eager: true - max-model-len: 16384 - max-num-seqs: 16 - max-num-batched-tokens: 32768 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - no-async-scheduling: true - block-size: 256 - gpu-memory-utilization: 0.8 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - numa-bind: true - offload-group-size: 3 - offload-num-in-group: 1 - offload-prefetch-step: 2 - # offload-params: "w13_weight w2_weight w13_weight_scale w2_weight_scale wq_b wo_a wo_b shared_experts" - tokenizer-mode: deepseek_v4 - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 8 - pipeline-parallel-size: 1 -# data-parallel-size: 8 -# data-parallel-rpc-port: 13345 -# enable-expert-parallel: true - max-model-len: 16384 - max-num-seqs: 256 - max-cudagraph-capture-size: 256 - max-num-batched-tokens: 256 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - tokenizer-mode: deepseek_v4 -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "256x512" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" - -identity: - container: - image: "vllm/vllm-openai:v0.20.0-ubuntu2404" - frameworks: - dynamo: "1.2.0.dev20260426" - vllm: "0.20.0" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-max-tpt-megamoe.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-max-tpt-megamoe.yaml deleted file mode 100644 index db3caef44..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-max-tpt-megamoe.yaml +++ /dev/null @@ -1,155 +0,0 @@ -name: "svf-vllm-disagg-gb200-max-tpt-megamoe" - -# Mirrored from NVIDIA/srt-slurm aflowers/vllm-gb200-v0.20.0 branch: -# recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-max-tpt-megamoe.yaml -# -# Topology: 3 prefill (DEP=8 each) + 1 decode (DEP=8). 9 nodes total with a -# dedicated NATS/etcd infra node. MegaMOE max-throughput point at concurrency -# 4096 with no CPU/NVMe offload. -# -# Local deltas vs upstream: -# * model.path alias renamed deepseekv4-fp4 -> deepseek-v4-pro to match -# SRT_SLURM_MODEL_PREFIX in runners/launch_gb200-nv.sh. -# * model.container set to vllm/vllm-openai:v0.20.0-ubuntu2404 to -# match nvidia-master.yaml image (which the launch script registers as -# the alias key in srtslurm.yaml). Upstream variants ship either the -# non-dynamo floating tag or a sha256 pin. -# * slurm.time_limit + health_check set to 8h / 1440 attempts to -# absorb cold-cache /mnt/numa1 model loads. -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.20.0-ubuntu2404" - precision: "fp4" - -dynamo: - install: true - wheel: "1.2.0.dev20260426" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 6 - decode_nodes: 2 - prefill_workers: 3 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: true - -frontend: - type: dynamo - enable_multiple_frontends: false -backend: - type: vllm - connector: null - prefill_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - TORCH_SYMMMEM: "NVSHMEM" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" - # VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - # VLLM_MOE_ROUTING_SIMULATION_STRATEGY: "uniform_random" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_TLS: "cuda_copy,cuda_ipc,tcp" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_P2P_LEVEL: NVL - decode_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - TORCH_SYMMMEM: "NVSHMEM" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - # VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - # VLLM_MOE_ROUTING_SIMULATION_STRATEGY: "uniform_random" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_TLS: "cuda_copy,cuda_ipc,tcp" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_P2P_LEVEL: NVL - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enable-ep-weight-filter: true - moe-backend: deep_gemm_mega_moe - enforce-eager: true - max-model-len: 9280 - max-num-seqs: 16 - max-num-batched-tokens: 32768 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - no-async-scheduling: true - block-size: 256 - gpu-memory-utilization: 0.95 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - numa-bind: true - tokenizer-mode: deepseek_v4 - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enable-ep-weight-filter: true - moe-backend: deep_gemm_mega_moe - max-model-len: 9280 - max-num-seqs: 512 - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 512 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - tokenizer-mode: deepseek_v4 -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "4096" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - revision: "0366e4e064385807ea86b088a5c6c878ff23343b" - container: - image: "vllm/vllm-openai:v0.20.0-ubuntu2404" - frameworks: - dynamo: "1.2.0.dev20260426" - vllm: "0.20.0" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-mid-curve-megamoe-mtp2.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-mid-curve-megamoe-mtp2.yaml deleted file mode 100644 index 813584eb9..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-mid-curve-megamoe-mtp2.yaml +++ /dev/null @@ -1,145 +0,0 @@ -name: "svf-vllm-disagg-gb200-mid-curve-megamoe-mtp2" - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.21.0-ubuntu2404" - precision: "fp4" - -dynamo: - install: true - wheel: "1.2.0.dev20260426" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 2 - decode_nodes: 2 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: true - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - TORCH_SYMMMEM: "NVSHMEM" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_TLS: "cuda_copy,cuda_ipc,tcp" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_P2P_LEVEL: NVL - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - TORCH_SYMMMEM: "NVSHMEM" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_TLS: "cuda_copy,cuda_ipc,tcp" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_P2P_LEVEL: NVL - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-hybrid-lb: true - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enable-ep-weight-filter: true - moe-backend: deep_gemm_mega_moe - enforce-eager: true - speculative-config: '{"method":"mtp","num_speculative_tokens":2}' - attention-config: '{"use_fp4_indexer_cache":true}' - max-model-len: 9280 - max-num-seqs: 8 - max-num-batched-tokens: 16384 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - no-async-scheduling: true - block-size: 256 - gpu-memory-utilization: 0.9 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - numa-bind: true - tokenizer-mode: deepseek_v4 - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-hybrid-lb: true - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enable-ep-weight-filter: true - moe-backend: deep_gemm_mega_moe - speculative-config: '{"method":"mtp","num_speculative_tokens":2}' - attention-config: '{"use_fp4_indexer_cache":true}' - max-model-len: 9280 - max-num-seqs: 512 - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 512 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - tokenizer-mode: deepseek_v4 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "128x256x512x1024" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - revision: "0366e4e064385807ea86b088a5c6c878ff23343b" - container: - image: "vllm/vllm-openai:v0.21.0-ubuntu2404" - frameworks: - dynamo: "1.2.0.dev20260426" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-mid-curve-megamoe.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-mid-curve-megamoe.yaml deleted file mode 100644 index a1f7cfd5a..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb200-mid-curve-megamoe.yaml +++ /dev/null @@ -1,155 +0,0 @@ -name: "svf-vllm-disagg-gb200-mid-curve-megamoe" - -# Mirrored from NVIDIA/srt-slurm aflowers/vllm-gb200-v0.20.0 branch: -# recipes/vllm/deepseek-v4-pro/GB200/8k1k/disagg-gb200-mid-curve-megamoe.yaml -# -# Topology: 1 prefill (DEP=8) + 1 decode (DEP=8). 5 nodes total with a -# dedicated NATS/etcd infra node. MegaMOE mid-curve points at concurrency -# 256/512/1024 with no CPU/NVMe offload. -# -# Local deltas vs upstream: -# * model.path alias renamed deepseekv4-fp4 -> deepseek-v4-pro to match -# SRT_SLURM_MODEL_PREFIX in runners/launch_gb200-nv.sh. -# * model.container set to vllm/vllm-openai:v0.20.0-ubuntu2404 to -# match nvidia-master.yaml image (which the launch script registers as -# the alias key in srtslurm.yaml). Upstream variants ship either the -# non-dynamo floating tag or a sha256 pin. -# * slurm.time_limit + health_check set to 8h / 1440 attempts to -# absorb cold-cache /mnt/numa1 model loads. -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.20.0-ubuntu2404" - precision: "fp4" - -dynamo: - install: true - wheel: "1.2.0.dev20260426" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 2 - decode_nodes: 2 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: true - -frontend: - type: dynamo - enable_multiple_frontends: false -backend: - type: vllm - connector: null - prefill_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - TORCH_SYMMMEM: "NVSHMEM" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" - # VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - # VLLM_MOE_ROUTING_SIMULATION_STRATEGY: "uniform_random" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_TLS: "cuda_copy,cuda_ipc,tcp" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_P2P_LEVEL: NVL - decode_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - TORCH_SYMMMEM: "NVSHMEM" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - # VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - # VLLM_MOE_ROUTING_SIMULATION_STRATEGY: "uniform_random" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_TLS: "cuda_copy,cuda_ipc,tcp" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_P2P_LEVEL: NVL - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enable-ep-weight-filter: true - moe-backend: deep_gemm_mega_moe - enforce-eager: true - max-model-len: 9280 - max-num-seqs: 16 - max-num-batched-tokens: 32768 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - no-async-scheduling: true - block-size: 256 - gpu-memory-utilization: 0.95 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - numa-bind: true - tokenizer-mode: deepseek_v4 - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enable-ep-weight-filter: true - moe-backend: deep_gemm_mega_moe - max-model-len: 9280 - max-num-seqs: 512 - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 512 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - tokenizer-mode: deepseek_v4 -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "256x512x1024" - req_rate: "inf" - use_chat_template: true - custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - revision: "0366e4e064385807ea86b088a5c6c878ff23343b" - container: - image: "vllm/vllm-openai:v0.20.0-ubuntu2404" - frameworks: - dynamo: "1.2.0.dev20260426" - vllm: "0.20.0" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb300-1p17d-tep4-tp4.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb300-1p17d-tep4-tp4.yaml deleted file mode 100644 index a2c3ab80a..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb300-1p17d-tep4-tp4.yaml +++ /dev/null @@ -1,106 +0,0 @@ -name: "svf-vllm-disagg-gb300-1p17d-tep4-tp4" - -# Topology: 1 prefill (TEP=4) + 17 decode (TP=4). 18 GB300 nodes (1P + 17D = 72 -# GPUs at 4 GPUs/node), NATS/etcd colocated on the prefill node. -# Wide-decode point at concurrency 18 — each decode worker holds a -# single replica. -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.20.0-ubuntu2404" - precision: "fp4" - -dynamo: - install: true - wheel: "1.2.0.dev20260426" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 17 - prefill_workers: 1 - decode_workers: 17 - gpus_per_prefill: 4 - gpus_per_decode: 4 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - prefill_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - decode_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - enable-expert-parallel: true - enforce-eager: true - max-model-len: 16384 - max-num-seqs: 256 - max-num-batched-tokens: 16384 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - no-async-scheduling: true - block-size: 256 - gpu-memory-utilization: 0.9 - enable-ep-weight-filter: true - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - max-model-len: 16384 - max-num-seqs: 512 - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 512 - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - no-disable-hybrid-kv-cache-manager: true - enable-ep-weight-filter: true - all2all-backend: "flashinfer_nvlink_one_sided" - no-enable-flashinfer-autotune: true - enable-sleep-mode: true - tokenizer-mode: deepseek_v4 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "18" - req_rate: "inf" - tokenizer_mode: "deepseek_v4" - use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb300-1p6d-dep4-tp4.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb300-1p6d-dep4-tp4.yaml deleted file mode 100644 index c3b25acc1..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb300-1p6d-dep4-tp4.yaml +++ /dev/null @@ -1,114 +0,0 @@ -name: "svf-vllm-disagg-gb300-1p6d-dep4-tp4" - -# Topology: 1 prefill (DEP=4) + 6 decode (TP=4). 7 GB300 nodes (1P + 6D = 28 -# GPUs at 4 GPUs/node) plus a dedicated NATS/etcd infra node. Low-mid curve -# point at concurrency 192. -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.20.0-ubuntu2404" - precision: "fp4" - -dynamo: - install: true - wheel: "1.2.0.dev20260426" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 6 - prefill_workers: 1 - decode_workers: 6 - gpus_per_prefill: 4 - gpus_per_decode: 4 - -infra: - etcd_nats_dedicated_node: true - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - prefill_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - TORCH_SYMMMEM: "NVSHMEM" - decode_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - TORCH_SYMMMEM: "NVSHMEM" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - attention-config: '{"use_fp4_indexer_cache": true}' - moe-backend: "deep_gemm_mega_moe" - enforce-eager: true - max-model-len: 16384 - max-num-seqs: 256 - max-num-batched-tokens: 16384 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - no-async-scheduling: true - block-size: 256 - gpu-memory-utilization: 0.9 - enable-ep-weight-filter: true - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - max-model-len: 16384 - max-num-seqs: 512 - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 512 - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - no-disable-hybrid-kv-cache-manager: true - enable-ep-weight-filter: true - all2all-backend: "flashinfer_nvlink_one_sided" - no-enable-flashinfer-autotune: true - enable-sleep-mode: true - tokenizer-mode: deepseek_v4 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "192" - req_rate: "inf" - tokenizer_mode: "deepseek_v4" - use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb300-4p1d-dep4-dep8-24-c4096.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb300-4p1d-dep4-dep8-24-c4096.yaml deleted file mode 100644 index b97ef0d5a..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb300-4p1d-dep4-dep8-24-c4096.yaml +++ /dev/null @@ -1,122 +0,0 @@ -name: "svf-vllm-disagg-gb300-4p1d-dep4-dep8-24" - -# Topology: 4 prefill (DEP=4 each) + 1 decode (DEP=8). 6 GB300 nodes (4P + 2D -# = 24 GPUs at 4 GPUs/node) plus a dedicated NATS/etcd infra node. -# Max-throughput point at concurrency 4096 with deep_gemm_mega_moe on -# both workers. -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.20.0-ubuntu2404" - precision: "fp4" - -dynamo: - wheel: "1.2.0.dev20260426" - install: true - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 4 - decode_nodes: 2 - prefill_workers: 4 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: true - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - TORCH_SYMMMEM: "NVSHMEM" - - decode_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - TORCH_SYMMMEM: "NVSHMEM" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enforce-eager: true - max-model-len: 16384 - max-num-seqs: 16 - max-num-batched-tokens: 16384 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - safetensors-load-strategy: "prefetch" - block-size: 256 - gpu-memory-utilization: 0.9 - no-disable-hybrid-kv-cache-manager: true - no-async-scheduling: true - tokenizer-mode: deepseek_v4 - enable-ep-weight-filter: true - enable-sleep-mode: true - moe-backend: "deep_gemm_mega_moe" - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - max-model-len: 16384 - max-num-seqs: 512 - max-cudagraph-capture-size: 512 - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: deepseek_v4 - enable-ep-weight-filter: true - enable-sleep-mode: true - moe-backend: "deep_gemm_mega_moe" - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "4096" - req_rate: "inf" - tokenizer_mode: "deepseek_v4" - use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb300-5p1d-dep4-dep8-28-c4096.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb300-5p1d-dep4-dep8-28-c4096.yaml deleted file mode 100644 index d83e6d771..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb300-5p1d-dep4-dep8-28-c4096.yaml +++ /dev/null @@ -1,122 +0,0 @@ -name: "svf-vllm-disagg-gb300-5p1d-dep4-dep8-28" - -# Topology: 5 prefill (DEP=4 each) + 1 decode (DEP=8). 7 GB300 nodes (5P + 2D -# = 28 GPUs at 4 GPUs/node) plus a dedicated NATS/etcd infra node. -# Max-throughput point at concurrency 4096 with deep_gemm_mega_moe on -# both workers. -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.20.0-ubuntu2404" - precision: "fp4" - -dynamo: - wheel: "1.2.0.dev20260426" - install: true - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 5 - decode_nodes: 2 - prefill_workers: 5 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: true - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - TORCH_SYMMMEM: "NVSHMEM" - - decode_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - TORCH_SYMMMEM: "NVSHMEM" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enforce-eager: true - max-model-len: 16384 - max-num-seqs: 16 - max-num-batched-tokens: 16384 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - safetensors-load-strategy: "prefetch" - block-size: 256 - gpu-memory-utilization: 0.9 - no-disable-hybrid-kv-cache-manager: true - no-async-scheduling: true - tokenizer-mode: deepseek_v4 - enable-ep-weight-filter: true - enable-sleep-mode: true - moe-backend: "deep_gemm_mega_moe" - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - max-model-len: 16384 - max-num-seqs: 512 - max-cudagraph-capture-size: 512 - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: deepseek_v4 - enable-ep-weight-filter: true - enable-sleep-mode: true - moe-backend: "deep_gemm_mega_moe" - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "4096" - req_rate: "inf" - tokenizer_mode: "deepseek_v4" - use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb300-6p1d-dep4-dep8-32-c4096.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb300-6p1d-dep4-dep8-32-c4096.yaml deleted file mode 100644 index 4b54cc13e..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb300-6p1d-dep4-dep8-32-c4096.yaml +++ /dev/null @@ -1,122 +0,0 @@ -name: "svf-vllm-disagg-gb300-6p1d-dep4-dep8-32" - -# Topology: 6 prefill (DEP=4 each) + 1 decode (DEP=8). 8 GB300 nodes (6P + 2D -# = 32 GPUs at 4 GPUs/node) plus a dedicated NATS/etcd infra node. -# Max-throughput point at concurrency 4096 with deep_gemm_mega_moe on -# both workers. -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.20.0-ubuntu2404" - precision: "fp4" - -dynamo: - wheel: "1.2.0.dev20260426" - install: true - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 6 - decode_nodes: 2 - prefill_workers: 6 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: true - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - TORCH_SYMMMEM: "NVSHMEM" - - decode_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - TORCH_SYMMMEM: "NVSHMEM" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enforce-eager: true - max-model-len: 16384 - max-num-seqs: 16 - max-num-batched-tokens: 16384 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - safetensors-load-strategy: "prefetch" - block-size: 256 - gpu-memory-utilization: 0.9 - no-disable-hybrid-kv-cache-manager: true - no-async-scheduling: true - tokenizer-mode: deepseek_v4 - enable-ep-weight-filter: true - enable-sleep-mode: true - moe-backend: "deep_gemm_mega_moe" - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - max-model-len: 16384 - max-num-seqs: 512 - max-cudagraph-capture-size: 512 - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: deepseek_v4 - enable-ep-weight-filter: true - enable-sleep-mode: true - moe-backend: "deep_gemm_mega_moe" - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "4096" - req_rate: "inf" - tokenizer_mode: "deepseek_v4" - use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb300-7p2d-dep4-dep16.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb300-7p2d-dep4-dep16.yaml deleted file mode 100644 index 43c2031a8..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/disagg-gb300-7p2d-dep4-dep16.yaml +++ /dev/null @@ -1,119 +0,0 @@ -name: "svf-vllm-disagg-gb300-7p2d-dep4-dep16" - -# Topology: 7 prefill (DEP=4) + 2 decode (DEP=16). 15 GB300 nodes (7P + 8D -# = 60 GPUs at 4 GPUs/node) plus a dedicated NATS/etcd infra node. -# Wide-EP decode max-throughput point at concurrency 3072. -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.20.0-ubuntu2404" - precision: "fp4" - -dynamo: - install: true - wheel: "1.2.0.dev20260426" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 7 - decode_nodes: 8 - prefill_workers: 7 - decode_workers: 2 - gpus_per_prefill: 4 - gpus_per_decode: 16 - -infra: - etcd_nats_dedicated_node: true - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - prefill_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - TORCH_SYMMMEM: "NVSHMEM" - decode_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - TORCH_SYMMMEM: "NVSHMEM" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - attention-config: '{"use_fp4_indexer_cache": true}' - moe-backend: "deep_gemm_mega_moe" - enforce-eager: true - max-model-len: 16384 - max-num-seqs: 256 - max-num-batched-tokens: 16384 - trust-remote-code: true - no-enable-prefix-caching: true - no-enable-flashinfer-autotune: true - no-async-scheduling: true - block-size: 256 - gpu-memory-utilization: 0.9 - enable-ep-weight-filter: true - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 16 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - attention-config: '{"use_fp4_indexer_cache": true}' - moe-backend: "deep_gemm_mega_moe" - max-model-len: 16384 - max-num-seqs: 512 - max-cudagraph-capture-size: 512 - max-num-batched-tokens: 512 - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - gpu-memory-utilization: 0.9 - stream-interval: 50 - no-disable-hybrid-kv-cache-manager: true - enable-ep-weight-filter: true - all2all-backend: "flashinfer_nvlink_one_sided" - no-enable-flashinfer-autotune: true - enable-sleep-mode: true - tokenizer-mode: deepseek_v4 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "3072" - req_rate: "inf" - tokenizer_mode: "deepseek_v4" - use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-agentic.yaml deleted file mode 100644 index b64727617..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-agentic.yaml +++ /dev/null @@ -1,161 +0,0 @@ -name: "svf-vllm-disagg-gb200-2p1d-dep8-dep8-agentic" - -# High-throughput topology: two DEP8 prefill workers and one -# DEP8 decode worker. All three workers use attention DP8 / expert EP8, for -# 24 inference GPUs total. This makes the previously manual-only DEP/DEP -# experiment reproducible while the June 21 AgentX frontier is retuned. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.23.0" - precision: "fp4" - -dynamo: - install: true - wheel: "1.3.0.dev20260618" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 4 - decode_nodes: 2 - prefill_workers: 2 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - env: - DYN_REQUEST_PLANE: "tcp" - # Dynamo's 10-second default can expire during brief etcd stalls while - # many long-loading DEP ranks initialize. Slurm still detects hard exits. - ETCD_LEASE_TTL: "120" - args: - router-mode: "kv" - router-reset-states: true - -backend: - type: vllm - connector: null - prefill_environment: - DYN_REQUEST_PLANE: "tcp" - ETCD_LEASE_TTL: "120" - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - TORCH_SYMMMEM: "NVSHMEM" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - # Long AgentX requests can make one DEP rank hold the EP step beyond - # vLLM's 300-second model-execution timeout. - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_RCACHE_MAX_UNRELEASED: "1024" - UCX_TLS: "cuda_copy,rc" - NCCL_P2P_LEVEL: NVL - decode_environment: - DYN_REQUEST_PLANE: "tcp" - ETCD_LEASE_TTL: "120" - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - TORCH_SYMMMEM: "NVSHMEM" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_RCACHE_MAX_UNRELEASED: "1024" - UCX_TLS: "cuda_copy,rc" - NCCL_P2P_LEVEL: NVL - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enable-ep-weight-filter: true - attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_prefill_query_quantization": true, "use_fp4_indexer_cache": true}' - # Let vLLM select max-num-seqs for the long-context AgentX trace. - # max-num-seqs: 256 - # DEP8 holds a full model replica per GPU. A 16K prefill step requires - # a 9.84 GiB FP4 MoE intermediate and OOMs before serving traffic. - max-num-batched-tokens: 8192 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"PIECEWISE"}' - gpu-memory-utilization: 0.9 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: deepseek_v4 - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enable-ep-weight-filter: true - attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_fp4_indexer_cache": true}' - # Let vLLM select max-num-seqs and max-num-batched-tokens. - # max-num-seqs: 512 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - gpu-memory-utilization: 0.9 - stream-interval: 10 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: deepseek_v4 - -sbatch_directives: - cpus-per-task: "72" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - aiperf_server_metrics: true - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p2d-tep8-tp8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p2d-tep8-tp8-agentic.yaml deleted file mode 100644 index 9e2b66584..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p2d-tep8-tp8-agentic.yaml +++ /dev/null @@ -1,155 +0,0 @@ -name: "svf-vllm-disagg-gb200-3p2d-tep8-tp8-agentic" - -# Middle/high-interactivity topology: three cache-affinitized TEP8 prefill -# workers and two independent TP8 decode workers, for 40 inference GPUs. -# The dense historical curve covered roughly 47--71 output tok/s/user through -# c16--c80. This recipe re-establishes that curve on the June 21 AgentX corpus -# with the current vLLM/Dynamo runtime contract. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.23.0" - precision: "fp4" - -dynamo: - install: true - wheel: "1.3.0.dev20260618" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 6 - decode_nodes: 4 - prefill_workers: 3 - decode_workers: 2 - gpus_per_prefill: 8 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - env: - DYN_REQUEST_PLANE: "tcp" - ETCD_LEASE_TTL: "120" - args: - router-mode: "kv" - router-reset-states: true - -backend: - type: vllm - connector: null - prefill_environment: - DYN_REQUEST_PLANE: "tcp" - ETCD_LEASE_TTL: "120" - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - TORCH_SYMMMEM: "NVSHMEM" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_RCACHE_MAX_UNRELEASED: "1024" - UCX_TLS: "cuda_copy,rc" - NCCL_P2P_LEVEL: NVL - decode_environment: - DYN_REQUEST_PLANE: "tcp" - ETCD_LEASE_TTL: "120" - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - TORCH_SYMMMEM: "NVSHMEM" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_RCACHE_MAX_UNRELEASED: "1024" - UCX_TLS: "cuda_copy,rc" - NCCL_P2P_LEVEL: NVL - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 8 - pipeline-parallel-size: 1 - enable-expert-parallel: true - enable-ep-weight-filter: true - attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_prefill_query_quantization": true, "use_fp4_indexer_cache": true}' - # Let vLLM select max-num-seqs for the long-context AgentX trace. - # max-num-seqs: 16 - max-num-batched-tokens: 16384 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - # FULL capture conflicts with NCCL symmetric-memory registration on the - # TP prefill path; PIECEWISE preserves compilation and safe graph regions. - compilation-config: '{"cudagraph_mode":"PIECEWISE"}' - gpu-memory-utilization: 0.9 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: deepseek_v4 - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 8 - pipeline-parallel-size: 1 - attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_fp4_indexer_cache": true}' - # Let vLLM select max-num-seqs and max-num-batched-tokens. - # max-num-seqs: 512 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - # TP8 all-reduce uses NCCL symmetric memory, which is incompatible with - # the full graph-capture path. Keep graph-incompatible collectives eager. - compilation-config: '{"cudagraph_mode":"PIECEWISE"}' - gpu-memory-utilization: 0.9 - stream-interval: 10 - all2all-backend: "flashinfer_nvlink_one_sided" - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: deepseek_v4 - -sbatch_directives: - cpus-per-task: "72" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - aiperf_server_metrics: true - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p6d-dep4-tp4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p6d-dep4-tp4-agentic.yaml deleted file mode 100644 index 13c8d353e..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p6d-dep4-tp4-agentic.yaml +++ /dev/null @@ -1,177 +0,0 @@ -name: "svf-vllm-disagg-gb300-1p6d-dep4-tp4-agentic" - -# Agentic-coding variant of vllm/deepseek-v4/8k1k/disagg-gb300-1p6d-dep4-tp4.yaml. -# Topology is identical (1 prefill DEP=4 + 6 decode TP=4, 28 GPUs across 7 -# GB300 nodes + 1 dedicated NATS/etcd infra node) so we can compare against -# the fixed-seq-len 1p6d baseline at the same concurrency point (192). -# -# Divergence vs the 8k1k sibling: -# - benchmark.type: sa-bench -> custom (hands off to agentic_srt.sh) -# - max-model-len: removed (let vLLM derive from model config; agentic -# trajectories blow past any small explicit cap) -# - no-enable-prefix-caching: dropped (prefix caching MUST be on for -# trajectory reuse — entire point of agentic) -# Note: --enable-auto-tool-choice / --tool-call-parser / --reasoning-parser -# are NOT set on the worker. The dynamo-vllm worker entrypoint doesn't -# accept them (different arg parser than `vllm serve`). In disagg, chat -# parsing happens at the dynamo frontend, not at the worker. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.21.0-ubuntu2404" - precision: "fp4" - -dynamo: - install: true - wheel: "1.2.0.dev20260426" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 6 - prefill_workers: 1 - decode_workers: 6 - gpus_per_prefill: 4 - gpus_per_decode: 4 - -infra: - etcd_nats_dedicated_node: true - # Raise NATS server max_payload from the 1 MiB default to 32 MiB. - # Agentic prompts at 50k-200k DSv4 tokens serialize to JSON at ~10-15 - # bytes/token, easily clearing 1-3 MB per request. Without this, every - # long-prompt prefill RPC gets rejected by the NATS server with - # "maximum payload exceeded" (visible in infra.out), and the dynamo - # frontend surfaces a misleading "NATS request ... deadline has elapsed" - # (it never gets a reply because the publish was rejected). 32 MiB gives - # ~10x headroom over the largest observed payload (3.2 MB) without - # crossing NATS's 64 MiB hard cap or Dynamo's 16 MiB advisory limit. - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - prefill_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - TORCH_SYMMMEM: "NVSHMEM" - decode_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - TORCH_SYMMMEM: "NVSHMEM" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - attention-config: '{"use_fp4_indexer_cache": true}' - moe-backend: "deep_gemm_mega_moe" - # enforce-eager: true - # max-num-seqs: 256 - max-num-batched-tokens: 16384 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - gpu-memory-utilization: 0.9 - enable-ep-weight-filter: true - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - tokenizer-mode: deepseek_v4 - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - # max-num-seqs: 512 - trust-remote-code: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - gpu-memory-utilization: 0.9 - stream-interval: 10 - no-disable-hybrid-kv-cache-manager: true - enable-ep-weight-filter: true - all2all-backend: "flashinfer_nvlink_one_sided" - no-enable-flashinfer-autotune: true - enable-sleep-mode: true - tokenizer-mode: deepseek_v4 - -# sbatch + srun resource grants for clusters without per-GPU defaults. -# -# mem=0: allocate all available node memory (~868 GB on CW gb300). Without -# this, sbatch only requests ntasks × DefMemPerCPU = 8 × 4 GB = 32 GB for -# the whole job and worker cgroups OOM-kill mid model load (R7-R11 hit -# this; sacct showed AllocTRES mem=4G per step). -# -# cpus-per-task=72: give each task one CW gb300 NUMA socket (144 cores -# split 2 × 72). Critical for the *infra step* (etcd + nats) which -# srtctl spawns without --gres=gpu — on CW that means DefMemPerCPU -# applies and the step gets 1 CPU by default. With 24 dynamo DP ranks -# all hammering etcd for lease keep-alives, single-CPU etcd can't keep -# up and dies (R12 hit this; etcd reported max-cpu-set=1, leases -# deadline-exceeded, infra SIGKILL'd at 16:35:49). 72 CPUs is plenty -# for both etcd + nats AND for vLLM worker auxiliary threads. -# -# nv gb300 doesn't need this because cluster default DefCpuPerGPU=35 -# auto-allocates 4*35=140 CPUs per GPU-bearing task; cw has no per-GPU -# default. Setting it here is safe on both because the value is ≤ node -# CPU count. -# -# srun_options.mem=0 forces each srun step to use the full node memory -# (without it, srun steps default back to cpus_per_task × DefMemPerCPU). -# Docs: docs/config-reference.md#sbatch_directives + #srun_options. -sbatch_directives: - mem: "0" - cpus-per-task: "72" -srun_options: - mem: "0" - # gb300-nv: pyxis maps the calling user (sa-shared) into the container as - # uid 345200007. dpkg refuses to run without EUID 0 even though - # ENROOT_ROOTFS_WRITABLE=1 makes the rootfs writable, so the agentic_srt - # apt-get install git step fails. --container-remap-root asks pyxis to - # remap us to uid 0 inside the container. srt-slurm renders empty-string - # values as flag-only srun args (see core/slurm.py:250). - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - # Container-side path of the aiperf mmap dataset cache; the host-side - # mount is wired via launch_gb300-*.sh's srtslurm.yaml default_mounts. - # Without this, aiperf re-tokenizes + re-writes ~65 GB of mmap files - # per dataset on every run. - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - # Persistent HF hub cache (also wired via default_mounts) so the trace - # dataset isn't re-downloaded on every run. Overrides the workflow-level - # HF_HUB_CACHE=/mnt/hf_hub_cache, which doesn't exist on these nodes. - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-4p1d-dep4-dep8-24-c4096-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-4p1d-dep4-dep8-24-c4096-agentic.yaml deleted file mode 100644 index 05b779d54..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-4p1d-dep4-dep8-24-c4096-agentic.yaml +++ /dev/null @@ -1,179 +0,0 @@ -name: "svf-vllm-disagg-gb300-4p1d-dep4-dep8-24-c4096-agentic" - -# Agentic-coding variant of vllm/deepseek-v4/8k1k/disagg-gb300-4p1d-dep4-dep8-24-c4096.yaml. -# Max-throughput shape: 4 prefill (DEP=4 each) + 1 decode (DEP=8). 6 GB300 -# nodes (4P + 2D = 24 GPUs at 4 GPUs/node) plus a dedicated NATS/etcd infra -# node. Sized for concurrency 4096 with deep_gemm_mega_moe on both workers. -# -# Divergence vs the 8k1k sibling: -# - benchmark.type: sa-bench -> custom (hands off to agentic_srt.sh) -# - max-model-len: removed (let vLLM derive from model config; agentic -# trajectories blow past any small explicit cap) -# - no-enable-prefix-caching: dropped (prefix caching MUST be on for -# trajectory reuse — entire point of agentic) -# Note: --enable-auto-tool-choice / --tool-call-parser / --reasoning-parser -# are NOT set on the worker. The dynamo-vllm worker entrypoint doesn't -# accept them (different arg parser than `vllm serve`). In disagg, chat -# parsing happens at the dynamo frontend, not at the worker. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.21.0-ubuntu2404" - precision: "fp4" - -dynamo: - install: true - wheel: "1.2.0.dev20260426" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 4 - decode_nodes: 2 - prefill_workers: 4 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: true - # See sibling 1p6d recipe for rationale — NATS 1 MiB default rejects - # agentic prompts; 32 MiB gives ~10x headroom over observed payloads. - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - TORCH_SYMMMEM: "NVSHMEM" - - decode_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - TORCH_SYMMMEM: "NVSHMEM" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - # enforce-eager: true - # Inherited from fixed-sequence recipes; let vLLM select the scheduler - # sequence limit until this is tuned explicitly for the agentic trace. - # max-num-seqs: 16 - max-num-batched-tokens: 16384 - trust-remote-code: true - no-enable-flashinfer-autotune: true - safetensors-load-strategy: "prefetch" - block-size: 256 - gpu-memory-utilization: 0.9 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: deepseek_v4 - enable-ep-weight-filter: true - enable-sleep-mode: true - moe-backend: "deep_gemm_mega_moe" - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - # max-num-seqs: 512 - trust-remote-code: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - gpu-memory-utilization: 0.9 - stream-interval: 10 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: deepseek_v4 - enable-ep-weight-filter: true - enable-sleep-mode: true - moe-backend: "deep_gemm_mega_moe" - -# sbatch + srun resource grants for clusters without per-GPU defaults. -# -# mem=0: allocate all available node memory (~868 GB on CW gb300). Without -# this, sbatch only requests ntasks × DefMemPerCPU = 8 × 4 GB = 32 GB for -# the whole job and worker cgroups OOM-kill mid model load (R7-R11 hit -# this; sacct showed AllocTRES mem=4G per step). -# -# cpus-per-task=72: give each task one CW gb300 NUMA socket (144 cores -# split 2 × 72). Critical for the *infra step* (etcd + nats) which -# srtctl spawns without --gres=gpu — on CW that means DefMemPerCPU -# applies and the step gets 1 CPU by default. With 24 dynamo DP ranks -# all hammering etcd for lease keep-alives, single-CPU etcd can't keep -# up and dies (R12 hit this; etcd reported max-cpu-set=1, leases -# deadline-exceeded, infra SIGKILL'd at 16:35:49). 72 CPUs is plenty -# for both etcd + nats AND for vLLM worker auxiliary threads. -# -# nv gb300 doesn't need this because cluster default DefCpuPerGPU=35 -# auto-allocates 4*35=140 CPUs per GPU-bearing task; cw has no per-GPU -# default. Setting it here is safe on both because the value is ≤ node -# CPU count. -# -# srun_options.mem=0 forces each srun step to use the full node memory -# (without it, srun steps default back to cpus_per_task × DefMemPerCPU). -# Docs: docs/config-reference.md#sbatch_directives + #srun_options. -sbatch_directives: - mem: "0" - cpus-per-task: "72" -srun_options: - mem: "0" - # gb300-nv: pyxis maps the calling user (sa-shared) into the container as - # uid 345200007. dpkg refuses to run without EUID 0 even though - # ENROOT_ROOTFS_WRITABLE=1 makes the rootfs writable, so the agentic_srt - # apt-get install git step fails. --container-remap-root asks pyxis to - # remap us to uid 0 inside the container. srt-slurm renders empty-string - # values as flag-only srun args (see core/slurm.py:250). - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - # Container-side path of the aiperf mmap dataset cache; the host-side - # mount is wired via launch_gb300-*.sh's srtslurm.yaml default_mounts. - # Without this, aiperf re-tokenizes + re-writes ~65 GB of mmap files - # per dataset on every run. - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - # Persistent HF hub cache (also wired via default_mounts) so the trace - # dataset isn't re-downloaded on every run. Overrides the workflow-level - # HF_HUB_CACHE=/mnt/hf_hub_cache, which doesn't exist on these nodes. - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/1k1k/1p1d-dep4-dep8-1k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/1k1k/1p1d-dep4-dep8-1k1k.yaml deleted file mode 100644 index 710c4d979..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/1k1k/1p1d-dep4-dep8-1k1k.yaml +++ /dev/null @@ -1,107 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb200-1p1d-dep4-dep8-fp8-1k1k" - -# 1P DEP4 prefill (TP1 DP4 EP, 4 GPU/worker) + 1D DEP8 decode (TP1 DP8 EP, 8 GPU/worker = 2 nodes each). -# GB200 has 4 GPUs/node. -# Nodes: 1 prefill + 2 decode (+ head/infra). - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" - precision: "fp8" - - -dynamo: - install: true - version: 1.3.0.dev20260614 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 2 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "1024x4096" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/1k1k/1p1d-dep4-tep8-1k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/1k1k/1p1d-dep4-tep8-1k1k.yaml deleted file mode 100644 index c1b9cf32d..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/1k1k/1p1d-dep4-tep8-1k1k.yaml +++ /dev/null @@ -1,105 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb200-1p1d-dep4-tep8-fp8-1k1k" - -# 1P DEP4 prefill (TP1 DP4 EP, 4 GPU/worker) + 1D TEP8 decode (TP8 EP, 8 GPU/worker = 2 nodes each). -# GB200 has 4 GPUs/node. -# Nodes: 1 prefill + 2 decode (+ head/infra). - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" - precision: "fp8" - - -dynamo: - install: true - version: 1.3.0.dev20260614 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 2 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 8 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 8196 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "128x256" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/1k1k/1p2d-dep4-dep8-1k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/1k1k/1p2d-dep4-dep8-1k1k.yaml deleted file mode 100644 index 907633ba7..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/1k1k/1p2d-dep4-dep8-1k1k.yaml +++ /dev/null @@ -1,107 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb200-1p2d-dep4-dep8-fp8-1k1k" - -# 1P DEP4 prefill (TP1 DP4 EP, 4 GPU/worker) + 2D DEP8 decode (TP1 DP8 EP, 8 GPU/worker = 2 nodes each). -# GB200 has 4 GPUs/node. -# Nodes: 1 prefill + 4 decode (+ head/infra). - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" - precision: "fp8" - - -dynamo: - install: true - version: 1.3.0.dev20260614 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 4 - prefill_workers: 1 - decode_workers: 2 - gpus_per_prefill: 4 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "1024x4096" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/1k1k/1p2d-dep4-tep8-1k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/1k1k/1p2d-dep4-tep8-1k1k.yaml deleted file mode 100644 index 2e2bbc12a..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/1k1k/1p2d-dep4-tep8-1k1k.yaml +++ /dev/null @@ -1,105 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb200-1p2d-dep4-tep8-fp8-1k1k" - -# 1P DEP4 prefill (TP1 DP4 EP, 4 GPU/worker) + 2D TEP8 decode (TP8 EP, 8 GPU/worker = 2 nodes each). -# GB200 has 4 GPUs/node. -# Nodes: 1 prefill + 4 decode (+ head/infra). - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" - precision: "fp8" - - -dynamo: - install: true - version: 1.3.0.dev20260614 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 4 - prefill_workers: 1 - decode_workers: 2 - gpus_per_prefill: 4 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 8 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 8196 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "4x16x64" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/1k1k/1p4d-dep4-tp4-marlin-1k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/1k1k/1p4d-dep4-tp4-marlin-1k1k.yaml deleted file mode 100644 index 429aa015e..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/1k1k/1p4d-dep4-tp4-marlin-1k1k.yaml +++ /dev/null @@ -1,107 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb200-1p4d-dep4-tp4-marlin-fp8-1k1k" - -# 1P DEP4 prefill (TP1 DP4 EP, 4 GPU/worker) + 4D TP4 decode (TP4, 4 GPU/worker = 1 node each). -# Decode uses Marlin MoE backend with expert-parallel disabled (1p4d only). -# GB200 has 4 GPUs/node. -# Nodes: 1 prefill + 4 decode (+ head/infra). - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" - precision: "fp8" - - -dynamo: - install: true - version: 1.3.0.dev20260614 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 4 - prefill_workers: 1 - decode_workers: 4 - gpus_per_prefill: 4 - gpus_per_decode: 4 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 4 - moe-backend: marlin - enable-expert-parallel: false - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 8196 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "32" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/1p2d-dep4-dep8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/1p2d-dep4-dep8-8k1k.yaml deleted file mode 100644 index 4129408eb..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/1p2d-dep4-dep8-8k1k.yaml +++ /dev/null @@ -1,107 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb200-1p2d-dep4-dep8-fp8-8k1k" - -# 1P DEP4 prefill (TP1 DP4 EP, 4 GPU/worker) + 2D DEP8 decode (TP1 DP8 EP, 8 GPU/worker = 2 nodes each). -# GB200 has 4 GPUs/node. -# Nodes: 1 prefill + 4 decode (+ head/infra). - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" - precision: "fp8" - - -dynamo: - install: true - version: 1.3.0.dev20260614 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 4 - prefill_workers: 1 - decode_workers: 2 - gpus_per_prefill: 4 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "512" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/1p2d-dep4-tep4-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/1p2d-dep4-tep4-8k1k.yaml deleted file mode 100644 index 5bde9ac9b..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/1p2d-dep4-tep4-8k1k.yaml +++ /dev/null @@ -1,105 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb200-1p2d-dep4-tep4-fp8-8k1k" - -# 1P DEP4 prefill (TP1 DP4 EP, 4 GPU/worker) + 2D TEP4 decode (TP4 EP, 4 GPU/worker = 1 node each). -# GB200 has 4 GPUs/node. -# Nodes: 1 prefill + 2 decode (+ head/infra). - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" - precision: "fp8" - - -dynamo: - install: true - version: 1.3.0.dev20260614 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 2 - prefill_workers: 1 - decode_workers: 2 - gpus_per_prefill: 4 - gpus_per_decode: 4 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 4 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-num-seqs: 512 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 4096 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "4" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/1p2d-dep4-tep8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/1p2d-dep4-tep8-8k1k.yaml deleted file mode 100644 index 56ed02e5e..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/1p2d-dep4-tep8-8k1k.yaml +++ /dev/null @@ -1,105 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb200-1p2d-dep4-tep8-fp8-8k1k" - -# 1P DEP4 prefill (TP1 DP4 EP, 4 GPU/worker) + 2D TEP8 decode (TP8 EP, 8 GPU/worker = 2 nodes each). -# GB200 has 4 GPUs/node. -# Nodes: 1 prefill + 4 decode (+ head/infra). - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" - precision: "fp8" - - -dynamo: - install: true - version: 1.3.0.dev20260614 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 4 - prefill_workers: 1 - decode_workers: 2 - gpus_per_prefill: 4 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 8 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 4096 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "4x16x32x64x128" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/2p2d-dep4-dep8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/2p2d-dep4-dep8-8k1k.yaml deleted file mode 100644 index 22ebfc783..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/2p2d-dep4-dep8-8k1k.yaml +++ /dev/null @@ -1,107 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb200-2p2d-dep4-dep8-fp8-8k1k" - -# 2P DEP4 prefill (TP1 DP4 EP, 4 GPU/worker) + 2D DEP8 decode (TP1 DP8 EP, 8 GPU/worker = 2 nodes each). -# GB200 has 4 GPUs/node. -# Nodes: 1 prefill + 4 decode (+ head/infra). - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" - precision: "fp8" - - -dynamo: - install: true - version: 1.3.0.dev20260614 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 2 - decode_nodes: 4 - prefill_workers: 2 - decode_workers: 2 - gpus_per_prefill: 4 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "512x1024" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/3p2d-dep4-dep8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/3p2d-dep4-dep8-8k1k.yaml deleted file mode 100644 index ba1db206e..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/3p2d-dep4-dep8-8k1k.yaml +++ /dev/null @@ -1,107 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb200-3p2d-dep4-dep8-fp8-8k1k" - -# 3P DEP4 prefill (TP1 DP4 EP, 4 GPU/worker) + 2D DEP8 decode (TP1 DP8 EP, 8 GPU/worker = 2 nodes each). -# GB200 has 4 GPUs/node. -# Nodes: 2 prefill + 4 decode (+ head/infra). - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" - precision: "fp8" - - -dynamo: - install: true - version: 1.3.0.dev20260614 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 3 - decode_nodes: 4 - prefill_workers: 3 - decode_workers: 2 - gpus_per_prefill: 4 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "4096" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/5p2d-dep4-dep8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/5p2d-dep4-dep8-8k1k.yaml deleted file mode 100644 index a54ddcb22..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb200-fp8/8k1k/5p2d-dep4-dep8-8k1k.yaml +++ /dev/null @@ -1,107 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb200-5p2d-dep4-dep8-fp8-8k1k" - -# 5P DEP4 prefill (TP1 DP4 EP, 4 GPU/worker) + 2D DEP8 decode (TP1 DP8 EP, 8 GPU/worker = 2 nodes each). -# GB200 has 4 GPUs/node. -# Nodes: 3 prefill + 4 decode (+ head/infra). - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" - precision: "fp8" - - -dynamo: - install: true - version: 1.3.0.dev20260614 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 5 - decode_nodes: 4 - prefill_workers: 5 - decode_workers: 2 - gpus_per_prefill: 4 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "4096" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/1k1k/1p1d-dep2-dep4-1k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/1k1k/1p1d-dep2-dep4-1k1k.yaml deleted file mode 100644 index f57d7af09..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/1k1k/1p1d-dep2-dep4-1k1k.yaml +++ /dev/null @@ -1,107 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb300-1p1d-dep2-dep4-fp8-1k1k" - -# 1P DEP2 prefill (TP1 DP2 EP, 2 GPU/worker) + 1D DEP4 decode (TP1 DP4 EP, 4 GPU/worker = 1 node each). -# GB300 has 4 GPUs/node. Adapted from NV B300 PR #1863. -# Nodes: 1 prefill + 1 decode (+ head/infra). - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" - precision: "fp8" - - -dynamo: - install: true - version: 1.3.0.dev20260614 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 2 - gpus_per_decode: 4 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "8192" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/1k1k/1p2d-dep2-tep8-1k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/1k1k/1p2d-dep2-tep8-1k1k.yaml deleted file mode 100644 index b4f457654..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/1k1k/1p2d-dep2-tep8-1k1k.yaml +++ /dev/null @@ -1,105 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb300-1p2d-dep2-tep8-fp8-1k1k" - -# 1P DEP2 prefill (TP1 DP2 EP, 2 GPU/worker) + 2D TEP8 decode (TP8 EP, 8 GPU/worker = 2 nodes each). -# GB300 has 4 GPUs/node. Adapted from NV B300 PR #1863. -# Nodes: 1 prefill + 4 decode (+ head/infra). - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" - precision: "fp8" - - -dynamo: - install: true - version: 1.3.0.dev20260614 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 4 - prefill_workers: 1 - decode_workers: 2 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 8 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 8196 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "4x16x64x128x256" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/1k1k/2p2d-dep2-tep8-1k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/1k1k/2p2d-dep2-tep8-1k1k.yaml deleted file mode 100644 index 6bba9ea86..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/1k1k/2p2d-dep2-tep8-1k1k.yaml +++ /dev/null @@ -1,105 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb300-2p2d-dep2-tep8-fp8-1k1k" - -# 2P DEP2 prefill (TP1 DP2 EP, 2 GPU/worker) + 2D TEP8 decode (TP8 EP, 8 GPU/worker = 2 nodes each). -# GB300 has 4 GPUs/node. Adapted from NV B300 PR #1863. -# Nodes: 1 prefill + 4 decode (+ head/infra). - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" - precision: "fp8" - - -dynamo: - install: true - version: 1.3.0.dev20260614 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 4 - prefill_workers: 2 - decode_workers: 2 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 8 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 8196 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "32" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/1k1k/2p3d-dep2-dep4-1k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/1k1k/2p3d-dep2-dep4-1k1k.yaml deleted file mode 100644 index de852e427..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/1k1k/2p3d-dep2-dep4-1k1k.yaml +++ /dev/null @@ -1,107 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb300-2p3d-dep2-dep4-fp8-1k1k" - -# 2P DEP2 prefill (TP1 DP2 EP, 2 GPU/worker) + 3D DEP4 decode (TP1 DP4 EP, 4 GPU/worker = 1 node each). -# GB300 has 4 GPUs/node. Adapted from NV B300 PR #1863. -# Nodes: 1 prefill + 3 decode (+ head/infra). - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" - precision: "fp8" - - -dynamo: - install: true - version: 1.3.0.dev20260614 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 3 - prefill_workers: 2 - decode_workers: 3 - gpus_per_prefill: 2 - gpus_per_decode: 4 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "8192" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/1k1k/2p4d-dep2-dep4-1k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/1k1k/2p4d-dep2-dep4-1k1k.yaml deleted file mode 100644 index 8f7b7b140..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/1k1k/2p4d-dep2-dep4-1k1k.yaml +++ /dev/null @@ -1,107 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb300-2p4d-dep2-dep4-fp8-1k1k" - -# 2P DEP2 prefill (TP1 DP2 EP, 2 GPU/worker) + 4D DEP4 decode (TP1 DP4 EP, 4 GPU/worker = 1 node each). -# GB300 has 4 GPUs/node. Adapted from NV B300 PR #1863. -# Nodes: 1 prefill + 4 decode (+ head/infra). - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" - precision: "fp8" - - -dynamo: - install: true - version: 1.3.0.dev20260614 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 4 - prefill_workers: 2 - decode_workers: 4 - gpus_per_prefill: 2 - gpus_per_decode: 4 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "8192" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/1k1k/4p2d-dep2-dep8-1k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/1k1k/4p2d-dep2-dep8-1k1k.yaml deleted file mode 100644 index f6cf6a59f..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/1k1k/4p2d-dep2-dep8-1k1k.yaml +++ /dev/null @@ -1,107 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb300-4p2d-dep2-dep8-fp8-1k1k" - -# 4P DEP2 prefill (TP1 DP2 EP, 2 GPU/worker) + 2D DEP8 decode (TP1 DP8 EP, 8 GPU/worker = 2 nodes each). -# GB300 has 4 GPUs/node. Adapted from NV B300 PR #1863. -# Nodes: 2 prefill + 4 decode (+ head/infra). - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" - precision: "fp8" - - -dynamo: - install: true - version: 1.3.0.dev20260614 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 2 - decode_nodes: 4 - prefill_workers: 4 - decode_workers: 2 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "1024x4096" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/1p1d-dep2-dep8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/1p1d-dep2-dep8-8k1k.yaml deleted file mode 100644 index d990d661b..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/1p1d-dep2-dep8-8k1k.yaml +++ /dev/null @@ -1,107 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb300-1p1d-dep2-dep8-fp8-8k1k" - -# 1P DEP2 prefill (TP1 DP2 EP, 2 GPU/worker) + 1D DEP8 decode (TP1 DP8 EP, 8 GPU/worker = 2 nodes each). -# GB300 has 4 GPUs/node. Adapted from NV B300 PR #1863. -# Nodes: 1 prefill + 2 decode (+ head/infra). - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" - precision: "fp8" - - -dynamo: - install: true - version: 1.3.0.dev20260614 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 2 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "256" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/1p1d-dep2-tep8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/1p1d-dep2-tep8-8k1k.yaml deleted file mode 100644 index d46133924..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/1p1d-dep2-tep8-8k1k.yaml +++ /dev/null @@ -1,105 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb300-1p1d-dep2-tep8-fp8-8k1k" - -# 1P DEP2 prefill (TP1 DP2 EP, 2 GPU/worker) + 1D TEP8 decode (TP8 EP, 8 GPU/worker = 2 nodes each). -# GB300 has 4 GPUs/node. Adapted from NV B300 PR #1863. -# Nodes: 1 prefill + 2 decode (+ head/infra). - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" - precision: "fp8" - - -dynamo: - install: true - version: 1.3.0.dev20260614 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 2 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 8 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 4096 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "128" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/1p2d-dep2-tep8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/1p2d-dep2-tep8-8k1k.yaml deleted file mode 100644 index e8c606e27..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/1p2d-dep2-tep8-8k1k.yaml +++ /dev/null @@ -1,105 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb300-1p2d-dep2-tep8-fp8-8k1k" - -# 1P DEP2 prefill (TP1 DP2 EP, 2 GPU/worker) + 2D TEP8 decode (TP8 EP, 8 GPU/worker = 2 nodes each). -# GB300 has 4 GPUs/node. Adapted from NV B300 PR #1863. -# Nodes: 1 prefill + 4 decode (+ head/infra). - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" - precision: "fp8" - - -dynamo: - install: true - version: 1.3.0.dev20260614 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 4 - prefill_workers: 1 - decode_workers: 2 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 8 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 4096 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "32x64x128" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/2p1d-dep2-dep8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/2p1d-dep2-dep8-8k1k.yaml deleted file mode 100644 index 02c3be14a..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/2p1d-dep2-dep8-8k1k.yaml +++ /dev/null @@ -1,107 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb300-2p1d-dep2-dep8-fp8-8k1k" - -# 2P DEP2 prefill (TP1 DP2 EP, 2 GPU/worker) + 1D DEP8 decode (TP1 DP8 EP, 8 GPU/worker = 2 nodes each). -# GB300 has 4 GPUs/node. Adapted from NV B300 PR #1863. -# Nodes: 1 prefill + 2 decode (+ head/infra). - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" - precision: "fp8" - - -dynamo: - install: true - version: 1.3.0.dev20260614 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 2 - prefill_workers: 2 - decode_workers: 1 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "512" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/2p2d-dep2-tep8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/2p2d-dep2-tep8-8k1k.yaml deleted file mode 100644 index 304650d6c..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/2p2d-dep2-tep8-8k1k.yaml +++ /dev/null @@ -1,105 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb300-2p2d-dep2-tep8-fp8-8k1k" - -# 2P DEP2 prefill (TP1 DP2 EP, 2 GPU/worker) + 2D TEP8 decode (TP8 EP, 8 GPU/worker = 2 nodes each). -# GB300 has 4 GPUs/node. Adapted from NV B300 PR #1863. -# Nodes: 1 prefill + 4 decode (+ head/infra). - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" - precision: "fp8" - - -dynamo: - install: true - version: 1.3.0.dev20260614 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 4 - prefill_workers: 2 - decode_workers: 2 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 8 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 4096 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "16" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/2p4d-dep2-tep4-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/2p4d-dep2-tep4-8k1k.yaml deleted file mode 100644 index efea8bfac..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/2p4d-dep2-tep4-8k1k.yaml +++ /dev/null @@ -1,105 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb300-2p4d-dep2-tep4-fp8-8k1k" - -# 2P DEP2 prefill (TP1 DP2 EP, 2 GPU/worker) + 4D TEP4 decode (TP4 EP, 4 GPU/worker = 1 node each). -# GB300 has 4 GPUs/node. Adapted from NV B300 PR #1863. -# Nodes: 1 prefill + 4 decode (+ head/infra). - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" - precision: "fp8" - - -dynamo: - install: true - version: 1.3.0.dev20260614 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 4 - prefill_workers: 2 - decode_workers: 4 - gpus_per_prefill: 2 - gpus_per_decode: 4 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 4 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 32 - max-num-seqs: 512 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 4096 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "4" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/3p1d-dep2-dep8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/3p1d-dep2-dep8-8k1k.yaml deleted file mode 100644 index 97e1ec88c..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/3p1d-dep2-dep8-8k1k.yaml +++ /dev/null @@ -1,107 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb300-3p1d-dep2-dep8-fp8-8k1k" - -# 3P DEP2 prefill (TP1 DP2 EP, 2 GPU/worker) + 1D DEP8 decode (TP1 DP8 EP, 8 GPU/worker = 2 nodes each). -# GB300 has 4 GPUs/node. Adapted from NV B300 PR #1863. -# Nodes: 2 prefill + 2 decode (+ head/infra). - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" - precision: "fp8" - - -dynamo: - install: true - version: 1.3.0.dev20260614 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 2 - decode_nodes: 2 - prefill_workers: 3 - decode_workers: 1 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "1024" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/3p2d-dep2-dep8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/3p2d-dep2-dep8-8k1k.yaml deleted file mode 100644 index 745b2fad4..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/3p2d-dep2-dep8-8k1k.yaml +++ /dev/null @@ -1,107 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb300-3p2d-dep2-dep8-fp8-8k1k" - -# 3P DEP2 prefill (TP1 DP2 EP, 2 GPU/worker) + 2D DEP8 decode (TP1 DP8 EP, 8 GPU/worker = 2 nodes each). -# GB300 has 4 GPUs/node. Adapted from NV B300 PR #1863. -# Nodes: 2 prefill + 4 decode (+ head/infra). - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" - precision: "fp8" - - -dynamo: - install: true - version: 1.3.0.dev20260614 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 2 - decode_nodes: 4 - prefill_workers: 3 - decode_workers: 2 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "512" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/6p1d-dep2-dep8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/6p1d-dep2-dep8-8k1k.yaml deleted file mode 100644 index 9be5cc177..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/6p1d-dep2-dep8-8k1k.yaml +++ /dev/null @@ -1,107 +0,0 @@ -name: "minimax-m3-vllm-disagg-gb300-6p1d-dep2-dep8-fp8-8k1k" - -# 6P DEP2 prefill (TP1 DP2 EP, 2 GPU/worker) + 1D DEP8 decode (TP1 DP8 EP, 8 GPU/worker = 2 nodes each). -# GB300 has 4 GPUs/node. Adapted from NV B300 PR #1863. -# Nodes: 3 prefill + 2 decode (+ head/infra). - -model: - path: "minimax-m3-mxfp8" - container: "vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223" - precision: "fp8" - - -dynamo: - install: true - version: 1.3.0.dev20260614 - -health_check: - max_attempts: 720 - interval_seconds: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "72" - -srun_options: - mem: "0" - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 3 - decode_nodes: 2 - prefill_workers: 6 - decode_workers: 1 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - decode_environment: - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - tensor-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - kv-cache-dtype: fp8 - stream-interval: 100 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "2048" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/1k1k/1p1d-dep2-tep8-1k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/1k1k/1p1d-dep2-tep8-1k1k.yaml deleted file mode 100644 index 486af0557..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/1k1k/1p1d-dep2-tep8-1k1k.yaml +++ /dev/null @@ -1,81 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-1p1d-fp4-dep2-tep8-1k1k" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41" - precision: "fp4" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -dynamo: - install: true - version: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - decode_environment: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 8 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 8192 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "4x16x64x128x4096" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/1k1k/1p1d-dep2-tp4-1k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/1k1k/1p1d-dep2-tp4-1k1k.yaml deleted file mode 100644 index 532b78a10..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/1k1k/1p1d-dep2-tp4-1k1k.yaml +++ /dev/null @@ -1,82 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-1p1d-fp4-dep2-tp4-1k1k" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41" - precision: "fp4" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 2 - gpus_per_decode: 4 - -dynamo: - install: true - version: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - allow_prefill_decode_colocation: true - - prefill_environment: - UCX_TLS: "cuda_ipc,cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - decode_environment: - UCX_TLS: "cuda_ipc,cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 4 - enable-expert-parallel: false - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "1x4x8x16" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/1k1k/1p2d-dep2-dep4-1k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/1k1k/1p2d-dep2-dep4-1k1k.yaml deleted file mode 100644 index fde8442a1..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/1k1k/1p2d-dep2-dep4-1k1k.yaml +++ /dev/null @@ -1,83 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-1p2d-fp4-dep2-dep4-1k1k" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41" - precision: "fp4" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 2 - gpus_per_prefill: 2 - gpus_per_decode: 4 - -dynamo: - install: true - version: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - decode_environment: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 8192 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "2048" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/1k1k/2p1d-dep2-dep8-1k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/1k1k/2p1d-dep2-dep8-1k1k.yaml deleted file mode 100644 index ed3b5f995..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/1k1k/2p1d-dep2-dep8-1k1k.yaml +++ /dev/null @@ -1,83 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-2p1d-fp4-dep2-dep8-1k1k" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41" - precision: "fp4" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 2 - decode_workers: 1 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -dynamo: - install: true - version: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - decode_environment: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 8192 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "512x4096" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/1k1k/2p1d-dep2-tep8-1k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/1k1k/2p1d-dep2-tep8-1k1k.yaml deleted file mode 100644 index 0784283b9..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/1k1k/2p1d-dep2-tep8-1k1k.yaml +++ /dev/null @@ -1,81 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-2p1d-fp4-dep2-tep8-1k1k" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41" - precision: "fp4" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 2 - decode_workers: 1 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -dynamo: - install: true - version: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - decode_environment: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 8 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 8192 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "32" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/1k1k/2p2d-dep2-tep8-1k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/1k1k/2p2d-dep2-tep8-1k1k.yaml deleted file mode 100644 index 59c52da00..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/1k1k/2p2d-dep2-tep8-1k1k.yaml +++ /dev/null @@ -1,81 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-2p2d-fp4-dep2-tep8-1k1k" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41" - precision: "fp4" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 2 - prefill_workers: 2 - decode_workers: 2 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -dynamo: - install: true - version: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - decode_environment: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 8 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 8192 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "16" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/1k1k/3p2d-dep2-tep8-1k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/1k1k/3p2d-dep2-tep8-1k1k.yaml deleted file mode 100644 index 7e9f7dec3..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/1k1k/3p2d-dep2-tep8-1k1k.yaml +++ /dev/null @@ -1,81 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-3p2d-fp4-dep2-tep8-1k1k" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41" - precision: "fp4" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 2 - prefill_workers: 3 - decode_workers: 2 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -dynamo: - install: true - version: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - decode_environment: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 8 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 8192 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "4" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/1p1d-dep2-tp4-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/1p1d-dep2-tp4-8k1k.yaml deleted file mode 100644 index be2683d0c..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/1p1d-dep2-tp4-8k1k.yaml +++ /dev/null @@ -1,86 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-1p1d-fp4-dep2-tp4-8k1k" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41" - precision: "fp4" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 2 - gpus_per_decode: 4 - -dynamo: - install: true - version: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - allow_prefill_decode_colocation: true - - prefill_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_ipc,cuda_copy,rc" - - decode_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_ipc,cuda_copy,rc" - - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 4 - enable-expert-parallel: false - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "1x4x8x16" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/1p2d-dep2-tep4-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/1p2d-dep2-tep4-8k1k.yaml deleted file mode 100644 index 5be198f11..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/1p2d-dep2-tep4-8k1k.yaml +++ /dev/null @@ -1,85 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-1p2d-fp4-dep2-tep4-8k1k" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41" - precision: "fp4" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 2 - gpus_per_prefill: 2 - gpus_per_decode: 4 - -dynamo: - install: true - version: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - - decode_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 4 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-num-seqs: 512 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 4096 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "16" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/1p4d-dep2-tep4-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/1p4d-dep2-tep4-8k1k.yaml deleted file mode 100644 index 90d688f61..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/1p4d-dep2-tep4-8k1k.yaml +++ /dev/null @@ -1,85 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-1p4d-fp4-dep2-tep4-8k1k" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41" - precision: "fp4" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 2 - prefill_workers: 1 - decode_workers: 4 - gpus_per_prefill: 2 - gpus_per_decode: 4 - -dynamo: - install: true - version: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - - decode_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 4 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-num-seqs: 512 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 4096 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "16x32x64x128" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/1p4d-dep2-tep8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/1p4d-dep2-tep8-8k1k.yaml deleted file mode 100644 index 215474282..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/1p4d-dep2-tep8-8k1k.yaml +++ /dev/null @@ -1,85 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-1p4d-fp4-dep2-tep8-8k1k" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41" - precision: "fp4" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 4 - prefill_workers: 1 - decode_workers: 4 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -dynamo: - install: true - version: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - - decode_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 8 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 4096 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "4" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/2p2d-dep2-dep8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/2p2d-dep2-dep8-8k1k.yaml deleted file mode 100644 index c49fd1ccb..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/2p2d-dep2-dep8-8k1k.yaml +++ /dev/null @@ -1,87 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-2p2d-fp4-dep2-dep8-8k1k" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41" - precision: "fp4" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 2 - prefill_workers: 2 - decode_workers: 2 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -dynamo: - install: true - version: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - - decode_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 4096 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "256x512" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/2p2d-dep2-tep8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/2p2d-dep2-tep8-8k1k.yaml deleted file mode 100644 index 1b8dfd627..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/2p2d-dep2-tep8-8k1k.yaml +++ /dev/null @@ -1,85 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-2p2d-fp4-dep2-tep8-8k1k" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41" - precision: "fp4" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 2 - prefill_workers: 2 - decode_workers: 2 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -dynamo: - install: true - version: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - - decode_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 8 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 4096 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "16" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/4p2d-dep2-dep8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/4p2d-dep2-dep8-8k1k.yaml deleted file mode 100644 index 73473aac9..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/4p2d-dep2-dep8-8k1k.yaml +++ /dev/null @@ -1,87 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-4p2d-fp4-dep2-dep8-8k1k" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41" - precision: "fp4" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 2 - prefill_workers: 4 - decode_workers: 2 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -dynamo: - install: true - version: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - - decode_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-num-seqs: 1024 # Per DP rank: 2 workers x DP8 = 16 ranks. - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 4096 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "4096" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/4p2d-dep2-tep4-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/4p2d-dep2-tep4-8k1k.yaml deleted file mode 100644 index 23c99d328..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp4/8k1k/4p2d-dep2-tep4-8k1k.yaml +++ /dev/null @@ -1,85 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-4p2d-fp4-dep2-tep4-8k1k" - -model: - path: "nvidia/MiniMax-M3-NVFP4" - container: "vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41" - precision: "fp4" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 4 - decode_workers: 2 - gpus_per_prefill: 2 - gpus_per_decode: 4 - -dynamo: - install: true - version: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - - decode_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - - vllm_config: - prefill: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - no-enable-flashinfer-autotune: true - tensor-parallel-size: 4 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 4096 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "4096" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/1k1k/1p1d-dep2-tep8-1k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/1k1k/1p1d-dep2-tep8-1k1k.yaml deleted file mode 100644 index 5bbb13362..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/1k1k/1p1d-dep2-tep8-1k1k.yaml +++ /dev/null @@ -1,79 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-1p1d-fp8-dep2-tep8-1k1k" - -model: - path: "MiniMaxAI/MiniMax-M3-MXFP8" - container: "vllm/vllm-openai:minimax-m3-0618-x86_64-cu130" - precision: "fp8" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -dynamo: - install: true - version: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - decode_environment: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - vllm_config: - prefill: - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - tensor-parallel-size: 8 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 8192 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "4x16x64x128x4096" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/1k1k/1p1d-dep2-tp4-marlin-1k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/1k1k/1p1d-dep2-tp4-marlin-1k1k.yaml deleted file mode 100644 index 49a60981e..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/1k1k/1p1d-dep2-tp4-marlin-1k1k.yaml +++ /dev/null @@ -1,81 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-1p1d-fp8-dep2-tp4-marlin-1k1k" - -model: - path: "MiniMaxAI/MiniMax-M3-MXFP8" - container: "vllm/vllm-openai:minimax-m3-0618-x86_64-cu130" - precision: "fp8" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 2 - gpus_per_decode: 4 - -dynamo: - install: true - version: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - allow_prefill_decode_colocation: true - - prefill_environment: - UCX_TLS: "cuda_ipc,cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - decode_environment: - UCX_TLS: "cuda_ipc,cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - vllm_config: - prefill: - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - tensor-parallel-size: 4 - enable-expert-parallel: false - moe-backend: marlin - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "1x4x8x16" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/1k1k/1p2d-dep2-dep4-1k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/1k1k/1p2d-dep2-dep4-1k1k.yaml deleted file mode 100644 index ef7e66d76..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/1k1k/1p2d-dep2-dep4-1k1k.yaml +++ /dev/null @@ -1,81 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-1p2d-fp8-dep2-dep4-1k1k" - -model: - path: "MiniMaxAI/MiniMax-M3-MXFP8" - container: "vllm/vllm-openai:minimax-m3-0618-x86_64-cu130" - precision: "fp8" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 2 - gpus_per_prefill: 2 - gpus_per_decode: 4 - -dynamo: - install: true - version: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - decode_environment: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - vllm_config: - prefill: - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - tensor-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 8192 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "2048" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/1k1k/2p1d-dep2-dep8-1k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/1k1k/2p1d-dep2-dep8-1k1k.yaml deleted file mode 100644 index 9f5aa341c..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/1k1k/2p1d-dep2-dep8-1k1k.yaml +++ /dev/null @@ -1,81 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-2p1d-fp8-dep2-dep8-1k1k" - -model: - path: "MiniMaxAI/MiniMax-M3-MXFP8" - container: "vllm/vllm-openai:minimax-m3-0618-x86_64-cu130" - precision: "fp8" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 2 - decode_workers: 1 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -dynamo: - install: true - version: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - decode_environment: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - vllm_config: - prefill: - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - tensor-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 8192 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "512x4096" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/1k1k/2p1d-dep2-tep8-1k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/1k1k/2p1d-dep2-tep8-1k1k.yaml deleted file mode 100644 index 42c6e7bbc..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/1k1k/2p1d-dep2-tep8-1k1k.yaml +++ /dev/null @@ -1,79 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-2p1d-fp8-dep2-tep8-1k1k" - -model: - path: "MiniMaxAI/MiniMax-M3-MXFP8" - container: "vllm/vllm-openai:minimax-m3-0618-x86_64-cu130" - precision: "fp8" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 2 - decode_workers: 1 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -dynamo: - install: true - version: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - decode_environment: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - vllm_config: - prefill: - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - tensor-parallel-size: 8 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 8192 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "32" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/1k1k/2p2d-dep2-tep8-1k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/1k1k/2p2d-dep2-tep8-1k1k.yaml deleted file mode 100644 index 3e701df05..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/1k1k/2p2d-dep2-tep8-1k1k.yaml +++ /dev/null @@ -1,79 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-2p2d-fp8-dep2-tep8-1k1k" - -model: - path: "MiniMaxAI/MiniMax-M3-MXFP8" - container: "vllm/vllm-openai:minimax-m3-0618-x86_64-cu130" - precision: "fp8" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 2 - prefill_workers: 2 - decode_workers: 2 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -dynamo: - install: true - version: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - decode_environment: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - vllm_config: - prefill: - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - tensor-parallel-size: 8 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 8192 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "16" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/1k1k/3p2d-dep2-tep8-1k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/1k1k/3p2d-dep2-tep8-1k1k.yaml deleted file mode 100644 index b9a1d1058..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/1k1k/3p2d-dep2-tep8-1k1k.yaml +++ /dev/null @@ -1,79 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-3p2d-fp8-dep2-tep8-1k1k" - -model: - path: "MiniMaxAI/MiniMax-M3-MXFP8" - container: "vllm/vllm-openai:minimax-m3-0618-x86_64-cu130" - precision: "fp8" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 2 - prefill_workers: 3 - decode_workers: 2 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -dynamo: - install: true - version: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - decode_environment: - UCX_TLS: "cuda_copy,rc" - VLLM_FLOAT32_MATMUL_PRECISION: "high" - - vllm_config: - prefill: - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 2048 - - decode: - tensor-parallel-size: 8 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 2304 - language-model-only: true - stream-interval: 32 - max-num-seqs: 4096 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 8192 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 1024 - osl: 1024 - concurrencies: "4" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/8k1k/1p1d-dep2-tp4-marlin-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/8k1k/1p1d-dep2-tp4-marlin-8k1k.yaml deleted file mode 100644 index 04aca6586..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/8k1k/1p1d-dep2-tp4-marlin-8k1k.yaml +++ /dev/null @@ -1,85 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-1p1d-fp8-dep2-tp4-marlin-8k1k" - -model: - path: "MiniMaxAI/MiniMax-M3-MXFP8" - container: "vllm/vllm-openai:minimax-m3-0618-x86_64-cu130" - precision: "fp8" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 2 - gpus_per_decode: 4 - -dynamo: - install: true - version: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - allow_prefill_decode_colocation: true - - prefill_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_ipc,cuda_copy,rc" - - decode_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_ipc,cuda_copy,rc" - - vllm_config: - prefill: - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - tensor-parallel-size: 4 - enable-expert-parallel: false - moe-backend: marlin - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 2048 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "1x4x8x16" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/8k1k/1p2d-dep2-tep4-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/8k1k/1p2d-dep2-tep4-8k1k.yaml deleted file mode 100644 index e48310898..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/8k1k/1p2d-dep2-tep4-8k1k.yaml +++ /dev/null @@ -1,83 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-1p2d-fp8-dep2-tep4-8k1k" - -model: - path: "MiniMaxAI/MiniMax-M3-MXFP8" - container: "vllm/vllm-openai:minimax-m3-0618-x86_64-cu130" - precision: "fp8" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 2 - gpus_per_prefill: 2 - gpus_per_decode: 4 - -dynamo: - install: true - version: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - - decode_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - - vllm_config: - prefill: - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - tensor-parallel-size: 4 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-num-seqs: 512 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 4096 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "16" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/8k1k/1p4d-dep2-tep4-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/8k1k/1p4d-dep2-tep4-8k1k.yaml deleted file mode 100644 index 30ac635a9..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/8k1k/1p4d-dep2-tep4-8k1k.yaml +++ /dev/null @@ -1,83 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-1p4d-fp8-dep2-tep4-8k1k" - -model: - path: "MiniMaxAI/MiniMax-M3-MXFP8" - container: "vllm/vllm-openai:minimax-m3-0618-x86_64-cu130" - precision: "fp8" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 2 - prefill_workers: 1 - decode_workers: 4 - gpus_per_prefill: 2 - gpus_per_decode: 4 - -dynamo: - install: true - version: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - - decode_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - - vllm_config: - prefill: - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - tensor-parallel-size: 4 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-num-seqs: 512 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 4096 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "16x32x64x128" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/8k1k/1p4d-dep2-tep8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/8k1k/1p4d-dep2-tep8-8k1k.yaml deleted file mode 100644 index 46af72e46..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/8k1k/1p4d-dep2-tep8-8k1k.yaml +++ /dev/null @@ -1,83 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-1p4d-fp8-dep2-tep8-8k1k" - -model: - path: "MiniMaxAI/MiniMax-M3-MXFP8" - container: "vllm/vllm-openai:minimax-m3-0618-x86_64-cu130" - precision: "fp8" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 4 - prefill_workers: 1 - decode_workers: 4 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -dynamo: - install: true - version: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - - decode_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - - vllm_config: - prefill: - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - tensor-parallel-size: 8 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 4096 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "4" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/8k1k/2p2d-dep2-dep8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/8k1k/2p2d-dep2-dep8-8k1k.yaml deleted file mode 100644 index b1558ae34..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/8k1k/2p2d-dep2-dep8-8k1k.yaml +++ /dev/null @@ -1,85 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-2p2d-fp8-dep2-dep8-8k1k" - -model: - path: "MiniMaxAI/MiniMax-M3-MXFP8" - container: "vllm/vllm-openai:minimax-m3-0618-x86_64-cu130" - precision: "fp8" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 2 - prefill_workers: 2 - decode_workers: 2 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -dynamo: - install: true - version: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - - decode_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - - vllm_config: - prefill: - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - tensor-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 4096 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "256x512" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/8k1k/2p2d-dep2-tep8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/8k1k/2p2d-dep2-tep8-8k1k.yaml deleted file mode 100644 index 46aaa045d..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/8k1k/2p2d-dep2-tep8-8k1k.yaml +++ /dev/null @@ -1,83 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-2p2d-fp8-dep2-tep8-8k1k" - -model: - path: "MiniMaxAI/MiniMax-M3-MXFP8" - container: "vllm/vllm-openai:minimax-m3-0618-x86_64-cu130" - precision: "fp8" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 2 - prefill_workers: 2 - decode_workers: 2 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -dynamo: - install: true - version: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - - decode_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - - vllm_config: - prefill: - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - tensor-parallel-size: 8 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 4096 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "16" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/8k1k/4p2d-dep2-dep8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/8k1k/4p2d-dep2-dep8-8k1k.yaml deleted file mode 100644 index 3756103ee..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/8k1k/4p2d-dep2-dep8-8k1k.yaml +++ /dev/null @@ -1,85 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-4p2d-fp8-dep2-dep8-8k1k" - -model: - path: "MiniMaxAI/MiniMax-M3-MXFP8" - container: "vllm/vllm-openai:minimax-m3-0618-x86_64-cu130" - precision: "fp8" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 2 - prefill_workers: 4 - decode_workers: 2 - gpus_per_prefill: 2 - gpus_per_decode: 8 - -dynamo: - install: true - version: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - - decode_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - - vllm_config: - prefill: - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - tensor-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-num-seqs: 1024 # Per DP rank: 2 workers x DP8 = 16 ranks. - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 4096 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "4096" - req_rate: "inf" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/8k1k/4p2d-dep2-tep4-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/8k1k/4p2d-dep2-tep4-8k1k.yaml deleted file mode 100644 index c9f29f785..000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b300-fp8/8k1k/4p2d-dep2-tep4-8k1k.yaml +++ /dev/null @@ -1,83 +0,0 @@ -name: "minimax-m3-vllm-disagg-b300-4p2d-fp8-dep2-tep4-8k1k" - -model: - path: "MiniMaxAI/MiniMax-M3-MXFP8" - container: "vllm/vllm-openai:minimax-m3-0618-x86_64-cu130" - precision: "fp8" - -resources: - gpu_type: "b300" - gpus_per_node: 8 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 4 - decode_workers: 2 - gpus_per_prefill: 2 - gpus_per_decode: 4 - -dynamo: - install: true - version: 1.3.0.dev20260614 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - - decode_environment: - VLLM_FLOAT32_MATMUL_PRECISION: high - UCX_TLS: "cuda_copy,rc" - - vllm_config: - prefill: - tensor-parallel-size: 1 - data-parallel-size: 2 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-cudagraph-capture-size: 2048 - max-num-batched-tokens: 16384 - - decode: - tensor-parallel-size: 4 - enable-expert-parallel: true - trust-remote-code: true - no-enable-prefix-caching: true - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' - kv-cache-dtype: fp8 - block-size: 128 - gpu-memory-utilization: 0.90 - max-model-len: 9472 - language-model-only: true - stream-interval: 32 - max-num-seqs: 1024 - max-num-batched-tokens: 16384 - max-cudagraph-capture-size: 4096 - -health_check: - max_attempts: 360 - interval_seconds: 10 - -benchmark: - type: "sa-bench" - isl: 8192 - osl: 1024 - concurrencies: "4096" - req_rate: "inf" diff --git a/benchmarks/single_node/agentic/README.md b/benchmarks/single_node/agentic/README.md deleted file mode 100644 index 0ad5ec512..000000000 --- a/benchmarks/single_node/agentic/README.md +++ /dev/null @@ -1,53 +0,0 @@ -# Agentic single-node benchmarks - -**MVP / experimental.** Nothing in this directory is an official InferenceX -benchmark. Results are not published on https://inferencex.com and are not -intended to be cited. - -These launchers exist to develop and validate the agentic-coding scenario -type before it is promoted to first-class status. The scripts themselves -are best-effort and mainly serve as a reference implementation of how the -plumbing (env vars, scenario routing, result paths) should work. Specific -models and configs may be broken at any given time — multi-node in -particular is not yet first-class. - -## DRAM KV offload memory policy - -Agentic scenarios use `kv-offloading` for the resource tier and -`kv-offload-backend` for the backend implementation. `kv-offloading` is -currently either `none` or `dram`; when it is `dram`, the backend must be set: - -```yaml -- dram-utilization: 0.80 - search-space: - - { tp: 4, kv-offloading: dram, kv-offload-backend: native, conc-list: [16, 32] } - - { tp: 8, kv-offloading: none, conc-list: [16, 32] } -``` - -Agentic matrix generation uses a 3600-second default duration. Reusable -workflow callers can still override the `duration` input explicitly. - -Agentic master configs must use an exact `cluster:` runner label so every -search-space point lands on the same hardware fleet. Machine-level host memory -is declared once in the `configs/runners.yaml` `hardware` entry matching -that runner label: - -```yaml -hardware: - cluster:b300-nv: - available-cpu-dram-mib: 2964436 - gpus-per-node: 8 -``` - -The matrix generator combines the master config utilization with runner -hardware metadata and emits the aggregate budget as -`floor(min(available MiB, 2,861,022) * 1,048,576 * utilization * tp / gpus-per-node / 1,000,000,000)`. -The `2,861,022 MiB` cap is the 3 TB decimal DRAM limit. For example, TP4 in -an eight-GPU B300 search at 80% utilization receives 1,199 GB while TP8 -receives 2,399 GB. - -Benchmark scripts must consume `TOTAL_CPU_DRAM_GB`; they must not replace it -with model-specific constants. Backends with per-rank or per-pool settings must -divide this aggregate budget accordingly. DSv4 SGLang is the exception because -it exposes only `--hicache-ratio`; its empirically measured ratios are capped -in the model launchers to remain below the generated byte budget. diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b200_sglang.sh b/benchmarks/single_node/agentic/dsv4_fp4_b200_sglang.sh deleted file mode 100755 index 297878af2..000000000 --- a/benchmarks/single_node/agentic/dsv4_fp4_b200_sglang.sh +++ /dev/null @@ -1,246 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -set -x - -# Agentic trace replay benchmark for DeepSeek-V4-Pro FP4 on B200 using SGLang. -# -# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=hicache. - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -INFERENCEX_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" -export INFMAX_CONTAINER_WORKSPACE="${INFMAX_CONTAINER_WORKSPACE:-/workspace}" - -# The B200 DeepSeek-V4 Blackwell image installs SGLang editable under -# /workspace, so its launcher mounts InferenceX at /ix instead. Resolve the -# agentic tooling and results against the actual repository mount so the image -# can keep its /workspace install and GitHub Actions can collect the outputs. -if [[ ! -d "$INFMAX_CONTAINER_WORKSPACE/utils/aiperf" ]]; then - export INFMAX_CONTAINER_WORKSPACE="$INFERENCEX_ROOT" -fi -if [[ "${RESULT_DIR:-}" == /workspace/* && "$INFMAX_CONTAINER_WORKSPACE" != /workspace ]]; then - export RESULT_DIR="$INFMAX_CONTAINER_WORKSPACE/${RESULT_DIR#/workspace/}" -fi - -source "$INFERENCEX_ROOT/benchmarks/benchmark_lib.sh" - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -nvidia-smi - -resolve_trace_source - -# Keep AIPerf's Transformers-main dependency from replacing the older -# Transformers build pinned by the B200-specialized SGLang image. The server -# always launches with the image's original interpreter; AIPerf and result -# processing use the isolated environment when InferenceX is mounted at /ix. -SGLANG_PYTHON="$(command -v python3)" -if [[ "$INFMAX_CONTAINER_WORKSPACE" != /workspace ]]; then - AGENTIC_VENV="${AGENTIC_VENV:-/tmp/inferencex-agentic-venv}" - "$SGLANG_PYTHON" -m venv "$AGENTIC_VENV" - export PATH="$AGENTIC_VENV/bin:$PATH" -fi -install_agentic_deps - -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -CACHE_ARGS=() -if require_agentic_kv_offload_backend hicache; then - # DeepSeek V4 HiCache currently rejects --hicache-size and supports - # capacity control only through a host/device token-capacity ratio. - # DSv4 exposes capacity as a host/device token ratio rather than bytes. - # B200 ratio=8 stays below the configured host-memory capacity for the - # currently supported TP8 shape. - DEFAULT_HICACHE_RATIO=8 - HICACHE_RATIO="${HICACHE_RATIO:-$DEFAULT_HICACHE_RATIO}" - if [ "$HICACHE_RATIO" -gt "$DEFAULT_HICACHE_RATIO" ]; then - echo "Error: HICACHE_RATIO=$HICACHE_RATIO exceeds configured limit $DEFAULT_HICACHE_RATIO" >&2 - exit 1 - fi - HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through}" - HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" - HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first_direct}" - export SGLANG_ENABLE_UNIFIED_RADIX_TREE=1 - CACHE_ARGS=( - --enable-hierarchical-cache - --hicache-ratio "$HICACHE_RATIO" - --hicache-write-policy "$HICACHE_WRITE_POLICY" - --hicache-io-backend "$HICACHE_IO_BACKEND" - --hicache-mem-layout "$HICACHE_MEM_LAYOUT" - ) - echo "HiCache DSv4 CPU tier: ratio=$HICACHE_RATIO, capacity=${TOTAL_CPU_DRAM_GB} GB, write_policy=$HICACHE_WRITE_POLICY, io_backend=$HICACHE_IO_BACKEND, mem_layout=$HICACHE_MEM_LAYOUT" -fi - -USE_SGLANG_ROUTER=false -SGLANG_BACKEND_PORT="$PORT" -ROUTER_LOG="$RESULT_DIR/router.log" -if [ "$DP_ATTENTION" = "true" ]; then - USE_SGLANG_ROUTER=true - export AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID=true - SGLANG_BACKEND_PORT=$((PORT + 1)) - SGLANG_ROUTER_METRICS_PORT=$((PORT + 10000)) - SGLANG_ROUTER_CMD=("$SGLANG_PYTHON" -m sglang_router.launch_router) -fi - -PARALLEL_ARGS=(--tp "$TP") -METRICS_ARGS=(--enable-metrics) -CHUNKED_PREFILL_SIZE=8192 -if [ "$DP_ATTENTION" = "true" ]; then - DEEPEP_CONFIG='{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' - export SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE=1 - export SGLANG_OPT_FIX_HASH_MEGA_MOE=1 - export SGLANG_OPT_USE_FAST_MASK_EP=1 - export SGLANG_OPT_FIX_MEGA_MOE_MEMORY=1 - export SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=4096 - export SGLANG_OPT_FIX_NEXTN_MEGA_MOE=1 - export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=0 - PARALLEL_ARGS+=( - --dp "$TP" - --tokenizer-worker-num "$TP" - --enable-dp-attention - --enable-dp-attention-local-control-broadcast - --incremental-streaming-output - --stream-interval 20 - --dist-init-addr "127.0.0.1:$((PORT + 2000))" - --ep-size "$EP_SIZE" - --moe-a2a-backend deepep - --deepep-config "$DEEPEP_CONFIG" - ) - CHUNKED_PREFILL_SIZE=32768 -else - PARALLEL_ARGS+=( - --moe-runner-backend flashinfer_mxfp4 - --disable-flashinfer-autotune - ) -fi - -MODEL_ARGS=() -# The B200-specialized image deadlocks immediately after weight loading when -# forced through the B300 compressed-attention/page-size overrides. -# DeepGEMM's DSv4 indexer needs a multi-GiB temporary allocation at long -# contexts. Leave the same HBM headroom used by the B300 recipe so a nearly -# full GPU KV cache does not OOM while HiCache is spilling to host memory. -MEM_FRACTION_STATIC=0.88 - -# AgentX concurrency counts live session trees, not individual requests. -# Allow subagent fan-out to exceed CONC without clipping request bursts. -MAX_RUNNING_REQUESTS=$((2 * CONC)) -CUDA_GRAPH_MAX_BS=$CONC -[ "$CUDA_GRAPH_MAX_BS" -gt 64 ] && CUDA_GRAPH_MAX_BS=64 - -export PYTHONNOUSERSITE=1 -export TORCH_CUDA_ARCH_LIST=10.0 -# Agentic warmup dispatches hundreds of large prompts at once. SGLang's -# tokenizer process can leave request bytes unacknowledged for longer than -# AIPerf's 30-second TCP_USER_TIMEOUT while it admits that initial burst, -# causing Linux to abort otherwise-live localhost connections. Keep the -# six-hour request timeout unchanged, but allow up to 15 minutes for TCP -# progress before declaring the connection dead. -export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 -export SGLANG_JIT_DEEPGEMM_FAST_WARMUP=1 -export SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT=1 -export SGLANG_OPT_USE_JIT_NORM=1 -export SGLANG_OPT_USE_JIT_INDEXER_METADATA=1 -export SGLANG_OPT_USE_TOPK_V2=1 -export SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2=1 -TRITON_PTXAS_PATH=$(find \ - /usr/local/cuda* \ - /usr/local/lib/python*/dist-packages/nvidia \ - /usr/local/lib/python*/site-packages/nvidia \ - -type f -name ptxas -perm -u+x -print -quit 2>/dev/null || true) -if [ -n "$TRITON_PTXAS_PATH" ]; then - export TRITON_PTXAS_PATH - echo "Using ptxas for Triton: $TRITON_PTXAS_PATH" -fi -SGLANG_CMD=( - "$SGLANG_PYTHON" -m sglang.launch_server - --model-path "$MODEL_PATH" - --served-model-name "$MODEL" - --host 0.0.0.0 - --port "$SGLANG_BACKEND_PORT" - --trust-remote-code - "${PARALLEL_ARGS[@]}" - --mem-fraction-static "$MEM_FRACTION_STATIC" - --swa-full-tokens-ratio 0.1 - --max-running-requests "$MAX_RUNNING_REQUESTS" - --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS" - --chunked-prefill-size "$CHUNKED_PREFILL_SIZE" - --tool-call-parser deepseekv4 - --reasoning-parser deepseek-v4 - --chat-template "$SCRIPT_DIR/../chat_templates/deepseek_v4_thinking.jinja" - --watchdog-timeout 1800 - # The B200 checkpoint lives on Lustre. Partition sequential prefetching - # across local ranks so post-load weight repacking reads from page cache - # instead of issuing redundant fragmented mmap faults from every rank. - --weight-loader-prefetch-checkpoints - "${MODEL_ARGS[@]}" - "${METRICS_ARGS[@]}" - "${CACHE_ARGS[@]}" -) - -printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt" -printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt" - -{ - echo "=== SGLANG_* env vars at launch ===" - env | grep -E '^SGLANG_' | sort - echo "===================================" -} | tee "$SERVER_LOG" - -echo "Starting SGLang server for B200..." -"${SGLANG_CMD[@]}" >> "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -capture_cache_metrics() { - { - echo "=== SGLang cache metrics snapshot $(date --iso-8601=seconds) ===" - curl -fsS "http://localhost:$SGLANG_BACKEND_PORT/metrics" 2>/dev/null \ - | grep -E '^(sglang:(cache_hit_rate|cached_tokens_total|prompt_tokens_total|hicache_host_used_tokens|hicache_host_total_tokens|token_usage|num_requests_running|num_requests_waiting))' \ - || true - echo "============================================================" - } >> "$SERVER_LOG" -} - -wait_for_server_ready --port "$SGLANG_BACKEND_PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [ "$USE_SGLANG_ROUTER" = "true" ]; then - echo "Starting SGLang router on port $PORT for $TP DP ranks..." - "${SGLANG_ROUTER_CMD[@]}" \ - --worker-urls "http://localhost:$SGLANG_BACKEND_PORT" \ - --policy consistent_hashing \ - --request-id-headers x-correlation-id \ - --dp-aware \ - --host 0.0.0.0 \ - --port "$PORT" \ - --prometheus-host 127.0.0.1 \ - --prometheus-port "$SGLANG_ROUTER_METRICS_PORT" \ - --connect-timeout-secs 900 \ - --request-timeout-secs 14400 \ - --disable-health-check \ - --disable-retries > "$ROUTER_LOG" 2>&1 & - ROUTER_PID=$! - echo "Router PID: $ROUTER_PID" - wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID" -fi - -if [ "${#METRICS_ARGS[@]}" -gt 0 ]; then - capture_cache_metrics - trap capture_cache_metrics EXIT -fi - -build_replay_cmd "$RESULT_DIR" -REPLAY_CMD+=" --server-metrics http://localhost:$SGLANG_BACKEND_PORT/metrics" -run_agentic_replay_and_write_outputs "$RESULT_DIR" diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh deleted file mode 100755 index fdebbc6f8..000000000 --- a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh +++ /dev/null @@ -1,226 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -set -x - -# Agentic trace replay benchmark for DeepSeek-V4-Pro FP4 on B200 using vLLM. -# Mirrors the fixed-seq-len parallelism options (pure TP and DEP) so the -# agentic sweep can probe both interactivity and throughput regimes: -# pure TP (DP_ATTENTION=false, EP_SIZE=1): attention TP-sharded across -# all $TP GPUs in a single engine. Lower TPOT, lower batch. -# TP+EP (DP_ATTENTION=false, EP_SIZE>1): attention TP-sharded, MoE -# experts EP-sharded within the TP group. -# DEP (DP_ATTENTION=true, EP_SIZE>1): per-DP-rank attention with -# experts EP-sharded across DP ranks (per the vLLM blog recipe). -# Highest aggregate throughput at large CONC. -# -# Image is configured in nvidia-master.yaml. block_size=256, -# kv-cache-dtype=fp8, FP4 indexer cache enabled, FULL_AND_PIECEWISE cudagraph -# capture with custom_ops=all (per the vLLM blog recipe at -# https://vllm.ai/blog/deepseek-v4). -# -# Required env vars: -# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR -# -# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=mooncake. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -nvidia-smi - -# ---- Resolve traces and install deps ---------------------------------------- -resolve_trace_source -install_agentic_deps - -export AIPERF_AGENTIC_CACHE_WARMUP_DURATION=600 - -# vLLM v0.22.1 can ship CUTLASS DSL 4.5.2 with stale native MLIR bindings, -# which fails DSV4 indexer compilation with mlir_global_dtors(..., data). -# Reinstall the matching native wheel until NVIDIA/cutlass#3259 is resolved. -agentic_pip_install --quiet --force-reinstall --no-deps \ - 'nvidia-cutlass-dsl-libs-cu13==4.5.2' - -# vllm-project/router expands the one HTTP backend into one logical worker per -# DP rank and sends X-data-parallel-rank on forwarded requests. aiperf's -# X-Correlation-ID is stable for every turn of a conversation; alias it to the -# router's preferred X-Session-ID header. -USE_VLLM_ROUTER=false -VLLM_BACKEND_PORT="$PORT" -if [ "$DP_ATTENTION" = "true" ]; then - USE_VLLM_ROUTER=true - VLLM_BACKEND_PORT=$((PORT + 1)) - VLLM_ROUTER_VERSION=0.1.14 - VLLM_ROUTER_POLICY=consistent_hash - VLLM_ROUTER_METRICS_PORT=$((PORT + 10000)) - export AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID=1 - agentic_pip_install --quiet "vllm-router==$VLLM_ROUTER_VERSION" -fi - -# DeepSeek-V4-Pro weights are large; engine startup can exceed default 600s. -export VLLM_ENGINE_READY_TIMEOUT_S=3600 - -# vllm-project/vllm#43447 keeps local SWA prefix-cache tails sparsely, while -# vllm-project/vllm#44774 applies the same reachability policy to Mooncake's -# store mask. 32k matches the trace-replay tuning validated for this workload. -export VLLM_PREFIX_CACHE_RETENTION_INTERVAL=32768 - -# ---- Server config ---------------------------------------------------------- -SERVER_LOG="$RESULT_DIR/server.log" -ROUTER_LOG="$RESULT_DIR/router.log" -MOONCAKE_MASTER_LOG="$RESULT_DIR/mooncake_master.log" -mkdir -p "$RESULT_DIR" - -SERVER_PID="" -ROUTER_PID="" -MOONCAKE_MASTER_PID="" - -OFFLOAD_ARGS=() - -if require_agentic_kv_offload_backend mooncake; then - # Embedded mode contributes one segment per GPU rank to a shared - # distributed store, so pre-divide the aggregate host-memory budget. - PER_RANK_GB=$((TOTAL_CPU_DRAM_GB / TP)) - - MOONCAKE_VERSION=0.3.11.post1 - agentic_pip_install --quiet --no-cache-dir --no-deps \ - --force-reinstall "mooncake-transfer-engine-cuda13==$MOONCAKE_VERSION" - python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null - - MOONCAKE_MASTER_PORT=$((PORT + 12000)) - MOONCAKE_CONFIG_PATH="$RESULT_DIR/mooncake_config.json" - cat > "$MOONCAKE_CONFIG_PATH" < "$MOONCAKE_MASTER_LOG" 2>&1 & - MOONCAKE_MASTER_PID=$! - sleep 2 - if ! kill -0 "$MOONCAKE_MASTER_PID" 2>/dev/null; then - echo "Mooncake master died during startup." >&2 - cat "$MOONCAKE_MASTER_LOG" >&2 - exit 1 - fi - unset VLLM_USE_SIMPLE_KV_OFFLOAD - OFFLOAD_ARGS=( - --kv-transfer-config - '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}' - ) -fi - -PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) -if [ "$DP_ATTENTION" = "true" ]; then - PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") -fi - -EP_ARGS=() -if [ "$EP_SIZE" -gt 1 ]; then - EP_ARGS=(--enable-expert-parallel) -fi - -# AgentX concurrency counts live session trees, not individual requests. -# Subagent fan-out can push instantaneous request concurrency above CONC, so -# leave 2x headroom rather than clipping those bursts at the scheduler. -MAX_NUM_SEQS=$((2 * CONC)) - -echo "Starting vllm server..." -export TORCH_CUDA_ARCH_LIST="10.0" -export PYTHONNOUSERSITE=1 -export VLLM_FLOAT32_MATMUL_PRECISION=high - -{ set +x; } 2>/dev/null -VLLM_CMD=( - vllm serve "$MODEL_PATH" --served-model-name "$MODEL" - --host 0.0.0.0 - --port "$VLLM_BACKEND_PORT" - --trust-remote-code - --kv-cache-dtype fp8 - --block-size 256 - "${PARALLEL_ARGS[@]}" - "${EP_ARGS[@]}" - --compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' - --attention_config.use_fp4_indexer_cache=True - --tokenizer-mode deepseek_v4 - --tool-call-parser deepseek_v4 - --enable-auto-tool-choice - --reasoning-parser deepseek_v4 - --enable-prefix-caching - --no-disable-hybrid-kv-cache-manager - --max-num-seqs "$MAX_NUM_SEQS" - "${OFFLOAD_ARGS[@]}" -) -printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" -printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt" -"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$VLLM_BACKEND_PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [ "$USE_VLLM_ROUTER" = "true" ]; then - echo "Starting native vLLM router on port $PORT for $TP DP ranks..." - vllm-router \ - --worker-urls "http://localhost:$VLLM_BACKEND_PORT" \ - --policy "$VLLM_ROUTER_POLICY" \ - --intra-node-data-parallel-size "$TP" \ - --host 0.0.0.0 \ - --port "$PORT" \ - --prometheus-host 127.0.0.1 \ - --prometheus-port "$VLLM_ROUTER_METRICS_PORT" \ - --request-timeout-secs 14400 \ - --disable-retries > "$ROUTER_LOG" 2>&1 & - ROUTER_PID=$! - echo "Router PID: $ROUTER_PID" - wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID" -fi - -# ---- Run benchmark ---------------------------------------------------------- -build_replay_cmd "$RESULT_DIR" - -run_agentic_replay_and_write_outputs "$RESULT_DIR" diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang.sh deleted file mode 100755 index 3600382ac..000000000 --- a/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang.sh +++ /dev/null @@ -1,240 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -set -x - -# Agentic trace replay benchmark for DeepSeek-V4-Pro FP4 on B300 using SGLang. -# -# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=hicache. - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -INFERENCEX_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" -export INFMAX_CONTAINER_WORKSPACE="${INFMAX_CONTAINER_WORKSPACE:-/workspace}" - -# The B200 DeepSeek-V4 Blackwell image installs SGLang editable under -# /workspace, so its launcher mounts InferenceX at /ix instead. Resolve the -# agentic tooling and results against the actual repository mount so the image -# can keep its /workspace install and GitHub Actions can collect the outputs. -if [[ ! -d "$INFMAX_CONTAINER_WORKSPACE/utils/aiperf" ]]; then - export INFMAX_CONTAINER_WORKSPACE="$INFERENCEX_ROOT" -fi -if [[ "${RESULT_DIR:-}" == /workspace/* && "$INFMAX_CONTAINER_WORKSPACE" != /workspace ]]; then - export RESULT_DIR="$INFMAX_CONTAINER_WORKSPACE/${RESULT_DIR#/workspace/}" -fi - -source "$INFERENCEX_ROOT/benchmarks/benchmark_lib.sh" - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -nvidia-smi - -resolve_trace_source - -# Keep AIPerf's Transformers-main dependency from replacing the older -# Transformers build pinned by the B200-specialized SGLang image. The server -# always launches with the image's original interpreter; AIPerf and result -# processing use the isolated environment when InferenceX is mounted at /ix. -SGLANG_PYTHON="$(command -v python3)" -if [[ "$INFMAX_CONTAINER_WORKSPACE" != /workspace ]]; then - AGENTIC_VENV="${AGENTIC_VENV:-/tmp/inferencex-agentic-venv}" - "$SGLANG_PYTHON" -m venv "$AGENTIC_VENV" - export PATH="$AGENTIC_VENV/bin:$PATH" -fi -install_agentic_deps - -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -CACHE_ARGS=() -if require_agentic_kv_offload_backend hicache; then - # DeepSeek V4 HiCache currently rejects --hicache-size and supports - # capacity control only through a host/device token-capacity ratio. - # DSv4 exposes capacity as a host/device token ratio rather than bytes. - # Measurements put TP8 ratio=2 near 950 GB and TP4 ratio=8 near 1 TB, - # both below their configured capacities. The old TP4 ratio=16 - # used roughly 2 TB and violated the half-node allocation rule. - if [ "$TP" -ge 8 ]; then - DEFAULT_HICACHE_RATIO=2 - else - DEFAULT_HICACHE_RATIO=8 - fi - HICACHE_RATIO="${HICACHE_RATIO:-$DEFAULT_HICACHE_RATIO}" - if [ "$HICACHE_RATIO" -gt "$DEFAULT_HICACHE_RATIO" ]; then - echo "Error: HICACHE_RATIO=$HICACHE_RATIO exceeds configured limit $DEFAULT_HICACHE_RATIO" >&2 - exit 1 - fi - HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_back}" - HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" - HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first_direct}" - export SGLANG_ENABLE_UNIFIED_RADIX_TREE=1 - CACHE_ARGS=( - --enable-hierarchical-cache - --hicache-ratio "$HICACHE_RATIO" - --hicache-write-policy "$HICACHE_WRITE_POLICY" - --hicache-io-backend "$HICACHE_IO_BACKEND" - --hicache-mem-layout "$HICACHE_MEM_LAYOUT" - ) - echo "HiCache DSv4 CPU tier: ratio=$HICACHE_RATIO, capacity=${TOTAL_CPU_DRAM_GB} GB, write_policy=$HICACHE_WRITE_POLICY, io_backend=$HICACHE_IO_BACKEND, mem_layout=$HICACHE_MEM_LAYOUT" -fi - -USE_SGLANG_ROUTER=false -SGLANG_BACKEND_PORT="$PORT" -ROUTER_LOG="$RESULT_DIR/router.log" -if [ "$DP_ATTENTION" = "true" ]; then - USE_SGLANG_ROUTER=true - export AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID=true - SGLANG_BACKEND_PORT=$((PORT + 1)) - SGLANG_ROUTER_METRICS_PORT=$((PORT + 10000)) - SGLANG_ROUTER_CMD=("$SGLANG_PYTHON" -m sglang_router.launch_router) -fi - -PARALLEL_ARGS=(--tp "$TP") -METRICS_ARGS=(--enable-metrics) -MEM_FRACTION_STATIC=0.88 -CHUNKED_PREFILL_SIZE=8192 -if [ "$DP_ATTENTION" = "true" ]; then - PARALLEL_ARGS+=( - --dp "$TP" - --tokenizer-worker-num "$TP" - --enable-dp-attention - --enable-dp-attention-local-control-broadcast - --incremental-streaming-output - --stream-interval 20 - --dist-init-addr "127.0.0.1:$((PORT + 2000))" - --ep-size "$EP_SIZE" - --moe-runner-backend flashinfer_mxfp4 - --disable-flashinfer-autotune - ) - MEM_FRACTION_STATIC=0.95 - CHUNKED_PREFILL_SIZE=16384 -else - PARALLEL_ARGS+=( - --moe-runner-backend flashinfer_mxfp4 - --disable-flashinfer-autotune - ) -fi - -MODEL_ARGS=( - --attention-backend compressed - --page-size 256 - --disable-shared-experts-fusion -) - -# AgentX concurrency counts live session trees, not individual requests. -# Allow subagent fan-out to exceed CONC without clipping request bursts. -MAX_RUNNING_REQUESTS=$((2 * CONC)) -CUDA_GRAPH_MAX_BS=$CONC -[ "$CUDA_GRAPH_MAX_BS" -gt 64 ] && CUDA_GRAPH_MAX_BS=64 - -export PYTHONNOUSERSITE=1 -export TORCH_CUDA_ARCH_LIST=10.0 -# Agentic warmup dispatches hundreds of large prompts at once. SGLang's -# tokenizer process can leave request bytes unacknowledged for longer than -# AIPerf's 30-second TCP_USER_TIMEOUT while it admits that initial burst, -# causing Linux to abort otherwise-live localhost connections. Keep the -# six-hour request timeout unchanged, but allow up to 15 minutes for TCP -# progress before declaring the connection dead. -export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 -export SGLANG_JIT_DEEPGEMM_FAST_WARMUP=1 -export SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT=1 -export SGLANG_OPT_USE_JIT_NORM=1 -export SGLANG_OPT_USE_JIT_INDEXER_METADATA=1 -export SGLANG_OPT_USE_TOPK_V2=1 -export SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2=1 -TRITON_PTXAS_PATH=$(find \ - /usr/local/cuda* \ - /usr/local/lib/python*/dist-packages/nvidia \ - /usr/local/lib/python*/site-packages/nvidia \ - -type f -name ptxas -perm -u+x -print -quit 2>/dev/null || true) -if [ -n "$TRITON_PTXAS_PATH" ]; then - export TRITON_PTXAS_PATH - echo "Using ptxas for Triton: $TRITON_PTXAS_PATH" -fi -SGLANG_CMD=( - "$SGLANG_PYTHON" -m sglang.launch_server - --model-path "$MODEL_PATH" - --served-model-name "$MODEL" - --host 0.0.0.0 - --port "$SGLANG_BACKEND_PORT" - --trust-remote-code - "${PARALLEL_ARGS[@]}" - --mem-fraction-static "$MEM_FRACTION_STATIC" - --swa-full-tokens-ratio 0.1 - --max-running-requests "$MAX_RUNNING_REQUESTS" - --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS" - --allow-auto-truncate - --chunked-prefill-size "$CHUNKED_PREFILL_SIZE" - --tool-call-parser deepseekv4 - --reasoning-parser deepseek-v4 - --chat-template "$SCRIPT_DIR/../chat_templates/deepseek_v4_thinking.jinja" - --watchdog-timeout 1800 - "${MODEL_ARGS[@]}" - "${METRICS_ARGS[@]}" - "${CACHE_ARGS[@]}" -) - -printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt" -printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt" - -{ - echo "=== SGLANG_* env vars at launch ===" - env | grep -E '^SGLANG_' | sort - echo "===================================" -} | tee "$SERVER_LOG" - -echo "Starting SGLang server for B300..." -"${SGLANG_CMD[@]}" >> "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -capture_cache_metrics() { - { - echo "=== SGLang cache metrics snapshot $(date --iso-8601=seconds) ===" - curl -fsS "http://localhost:$SGLANG_BACKEND_PORT/metrics" 2>/dev/null \ - | grep -E '^(sglang:(cache_hit_rate|cached_tokens_total|prompt_tokens_total|hicache_host_used_tokens|hicache_host_total_tokens|token_usage|num_requests_running|num_requests_waiting))' \ - || true - echo "============================================================" - } >> "$SERVER_LOG" -} - -wait_for_server_ready --port "$SGLANG_BACKEND_PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [ "$USE_SGLANG_ROUTER" = "true" ]; then - echo "Starting SGLang router on port $PORT for $TP DP ranks..." - "${SGLANG_ROUTER_CMD[@]}" \ - --worker-urls "http://localhost:$SGLANG_BACKEND_PORT" \ - --policy consistent_hashing \ - --request-id-headers x-correlation-id \ - --dp-aware \ - --host 0.0.0.0 \ - --port "$PORT" \ - --prometheus-host 127.0.0.1 \ - --prometheus-port "$SGLANG_ROUTER_METRICS_PORT" \ - --connect-timeout-secs 900 \ - --request-timeout-secs 14400 \ - --disable-health-check \ - --disable-retries > "$ROUTER_LOG" 2>&1 & - ROUTER_PID=$! - echo "Router PID: $ROUTER_PID" - wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID" -fi - -if [ "${#METRICS_ARGS[@]}" -gt 0 ]; then - capture_cache_metrics - trap capture_cache_metrics EXIT -fi - -build_replay_cmd "$RESULT_DIR" -REPLAY_CMD+=" --server-metrics http://localhost:$SGLANG_BACKEND_PORT/metrics" -run_agentic_replay_and_write_outputs "$RESULT_DIR" diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh deleted file mode 100755 index 531a081be..000000000 --- a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm.sh +++ /dev/null @@ -1,225 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -set -x - -# Agentic trace replay benchmark for DeepSeek-V4-Pro FP4 on B300 using vLLM. -# Mirrors the fixed-seq-len parallelism options (pure TP and DEP) so the -# agentic sweep can probe both interactivity and throughput regimes: -# pure TP (DP_ATTENTION=false, EP_SIZE=1): attention TP-sharded across -# all $TP GPUs in a single engine. Lower TPOT, lower batch. -# TP+EP (DP_ATTENTION=false, EP_SIZE>1): attention TP-sharded, MoE -# experts EP-sharded within the TP group. -# DEP (DP_ATTENTION=true, EP_SIZE>1): per-DP-rank attention with -# experts EP-sharded across DP ranks (per the vLLM blog recipe). -# Highest aggregate throughput at large CONC. -# -# Image is vllm/vllm-openai:v0.20.0-cu130. block_size=256, kv-cache-dtype=fp8, -# FP4 indexer cache enabled, FULL_AND_PIECEWISE cudagraph capture with -# custom_ops=all (per the vLLM blog recipe at https://vllm.ai/blog/deepseek-v4). -# -# Required env vars: -# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR -# -# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=mooncake. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION - -if declare -p SLURM_JOB_ID >/dev/null 2>&1 && [ -n "$SLURM_JOB_ID" ]; then - SLURM_NODE=unknown - if declare -p SLURMD_NODENAME >/dev/null 2>&1 && [ -n "$SLURMD_NODENAME" ]; then - SLURM_NODE="$SLURMD_NODENAME" - fi - echo "JOB $SLURM_JOB_ID running on $SLURM_NODE" -fi - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if declare -p MODEL_PATH >/dev/null 2>&1 && [ -n "$MODEL_PATH" ]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -nvidia-smi - -# ---- Resolve traces and install deps ---------------------------------------- -resolve_trace_source -install_agentic_deps - -export AIPERF_AGENTIC_CACHE_WARMUP_DURATION=600 - -# vLLM v0.22.1 can ship CUTLASS DSL 4.5.2 with stale native MLIR bindings, -# which fails DSV4 indexer compilation with mlir_global_dtors(..., data). -# Reinstall the matching native wheel until NVIDIA/cutlass#3259 is resolved. -agentic_pip_install --quiet --force-reinstall --no-deps \ - 'nvidia-cutlass-dsl-libs-cu13==4.5.2' - -# vllm-project/router expands the one HTTP backend into one logical worker per -# DP rank and sends X-data-parallel-rank on forwarded requests. aiperf's -# X-Correlation-ID is stable for every turn of a conversation; alias it to the -# router's preferred X-Session-ID header. This also keeps affinity correct when -# testing older wheels that prioritize per-request X-Request-ID. -USE_VLLM_ROUTER=false -VLLM_BACKEND_PORT="$PORT" -if [ "$DP_ATTENTION" = "true" ]; then - USE_VLLM_ROUTER=true - VLLM_BACKEND_PORT=$((PORT + 1)) - VLLM_ROUTER_VERSION=0.1.14 - VLLM_ROUTER_POLICY=consistent_hash - VLLM_ROUTER_METRICS_PORT=$((PORT + 10000)) - export AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID=1 - agentic_pip_install --quiet "vllm-router==$VLLM_ROUTER_VERSION" -fi - -# DeepSeek-V4-Pro weights are large; engine startup can exceed default 600s. -export VLLM_ENGINE_READY_TIMEOUT_S=3600 - -# vllm-project/vllm#43447 keeps local SWA prefix-cache tails sparsely, while -# vllm-project/vllm#44774 applies the same reachability policy to Mooncake's -# store mask. 32k matches the trace-replay tuning validated for this workload. -export VLLM_PREFIX_CACHE_RETENTION_INTERVAL=32768 - -# ---- Server config ---------------------------------------------------------- -SERVER_LOG="$RESULT_DIR/server.log" -ROUTER_LOG="$RESULT_DIR/router.log" -MOONCAKE_MASTER_LOG="$RESULT_DIR/mooncake_master.log" -mkdir -p "$RESULT_DIR" - -SERVER_PID="" -ROUTER_PID="" -MOONCAKE_MASTER_PID="" - -OFFLOAD_ARGS=() -if require_agentic_kv_offload_backend mooncake; then - # Mooncake embedded mode contributes one global segment per GPU rank to - # a shared distributed store. Pre-divide the aggregate host budget - # across those rank-contributed segments. - PER_RANK_GB=$((TOTAL_CPU_DRAM_GB / TP)) - - MOONCAKE_VERSION=0.3.11.post1 - agentic_pip_install --quiet --no-cache-dir --no-deps \ - --force-reinstall "mooncake-transfer-engine-cuda13==$MOONCAKE_VERSION" - python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null - - MOONCAKE_MASTER_PORT=$((PORT + 12000)) - MOONCAKE_CONFIG_PATH="$RESULT_DIR/mooncake_config.json" - cat > "$MOONCAKE_CONFIG_PATH" < "$MOONCAKE_MASTER_LOG" 2>&1 & - MOONCAKE_MASTER_PID=$! - sleep 2 - if ! kill -0 "$MOONCAKE_MASTER_PID" 2>/dev/null; then - echo "Mooncake master died during startup." >&2 - cat "$MOONCAKE_MASTER_LOG" >&2 - exit 1 - fi - - unset VLLM_USE_SIMPLE_KV_OFFLOAD - OFFLOAD_ARGS=( - --kv-transfer-config - '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}' - ) -fi - -PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) -if [ "$DP_ATTENTION" = "true" ]; then - PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") -fi - -EP_ARGS=() -if [ "$EP_SIZE" -gt 1 ]; then - EP_ARGS=(--enable-expert-parallel) -fi - -# AgentX concurrency counts live session trees, not individual requests. -# Subagent fan-out can push instantaneous request concurrency above CONC, so -# leave 2x headroom rather than clipping those bursts at the scheduler. -MAX_NUM_SEQS=$((2 * CONC)) -if [ "$MAX_NUM_SEQS" -eq 128 ]; then - MAX_NUM_SEQS=136 -fi - -echo "Starting vllm server..." -export TORCH_CUDA_ARCH_LIST="10.0" -export PYTHONNOUSERSITE=1 -export VLLM_FLOAT32_MATMUL_PRECISION=high - -vllm serve "$MODEL_PATH" --served-model-name "$MODEL" \ ---host 0.0.0.0 \ ---port "$VLLM_BACKEND_PORT" \ ---trust-remote-code \ ---kv-cache-dtype fp8 \ ---block-size 256 \ -"${PARALLEL_ARGS[@]}" \ -"${EP_ARGS[@]}" \ ---compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' \ ---attention_config.use_fp4_indexer_cache=True \ ---tokenizer-mode deepseek_v4 \ ---tool-call-parser deepseek_v4 \ ---enable-auto-tool-choice \ ---reasoning-parser deepseek_v4 \ ---enable-prefix-caching \ ---no-disable-hybrid-kv-cache-manager \ ---max-num-seqs "$MAX_NUM_SEQS" \ -"${OFFLOAD_ARGS[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$VLLM_BACKEND_PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [ "$USE_VLLM_ROUTER" = "true" ]; then - echo "Starting native vLLM router on port $PORT for $TP DP ranks..." - vllm-router \ - --worker-urls "http://localhost:$VLLM_BACKEND_PORT" \ - --policy "$VLLM_ROUTER_POLICY" \ - --intra-node-data-parallel-size "$TP" \ - --host 0.0.0.0 \ - --port "$PORT" \ - --prometheus-host 127.0.0.1 \ - --prometheus-port "$VLLM_ROUTER_METRICS_PORT" \ - --request-timeout-secs 14400 \ - --disable-retries > "$ROUTER_LOG" 2>&1 & - ROUTER_PID=$! - echo "Router PID: $ROUTER_PID" - wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID" -fi - -# ---- Run benchmark ---------------------------------------------------------- -build_replay_cmd "$RESULT_DIR" - -run_agentic_replay_and_write_outputs "$RESULT_DIR" diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang.sh deleted file mode 100755 index d8cba43f0..000000000 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang.sh +++ /dev/null @@ -1,157 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -set -x - -# Agentic trace replay benchmark for DeepSeek-V4-Pro FP4 on MI355X using SGLang. -# Adapted from benchmarks/single_node/dsv4_fp4_mi355x_sglang.sh (fixed-seq-len -# sibling) with the agentic harness (build_replay_cmd / write_agentic_result_json -# / analyze_benchmark_distributions) swapped in for run_benchmark_serving. -# -# This launcher only supports on-device KV cache. -# -# Required env vars: -# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -# ROCR/HIP visibility under slurm cgroups. -if [ -n "${ROCR_VISIBLE_DEVICES:-}" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -rocm-smi || true -amd-smi || true - -# ---- Resolve traces and install deps ---------------------------------------- -resolve_trace_source -install_agentic_deps - -require_agentic_kv_offload_none - -# Transformers in the container doesn't recognize the `deepseek_v4` model_type. -# PR #23608's fallback in hf_transformers_utils.get_config tries to handle this -# by writing a patched config to /tmp, but in practice isn't catching the error -# in this image. Patch the cached config.json directly instead: set model_type -# to `deepseek_v3` so AutoConfig.from_pretrained succeeds, and keep -# architectures=['DeepseekV4ForCausalLM'] so SGLang dispatches to its native -# DSv4 model class (python/sglang/srt/models/deepseek_v4.py). -python3 << PYEOF -import json -from huggingface_hub import hf_hub_download -path = hf_hub_download(repo_id="$MODEL", filename="config.json") -with open(path) as f: - config = json.load(f) -if config.get("model_type") == "deepseek_v4": - config["model_type"] = "deepseek_v3" - with open(path, "w") as f: - json.dump(config, f, indent=2) - print(f"Patched {path}: model_type deepseek_v4 -> deepseek_v3") -else: - print(f"No patch needed: model_type is {config.get('model_type')!r}") -PYEOF - -# DSv4 FP4-experts path. Mirrors the env block in the fixed-seq-len sibling -# (benchmarks/single_node/dsv4_fp4_mi355x_sglang.sh), which tracks the active -# block in python/run_dsv4.sh on the amd/deepseek_v4 branch: -# SGLANG_DSV4_FP4_EXPERTS=True -> route experts through FP4 kernels -# SGLANG_FORCE_TRITON_MOE_FP8=0 -> dispatch MoE through aiter and apply -# the swiglu_limit clamp in the triton -# MoE fallback path. -export SGLANG_REASONING_EFFORT=max -export SGLANG_OPT_USE_FUSED_COMPRESS=true -export SGLANG_OPT_USE_OLD_COMPRESSOR=true -export SGLANG_OPT_USE_TILELANG_SWA_PREPARE=false -export SGLANG_OPT_USE_JIT_KERNEL_FUSED_TOPK=false -export SGLANG_OPT_USE_FUSED_HASH_TOPK=false -export SGLANG_OPT_DEEPGEMM_HC_PRENORM=false -export SGLANG_OPT_USE_TILELANG_MHC_PRE=false -export SGLANG_OPT_USE_TILELANG_MHC_POST=false -export SGLANG_OPT_USE_AITER_MHC_PRE=true -export SGLANG_OPT_USE_AITER_MHC_POST=true -export SGLANG_ENABLE_THINKING=1 -export SGLANG_USE_AITER=1 -export SGLANG_USE_ROCM700A=1 -export SGLANG_TOPK_TRANSFORM_512_TORCH=0 -export SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1 -export SGLANG_DSV4_FP4_EXPERTS=True -export SGLANG_OPT_DPSK_V4_RADIX=0 -export SGLANG_OPT_USE_OVERLAP_STORE_CACHE=false -export SGLANG_OPT_USE_FUSED_STORE_CACHE=false -export SGLANG_FORCE_TRITON_MOE_FP8=0 -export SGLANG_HACK_FLASHMLA_BACKEND=tilelang -export SGLANG_OPT_USE_TILELANG_INDEXER=true -export SGLANG_OPT_USE_TRITON_SWA_PREPARE=true - -# ---- Server config ---------------------------------------------------------- -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -# Parallelism: pure TP, TP+EP, or DEP (DP-attn + EP). Matches the dsv4 b200 -# vllm agentic launcher so the agentic sweep can probe both interactivity and -# throughput regimes. -PARALLEL_ARGS=(--tensor-parallel-size "$TP") -if [ "$DP_ATTENTION" = "true" ]; then - PARALLEL_ARGS+=( - --dp "$TP" - --enable-dp-attention - --enable-prefill-delayer - ) -fi -if [ "${EP_SIZE:-1}" -gt 1 ]; then - PARALLEL_ARGS+=(--ep-size "$EP_SIZE") -fi - -# --max-running-requests is per-engine. With DP-attn each DP engine handles -# only CONC/$TP sequences in steady state (the agentic harness load-balances -# users across DP ranks), so size the per-engine cap to that. -# Pure TP is a single engine and sees all CONC sequences itself. -if [ "$DP_ATTENTION" = "true" ]; then - PER_ENGINE_MAX_RUNNING=$(( CONC / TP )) - [ "$PER_ENGINE_MAX_RUNNING" -lt 1 ] && PER_ENGINE_MAX_RUNNING=1 -else - PER_ENGINE_MAX_RUNNING=$CONC -fi - -echo "Starting sglang server..." -python3 -m sglang.launch_server \ - --model-path "$MODEL_PATH" --served-model-name "$MODEL" \ - --host=0.0.0.0 \ - --port "$PORT" \ - "${PARALLEL_ARGS[@]}" \ - --trust-remote-code \ - --attention-backend compressed \ - --max-running-requests "$PER_ENGINE_MAX_RUNNING" \ - --cuda-graph-max-bs "$PER_ENGINE_MAX_RUNNING" \ - --page-size 256 \ - --chunked-prefill-size 8192 \ - --disable-shared-experts-fusion \ - --tool-call-parser deepseekv4 \ - --reasoning-parser deepseek-v4 \ - --chat-template "$(dirname "$0")/../chat_templates/deepseek_v4_thinking.jinja" \ - --watchdog-timeout 1800 > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -# ---- Run benchmark ---------------------------------------------------------- -build_replay_cmd "$RESULT_DIR" - -run_agentic_replay_and_write_outputs "$RESULT_DIR" diff --git a/benchmarks/single_node/agentic/dsv4_fp8_h200.sh b/benchmarks/single_node/agentic/dsv4_fp8_h200.sh deleted file mode 100755 index 61df70184..000000000 --- a/benchmarks/single_node/agentic/dsv4_fp8_h200.sh +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -set -x - -# Agentic trace replay benchmark for DeepSeek-V4-Pro FP8 on H200 using vLLM. -# Uses the cu129 image; H200 has no FP4 path so the FP4 indexer cache flag -# is omitted. Max-model-len pinned at 800k per the recipe. -# -# Required env vars: -# MODEL, TP, CONC, RESULT_DIR - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC RESULT_DIR DURATION - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -nvidia-smi - -# ---- Resolve traces and install deps ---------------------------------------- -resolve_trace_source -install_agentic_deps - -# DeepSeek-V4-Pro weights are large; engine startup can exceed default 600s. -export VLLM_ENGINE_READY_TIMEOUT_S=3600 - -# ---- Start vLLM server ------------------------------------------------------ -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -echo "Starting vLLM server..." -export PYTHONNOUSERSITE=1 - -# Per recipe: EP + DP=8 (no --tensor-parallel-size). TP from search space is -# used for GPU allocation by the runner and as the DP size. -vllm serve "$MODEL_PATH" --served-model-name "$MODEL" \ ---host 0.0.0.0 \ ---port $PORT \ ---trust-remote-code \ ---kv-cache-dtype fp8 \ ---block-size 256 \ ---enable-expert-parallel \ ---data-parallel-size $TP \ ---gpu-memory-utilization 0.95 \ ---max-num-seqs $CONC \ ---max-num-batched-tokens 512 \ ---no-enable-flashinfer-autotune \ ---compilation-config '{"mode":0,"cudagraph_mode":"FULL_DECODE_ONLY"}' \ ---tokenizer-mode deepseek_v4 \ ---tool-call-parser deepseek_v4 \ ---enable-auto-tool-choice \ ---reasoning-parser deepseek_v4 > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -# ---- Run benchmark ---------------------------------------------------------- -build_replay_cmd "$RESULT_DIR" - -run_agentic_replay_and_write_outputs "$RESULT_DIR" diff --git a/benchmarks/single_node/agentic/kimik2.5_fp4_b200.sh b/benchmarks/single_node/agentic/kimik2.5_fp4_b200.sh deleted file mode 100755 index b4ac77f2b..000000000 --- a/benchmarks/single_node/agentic/kimik2.5_fp4_b200.sh +++ /dev/null @@ -1,196 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -set -x - -# Agentic trace replay benchmark for Kimi-K2.5 NVFP4 on B200 using vLLM. -# -# Required env vars: -# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR -# -# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=lmcache. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION - - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -nvidia-smi - -# ---- Resolve traces and install deps ---------------------------------------- -resolve_trace_source -install_agentic_deps - -# ---- Server config ---------------------------------------------------------- -SERVER_LOG="$RESULT_DIR/server.log" -LMCACHE_LOG="$RESULT_DIR/lmcache_server.log" -mkdir -p "$RESULT_DIR" - -OFFLOAD_ARGS=() -PREFIX_CACHE_ARGS=() -LMCACHE_PID="" - -cleanup_lmcache_server() { - if [[ -n "$LMCACHE_PID" ]] && kill -0 "$LMCACHE_PID" 2>/dev/null; then - kill "$LMCACHE_PID" 2>/dev/null || true - wait "$LMCACHE_PID" 2>/dev/null || true - fi -} - -trap cleanup_lmcache_server EXIT - -wait_for_lmcache_ready() { - { set +x; } 2>/dev/null - local attempts="${LMCACHE_READY_ATTEMPTS:-120}" - local tail_pid="" - - while [ ! -f "$LMCACHE_LOG" ]; do - if [[ -n "$LMCACHE_PID" ]] && ! kill -0 "$LMCACHE_PID" 2>/dev/null; then - echo "LMCache server died before creating log file. Exiting." >&2 - exit 1 - fi - sleep 1 - done - - tail -f -n +1 "$LMCACHE_LOG" & - tail_pid=$! - - for ((i = 1; i <= attempts; i++)); do - if curl --output /dev/null --silent --fail "http://127.0.0.1:${LMCACHE_HTTP_PORT}/healthcheck"; then - kill "$tail_pid" 2>/dev/null || true - wait "$tail_pid" 2>/dev/null || true - return 0 - fi - if [[ -n "$LMCACHE_PID" ]] && ! kill -0 "$LMCACHE_PID" 2>/dev/null; then - echo "LMCache server died before becoming healthy. Log follows:" >&2 - kill "$tail_pid" 2>/dev/null || true - wait "$tail_pid" 2>/dev/null || true - cat "$LMCACHE_LOG" >&2 || true - exit 1 - fi - sleep 1 - done - - echo "Timed out waiting for LMCache server healthcheck. Log follows:" >&2 - kill "$tail_pid" 2>/dev/null || true - wait "$tail_pid" 2>/dev/null || true - cat "$LMCACHE_LOG" >&2 || true - exit 1 -} - -if require_agentic_kv_offload_backend lmcache; then - { set +x; } 2>/dev/null - unset VLLM_USE_SIMPLE_KV_OFFLOAD - - agentic_pip_install --quiet --no-cache-dir lmcache - python3 -c "import lmcache.integration.vllm.lmcache_mp_connector" >/dev/null - - # MP mode owns the configured CPU pool in the external LMCache - # server instead of passing - # --kv-offloading-size through vLLM's integrated LMCache convenience - # path, which divides the value by TP and then hits a large single-shot - # cudaHostAlloc in LMCache 0.4.5's single-process local CPU backend. - LMCACHE_HOST="${LMCACHE_HOST:-127.0.0.1}" - LMCACHE_PORT="${LMCACHE_PORT:-5555}" - LMCACHE_HTTP_PORT="${LMCACHE_HTTP_PORT:-8080}" - # LMCacheMPConnector builds its ZMQ endpoint by concatenating - # lmcache.mp.host and lmcache.mp.port, and its default host already - # includes the tcp:// scheme. Keep the server bind host raw, but pass - # a ZMQ-style host string to the connector. - LMCACHE_CONNECT_HOST="${LMCACHE_CONNECT_HOST:-tcp://$LMCACHE_HOST}" - LMCACHE_L1_SIZE_GB="${LMCACHE_L1_SIZE_GB:-$TOTAL_CPU_DRAM_GB}" - if [ "$LMCACHE_L1_SIZE_GB" -gt "$TOTAL_CPU_DRAM_GB" ]; then - echo "Error: LMCACHE_L1_SIZE_GB=$LMCACHE_L1_SIZE_GB exceeds configured capacity $TOTAL_CPU_DRAM_GB" >&2 - exit 1 - fi - # Initial allocation is deliberately small; --l1-size-gb above is the - # actual pool capacity and grows lazily as the run fills the cache. - LMCACHE_L1_INIT_SIZE_GB="${LMCACHE_L1_INIT_SIZE_GB:-20}" - LMCACHE_CHUNK_SIZE="${LMCACHE_CHUNK_SIZE:-256}" - LMCACHE_MAX_WORKERS="${LMCACHE_MAX_WORKERS:-$TP}" - export PYTHONHASHSEED="${PYTHONHASHSEED:-0}" - - echo "Starting LMCache MP server..." - LMCACHE_CMD=( - lmcache server - --host "$LMCACHE_HOST" - --port "$LMCACHE_PORT" - --http-host "$LMCACHE_HOST" - --http-port "$LMCACHE_HTTP_PORT" - --l1-size-gb "$LMCACHE_L1_SIZE_GB" - --l1-init-size-gb "$LMCACHE_L1_INIT_SIZE_GB" - --chunk-size "$LMCACHE_CHUNK_SIZE" - --max-workers "$LMCACHE_MAX_WORKERS" - --eviction-policy LRU - ) - printf '%q ' "${LMCACHE_CMD[@]}" > "$RESULT_DIR/lmcache_command.txt" - printf '\n' >> "$RESULT_DIR/lmcache_command.txt" - "${LMCACHE_CMD[@]}" > "$LMCACHE_LOG" 2>&1 & - LMCACHE_PID=$! - echo "LMCache server PID: $LMCACHE_PID" - wait_for_lmcache_ready - - PREFIX_CACHE_ARGS=(--enable-prefix-caching) - OFFLOAD_ARGS=( - --kv-transfer-config - "{\"kv_connector\":\"LMCacheMPConnector\",\"kv_connector_module_path\":\"lmcache.integration.vllm.lmcache_mp_connector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"lmcache.mp.host\":\"$LMCACHE_CONNECT_HOST\",\"lmcache.mp.port\":$LMCACHE_PORT}}" - --disable-hybrid-kv-cache-manager - ) -fi - -echo "Starting vllm server..." -export TORCH_CUDA_ARCH_LIST="10.0" -export PYTHONNOUSERSITE=1 -# Disable vLLM v0.21+ CUDA-graph memory estimator. Its pre-reservation -# eats ~32% of HBM upfront which, combined with FP4 weights at TP=4 -# (~62 GB/GPU), leaves no room for KV blocks -- _check_enough_kv_cache_memory -# trips before the engine starts. Our --gpu-memory-utilization=0.90 already -# leaves ~18 GB/GPU slack outside vLLM's budget, which is the same safety -# net the estimator provides, so disabling it is redundant rather than -# unsafe. -export VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0 - -{ set +x; } 2>/dev/null -VLLM_CMD=( - vllm serve "$MODEL_PATH" --served-model-name "$MODEL" - --host 0.0.0.0 - --port "$PORT" - --tensor-parallel-size="$TP" - --gpu-memory-utilization 0.90 - --max-num-seqs "$CONC" - --reasoning-parser kimi_k2 - --tool-call-parser kimi_k2 - --compilation_config.pass_config.fuse_allreduce_rms true - --kv-cache-dtype fp8 - --max-cudagraph-capture-size 2048 - --stream-interval 20 - --trust-remote-code - "${PREFIX_CACHE_ARGS[@]}" - "${OFFLOAD_ARGS[@]}" -) -printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" -printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt" -"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -# ---- Run benchmark ---------------------------------------------------------- -build_replay_cmd "$RESULT_DIR" - -run_agentic_replay_and_write_outputs "$RESULT_DIR" diff --git a/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh b/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh deleted file mode 100755 index b99299b12..000000000 --- a/benchmarks/single_node/agentic/kimik2.5_fp4_b300.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -set -x - -# Agentic trace replay benchmark for Kimi-K2.5 NVFP4 on B300 using vLLM. -# -# Required env vars: -# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR -# -# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=native. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION - - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -nvidia-smi - -# ---- Resolve traces and install deps ---------------------------------------- -resolve_trace_source -install_agentic_deps - -# ---- Server config ---------------------------------------------------------- -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -OFFLOAD_ARGS=() -PREFIX_CACHE_ARGS=() - -if require_agentic_kv_offload_backend native; then - export VLLM_USE_SIMPLE_KV_OFFLOAD=1 - OFFLOAD_ARGS=( - --kv_offloading_backend native - --kv_offloading_size "$TOTAL_CPU_DRAM_GB" - --disable-hybrid-kv-cache-manager - ) -fi - -echo "Starting vllm server..." -export PYTHONNOUSERSITE=1 - -{ set +x; } 2>/dev/null -VLLM_CMD=( - vllm serve "$MODEL_PATH" --served-model-name "$MODEL" - --host 0.0.0.0 - --port "$PORT" - --tensor-parallel-size="$TP" - --gpu-memory-utilization 0.90 - --max-num-seqs "$CONC" - --reasoning-parser kimi_k2 - --tool-call-parser kimi_k2 - --compilation_config.pass_config.fuse_allreduce_rms true - --kv-cache-dtype fp8 - --max-cudagraph-capture-size 2048 - --stream-interval 20 - --trust-remote-code - "${PREFIX_CACHE_ARGS[@]}" - "${OFFLOAD_ARGS[@]}" -) -printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" -printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt" -"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -# ---- Run benchmark ---------------------------------------------------------- -build_replay_cmd "$RESULT_DIR" - -run_agentic_replay_and_write_outputs "$RESULT_DIR" diff --git a/benchmarks/single_node/agentic/kimik2.5_fp4_mi355x.sh b/benchmarks/single_node/agentic/kimik2.5_fp4_mi355x.sh deleted file mode 100755 index 8a525d623..000000000 --- a/benchmarks/single_node/agentic/kimik2.5_fp4_mi355x.sh +++ /dev/null @@ -1,118 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -set -x - -# Agentic trace replay benchmark for Kimi-K2.5 FP4 on MI355X using vLLM. -# -# Required env vars: -# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR -# -# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=native. - - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -# ROCR/HIP visibility for vLLM 0.14+ -if [ -n "${ROCR_VISIBLE_DEVICES:-}" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -rocm-smi || true -amd-smi || true - -# ---- Resolve traces and install deps ---------------------------------------- -resolve_trace_source -install_agentic_deps - -# Install amd-quark for MXFP4 (manual install due to ROCm vLLM bug) -pip install amd-quark - -# Disable AITER RMSNorm for TP < 8 due to accuracy issues -if [ "${TP}" -lt 8 ]; then - export VLLM_ROCM_USE_AITER_RMSNORM=0 -fi -# Workaround for MEC FW <177 RCCL memory reclaim issue -version=$(rocm-smi --showfw 2>/dev/null | grep MEC | head -n 1 | awk '{print $NF}') -if [[ "$version" == "" || ${version:-0} -lt 177 ]]; then - export HSA_NO_SCRATCH_RECLAIM=1 -fi - -export VLLM_ROCM_USE_AITER=1 -export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 - -# ---- Server config ---------------------------------------------------------- -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -OFFLOAD_ARGS=() -PREFIX_CACHE_ARGS=() - -if require_agentic_kv_offload_backend native; then - unset VLLM_USE_SIMPLE_KV_OFFLOAD - # Use vLLM's regular native KV-offload path (OffloadingConnector), - # NOT the SimpleCPUOffloadConnector. The "native" backend resolves to - # OffloadingConnector by default; setting VLLM_USE_SIMPLE_KV_OFFLOAD=1 - # would switch it to SimpleCPUOffloadConnector. We intentionally leave - # that env var UNSET here so the regular OffloadingConnector path is - # used. The shortcut --kv_offloading_backend native + --kv_offloading_size - # form constructs the KVTransferConfig at engine startup - # (vllm/config/vllm.py:662). - OFFLOAD_ARGS=( - --kv_offloading_backend native - --kv_offloading_size "$TOTAL_CPU_DRAM_GB" - --disable-hybrid-kv-cache-manager - ) -fi - -EP_ARGS=() -if [ "$EP_SIZE" -gt 1 ]; then - EP_ARGS=(--enable-expert-parallel) -fi - -echo "Starting vllm server..." -export PYTHONNOUSERSITE=1 - -{ set +x; } 2>/dev/null -VLLM_CMD=( - vllm serve "$MODEL_PATH" --served-model-name "$MODEL" - --host 0.0.0.0 - --port "$PORT" - --tensor-parallel-size="$TP" - "${EP_ARGS[@]}" - --gpu-memory-utilization 0.90 - --block-size=1 - --trust-remote-code - --max-num-seqs "$CONC" - --mm-encoder-tp-mode data - "${PREFIX_CACHE_ARGS[@]}" - "${OFFLOAD_ARGS[@]}" -) -printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" -printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt" -"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -# ---- Run benchmark ---------------------------------------------------------- -build_replay_cmd "$RESULT_DIR" - -run_agentic_replay_and_write_outputs "$RESULT_DIR" diff --git a/benchmarks/single_node/agentic/kimik2.5_int4_b200.sh b/benchmarks/single_node/agentic/kimik2.5_int4_b200.sh deleted file mode 100755 index 8c5ab5328..000000000 --- a/benchmarks/single_node/agentic/kimik2.5_int4_b200.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -set -x - -# Agentic trace replay benchmark for Kimi-K2.5 INT4 on B200 using vLLM. -# -# Required env vars: -# MODEL, TP, CONC, RESULT_DIR - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION - - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -nvidia-smi - -# ---- Resolve traces and install deps ---------------------------------------- -resolve_trace_source -install_agentic_deps - -# ---- Server config ---------------------------------------------------------- -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -OFFLOAD_ARGS="" -if require_agentic_kv_offload_backend native; then - export VLLM_USE_SIMPLE_KV_OFFLOAD=1 - OFFLOAD_ARGS="--kv_offloading_backend native --kv_offloading_size $TOTAL_CPU_DRAM_GB --disable-hybrid-kv-cache-manager" -fi - -echo "Starting vllm server..." -export TORCH_CUDA_ARCH_LIST="10.0" -export PYTHONNOUSERSITE=1 -export VLLM_USE_FLASHINFER_MOE_INT4=1 - -vllm serve "$MODEL_PATH" --served-model-name "$MODEL" \ ---host 0.0.0.0 \ ---port $PORT \ ---gpu-memory-utilization 0.95 \ ---tensor-parallel-size $TP \ ---max-num-seqs $CONC \ ---reasoning-parser kimi_k2 \ ---tool-call-parser kimi_k2 \ ---compilation_config.pass_config.fuse_allreduce_rms true \ ---trust-remote-code \ -$OFFLOAD_ARGS > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -# ---- Run benchmark ---------------------------------------------------------- -build_replay_cmd "$RESULT_DIR" - -run_agentic_replay_and_write_outputs "$RESULT_DIR" diff --git a/benchmarks/single_node/agentic/kimik2.5_int4_h100.sh b/benchmarks/single_node/agentic/kimik2.5_int4_h100.sh deleted file mode 100755 index d5ade390b..000000000 --- a/benchmarks/single_node/agentic/kimik2.5_int4_h100.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -set -x - -# Agentic trace replay benchmark for Kimi-K2.5 INT4 on H100 using vLLM. -# -# Required env vars: -# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION - - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -nvidia-smi - -# ---- Resolve traces and install deps ---------------------------------------- -resolve_trace_source -install_agentic_deps - -# ---- Server config ---------------------------------------------------------- -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -OFFLOAD_ARGS="" -if require_agentic_kv_offload_backend native; then - export VLLM_USE_SIMPLE_KV_OFFLOAD=1 - OFFLOAD_ARGS="--kv_offloading_backend native --kv_offloading_size $TOTAL_CPU_DRAM_GB --disable-hybrid-kv-cache-manager" -fi - -echo "Starting vllm server..." -export PYTHONNOUSERSITE=1 -export VLLM_USE_FLASHINFER_MOE_INT4=1 - -vllm serve "$MODEL_PATH" --served-model-name "$MODEL" \ ---host 0.0.0.0 \ ---port $PORT \ ---gpu-memory-utilization 0.95 \ ---tensor-parallel-size $TP \ ---max-num-seqs $CONC \ ---reasoning-parser kimi_k2 \ ---tool-call-parser kimi_k2 \ ---compilation_config.pass_config.fuse_allreduce_rms true \ ---kv-cache-dtype fp8 \ ---trust-remote-code \ -$OFFLOAD_ARGS > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -# ---- Run benchmark ---------------------------------------------------------- -build_replay_cmd "$RESULT_DIR" - -run_agentic_replay_and_write_outputs "$RESULT_DIR" diff --git a/benchmarks/single_node/agentic/kimik2.5_int4_h200.sh b/benchmarks/single_node/agentic/kimik2.5_int4_h200.sh deleted file mode 100755 index 3b9411d1c..000000000 --- a/benchmarks/single_node/agentic/kimik2.5_int4_h200.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -set -x - -# Agentic trace replay benchmark for Kimi-K2.5 INT4 on H200 using vLLM. -# -# Required env vars: -# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION - - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -nvidia-smi - -# ---- Resolve traces and install deps ---------------------------------------- -resolve_trace_source -install_agentic_deps - -# ---- Server config ---------------------------------------------------------- -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -OFFLOAD_ARGS="" -if require_agentic_kv_offload_backend native; then - # Kimi K2.5 is pure TP (no DP-attn): single engine, world_size=TP. - # SimpleCPUOffloadConnector internally divides cpu_bytes_to_use by - # world_size, so pass the full TOTAL_CPU_DRAM_GB; TP-shared mmap - # keeps the aggregate at TOTAL. - PER_ENGINE_BYTES=$((TOTAL_CPU_DRAM_GB * 1024 * 1024 * 1024)) - # JSON form (rather than --kv_offloading_backend native shortcut) so we can - # pass lazy_offload=true. Eager mode hits a popleft_n AssertionError at - # low/mid CONC on DSv4 + SimpleCPUOffloadConnector. - export VLLM_USE_SIMPLE_KV_OFFLOAD=1 - OFFLOAD_ARGS="--kv-transfer-config {\"kv_connector\":\"SimpleCPUOffloadConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"cpu_bytes_to_use\":$PER_ENGINE_BYTES,\"lazy_offload\":true}}" -fi - -echo "Starting vllm server..." -export PYTHONNOUSERSITE=1 -export VLLM_USE_FLASHINFER_MOE_INT4=1 - -vllm serve "$MODEL_PATH" --served-model-name "$MODEL" \ ---host 0.0.0.0 \ ---port $PORT \ ---gpu-memory-utilization 0.95 \ ---tensor-parallel-size $TP \ ---max-num-seqs $CONC \ ---reasoning-parser kimi_k2 \ ---tool-call-parser kimi_k2 \ ---compilation_config.pass_config.fuse_allreduce_rms true \ ---trust-remote-code \ -$OFFLOAD_ARGS > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -# ---- Run benchmark ---------------------------------------------------------- -build_replay_cmd "$RESULT_DIR" - -run_agentic_replay_and_write_outputs "$RESULT_DIR" diff --git a/benchmarks/single_node/agentic/minimaxm3_fp8_h100.sh b/benchmarks/single_node/agentic/minimaxm3_fp8_h100.sh deleted file mode 100755 index 94f11afe6..000000000 --- a/benchmarks/single_node/agentic/minimaxm3_fp8_h100.sh +++ /dev/null @@ -1,135 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -set -x - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -nvidia-smi - -export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_062126 -resolve_trace_source -install_agentic_deps - -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export PYTHONNOUSERSITE=1 - -SERVER_LOG="$RESULT_DIR/server.log" -ROUTER_LOG="$RESULT_DIR/router.log" -MOONCAKE_MASTER_LOG="$RESULT_DIR/mooncake_master.log" -mkdir -p "$RESULT_DIR" - -OFFLOAD_ARGS=() -MODEL_CPU_OFFLOAD_GB=26 -MODEL_CHECKPOINT_PAGE_CACHE_GIB=414 -MOONCAKE_LOCAL_BUFFER_GIB=4 -if require_agentic_kv_offload_backend mooncake; then - TOTAL_CPU_DRAM_GIB=$((TOTAL_CPU_DRAM_GB * 1000000000 / 1073741824)) - PER_RANK_GIB=$(((TOTAL_CPU_DRAM_GIB - MODEL_CHECKPOINT_PAGE_CACHE_GIB) / TP - MODEL_CPU_OFFLOAD_GB - MOONCAKE_LOCAL_BUFFER_GIB)) - if (( PER_RANK_GIB <= 0 )); then - echo "Error: CPU DRAM budget is too small for checkpoint cache, model, and KV offload" >&2 - exit 1 - fi - MOONCAKE_VERSION=0.3.11.post1 - agentic_pip_install --quiet --no-cache-dir --no-deps \ - --force-reinstall "mooncake-transfer-engine-cuda13==$MOONCAKE_VERSION" - python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null - MOONCAKE_MASTER_PORT=$((PORT + 12000)) - MOONCAKE_CONFIG_PATH="$RESULT_DIR/mooncake_config.json" - cat > "$MOONCAKE_CONFIG_PATH" < "$MOONCAKE_MASTER_LOG" 2>&1 & - MOONCAKE_MASTER_PID=$! - sleep 2 - kill -0 "$MOONCAKE_MASTER_PID" - OFFLOAD_ARGS=( - --kv-transfer-config - '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}' - ) -fi - -PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) -if [[ "$DP_ATTENTION" == "true" ]]; then - PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") -fi - -EP_ARGS=() -if (( EP_SIZE > 1 )); then - EP_ARGS=(--enable-expert-parallel) -fi - -VLLM_BACKEND_PORT="$PORT" -if [[ "$DP_ATTENTION" == "true" ]]; then - VLLM_BACKEND_PORT=$((PORT + 1)) - export AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID=1 - agentic_pip_install --quiet 'vllm-router==0.1.14' -fi - -MAX_NUM_SEQS=$((2 * CONC)) -vllm serve "$MODEL_PATH" --served-model-name "$MODEL" \ - --host 0.0.0.0 \ - --port "$VLLM_BACKEND_PORT" \ - "${PARALLEL_ARGS[@]}" \ - "${EP_ARGS[@]}" \ - --gpu-memory-utilization 0.95 \ - --cpu-offload-gb "$MODEL_CPU_OFFLOAD_GB" \ - --kv-cache-dtype fp8 \ - --attention-backend TRITON_ATTN \ - --block-size 128 \ - --language-model-only \ - --enable-prefix-caching \ - --max-num-seqs "$MAX_NUM_SEQS" \ - --tool-call-parser minimax_m3 \ - --reasoning-parser minimax_m3 \ - --enable-auto-tool-choice \ - --safetensors-load-strategy lazy \ - --trust-remote-code \ - "${OFFLOAD_ARGS[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! - -wait_for_server_ready --port "$VLLM_BACKEND_PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [[ "$DP_ATTENTION" == "true" ]]; then - vllm-router \ - --worker-urls "http://localhost:$VLLM_BACKEND_PORT" \ - --policy consistent_hash \ - --intra-node-data-parallel-size "$TP" \ - --host 0.0.0.0 \ - --port "$PORT" \ - --prometheus-host 127.0.0.1 \ - --prometheus-port "$((PORT + 10000))" \ - --request-timeout-secs 14400 \ - --disable-retries > "$ROUTER_LOG" 2>&1 & - ROUTER_PID=$! - wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID" -fi - -build_replay_cmd "$RESULT_DIR" -run_agentic_replay_and_write_outputs "$RESULT_DIR" diff --git a/benchmarks/single_node/agentic/minimaxm3_fp8_h200.sh b/benchmarks/single_node/agentic/minimaxm3_fp8_h200.sh deleted file mode 100755 index c74926a52..000000000 --- a/benchmarks/single_node/agentic/minimaxm3_fp8_h200.sh +++ /dev/null @@ -1,177 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -set -x - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -resolve_complete_model_snapshot() { - python3 - "$1" <<'PY' -import json -import sys -from pathlib import Path - -model_cache_dir = Path(sys.argv[1]) -try: - revision = model_cache_dir.joinpath("refs/main").read_text().strip() -except OSError: - raise SystemExit - -if not revision or Path(revision).name != revision: - raise SystemExit - -snapshot = model_cache_dir / "snapshots" / revision -index_path = snapshot / "model.safetensors.index.json" -required_files = ( - snapshot / "config.json", - snapshot / "tokenizer_config.json", - index_path, -) -if not all(path.is_file() for path in required_files): - raise SystemExit -try: - weight_map = json.loads(index_path.read_text())["weight_map"] -except (KeyError, json.JSONDecodeError, OSError): - raise SystemExit -shards = {snapshot / filename for filename in weight_map.values()} -if shards and all(path.is_file() for path in shards): - print(snapshot) -PY -} - -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - MODEL_CACHE_ROOT="${HF_HUB_CACHE:-${HF_HOME:-$HOME/.cache/huggingface/hub}}" - MODEL_CACHE_DIR="$MODEL_CACHE_ROOT/models--${MODEL//\//--}" - mkdir -p "$MODEL_CACHE_ROOT" - MODEL_PATH=$(resolve_complete_model_snapshot "$MODEL_CACHE_DIR") - if [[ -z "$MODEL_PATH" ]]; then - exec 9>"$MODEL_CACHE_ROOT/.minimaxm3-download.lock" - flock -w 3600 9 - MODEL_PATH=$(resolve_complete_model_snapshot "$MODEL_CACHE_DIR") - if [[ -z "$MODEL_PATH" ]]; then - DOWNLOADED_MODEL_PATH=$(hf download "$MODEL") - MODEL_PATH=$(resolve_complete_model_snapshot "$MODEL_CACHE_DIR") - if [[ -z "$MODEL_PATH" ]]; then - echo "Downloaded model snapshot is incomplete: $DOWNLOADED_MODEL_PATH" >&2 - exit 1 - fi - fi - flock -u 9 - fi - echo "Using complete cached model snapshot: $MODEL_PATH" - export MODEL_PATH -fi -nvidia-smi - -export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_062126 -resolve_trace_source -install_agentic_deps - -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export PYTHONNOUSERSITE=1 - -SERVER_LOG="$RESULT_DIR/server.log" -ROUTER_LOG="$RESULT_DIR/router.log" -MOONCAKE_MASTER_LOG="$RESULT_DIR/mooncake_master.log" -mkdir -p "$RESULT_DIR" - -OFFLOAD_ARGS=() -if require_agentic_kv_offload_backend mooncake; then - PER_RANK_GB=$((TOTAL_CPU_DRAM_GB / TP)) - MOONCAKE_VERSION=0.3.11.post1 - agentic_pip_install --quiet --no-cache-dir --no-deps \ - --force-reinstall "mooncake-transfer-engine-cuda13==$MOONCAKE_VERSION" - python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null - MOONCAKE_MASTER_PORT=$((PORT + 12000)) - MOONCAKE_CONFIG_PATH="$RESULT_DIR/mooncake_config.json" - cat > "$MOONCAKE_CONFIG_PATH" < "$MOONCAKE_MASTER_LOG" 2>&1 & - MOONCAKE_MASTER_PID=$! - sleep 2 - kill -0 "$MOONCAKE_MASTER_PID" - OFFLOAD_ARGS=( - --kv-transfer-config - '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}' - ) -fi - -PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) -if [[ "$DP_ATTENTION" == "true" ]]; then - PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") -fi - -EP_ARGS=() -if (( EP_SIZE > 1 )); then - EP_ARGS=(--enable-expert-parallel) -fi - -VLLM_BACKEND_PORT="$PORT" -if [[ "$DP_ATTENTION" == "true" ]]; then - VLLM_BACKEND_PORT=$((PORT + 1)) - export AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID=1 - agentic_pip_install --quiet 'vllm-router==0.1.14' -fi - -MAX_NUM_SEQS=$((2 * CONC)) -vllm serve "$MODEL_PATH" --served-model-name "$MODEL" \ - --host 0.0.0.0 \ - --port "$VLLM_BACKEND_PORT" \ - "${PARALLEL_ARGS[@]}" \ - "${EP_ARGS[@]}" \ - --gpu-memory-utilization 0.92 \ - --kv-cache-dtype fp8 \ - --attention-backend TRITON_ATTN \ - --block-size 128 \ - --language-model-only \ - --enable-prefix-caching \ - --max-num-seqs "$MAX_NUM_SEQS" \ - --tool-call-parser minimax_m3 \ - --reasoning-parser minimax_m3 \ - --enable-auto-tool-choice \ - --trust-remote-code \ - "${OFFLOAD_ARGS[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! - -wait_for_server_ready --port "$VLLM_BACKEND_PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [[ "$DP_ATTENTION" == "true" ]]; then - vllm-router \ - --worker-urls "http://localhost:$VLLM_BACKEND_PORT" \ - --policy consistent_hash \ - --intra-node-data-parallel-size "$TP" \ - --host 0.0.0.0 \ - --port "$PORT" \ - --prometheus-host 127.0.0.1 \ - --prometheus-port "$((PORT + 10000))" \ - --request-timeout-secs 14400 \ - --disable-retries > "$ROUTER_LOG" 2>&1 & - ROUTER_PID=$! - wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID" -fi - -build_replay_cmd "$RESULT_DIR" -run_agentic_replay_and_write_outputs "$RESULT_DIR" diff --git a/benchmarks/single_node/agentic/minimaxm3_fp8_mi300x.sh b/benchmarks/single_node/agentic/minimaxm3_fp8_mi300x.sh deleted file mode 100755 index f6e98ea04..000000000 --- a/benchmarks/single_node/agentic/minimaxm3_fp8_mi300x.sh +++ /dev/null @@ -1,204 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -set -x - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -if [[ -n "${ROCR_VISIBLE_DEVICES:-}" ]]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -rocm-smi || true -amd-smi || true - -export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_062126 -resolve_trace_source -install_agentic_deps - -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export VLLM_USE_BREAKABLE_CUDAGRAPH=0 -export PYTHONNOUSERSITE=1 - -SERVER_LOG="$RESULT_DIR/server.log" -ROUTER_LOG="$RESULT_DIR/router.log" -MOONCAKE_MASTER_LOG="$RESULT_DIR/mooncake_master.log" -mkdir -p "$RESULT_DIR" - -install_mooncake_rocm() { - local mooncake_tag="v0.3.11.post1" - local mooncake_src="/tmp/Mooncake-$mooncake_tag" - local mooncake_stage="/tmp/mooncake-stage-$mooncake_tag" - local build_jobs - local cache_root - local cache_key - local cache_archive - local cache_tmp - local engine_path - local os_version - local python_abi - local rocm_version - - build_jobs=$(nproc) - if ((build_jobs > 32)); then - build_jobs=32 - fi - - os_version=$(. /etc/os-release && printf '%s-%s' "$ID" "$VERSION_ID") - python_abi=$(python3 -c 'import sys; print(f"cp{sys.version_info.major}{sys.version_info.minor}")') - rocm_version=$(sed -n '1p' /opt/rocm/.info/version 2>/dev/null || true) - if [[ -z "$rocm_version" ]]; then - rocm_version=$(hipconfig --version) - fi - rocm_version=${rocm_version//[^[:alnum:]._-]/_} - cache_root="${HF_HUB_CACHE:?HF_HUB_CACHE must be set}/inferencex/mooncake" - cache_key="${mooncake_tag}-${os_version}-${python_abi}-${rocm_version}-$(uname -m)-hip" - cache_archive="$cache_root/$cache_key.tar.gz" - mkdir -p "$cache_root" - - apt-get update - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - build-essential cmake git libasio-dev libboost-dev libcurl4-openssl-dev \ - libgflags-dev libgoogle-glog-dev libibverbs-dev libjsoncpp-dev \ - libnuma-dev libpython3-dev libssl-dev libunwind-dev liburing-dev \ - libxxhash-dev libyaml-cpp-dev libzstd-dev ninja-build pybind11-dev - - exec 9>"$cache_archive.lock" - flock -w 1800 9 - if [[ -f "$cache_archive" ]] && ! tar -tzf "$cache_archive" >/dev/null 2>&1; then - rm -f "$cache_archive" - fi - if [[ ! -f "$cache_archive" ]]; then - echo "Building HIP Mooncake cache artifact: $cache_archive" - rm -rf "$mooncake_src" "$mooncake_stage" - git clone --depth 1 --branch "$mooncake_tag" --recurse-submodules \ - --shallow-submodules https://github.com/kvcache-ai/Mooncake.git "$mooncake_src" - cmake -S "$mooncake_src/extern/yalantinglibs" \ - -B "$mooncake_src/extern/yalantinglibs/build" \ - -DBUILD_EXAMPLES=OFF -DBUILD_BENCHMARK=OFF -DBUILD_UNIT_TESTS=OFF - cmake --build "$mooncake_src/extern/yalantinglibs/build" -j "$build_jobs" - cmake --install "$mooncake_src/extern/yalantinglibs/build" - cmake -S "$mooncake_src" -B "$mooncake_src/build" -G Ninja \ - -DCMAKE_BUILD_TYPE=Release -DUSE_CUDA=OFF -DUSE_HIP=ON \ - -DWITH_EP=OFF -DWITH_STORE=ON -DWITH_STORE_RUST=OFF \ - -DWITH_RUST_EXAMPLE=OFF -DBUILD_EXAMPLES=OFF -DBUILD_UNIT_TESTS=OFF - cmake --build "$mooncake_src/build" -j "$build_jobs" - mkdir -p "$mooncake_stage" - DESTDIR="$mooncake_stage" cmake --install "$mooncake_src/build" - cache_tmp=$(mktemp "$cache_root/$cache_key.tmp.XXXXXX") - tar -C "$mooncake_stage" -czf "$cache_tmp" . - mv -f "$cache_tmp" "$cache_archive" - else - echo "Using HIP Mooncake cache artifact: $cache_archive" - fi - tar -C / -xzf "$cache_archive" - engine_path=$(python3 -c 'import mooncake.engine; print(mooncake.engine.__file__)') - ldd "$engine_path" | grep -q 'libamdhip64.so' - exec 9>&- -} - -OFFLOAD_ARGS=() -if require_agentic_kv_offload_backend mooncake; then - PER_RANK_GB=$((TOTAL_CPU_DRAM_GB / TP)) - if ! python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null 2>&1; then - install_mooncake_rocm - fi - python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null - MOONCAKE_MASTER_PORT=$((PORT + 12000)) - MOONCAKE_CONFIG_PATH="$RESULT_DIR/mooncake_config.json" - cat > "$MOONCAKE_CONFIG_PATH" < "$MOONCAKE_MASTER_LOG" 2>&1 & - MOONCAKE_MASTER_PID=$! - sleep 2 - kill -0 "$MOONCAKE_MASTER_PID" - OFFLOAD_ARGS=( - --kv-transfer-config - '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}' - ) -fi - -PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) -if [[ "$DP_ATTENTION" == "true" ]]; then - PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") -fi - -EP_ARGS=() -if (( EP_SIZE > 1 )); then - EP_ARGS=(--enable-expert-parallel) -fi - -VLLM_BACKEND_PORT="$PORT" -if [[ "$DP_ATTENTION" == "true" ]]; then - VLLM_BACKEND_PORT=$((PORT + 1)) - export AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID=1 - agentic_pip_install --quiet 'vllm-router==0.1.14' -fi - -MAX_NUM_SEQS=$((2 * CONC)) -vllm serve "$MODEL_PATH" --served-model-name "$MODEL" \ - --host 0.0.0.0 \ - --port "$VLLM_BACKEND_PORT" \ - "${PARALLEL_ARGS[@]}" \ - "${EP_ARGS[@]}" \ - --gpu-memory-utilization 0.95 \ - --block-size 128 \ - --language-model-only \ - --attention-backend TRITON_ATTN \ - --kv-cache-dtype fp8 \ - --enable-prefix-caching \ - --max-num-seqs "$MAX_NUM_SEQS" \ - --tool-call-parser minimax_m3 \ - --reasoning-parser minimax_m3 \ - --enable-auto-tool-choice \ - --trust-remote-code \ - "${OFFLOAD_ARGS[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! - -wait_for_server_ready --port "$VLLM_BACKEND_PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [[ "$DP_ATTENTION" == "true" ]]; then - vllm-router \ - --worker-urls "http://localhost:$VLLM_BACKEND_PORT" \ - --policy consistent_hash \ - --intra-node-data-parallel-size "$TP" \ - --host 0.0.0.0 \ - --port "$PORT" \ - --prometheus-host 127.0.0.1 \ - --prometheus-port "$((PORT + 10000))" \ - --request-timeout-secs 14400 \ - --disable-retries > "$ROUTER_LOG" 2>&1 & - ROUTER_PID=$! - wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID" -fi - -build_replay_cmd "$RESULT_DIR" -run_agentic_replay_and_write_outputs "$RESULT_DIR" diff --git a/benchmarks/single_node/agentic/minimaxm3_fp8_mi325x.sh b/benchmarks/single_node/agentic/minimaxm3_fp8_mi325x.sh deleted file mode 100755 index 324d98b7c..000000000 --- a/benchmarks/single_node/agentic/minimaxm3_fp8_mi325x.sh +++ /dev/null @@ -1,214 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -set -x - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars MODEL IMAGE TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -if [[ -n "${ROCR_VISIBLE_DEVICES:-}" ]]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -rocm-smi || true -amd-smi || true - -export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_062126 -resolve_trace_source -install_agentic_deps - -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export VLLM_USE_BREAKABLE_CUDAGRAPH=0 -export PYTHONNOUSERSITE=1 - -SERVER_LOG="$RESULT_DIR/server.log" -ROUTER_LOG="$RESULT_DIR/router.log" -MOONCAKE_MASTER_LOG="$RESULT_DIR/mooncake_master.log" -mkdir -p "$RESULT_DIR" - -install_mooncake_rocm() { - local mooncake_tag="v0.3.11.post1" - local mooncake_src="/tmp/Mooncake-$mooncake_tag" - local mooncake_stage="/tmp/mooncake-stage-$mooncake_tag" - local build_jobs - local cache_root - local cache_key - local cache_archive - local cache_tmp - local engine_path - local os_version - local python_abi - local rocm_version - - build_jobs=$(nproc) - if ((build_jobs > 32)); then - build_jobs=32 - fi - - os_version=$(. /etc/os-release && printf '%s-%s' "$ID" "$VERSION_ID") - python_abi=$(python3 -c 'import sys; print(f"cp{sys.version_info.major}{sys.version_info.minor}")') - rocm_version=$(sed -n '1p' /opt/rocm/.info/version 2>/dev/null || true) - if [[ -z "$rocm_version" ]]; then - rocm_version=$(hipconfig --version) - fi - rocm_version=${rocm_version//[^[:alnum:]._-]/_} - cache_root="${HF_HUB_CACHE:?HF_HUB_CACHE must be set}/inferencex/mooncake" - cache_key="${mooncake_tag}-${os_version}-${python_abi}-${rocm_version}-$(uname -m)-hip" - cache_archive="$cache_root/$cache_key.tar.gz" - mkdir -p "$cache_root" - - apt-get update - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - build-essential cmake git libasio-dev libboost-dev libcurl4-openssl-dev \ - libgflags-dev libgoogle-glog-dev libibverbs-dev libjsoncpp-dev \ - libnuma-dev libpython3-dev libssl-dev libunwind-dev liburing-dev \ - libxxhash-dev libyaml-cpp-dev libzstd-dev ninja-build pybind11-dev - - exec 9>"$cache_archive.lock" - flock -w 1800 9 - if [[ -f "$cache_archive" ]] && ! tar -tzf "$cache_archive" >/dev/null 2>&1; then - rm -f "$cache_archive" - fi - if [[ ! -f "$cache_archive" ]]; then - echo "Building HIP Mooncake cache artifact: $cache_archive" - rm -rf "$mooncake_src" "$mooncake_stage" - git clone --depth 1 --branch "$mooncake_tag" --recurse-submodules \ - --shallow-submodules https://github.com/kvcache-ai/Mooncake.git "$mooncake_src" - cmake -S "$mooncake_src/extern/yalantinglibs" \ - -B "$mooncake_src/extern/yalantinglibs/build" \ - -DBUILD_EXAMPLES=OFF -DBUILD_BENCHMARK=OFF -DBUILD_UNIT_TESTS=OFF - cmake --build "$mooncake_src/extern/yalantinglibs/build" -j "$build_jobs" - cmake --install "$mooncake_src/extern/yalantinglibs/build" - cmake -S "$mooncake_src" -B "$mooncake_src/build" -G Ninja \ - -DCMAKE_BUILD_TYPE=Release -DUSE_CUDA=OFF -DUSE_HIP=ON \ - -DWITH_EP=OFF -DWITH_STORE=ON -DWITH_STORE_RUST=OFF \ - -DWITH_RUST_EXAMPLE=OFF -DBUILD_EXAMPLES=OFF -DBUILD_UNIT_TESTS=OFF - cmake --build "$mooncake_src/build" -j "$build_jobs" - mkdir -p "$mooncake_stage" - DESTDIR="$mooncake_stage" cmake --install "$mooncake_src/build" - cache_tmp=$(mktemp "$cache_root/$cache_key.tmp.XXXXXX") - tar -C "$mooncake_stage" -czf "$cache_tmp" . - mv -f "$cache_tmp" "$cache_archive" - else - echo "Using HIP Mooncake cache artifact: $cache_archive" - fi - tar -C / -xzf "$cache_archive" - engine_path=$(python3 -c 'import mooncake.engine; print(mooncake.engine.__file__)') - ldd "$engine_path" | grep -q 'libamdhip64.so' - exec 9>&- -} - -OFFLOAD_ARGS=() -if require_agentic_kv_offload_backend mooncake; then - PER_RANK_GB=$((TOTAL_CPU_DRAM_GB / TP)) - if ! python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null 2>&1; then - install_mooncake_rocm - fi - python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null - MOONCAKE_MASTER_PORT=$((PORT + 12000)) - MOONCAKE_CONFIG_PATH="$RESULT_DIR/mooncake_config.json" - cat > "$MOONCAKE_CONFIG_PATH" < "$MOONCAKE_MASTER_LOG" 2>&1 & - MOONCAKE_MASTER_PID=$! - sleep 2 - kill -0 "$MOONCAKE_MASTER_PID" - OFFLOAD_ARGS=( - --kv-transfer-config - '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}' - ) -fi - -PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) -if [[ "$DP_ATTENTION" == "true" ]]; then - PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") -fi - -EP_ARGS=() -if (( EP_SIZE > 1 )); then - EP_ARGS=(--enable-expert-parallel) -fi - -KV_CACHE_ARGS=() -if [[ "$IMAGE" == vllm/vllm-openai-rocm:nightly-* ]]; then - KV_CACHE_ARGS=(--kv-cache-dtype fp8) -fi - -VLLM_BACKEND_PORT="$PORT" -if [[ "$DP_ATTENTION" == "true" ]]; then - VLLM_BACKEND_PORT=$((PORT + 1)) - export AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID=1 - agentic_pip_install --quiet 'vllm-router==0.1.14' -fi - -MAX_NUM_SEQS=$((2 * CONC)) -GPU_MEMORY_UTILIZATION=0.95 -if (( TP == 4 )); then - GPU_MEMORY_UTILIZATION=0.98 -fi - -vllm serve "$MODEL_PATH" --served-model-name "$MODEL" \ - --host 0.0.0.0 \ - --port "$VLLM_BACKEND_PORT" \ - "${PARALLEL_ARGS[@]}" \ - "${EP_ARGS[@]}" \ - --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" \ - --block-size 128 \ - --language-model-only \ - --attention-backend TRITON_ATTN \ - --enable-prefix-caching \ - --max-num-seqs "$MAX_NUM_SEQS" \ - --tool-call-parser minimax_m3 \ - --reasoning-parser minimax_m3 \ - --enable-auto-tool-choice \ - --trust-remote-code \ - "${KV_CACHE_ARGS[@]}" \ - "${OFFLOAD_ARGS[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! - -wait_for_server_ready --port "$VLLM_BACKEND_PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [[ "$DP_ATTENTION" == "true" ]]; then - vllm-router \ - --worker-urls "http://localhost:$VLLM_BACKEND_PORT" \ - --policy consistent_hash \ - --intra-node-data-parallel-size "$TP" \ - --host 0.0.0.0 \ - --port "$PORT" \ - --prometheus-host 127.0.0.1 \ - --prometheus-port "$((PORT + 10000))" \ - --request-timeout-secs 14400 \ - --disable-retries > "$ROUTER_LOG" 2>&1 & - ROUTER_PID=$! - wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID" -fi - -build_replay_cmd "$RESULT_DIR" -run_agentic_replay_and_write_outputs "$RESULT_DIR" diff --git a/benchmarks/single_node/agentic/qwen3.5_bf16_b200.sh b/benchmarks/single_node/agentic/qwen3.5_bf16_b200.sh deleted file mode 100755 index 722770765..000000000 --- a/benchmarks/single_node/agentic/qwen3.5_bf16_b200.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -set -x - -# Agentic trace replay benchmark for Qwen3.5 BF16 on B200 using SGLang. -# -# Required env vars: -# MODEL, TP, CONC, RESULT_DIR - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC RESULT_DIR DURATION EP_SIZE - -SCHEDULER_RECV_INTERVAL=${SCHEDULER_RECV_INTERVAL:-10} - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -nvidia-smi - -# ---- Resolve traces and install deps ---------------------------------------- -resolve_trace_source -install_agentic_deps - -# ---- Start SGLang server ---------------------------------------------------- -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -echo "Starting SGLang server..." -export TORCH_CUDA_ARCH_LIST="10.0" -export PYTHONNOUSERSITE=1 -export NCCL_NVLS_ENABLE=1 -export SGL_ENABLE_JIT_DEEPGEMM=false -export SGLANG_ENABLE_FLASHINFER_GEMM=true - -python3 -m sglang.launch_server \ ---model-path=$MODEL_PATH --served-model-name=$MODEL \ ---host=0.0.0.0 \ ---port=$PORT \ ---served-model-name "Qwen/Qwen3.5-397B-A17B" \ ---trust-remote-code \ ---tensor-parallel-size=$TP \ ---data-parallel-size=1 \ ---ep-size $EP_SIZE \ ---cuda-graph-max-bs $CONC \ ---max-running-requests $CONC \ ---mem-fraction-static 0.82 \ ---chunked-prefill-size 32768 \ ---max-prefill-tokens 32768 \ ---attention-backend trtllm_mha \ ---moe-runner-backend flashinfer_trtllm \ ---enable-flashinfer-allreduce-fusion \ ---scheduler-recv-interval $SCHEDULER_RECV_INTERVAL \ ---tokenizer-worker-num 6 \ ---stream-interval 30 \ ---enable-metrics > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -# ---- Run benchmark ---------------------------------------------------------- -build_replay_cmd "$RESULT_DIR" - -run_agentic_replay_and_write_outputs "$RESULT_DIR" diff --git a/benchmarks/single_node/agentic/qwen3.5_fp8_b200.sh b/benchmarks/single_node/agentic/qwen3.5_fp8_b200.sh deleted file mode 100755 index 04cb7b642..000000000 --- a/benchmarks/single_node/agentic/qwen3.5_fp8_b200.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -set -x - -# Agentic trace replay benchmark for Qwen3.5 FP8 on B200 using SGLang. -# -# Required env vars: -# MODEL, TP, CONC, RESULT_DIR - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC RESULT_DIR DURATION EP_SIZE - -SCHEDULER_RECV_INTERVAL=${SCHEDULER_RECV_INTERVAL:-10} - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -nvidia-smi - -# ---- Resolve traces and install deps ---------------------------------------- -resolve_trace_source -install_agentic_deps - -# ---- Start SGLang server ---------------------------------------------------- -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -echo "Starting SGLang server..." -export TORCH_CUDA_ARCH_LIST="10.0" -export PYTHONNOUSERSITE=1 -export NCCL_NVLS_ENABLE=1 -export SGL_ENABLE_JIT_DEEPGEMM=false -export SGLANG_ENABLE_FLASHINFER_GEMM=true - -python3 -m sglang.launch_server \ ---model-path=$MODEL_PATH --served-model-name=$MODEL \ ---host=0.0.0.0 \ ---port=$PORT \ ---served-model-name "Qwen/Qwen3.5-397B-A17B-FP8" \ ---trust-remote-code \ ---tensor-parallel-size=$TP \ ---data-parallel-size=1 \ ---ep-size $EP_SIZE \ ---cuda-graph-max-bs $CONC \ ---max-running-requests $CONC \ ---mem-fraction-static 0.82 \ ---chunked-prefill-size 32768 \ ---max-prefill-tokens 32768 \ ---attention-backend trtllm_mha \ ---moe-runner-backend flashinfer_trtllm \ ---enable-flashinfer-allreduce-fusion \ ---scheduler-recv-interval $SCHEDULER_RECV_INTERVAL \ ---tokenizer-worker-num 6 \ ---stream-interval 30 \ ---enable-metrics > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -# ---- Run benchmark ---------------------------------------------------------- -build_replay_cmd "$RESULT_DIR" - -run_agentic_replay_and_write_outputs "$RESULT_DIR" diff --git a/benchmarks/single_node/agentic/qwen3.5_fp8_b300_sglang.sh b/benchmarks/single_node/agentic/qwen3.5_fp8_b300_sglang.sh deleted file mode 100755 index d52c52bf6..000000000 --- a/benchmarks/single_node/agentic/qwen3.5_fp8_b300_sglang.sh +++ /dev/null @@ -1,127 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -set -x - -# Agentic trace replay benchmark for Qwen3.5 FP8 on B300 using SGLang. -# -# Required env vars: -# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR -# -# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=hicache. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE - -SCHEDULER_RECV_INTERVAL=${SCHEDULER_RECV_INTERVAL:-10} - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -nvidia-smi - -# ---- Resolve traces and install deps ---------------------------------------- -resolve_trace_source -install_agentic_deps - -# ---- Server config ---------------------------------------------------------- -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -CACHE_ARGS=() -if require_agentic_kv_offload_backend hicache; then - # HiCache extends RadixAttention, so do not pass --disable-radix-cache. - # Qwen3.5's hybrid GDN/Mamba path allocates two HiCache host pools per TP - # rank: one for hierarchical KV cache and one for hierarchical Mamba cache. - REQUESTED_HICACHE_TOTAL_GB="${HICACHE_TOTAL_CPU_DRAM_GB:-$TOTAL_CPU_DRAM_GB}" - if [ "$REQUESTED_HICACHE_TOTAL_GB" -gt "$TOTAL_CPU_DRAM_GB" ]; then - echo "Error: requested HiCache pool ${REQUESTED_HICACHE_TOTAL_GB} GB exceeds configured capacity ${TOTAL_CPU_DRAM_GB} GB" >&2 - exit 1 - fi - TOTAL_CPU_DRAM_GB="$REQUESTED_HICACHE_TOTAL_GB" - HICACHE_HOST_POOL_COUNT="${HICACHE_HOST_POOL_COUNT:-2}" - HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through_selective}" - # SGLang --hicache-size is per rank per host pool, while the workflow - # input is a node-total DRAM budget. Divide by TP and the number of - # host pools unless HICACHE_SIZE_GB is set directly for one-off tuning. - MAX_HICACHE_SIZE_GB=$((TOTAL_CPU_DRAM_GB / TP / HICACHE_HOST_POOL_COUNT)) - HICACHE_SIZE_GB="${HICACHE_SIZE_GB:-$MAX_HICACHE_SIZE_GB}" - if [ "$HICACHE_SIZE_GB" -gt "$MAX_HICACHE_SIZE_GB" ]; then - echo "Error: HICACHE_SIZE_GB=$HICACHE_SIZE_GB exceeds configured per-pool limit $MAX_HICACHE_SIZE_GB" >&2 - exit 1 - fi - if [ "$HICACHE_SIZE_GB" -lt 1 ]; then - echo "Error: computed HICACHE_SIZE_GB=$HICACHE_SIZE_GB from TOTAL_CPU_DRAM_GB=$TOTAL_CPU_DRAM_GB, TP=$TP, HICACHE_HOST_POOL_COUNT=$HICACHE_HOST_POOL_COUNT" >&2 - exit 1 - fi - echo "HiCache CPU pool: ${HICACHE_SIZE_GB} GB per rank per host pool across TP=${TP}, host_pool_count=${HICACHE_HOST_POOL_COUNT}" - CACHE_ARGS=( - --page-size 64 - --enable-hierarchical-cache - --hicache-size "$HICACHE_SIZE_GB" - --hicache-io-backend kernel - --hicache-mem-layout page_first - --hicache-write-policy "$HICACHE_WRITE_POLICY" - ) -fi - -echo "Starting SGLang server..." -export TORCH_CUDA_ARCH_LIST="10.0" -export PYTHONNOUSERSITE=1 -export NCCL_NVLS_ENABLE=1 -export SGL_ENABLE_JIT_DEEPGEMM=false -export SGLANG_ENABLE_FLASHINFER_GEMM=true - -{ set +x; } 2>/dev/null -SGLANG_CMD=( - python3 -m sglang.launch_server - --model-path="$MODEL_PATH" --served-model-name="$MODEL" - --host=0.0.0.0 - --port="$PORT" - --served-model-name "Qwen/Qwen3.5-397B-A17B-FP8" - --trust-remote-code - --tensor-parallel-size="$TP" - --data-parallel-size=1 - --expert-parallel-size="$EP_SIZE" - --enable-symm-mem - --quantization fp8 - --kv-cache-dtype fp8_e4m3 - --mamba-ssm-dtype bfloat16 - --attention-backend trtllm_mha - --moe-runner-backend flashinfer_trtllm - --cuda-graph-max-bs "$CONC" - --max-running-requests "$CONC" - --max-prefill-tokens 16384 - --chunked-prefill-size 16384 - --mem-fraction-static 0.80 - --stream-interval 50 - --scheduler-recv-interval "$SCHEDULER_RECV_INTERVAL" - --tokenizer-worker-num 6 - --tokenizer-path "$MODEL" - --enable-metrics - "${CACHE_ARGS[@]}" -) -printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt" -printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt" -"${SGLANG_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -# ---- Run benchmark ---------------------------------------------------------- -build_replay_cmd "$RESULT_DIR" - -run_agentic_replay_and_write_outputs "$RESULT_DIR" diff --git a/benchmarks/single_node/agentic/qwen3.5_fp8_h100.sh b/benchmarks/single_node/agentic/qwen3.5_fp8_h100.sh deleted file mode 100755 index 9a548e80d..000000000 --- a/benchmarks/single_node/agentic/qwen3.5_fp8_h100.sh +++ /dev/null @@ -1,141 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -set -x - -# Agentic trace replay benchmark for Qwen3.5 FP8 on H100 using SGLang. -# -# H100 has 80 GB HBM3 (vs B300's 192 GB), so weights + KV fit tighter. -# Mem-fraction-static lowered to 0.75 and chunked-prefill-size halved to -# 8192 (mirrors fixed_seq_len/qwen3.5_fp8_h100.sh). Attention backend is -# flashinfer (sm_90); the trtllm_mha path is Blackwell-only. -# -# Required env vars: -# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR -# -# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=hicache. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE - -SCHEDULER_RECV_INTERVAL=${SCHEDULER_RECV_INTERVAL:-10} - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -nvidia-smi - -# ---- Resolve traces and install deps ---------------------------------------- -# H100 max_model_len caps at 131k (HBM-bound). The unfiltered with-subagents -# corpus has requests up to ~1M proxy tokens that the server would reject. -# Switch to the 256k-capped variant (470 traces, max in+out <= 256k); even -# at 131k context, the rejection rate is much lower than against the -# unfiltered corpus. -export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_with_subagents_256k - -resolve_trace_source -install_agentic_deps - -# ---- Server config ---------------------------------------------------------- -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -CACHE_ARGS=() -if require_agentic_kv_offload_backend hicache; then - # HiCache extends RadixAttention, so do not pass --disable-radix-cache. - # Hybrid GDN/Mamba allocates one KV and one Mamba host pool per rank. - REQUESTED_HICACHE_TOTAL_GB="${HICACHE_TOTAL_CPU_DRAM_GB:-$TOTAL_CPU_DRAM_GB}" - if [ "$REQUESTED_HICACHE_TOTAL_GB" -gt "$TOTAL_CPU_DRAM_GB" ]; then - echo "Error: requested HiCache pool ${REQUESTED_HICACHE_TOTAL_GB} GB exceeds configured capacity ${TOTAL_CPU_DRAM_GB} GB" >&2 - exit 1 - fi - TOTAL_CPU_DRAM_GB="$REQUESTED_HICACHE_TOTAL_GB" - HICACHE_HOST_POOL_COUNT="${HICACHE_HOST_POOL_COUNT:-2}" - HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through_selective}" - MAX_HICACHE_SIZE_GB=$((TOTAL_CPU_DRAM_GB / TP / HICACHE_HOST_POOL_COUNT)) - HICACHE_SIZE_GB="${HICACHE_SIZE_GB:-$MAX_HICACHE_SIZE_GB}" - if [ "$HICACHE_SIZE_GB" -gt "$MAX_HICACHE_SIZE_GB" ]; then - echo "Error: HICACHE_SIZE_GB=$HICACHE_SIZE_GB exceeds configured per-pool limit $MAX_HICACHE_SIZE_GB" >&2 - exit 1 - fi - if [ "$HICACHE_SIZE_GB" -lt 1 ]; then - echo "Error: computed HICACHE_SIZE_GB=$HICACHE_SIZE_GB from TOTAL_CPU_DRAM_GB=$TOTAL_CPU_DRAM_GB, TP=$TP, HICACHE_HOST_POOL_COUNT=$HICACHE_HOST_POOL_COUNT" >&2 - exit 1 - fi - echo "HiCache CPU pool: ${HICACHE_SIZE_GB} GB per rank per host pool across TP=${TP}, host_pool_count=${HICACHE_HOST_POOL_COUNT}" - CACHE_ARGS=( - --page-size 64 - --enable-hierarchical-cache - --hicache-size "$HICACHE_SIZE_GB" - --hicache-io-backend kernel - --hicache-mem-layout page_first - --hicache-write-policy "$HICACHE_WRITE_POLICY" - ) -fi - -echo "Starting SGLang server..." -export PYTHONNOUSERSITE=1 - -SGLANG_MULTI_TOKENIZER=/sgl-workspace/sglang/python/sglang/srt/managers/multi_tokenizer_mixin.py -if ! sed -n '/elif isinstance(output, BatchStrOutput):/,/input_token_logprobs_val=_extract_field_by_index/p' "$SGLANG_MULTI_TOKENIZER" \ - | grep -q 'cached_tokens_details=_extract_field_by_index'; then - sed -i '/elif isinstance(output, BatchStrOutput):/,/input_token_logprobs_val=_extract_field_by_index/ { - /cached_tokens=_extract_field_by_index(output, "cached_tokens", i),/a\ - cached_tokens_details=_extract_field_by_index(\ - output, "cached_tokens_details", i\ - ), - }' "$SGLANG_MULTI_TOKENIZER" -fi - -{ set +x; } 2>/dev/null -SGLANG_CMD=( - python3 -m sglang.launch_server - --model-path="$MODEL_PATH" --served-model-name="$MODEL" - --host=0.0.0.0 - --port="$PORT" - --served-model-name "Qwen/Qwen3.5-397B-A17B-FP8" - --trust-remote-code - --tensor-parallel-size="$TP" - --data-parallel-size=1 - --expert-parallel-size="$EP_SIZE" - --quantization fp8 - --kv-cache-dtype fp8_e4m3 - --mamba-ssm-dtype bfloat16 - --attention-backend flashinfer - --enable-flashinfer-allreduce-fusion - # --cuda-graph-max-bs "$CONC" - # --max-running-requests "$CONC" - # --max-prefill-tokens 8192 - # --chunked-prefill-size 8192 - --mem-fraction-static 0.75 - --stream-interval 50 - --scheduler-recv-interval "$SCHEDULER_RECV_INTERVAL" - --tokenizer-worker-num 6 - --tokenizer-path "$MODEL" - --enable-metrics - "${CACHE_ARGS[@]}" -) -printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt" -printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt" -"${SGLANG_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -# ---- Run benchmark ---------------------------------------------------------- -build_replay_cmd "$RESULT_DIR" - -run_agentic_replay_and_write_outputs "$RESULT_DIR" diff --git a/benchmarks/single_node/agentic/qwen3.5_fp8_mi355x.sh b/benchmarks/single_node/agentic/qwen3.5_fp8_mi355x.sh deleted file mode 100755 index 378a4204a..000000000 --- a/benchmarks/single_node/agentic/qwen3.5_fp8_mi355x.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -set -x - -# Agentic trace replay benchmark for Qwen3.5 FP8 on MI355X using SGLang. -# -# Required env vars: -# MODEL, TP, CONC, RESULT_DIR - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC RESULT_DIR DURATION EP_SIZE - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -rocm-smi || true -amd-smi || true - -# ---- Resolve traces and install deps ---------------------------------------- -resolve_trace_source -install_agentic_deps - -# ---- Start SGLang server ---------------------------------------------------- -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -echo "Starting SGLang server..." -export PYTHONNOUSERSITE=1 - -python3 -m sglang.launch_server \ - --attention-backend triton \ - --model-path "$MODEL_PATH" --served-model-name "$MODEL" \ - --host=0.0.0.0 \ - --port $PORT \ - --tensor-parallel-size $TP \ - --ep-size $EP_SIZE \ - --trust-remote-code \ - --tokenizer-worker-num 6 \ - --enable-aiter-allreduce-fusion \ - --cuda-graph-max-bs $CONC \ - --max-running-requests $CONC \ - --max-prefill-tokens 32768 \ - --scheduler-recv-interval 30 \ - --mem-fraction-static 0.8 \ - --enable-metrics > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -# ---- Run benchmark ---------------------------------------------------------- -build_replay_cmd "$RESULT_DIR" - -run_agentic_replay_and_write_outputs "$RESULT_DIR" diff --git a/benchmarks/single_node/agentic/qwen3.5_fp8_mi355x_sglang.sh b/benchmarks/single_node/agentic/qwen3.5_fp8_mi355x_sglang.sh deleted file mode 100755 index 54d58c52b..000000000 --- a/benchmarks/single_node/agentic/qwen3.5_fp8_mi355x_sglang.sh +++ /dev/null @@ -1,137 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -set -x - -# Agentic trace replay benchmark for Qwen3.5 FP8 on MI355X using SGLang. -# -# Required env vars: -# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR -# -# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=hicache. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE - -SCHEDULER_RECV_INTERVAL=${SCHEDULER_RECV_INTERVAL:-30} - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -rocm-smi || true -amd-smi || true - -# ---- Resolve traces and install deps ---------------------------------------- -resolve_trace_source -install_agentic_deps - -# ---- Server config ---------------------------------------------------------- -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -CACHE_ARGS=() -WARMUP_ARGS=() -CUDA_GRAPH_MAX_BS="$CONC" -if require_agentic_kv_offload_backend hicache; then - # Qwen3.5 allocates one KV and one Mamba host pool per TP rank. - REQUESTED_HICACHE_TOTAL_GB="${HICACHE_TOTAL_CPU_DRAM_GB:-$TOTAL_CPU_DRAM_GB}" - if [ "$REQUESTED_HICACHE_TOTAL_GB" -gt "$TOTAL_CPU_DRAM_GB" ]; then - echo "Error: requested HiCache pool ${REQUESTED_HICACHE_TOTAL_GB} GB exceeds configured capacity ${TOTAL_CPU_DRAM_GB} GB" >&2 - exit 1 - fi - TOTAL_CPU_DRAM_GB="$REQUESTED_HICACHE_TOTAL_GB" - HICACHE_HOST_POOL_COUNT="${HICACHE_HOST_POOL_COUNT:-2}" - HICACHE_MAX_SIZE_GB_PER_RANK_POOL="${HICACHE_MAX_SIZE_GB_PER_RANK_POOL:-${HICACHE_MAX_SIZE_GB_PER_RANK:-180}}" - HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through_selective}" - # Qwen3.5's hybrid Mamba path runs SGLang's no_buffer scheduler on MI355X, - # which requires page_size=1. The kernel/page_first HiCache transfer path - # faults on first prefill in this mode on ROCm, so keep the default on the - # safer direct/layer_first copy path. These remain env-overridable. - HICACHE_PAGE_SIZE="${HICACHE_PAGE_SIZE:-1}" - HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" - HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-layer_first}" - # SGLang --hicache-size is per rank per host pool, while the workflow - # input is a node-total DRAM budget. Divide by TP and the number of - # host pools unless HICACHE_SIZE_GB is set directly for one-off tuning. - MAX_HICACHE_SIZE_GB=$((TOTAL_CPU_DRAM_GB / TP / HICACHE_HOST_POOL_COUNT)) - HICACHE_SIZE_GB="${HICACHE_SIZE_GB:-$MAX_HICACHE_SIZE_GB}" - if [ "$HICACHE_SIZE_GB" -gt "$MAX_HICACHE_SIZE_GB" ]; then - echo "Error: HICACHE_SIZE_GB=$HICACHE_SIZE_GB exceeds configured per-pool limit $MAX_HICACHE_SIZE_GB" >&2 - exit 1 - fi - if [ "$HICACHE_SIZE_GB" -gt "$HICACHE_MAX_SIZE_GB_PER_RANK_POOL" ]; then - HICACHE_SIZE_GB="$HICACHE_MAX_SIZE_GB_PER_RANK_POOL" - fi - if [ "$HICACHE_SIZE_GB" -lt 1 ]; then - echo "Error: computed HICACHE_SIZE_GB=$HICACHE_SIZE_GB from TOTAL_CPU_DRAM_GB=$TOTAL_CPU_DRAM_GB, TP=$TP, HICACHE_HOST_POOL_COUNT=$HICACHE_HOST_POOL_COUNT" >&2 - exit 1 - fi - echo "HiCache CPU pool: ${HICACHE_SIZE_GB} GB per rank per host pool across TP=${TP}, host_pool_count=${HICACHE_HOST_POOL_COUNT}" - CACHE_ARGS=( - --page-size "$HICACHE_PAGE_SIZE" - --enable-hierarchical-cache - --hicache-size "$HICACHE_SIZE_GB" - --hicache-io-backend "$HICACHE_IO_BACKEND" - --hicache-mem-layout "$HICACHE_MEM_LAYOUT" - --hicache-write-policy "$HICACHE_WRITE_POLICY" - ) - # HiCache startup reaches API readiness, but SGLang's internal warmup - # request has timed out after 600s on this Qwen MI355X path. Let aiperf - # own benchmark traffic instead of blocking server readiness on it. - WARMUP_ARGS=(--skip-server-warmup) - # Keep request concurrency as the swept variable, but do not force HiCache - # runs to capture ROCm graphs at every high concurrency point. - HICACHE_CUDA_GRAPH_MAX_BS="${HICACHE_CUDA_GRAPH_MAX_BS:-16}" - if [ "$HICACHE_CUDA_GRAPH_MAX_BS" -lt "$CUDA_GRAPH_MAX_BS" ]; then - CUDA_GRAPH_MAX_BS="$HICACHE_CUDA_GRAPH_MAX_BS" - fi -fi - -echo "Starting SGLang server..." -export PYTHONNOUSERSITE=1 - -{ set +x; } 2>/dev/null -SGLANG_CMD=( - python3 -m sglang.launch_server - --attention-backend triton - --model-path "$MODEL_PATH" --served-model-name "$MODEL" - --host=0.0.0.0 - --port "$PORT" - --tensor-parallel-size "$TP" - --ep-size "$EP_SIZE" - --trust-remote-code - --tokenizer-worker-num 6 - --enable-aiter-allreduce-fusion - --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS" - --max-running-requests "$CONC" - --max-prefill-tokens 32768 - --scheduler-recv-interval "$SCHEDULER_RECV_INTERVAL" - --mem-fraction-static 0.8 - --enable-metrics - "${CACHE_ARGS[@]}" - "${WARMUP_ARGS[@]}" -) -printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt" -printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt" -"${SGLANG_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -# ---- Run benchmark ---------------------------------------------------------- -build_replay_cmd "$RESULT_DIR" - -run_agentic_replay_and_write_outputs "$RESULT_DIR" diff --git a/benchmarks/single_node/chat_templates/deepseek_v4_thinking.jinja b/benchmarks/single_node/chat_templates/deepseek_v4_thinking.jinja deleted file mode 100644 index 7aa5486ff..000000000 --- a/benchmarks/single_node/chat_templates/deepseek_v4_thinking.jinja +++ /dev/null @@ -1,9 +0,0 @@ -{%- if add_generation_prompt is not defined -%}{%- set add_generation_prompt = true -%}{%- endif -%} -{{- '<｜begin▁of▁sentence｜>' -}} -{%- for m in messages -%} -{%- if m['role'] == 'system' -%}{{- m['content'] -}} -{%- elif m['role'] == 'user' -%}<｜User｜>{{- m['content'] -}} -{%- elif m['role'] == 'assistant' -%}<｜Assistant｜>{{- m['content'] -}}<｜end▁of▁sentence｜> -{%- endif -%} -{%- endfor -%} -{%- if add_generation_prompt -%}<｜Assistant｜>{%- endif -%} diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp4_b200.sh b/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp4_b200.sh deleted file mode 100755 index fc7877a1c..000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp4_b200.sh +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -SERVER_LOG=/workspace/server.log - -export VLLM_FLOAT32_MATMUL_PRECISION=high - -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS="--tensor-parallel-size=1 --data-parallel-size=$TP --enable-expert-parallel" -elif [ "$EP_SIZE" -gt 1 ]; then - PARALLEL_ARGS="--tensor-parallel-size=$TP --enable-expert-parallel" -else - PARALLEL_ARGS="--tensor-parallel-size=$TP" -fi - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve $MODEL --port $PORT \ -$PARALLEL_ARGS \ ---gpu-memory-utilization 0.90 \ ---max-model-len $MAX_MODEL_LEN \ ---kv-cache-dtype fp8 \ ---max-cudagraph-capture-size 2048 \ ---max-num-batched-tokens "$((ISL * 2 ))" \ ---stream-interval 20 --no-enable-prefix-caching \ ---trust-remote-code > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp4_b200_trt.sh b/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp4_b200_trt.sh deleted file mode 100755 index 2fd9d7997..000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp4_b200_trt.sh +++ /dev/null @@ -1,139 +0,0 @@ -#!/usr/bin/env bash - -# Source benchmark utilities early -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - DP_ATTENTION \ - EP_SIZE - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -MAX_NUM_TOKENS=16384 -MAX_CAPTURE_TOKENS=$(( MAX_NUM_TOKENS < CONC * ISL ? MAX_NUM_TOKENS : CONC * ISL )) -CAPTURE_TOKENS_LIST=(1 2 4 8 12 16 24 32 48 64 96 128 192 256 384 512 768) -CAPTURE_TOKENS_LIST+=( $(seq 1024 128 2047)) -CAPTURE_TOKENS_LIST+=( $(seq 2048 256 4095)) -if [[ $MAX_CAPTURE_TOKENS -ge 4096 ]]; then - CAPTURE_TOKENS_LIST+=( $(seq 4096 512 $MAX_CAPTURE_TOKENS)) -fi -CAPTURE_TOKENS_LIST=$(printf "%s, " "${CAPTURE_TOKENS_LIST[@]}") - -CAPTURE_BATCH_LIST=(1 2 4 8 12 ) -if [[ $CONC -ge 16 ]]; then - MAX_CAPTURE_BATCH=$(( CONC < 256 ? CONC : 255 )) - CAPTURE_BATCH_LIST+=( $(seq 16 8 $MAX_CAPTURE_BATCH )) -fi -if [[ $CONC -ge 256 ]]; then - MAX_CAPTURE_BATCH=$(( CONC < 512 ? CONC : 511 )) - CAPTURE_BATCH_LIST+=( $(seq 256 16 $MAX_CAPTURE_BATCH)) -fi -if [[ $CONC -ge 512 ]]; then - MAX_CAPTURE_BATCH=$(( CONC < 768 ? CONC : 767 )) - CAPTURE_BATCH_LIST+=( $(seq 512 32 $MAX_CAPTURE_BATCH)) -fi -if [[ $CONC -ge 1024 ]]; then - CAPTURE_BATCH_LIST+=( $(seq 768 64 $CONC)) -fi -CAPTURE_BATCH_LIST=$(printf "%s, " "${CAPTURE_BATCH_LIST[@]}") -MAX_CAPTURE_TOKENS=$(( CONC < 16 ? 4096 : MAX_NUM_TOKENS )) - -CONFIG_FILE="minimax-fp4.yaml" -cat << EOF > $CONFIG_FILE -cuda_graph_config: - enable_padding: true - batch_sizes: [${CAPTURE_BATCH_LIST%, }] -moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true -enable_attention_dp: $DP_ATTENTION -torch_compile_config: - capture_num_tokens: [${CAPTURE_TOKENS_LIST%, }] - enable_piecewise_cuda_graph: true -stream_interval: 100 -print_iter_log: true -max_num_tokens: $MAX_NUM_TOKENS -kv_cache_config: - free_gpu_memory_fraction: 0.9 - enable_block_reuse: False - dtype: fp8 -scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - context_chunking_policy: FIRST_COME_FIRST_SERVED -nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core -max_seq_len: $MAX_MODEL_LEN -num_postprocess_workers: 4 -EOF - -if [[ $DP_ATTENTION == true ]]; then -cat << EOF >> $CONFIG_FILE -attention_dp_config: - enable_balance: true -EOF -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi -SERVER_LOG=/workspace/server.log -PORT=${PORT:-8888} - -echo "Generated config file contents:" -cat $CONFIG_FILE - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x - -# Launch TRT-LLM server -mpirun -n 1 --oversubscribe --allow-run-as-root \ - trtllm-serve $MODEL --port=$PORT \ - --trust_remote_code \ - --backend=pytorch \ - --max_batch_size $CONC \ - --tp_size=$TP --ep_size=$EP_SIZE \ - --config=$CONFIG_FILE \ - > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend openai \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $(( $CONC * 10 )) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp4_b300.sh b/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp4_b300.sh deleted file mode 100755 index 33492aada..000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp4_b300.sh +++ /dev/null @@ -1,96 +0,0 @@ -#!/usr/bin/env bash - -# NOTE: At the time of submission, https://docs.vllm.ai/projects/recipes/en/latest/MiniMax/MiniMax-M2.html -# does not have a B300-specific recipe, so this script reuses the existing -# MiniMax-M2.5 FP4 B200 vLLM recipe as-is until B300-specific tuning is available. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - - -SERVER_LOG=/workspace/server.log - -export VLLM_FLOAT32_MATMUL_PRECISION=high - -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS="--tensor-parallel-size 1 --data-parallel-size $TP --enable-expert-parallel" -elif [ "$EP_SIZE" -gt 1 ]; then - PARALLEL_ARGS="--tensor-parallel-size $TP --enable-expert-parallel" -else - PARALLEL_ARGS="--tensor-parallel-size $TP" -fi - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve $MODEL_PATH --served-model-name $MODEL --port $PORT \ -$PARALLEL_ARGS \ ---gpu-memory-utilization 0.90 \ ---max-model-len $MAX_MODEL_LEN \ ---kv-cache-dtype fp8 \ ---max-cudagraph-capture-size 2048 \ ---max-num-batched-tokens "$((ISL * 2 ))" \ ---stream-interval 20 --no-enable-prefix-caching \ ---trust-remote-code > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp4_b300_trt.sh b/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp4_b300_trt.sh deleted file mode 100755 index a4961b0c1..000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp4_b300_trt.sh +++ /dev/null @@ -1,150 +0,0 @@ -#!/usr/bin/env bash - -# Source benchmark utilities early -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - DP_ATTENTION \ - EP_SIZE - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -MAX_NUM_TOKENS=16384 -MAX_CAPTURE_TOKENS=$(( MAX_NUM_TOKENS < CONC * ISL ? MAX_NUM_TOKENS : CONC * ISL )) -CAPTURE_TOKENS_LIST=(1 2 4 8 12 16 24 32 48 64 96 128 192 256 384 512 768) -CAPTURE_TOKENS_LIST+=( $(seq 1024 128 2047)) -CAPTURE_TOKENS_LIST+=( $(seq 2048 256 4095)) -if [[ $MAX_CAPTURE_TOKENS -ge 4096 ]]; then - CAPTURE_TOKENS_LIST+=( $(seq 4096 512 $MAX_CAPTURE_TOKENS)) -fi -CAPTURE_TOKENS_LIST=$(printf "%s, " "${CAPTURE_TOKENS_LIST[@]}") - -CAPTURE_BATCH_LIST=(1 2 4 8 12 ) -if [[ $CONC -ge 16 ]]; then - MAX_CAPTURE_BATCH=$(( CONC < 256 ? CONC : 255 )) - CAPTURE_BATCH_LIST+=( $(seq 16 8 $MAX_CAPTURE_BATCH )) -fi -if [[ $CONC -ge 256 ]]; then - MAX_CAPTURE_BATCH=$(( CONC < 512 ? CONC : 511 )) - CAPTURE_BATCH_LIST+=( $(seq 256 16 $MAX_CAPTURE_BATCH)) -fi -if [[ $CONC -ge 512 ]]; then - MAX_CAPTURE_BATCH=$(( CONC < 768 ? CONC : 767 )) - CAPTURE_BATCH_LIST+=( $(seq 512 32 $MAX_CAPTURE_BATCH)) -fi -if [[ $CONC -ge 1024 ]]; then - CAPTURE_BATCH_LIST+=( $(seq 768 64 $CONC)) -fi -CAPTURE_BATCH_LIST=$(printf "%s, " "${CAPTURE_BATCH_LIST[@]}") -MAX_CAPTURE_TOKENS=$(( CONC < 16 ? 4096 : MAX_NUM_TOKENS )) - -CONFIG_FILE="minimax-fp4.yaml" -cat << EOF > $CONFIG_FILE -cuda_graph_config: - enable_padding: true - batch_sizes: [${CAPTURE_BATCH_LIST%, }] -moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true -enable_attention_dp: $DP_ATTENTION -torch_compile_config: - capture_num_tokens: [${CAPTURE_TOKENS_LIST%, }] - enable_piecewise_cuda_graph: true -stream_interval: 100 -print_iter_log: true -max_num_tokens: $MAX_NUM_TOKENS -kv_cache_config: - free_gpu_memory_fraction: 0.9 - enable_block_reuse: False - dtype: fp8 -scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - context_chunking_policy: FIRST_COME_FIRST_SERVED -nvfp4_gemm_config: - allowed_backends: - - cutlass - - cublaslt - - cutedsl - - cuda_core -max_seq_len: $MAX_MODEL_LEN -num_postprocess_workers: 4 -EOF - -if [[ $DP_ATTENTION == true ]]; then -cat << EOF >> $CONFIG_FILE -attention_dp_config: - enable_balance: true -EOF -fi - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE. -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -SERVER_LOG=/workspace/server.log -PORT=${PORT:-8888} - -echo "Generated config file contents:" -cat $CONFIG_FILE - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x - -# Launch TRT-LLM server -mpirun -n 1 --oversubscribe --allow-run-as-root \ - trtllm-serve "$MODEL_PATH" --port=$PORT \ - --trust_remote_code \ - --backend=pytorch \ - --max_batch_size $CONC \ - --tp_size=$TP --ep_size=$EP_SIZE \ - --config=$CONFIG_FILE \ - > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend openai \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $(( $CONC * 10 )) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp4_mi355x.sh b/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp4_mi355x.sh deleted file mode 100755 index 806c59278..000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp4_mi355x.sh +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -# Set HIP_VISIBLE_DEVICES to match ROCR_VISIBLE_DEVICES for Ray compatibility in vLLM 0.14+ -if [ -n "$ROCR_VISIBLE_DEVICES" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -export VLLM_ROCM_USE_AITER=1 -export VLLM_USE_RUST_FRONTEND=1 -EXTRA_VLLM_ARGS="" -# if [ "$TP" -ge 4 ]; then -# # AITER CK fused MoE kernels lack compiled tiles for N=intermediate_size/TP -# # when TP>=4 (TP=4, N=384). Disable AITER MoE to fall back to triton, but keep -# # AITER attention. See: https://github.com/vllm-project/vllm/issues/35637 -# export VLLM_ROCM_USE_AITER_MOE=0 -# EXTRA_VLLM_ARGS="--attention-backend ROCM_AITER_UNIFIED_ATTN" -# pip install amd-quark 2>/dev/null || true -# fi - -SERVER_LOG=/workspace/server.log - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi - -if [ "$EP_SIZE" -gt 1 ]; then - EP=" --enable-expert-parallel" -else - EP=" " -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve $MODEL --port $PORT \ ---tensor-parallel-size=$TP \ -$EP \ ---gpu-memory-utilization 0.95 \ ---max-model-len $MAX_MODEL_LEN \ ---kv-cache-dtype fp8 \ ---block-size=32 \ ---no-enable-prefix-caching \ ---attention-backend "ROCM_AITER_FA" \ ---trust-remote-code \ -$EXTRA_VLLM_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp4_mi355x_atom.sh b/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp4_mi355x_atom.sh deleted file mode 100644 index 6730aded2..000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp4_mi355x_atom.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE \ - DP_ATTENTION - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -SERVER_LOG=/workspace/server.log - -export OMP_NUM_THREADS=1 - -# Calculate max-model-len based on ISL and OSL -if [ "$ISL" = "1024" ] && [ "$OSL" = "1024" ]; then - CALCULATED_MAX_MODEL_LEN="" -else - CALCULATED_MAX_MODEL_LEN=" --max-model-len 10240 " -fi - -if [ "$EP_SIZE" -gt 1 ]; then - EP=" --enable-expert-parallel" -else - EP=" " -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x - -python3 -m atom.entrypoints.openai_server \ - --model $MODEL \ - --server-port $PORT \ - -tp $TP \ - --kv_cache_dtype fp8 $CALCULATED_MAX_MODEL_LEN $EP \ - --trust-remote-code \ - > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -export PYTHONDONTWRITEBYTECODE=1 -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp8_b200.sh b/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp8_b200.sh deleted file mode 100755 index 9897afca3..000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp8_b200.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -SERVER_LOG=/workspace/server.log - -export VLLM_FLOAT32_MATMUL_PRECISION=high - -if [ "$EP_SIZE" -gt 1 ]; then - EP=" --enable-expert-parallel" -else - EP=" " -fi - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve $MODEL --port $PORT \ ---tensor-parallel-size=$TP \ -$EP \ ---gpu-memory-utilization 0.90 \ ---max-model-len $MAX_MODEL_LEN \ ---block-size=32 \ ---kv-cache-dtype fp8 \ ---max-cudagraph-capture-size 2048 \ ---max-num-batched-tokens "$((ISL * 2 ))" \ ---stream-interval 20 --no-enable-prefix-caching \ ---trust-remote-code > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp8_b300.sh b/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp8_b300.sh deleted file mode 100755 index 14e853ce9..000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp8_b300.sh +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/bin/env bash - -# NOTE: At the time of submission, https://docs.vllm.ai/projects/recipes/en/latest/MiniMax/MiniMax-M2.html -# does not have a B300-specific recipe, so this script reuses the existing -# MiniMax-M2.5 FP8 B200 vLLM recipe as-is until B300-specific tuning is available. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - - -SERVER_LOG=/workspace/server.log - -export VLLM_FLOAT32_MATMUL_PRECISION=high - -if [ "$EP_SIZE" -gt 1 ]; then - EP=" --enable-expert-parallel" -else - EP=" " -fi - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve $MODEL_PATH --served-model-name $MODEL --port $PORT \ ---tensor-parallel-size $TP \ -$EP \ ---gpu-memory-utilization 0.90 \ ---max-model-len $MAX_MODEL_LEN \ ---block-size 32 \ ---kv-cache-dtype fp8 \ ---max-cudagraph-capture-size 2048 \ ---max-num-batched-tokens "$((ISL * 2 ))" \ ---stream-interval 20 --no-enable-prefix-caching \ ---trust-remote-code > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp8_h100.sh b/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp8_h100.sh deleted file mode 100755 index 012c8b535..000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp8_h100.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -nvidia-smi - -export PYTHONNOUSERSITE=1 - -SERVER_LOG=/workspace/server.log - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context -fi - -if [ "$EP_SIZE" -gt 1 ]; then - EP=" --enable-expert-parallel" -else - EP=" " -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve $MODEL --host 0.0.0.0 --port $PORT \ ---tensor-parallel-size=$TP \ -$EP \ ---trust-remote-code \ ---enable-auto-tool-choice \ ---tool-call-parser minimax_m2 \ ---reasoning-parser minimax_m2_append_think \ ---compilation-config '{"mode":3,"pass_config":{"fuse_minimax_qk_norm":true}}' \ ---gpu-memory-utilization 0.9 \ -> $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp8_h200.sh b/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp8_h200.sh deleted file mode 100755 index bb53e0a06..000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp8_h200.sh +++ /dev/null @@ -1,98 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -SERVER_LOG=/workspace/server.log - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi - -export PYTHONNOUSERSITE=1 -export SAFETENSORS_FAST_GPU=1 -export VLLM_USE_DEEP_GEMM=0 -export VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER=0 -export VLLM_FLOAT32_MATMUL_PRECISION=high - -COMPILATION_CONFIG=${COMPILATION_CONFIG:-'{"mode":3,"cudagraph_mode":"PIECEWISE","pass_config":{"fuse_minimax_qk_norm":true}}'} -MAX_NUM_SEQS=${MAX_NUM_SEQS:-512} -MAX_NUM_BATCHED_TOKENS=${MAX_NUM_BATCHED_TOKENS:-32768} - -if [ "$EP_SIZE" -gt 1 ]; then - EP=(--enable-expert-parallel) -else - EP=() -fi - -if [ "$ISL" = "8192" ]; then - ATTN_BACKEND="FLASH_ATTN" - AUTOTUNE_FLAG=() -else - ATTN_BACKEND="FLASHINFER" - AUTOTUNE_FLAG=(--enable-flashinfer-autotune) -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve "$MODEL" --port "$PORT" \ ---tensor-parallel-size="$TP" \ -"${EP[@]}" \ ---gpu-memory-utilization 0.95 \ ---max-model-len "$MAX_MODEL_LEN" \ ---max-num-seqs "$MAX_NUM_SEQS" \ ---max-num-batched-tokens "$MAX_NUM_BATCHED_TOKENS" \ ---kv-cache-dtype fp8 \ ---moe-backend triton \ ---attention-backend "$ATTN_BACKEND" \ -"${AUTOTUNE_FLAG[@]}" \ ---compilation-config "$COMPILATION_CONFIG" \ ---no-enable-prefix-caching \ ---trust-remote-code > "$SERVER_LOG" 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp8_mi300x.sh b/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp8_mi300x.sh deleted file mode 100755 index 8a95dc138..000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp8_mi300x.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -# Set HIP_VISIBLE_DEVICES to match ROCR_VISIBLE_DEVICES for Ray compatibility in vLLM 0.14+ -if [ -n "$ROCR_VISIBLE_DEVICES" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -export VLLM_ROCM_USE_AITER=1 - -SERVER_LOG=/workspace/server.log - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve $MODEL --port $PORT \ ---tensor-parallel-size=$TP \ ---gpu-memory-utilization 0.95 \ ---max-model-len $MAX_MODEL_LEN \ ---block-size=32 \ ---no-enable-prefix-caching \ ---trust-remote-code > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp8_mi325x.sh b/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp8_mi325x.sh deleted file mode 100755 index 3b74e845e..000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp8_mi325x.sh +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -# Set HIP_VISIBLE_DEVICES to match ROCR_VISIBLE_DEVICES for Ray compatibility in vLLM 0.14+ -if [ -n "$ROCR_VISIBLE_DEVICES" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -export VLLM_ROCM_USE_AITER=1 - -ENABLE_SHUFFLE_KV_CACHE_LAYOUT=0 -if [[ "$TP" == "2" && "$EP_SIZE" == "1" ]] && (( CONC <= 16 )); then - ENABLE_SHUFFLE_KV_CACHE_LAYOUT=1 -elif [[ "$TP" == "8" && "$EP_SIZE" == "8" ]] && (( CONC <= 64 )); then - ENABLE_SHUFFLE_KV_CACHE_LAYOUT=1 -fi -if (( ENABLE_SHUFFLE_KV_CACHE_LAYOUT )); then - export VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=1 -fi - -SERVER_LOG=/workspace/server.log - -if [ "$EP_SIZE" -gt 1 ]; then - EP=" --enable-expert-parallel" -else - EP=" " -fi - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve $MODEL --port $PORT \ ---tensor-parallel-size=$TP \ -$EP \ ---gpu-memory-utilization 0.95 \ ---max-model-len $MAX_MODEL_LEN \ ---block-size=32 \ ---no-enable-prefix-caching \ ---attention-backend ROCM_AITER_FA \ ---compilation-config '{"mode":3,"cudagraph_mode":"PIECEWISE"}' \ ---trust-remote-code > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp8_mi355x.sh b/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp8_mi355x.sh deleted file mode 100755 index 5093a56d6..000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp8_mi355x.sh +++ /dev/null @@ -1,122 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -# Set HIP_VISIBLE_DEVICES to match ROCR_VISIBLE_DEVICES for Ray compatibility in vLLM 0.14+ -if [ -n "$ROCR_VISIBLE_DEVICES" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -export VLLM_ROCM_USE_AITER=1 -export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 -export VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=0 -VLLM_BLOCK_SIZE=32 -ASYNC_SCHEDULING_ARGS="" - -if [[ "$ISL" == "1024" && "$OSL" == "1024" ]]; then - if [[ "$TP" == "8" && "$EP_SIZE" == "8" ]]; then - ASYNC_SCHEDULING_ARGS="--no-async-scheduling" - echo "1k1k TP8/EP8: using block size 32, shuffle disabled, async scheduling disabled." - elif (( CONC <= 128 )); then - export VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=1 - VLLM_BLOCK_SIZE=16 - ASYNC_SCHEDULING_ARGS="--no-async-scheduling" - echo "1k1k c${CONC}: using block size 16, shuffle enabled, async scheduling disabled." - else - export VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=1 - VLLM_BLOCK_SIZE=16 - echo "1k1k c${CONC}: using block size 16, shuffle enabled, async scheduling enabled." - fi -elif [[ "$ISL" == "8192" && "$OSL" == "1024" ]]; then - if [[ "$TP" == "8" && "$EP_SIZE" == "8" ]]; then - export VLLM_ROCM_USE_AITER_MOE=0 - ASYNC_SCHEDULING_ARGS="--no-async-scheduling" - echo "8k1k TP8/EP8: using block size 32, shuffle disabled, AITER MoE disabled, async scheduling disabled." - elif (( CONC < 64 )); then - ASYNC_SCHEDULING_ARGS="--no-async-scheduling" - echo "8k1k c${CONC}: using block size 32, shuffle disabled, async scheduling disabled." - elif (( CONC == 64 )); then - ASYNC_SCHEDULING_ARGS="--no-async-scheduling" - export VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=1 - VLLM_BLOCK_SIZE=16 - echo "8k1k c64: using block size 16, shuffle enabled, async scheduling disabled." - else - export VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=1 - VLLM_BLOCK_SIZE=16 - echo "8k1k c${CONC}: using block size 16, shuffle enabled, async scheduling enabled." - fi -fi - -SERVER_LOG=/workspace/server.log - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi - -if [ "$EP_SIZE" -gt 1 ]; then - EP=" --enable-expert-parallel" -else - EP=" " -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve $MODEL --port $PORT \ ---tensor-parallel-size=$TP \ -$EP \ ---gpu-memory-utilization 0.95 \ ---max-model-len $MAX_MODEL_LEN \ ---kv-cache-dtype fp8 \ ---block-size=$VLLM_BLOCK_SIZE \ ---no-enable-prefix-caching \ ---attention-backend "ROCM_AITER_FA" \ -$ASYNC_SCHEDULING_ARGS \ ---trust-remote-code > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp8_mi355x_atom.sh b/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp8_mi355x_atom.sh deleted file mode 100755 index 325c97726..000000000 --- a/benchmarks/single_node/fixed_seq_len/deprecated/minimaxm2.5_fp8_mi355x_atom.sh +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE \ - DP_ATTENTION - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -SERVER_LOG=/workspace/server.log - -export OMP_NUM_THREADS=1 - -# Calculate max-model-len based on ISL and OSL -if [ "$ISL" = "1024" ] && [ "$OSL" = "1024" ]; then - CALCULATED_MAX_MODEL_LEN="" -else - CALCULATED_MAX_MODEL_LEN=" --max-model-len 10240 " -fi - -if [ "$EP_SIZE" -gt 1 ]; then - EP=" --enable-expert-parallel" -else - EP=" " -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor -MEM_FRAC_STATIC=0.9 - -set -x - -python3 -m atom.entrypoints.openai_server \ - --model $MODEL \ - --server-port $PORT \ - -tp $TP \ - --kv_cache_dtype fp8 $CALCULATED_MAX_MODEL_LEN $EP \ - --gpu-memory-utilization $MEM_FRAC_STATIC \ - --trust-remote-code \ - > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -export PYTHONDONTWRITEBYTECODE=1 -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/dsr1_fp4_b200.sh b/benchmarks/single_node/fixed_seq_len/dsr1_fp4_b200.sh deleted file mode 100644 index b1426a6c2..000000000 --- a/benchmarks/single_node/fixed_seq_len/dsr1_fp4_b200.sh +++ /dev/null @@ -1,109 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -DP_ATTENTION="${DP_ATTENTION:-false}" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE \ - DP_ATTENTION - -if [[ "$DP_ATTENTION" != "true" && "$DP_ATTENTION" != "false" ]]; then - echo "DP_ATTENTION must be true or false; got '$DP_ATTENTION'" >&2 - exit 1 -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -nvidia-smi - -SERVER_LOG=/workspace/server.log - -# Default: recv every ~10 requests; if CONC >= 16, relax to ~30 requests between scheduler recv polls. -if [[ $CONC -ge 16 ]]; then - SCHEDULER_RECV_INTERVAL=30 -else - SCHEDULER_RECV_INTERVAL=10 -fi - -CHUNKED_PREFILL_SIZE=16384 -SGLANG_PARALLEL_ARGS=( - --tensor-parallel-size="$TP" - --data-parallel-size=1 -) -SGLANG_DPA_ARGS=() - -if [[ "$DP_ATTENTION" == "true" ]]; then - SCHEDULER_RECV_INTERVAL=1 - CHUNKED_PREFILL_SIZE=32768 - SGLANG_PARALLEL_ARGS=( - --tensor-parallel-size="$TP" - --data-parallel-size="$TP" - --enable-dp-attention - --enable-dp-attention-local-control-broadcast - --enable-dp-lm-head - ) - SGLANG_DPA_ARGS=( - --schedule-conservativeness 3.33 - --enable-prefill-delayer - ) -fi - -echo "TP: $TP, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION, CONC: $CONC, ISL: $ISL, OSL: $OSL" -echo "SCHEDULER_RECV_INTERVAL: $SCHEDULER_RECV_INTERVAL, CHUNKED_PREFILL_SIZE: $CHUNKED_PREFILL_SIZE" - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -SGLANG_RADIX_FORCE_MISS=1 PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path $MODEL --host 0.0.0.0 --port $PORT --trust-remote-code \ -"${SGLANG_PARALLEL_ARGS[@]}" \ ---cuda-graph-max-bs 256 --max-running-requests 256 --mem-fraction-static 0.85 --kv-cache-dtype fp8_e4m3 \ ---chunked-prefill-size "$CHUNKED_PREFILL_SIZE" \ ---ep-size $EP_SIZE --quantization modelopt_fp4 --enable-flashinfer-allreduce-fusion --scheduler-recv-interval $SCHEDULER_RECV_INTERVAL \ ---enable-symm-mem --disable-piecewise-cuda-graph --attention-backend trtllm_mla --moe-runner-backend flashinfer_trtllm --stream-interval 10 "${SGLANG_DPA_ARGS[@]}" $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $((CONC * 10)) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/dsr1_fp4_b200_mtp.sh b/benchmarks/single_node/fixed_seq_len/dsr1_fp4_b200_mtp.sh deleted file mode 100755 index df08e52e9..000000000 --- a/benchmarks/single_node/fixed_seq_len/dsr1_fp4_b200_mtp.sh +++ /dev/null @@ -1,121 +0,0 @@ -#!/usr/bin/env bash - -# DeepSeek-R1-0528 FP4 on B200 with EAGLE/MTP speculative decoding. -# Mirrors dsr1_fp4_b200.sh and adds the speculative-* flags from -# dsr1_fp8_b200_mtp.sh (the production B200 sglang MTP template). - -source "$(dirname "$0")/../../benchmark_lib.sh" - -DP_ATTENTION="${DP_ATTENTION:-false}" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE \ - DP_ATTENTION - -if [[ "$DP_ATTENTION" != "true" && "$DP_ATTENTION" != "false" ]]; then - echo "DP_ATTENTION must be true or false; got '$DP_ATTENTION'" >&2 - exit 1 -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -nvidia-smi - -SERVER_LOG=/workspace/server.log - -if [[ $CONC -ge 16 ]]; then - SCHEDULER_RECV_INTERVAL=30 -else - SCHEDULER_RECV_INTERVAL=10 -fi - -CHUNKED_PREFILL_SIZE=16384 -SGLANG_PARALLEL_ARGS=( - --tensor-parallel-size="$TP" - --data-parallel-size=1 -) -SGLANG_DPA_ARGS=() - -if [[ "$DP_ATTENTION" == "true" ]]; then - SCHEDULER_RECV_INTERVAL=1 - CHUNKED_PREFILL_SIZE=32768 - SGLANG_PARALLEL_ARGS=( - --tensor-parallel-size="$TP" - --data-parallel-size="$TP" - --enable-dp-attention - --enable-dp-attention-local-control-broadcast - --enable-dp-lm-head - ) - SGLANG_DPA_ARGS=( - --schedule-conservativeness 3.33 - --enable-prefill-delayer - ) -fi - -echo "TP: $TP, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION, CONC: $CONC, ISL: $ISL, OSL: $OSL" -echo "SCHEDULER_RECV_INTERVAL: $SCHEDULER_RECV_INTERVAL, CHUNKED_PREFILL_SIZE: $CHUNKED_PREFILL_SIZE" - -# MTP (Multi-Token Prediction) Config - EAGLE speculative decoding -SPECULATIVE_NUM_STEPS=2 -SPECULATIVE_DRAFT_TOKENS=3 -SPECULATIVE_EAGLE_TOPK=1 - -export SGLANG_ENABLE_SPEC_V2=1 - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi -start_gpu_monitor - -set -x -SGLANG_RADIX_FORCE_MISS=1 PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path $MODEL --host 0.0.0.0 --port $PORT --trust-remote-code \ -"${SGLANG_PARALLEL_ARGS[@]}" \ ---cuda-graph-max-bs 256 --max-running-requests 256 --mem-fraction-static 0.85 --kv-cache-dtype fp8_e4m3 \ ---chunked-prefill-size "$CHUNKED_PREFILL_SIZE" \ ---ep-size $EP_SIZE --quantization modelopt_fp4 --enable-flashinfer-allreduce-fusion --scheduler-recv-interval $SCHEDULER_RECV_INTERVAL \ ---disable-piecewise-cuda-graph --attention-backend trtllm_mla --moe-runner-backend flashinfer_trtllm --stream-interval 10 \ ---speculative-algorithm EAGLE \ ---speculative-num-steps $SPECULATIVE_NUM_STEPS \ ---speculative-num-draft-tokens $SPECULATIVE_DRAFT_TOKENS \ ---speculative-eagle-topk $SPECULATIVE_EAGLE_TOPK \ -"${SGLANG_DPA_ARGS[@]}" $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $((CONC * 10)) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --use-chat-template - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/dsr1_fp4_b200_trt.sh b/benchmarks/single_node/fixed_seq_len/dsr1_fp4_b200_trt.sh deleted file mode 100644 index d2186df2c..000000000 --- a/benchmarks/single_node/fixed_seq_len/dsr1_fp4_b200_trt.sh +++ /dev/null @@ -1,134 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - DP_ATTENTION \ - EP_SIZE - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -# ========= Determine other parameters based on ISL, OSL, CONC ========= -CUDA_GRAPH_MAX_BATCH_SIZE=$CONC -MOE_BACKEND="TRTLLM" -PIECEWISE_CUDA_GRAPHS="false" - -if [[ "$ISL" == "1024" && "$OSL" == "1024" ]]; then - if [[ "$TP" == "8" && "$EP_SIZE" == "8" ]]; then - PIECEWISE_CUDA_GRAPHS="true" - fi -fi - -if [[ "$DP_ATTENTION" == "true" ]]; then - MOE_BACKEND="CUTLASS" - CUDA_GRAPH_MAX_BATCH_SIZE=$(( CONC < 4 ? CONC : CONC / 4 )) -fi - -echo "MOE_BACKEND set to '$MOE_BACKEND'" - -SERVER_LOG=/workspace/server.log -EXTRA_CONFIG_FILE="dsr1-fp4.yml" - -cat > $EXTRA_CONFIG_FILE << EOF -cuda_graph_config: - enable_padding: true - max_batch_size: $CUDA_GRAPH_MAX_BATCH_SIZE -enable_attention_dp: $DP_ATTENTION -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: $MOE_BACKEND -EOF - -if [[ "$DP_ATTENTION" == "true" ]]; then - cat << EOF >> $EXTRA_CONFIG_FILE -attention_dp_config: - batching_wait_iters: 0 - enable_balance: true - timeout_iters: 60 -EOF -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x - -MAX_NUM_TOKENS=$(( ($CONC+$ISL+64+63)/64*64 )) -MAX_MODEL_LEN=$(( MAX_MODEL_LEN > 8192 ? MAX_MODEL_LEN : 8192 )) -MAX_NUM_TOKENS=$(( MAX_NUM_TOKENS > 8192 ? MAX_NUM_TOKENS : 8192 )) - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" - MAX_NUM_TOKENS="$EVAL_MAX_MODEL_LEN" -fi - -if [[ "$PIECEWISE_CUDA_GRAPHS" == "true" ]]; then - # [2^i for i in range(8)] + [i for i in range(256, max_num_tokens, 256)] + [max_num_tokens] - capture_tokens=(1 2 4 8 16 32 64 128) - capture_tokens+=( $(seq 256 256 $MAX_NUM_TOKENS)) - CAPTURE_TOKENS_LIST=$(printf "%s, " "${capture_tokens[@]}") - - cat << EOF >> $EXTRA_CONFIG_FILE -torch_compile_config: - capture_num_tokens: [${CAPTURE_TOKENS_LIST%, }] - enable_piecewise_cuda_graph: true -EOF -fi - -# Launch TRT-LLM server -mpirun -n 1 --oversubscribe --allow-run-as-root \ - trtllm-serve $MODEL --port=$PORT \ - --trust_remote_code \ - --backend=pytorch \ - --max_seq_len=$MAX_MODEL_LEN \ - --max_num_tokens=$MAX_NUM_TOKENS \ - --tp_size=$TP --ep_size=$EP_SIZE \ - --extra_llm_api_options=$EXTRA_CONFIG_FILE \ - > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend openai \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $(( $CONC * 10 )) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x \ No newline at end of file diff --git a/benchmarks/single_node/fixed_seq_len/dsr1_fp4_b200_trt_mtp.sh b/benchmarks/single_node/fixed_seq_len/dsr1_fp4_b200_trt_mtp.sh deleted file mode 100644 index 15d93458a..000000000 --- a/benchmarks/single_node/fixed_seq_len/dsr1_fp4_b200_trt_mtp.sh +++ /dev/null @@ -1,144 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - DP_ATTENTION \ - EP_SIZE - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -# ========= Determine MOE_BACKEND and MTP based on DP_ATTENTION ========= -MOE_BACKEND="TRTLLM" -PIECEWISE_CUDA_GRAPHS="false" -MAX_BATCH_SIZE=$CONC -MTP=3 - -if [[ "$DP_ATTENTION" == "true" ]]; then - MAX_BATCH_SIZE=$(( CONC < 4 ? CONC : CONC / 4 )) - MOE_BACKEND="CUTLASS" - MTP=1 -fi - -echo "MOE_BACKEND='$MOE_BACKEND', MTP='$MTP'" - -SERVER_LOG=/workspace/server.log -EXTRA_CONFIG_FILE="dsr1-fp4-mtp.yml" - -cat > $EXTRA_CONFIG_FILE << EOF -cuda_graph_config: - enable_padding: true - max_batch_size: $MAX_BATCH_SIZE -enable_attention_dp: $DP_ATTENTION -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: $MOE_BACKEND -speculative_config: - decoding_type: MTP - num_nextn_predict_layers: ${MTP} -EOF - -if [[ "$DP_ATTENTION" == "true" ]]; then - cat << EOF >> $EXTRA_CONFIG_FILE -attention_dp_config: - batching_wait_iters: 0 - enable_balance: true - timeout_iters: 60 -EOF -fi - -MAX_NUM_TOKENS=$(( ((MTP+1)*MAX_BATCH_SIZE+ISL+64+63)/64*64 )) - -# set of configs using piecewise_cuda_graphs -if [[ "$ISL" == "1024" && "$OSL" == "1024" ]]; then - if [[ $CONC == 32 || $CONC == 64 ]]; then - PIECEWISE_CUDA_GRAPHS="true" - elif [[ $CONC == 128 && $DP_ATTENTION == "false" ]]; then - PIECEWISE_CUDA_GRAPHS="true" - fi -fi - -if [[ "$PIECEWISE_CUDA_GRAPHS" == "true" ]]; then - # [2^i for i in range(8)] + [i for i in range(256, max_num_tokens, 256)] + [max_num_tokens] - capture_tokens=(1 2 4 8 16 32 64 128) - capture_tokens+=( $(seq 256 256 $MAX_NUM_TOKENS)) - if [ $((MAX_NUM_TOKENS%256)) -ne 0 ]; then - capture_tokens+=($MAX_NUM_TOKENS) - fi - CAPTURE_TOKENS_LIST=$(printf "%s, " "${capture_tokens[@]}") - - cat << EOF >> $EXTRA_CONFIG_FILE -torch_compile_config: - capture_num_tokens: [${CAPTURE_TOKENS_LIST%, }] - enable_piecewise_cuda_graph: true -EOF -fi # end of set of configs using piecewise_cuda_graphs - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" - MAX_NUM_TOKENS="$EVAL_MAX_MODEL_LEN" -fi - -set -x -# Launch TRT-LLM server -mpirun -n 1 --oversubscribe --allow-run-as-root \ - trtllm-serve $MODEL --port=$PORT \ - --trust_remote_code \ - --backend=pytorch \ - --max_batch_size=$MAX_BATCH_SIZE \ - --max_seq_len=$MAX_MODEL_LEN \ - --max_num_tokens=$MAX_NUM_TOKENS \ - --tp_size=$TP --ep_size=$EP_SIZE \ - --extra_llm_api_options=$EXTRA_CONFIG_FILE \ - > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend openai \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $(( $CONC * 10 )) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --use-chat-template - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x \ No newline at end of file diff --git a/benchmarks/single_node/fixed_seq_len/dsr1_fp4_b300.sh b/benchmarks/single_node/fixed_seq_len/dsr1_fp4_b300.sh deleted file mode 100644 index 96c9975bf..000000000 --- a/benchmarks/single_node/fixed_seq_len/dsr1_fp4_b300.sh +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env bash - -# NOTE: At the time of submission, https://cookbook.sglang.io/autoregressive/DeepSeek/DeepSeek-R1 -# does not have a B300-specific recipe, so this script reuses the existing -# DSR1 FP4 B200 SGLang recipe as-is until B300-specific tuning is available. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - - -nvidia-smi - -SERVER_LOG=/workspace/server.log - -# Default: recv every ~10 requests; if CONC ≥ 16, relax to ~30 requests between scheduler recv polls. -if [[ $CONC -ge 16 ]]; then - SCHEDULER_RECV_INTERVAL=30 -else - SCHEDULER_RECV_INTERVAL=10 -fi -echo "SCHEDULER_RECV_INTERVAL: $SCHEDULER_RECV_INTERVAL, CONC: $CONC, ISL: $ISL, OSL: $OSL" - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path $MODEL_PATH --served-model-name $MODEL --host 0.0.0.0 --port $PORT --trust-remote-code \ ---tensor-parallel-size $TP --data-parallel-size 1 \ ---cuda-graph-max-bs 256 --max-running-requests 256 --mem-fraction-static 0.85 --kv-cache-dtype fp8_e4m3 \ ---chunked-prefill-size 16384 \ ---ep-size $EP_SIZE --quantization modelopt_fp4 --enable-flashinfer-allreduce-fusion --scheduler-recv-interval $SCHEDULER_RECV_INTERVAL \ ---enable-symm-mem --disable-radix-cache --attention-backend trtllm_mla --moe-runner-backend flashinfer_trtllm --stream-interval 10 $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $((CONC * 10)) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/dsr1_fp4_mi355x.sh b/benchmarks/single_node/fixed_seq_len/dsr1_fp4_mi355x.sh deleted file mode 100644 index bb6ce75cb..000000000 --- a/benchmarks/single_node/fixed_seq_len/dsr1_fp4_mi355x.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -export SGLANG_USE_AITER=1 -export ROCM_QUICK_REDUCE_QUANTIZATION=INT4 - -PREFILL_SIZE=196608 -if [[ "$ISL" == "8192" && "$OSL" == "1024" ]]; then - if [[ "$CONC" -gt "32" ]]; then - PREFILL_SIZE=32768 - fi -fi - -SERVER_LOG=/workspace/server.log - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -python3 -m sglang.launch_server --model-path=$MODEL --trust-remote-code \ ---host=0.0.0.0 --port=$PORT \ ---tensor-parallel-size=$TP \ ---chunked-prefill-size=$PREFILL_SIZE \ ---mem-fraction-static=0.8 \ ---disable-radix-cache \ ---num-continuous-decode-steps=4 \ ---max-prefill-tokens=$PREFILL_SIZE \ ---cuda-graph-max-bs=128 \ ---attention-backend aiter \ ---kv-cache-dtype fp8_e4m3 $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/dsr1_fp4_mi355x_atom.sh b/benchmarks/single_node/fixed_seq_len/dsr1_fp4_mi355x_atom.sh deleted file mode 100644 index 6ae8f92ba..000000000 --- a/benchmarks/single_node/fixed_seq_len/dsr1_fp4_mi355x_atom.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE \ - DP_ATTENTION - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -SERVER_LOG=/workspace/server.log - -export OMP_NUM_THREADS=1 - -# Calculate max-model-len based on ISL and OSL -if [ "$ISL" = "1024" ] && [ "$OSL" = "1024" ]; then - CALCULATED_MAX_MODEL_LEN="" -else - CALCULATED_MAX_MODEL_LEN=" --max-model-len 10240 " -fi - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - CALCULATED_MAX_MODEL_LEN=" --max-model-len $EVAL_MAX_MODEL_LEN " -fi - -if [ "$EP_SIZE" -gt 1 ]; then - EP=" --enable-expert-parallel" -else - EP=" " -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x - -BLOCK_SIZE=${BLOCK_SIZE:-16} -python3 -m atom.entrypoints.openai_server \ - --model $MODEL \ - --server-port $PORT \ - -tp $TP \ - --kv_cache_dtype fp8 $CALCULATED_MAX_MODEL_LEN $EP \ - --block-size $BLOCK_SIZE > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x \ No newline at end of file diff --git a/benchmarks/single_node/fixed_seq_len/dsr1_fp4_mi355x_atom_mtp.sh b/benchmarks/single_node/fixed_seq_len/dsr1_fp4_mi355x_atom_mtp.sh deleted file mode 100644 index 8447a8b2a..000000000 --- a/benchmarks/single_node/fixed_seq_len/dsr1_fp4_mi355x_atom_mtp.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE \ - DP_ATTENTION - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -SERVER_LOG=/workspace/server.log - -export OMP_NUM_THREADS=1 - -# Calculate max-model-len based on ISL and OSL -if [ "$ISL" = "1024" ] && [ "$OSL" = "1024" ]; then - CALCULATED_MAX_MODEL_LEN="" -else - CALCULATED_MAX_MODEL_LEN=" --max-model-len 10240 " -fi - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - CALCULATED_MAX_MODEL_LEN=" --max-model-len $EVAL_MAX_MODEL_LEN " -fi - -if [ "$EP_SIZE" -gt 1 ]; then - EP=" --enable-expert-parallel" -else - EP=" " -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x - -export AMDGCN_USE_BUFFER_OPS=1 - -python3 -m atom.entrypoints.openai_server \ - --model $MODEL \ - --server-port $PORT \ - -tp $TP \ - --kv_cache_dtype fp8 $CALCULATED_MAX_MODEL_LEN $EP \ - --method mtp \ - > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --use-chat-template - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/dsr1_fp4_mi355x_mtp.sh b/benchmarks/single_node/fixed_seq_len/dsr1_fp4_mi355x_mtp.sh deleted file mode 100755 index 4499736e2..000000000 --- a/benchmarks/single_node/fixed_seq_len/dsr1_fp4_mi355x_mtp.sh +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env bash - -# DeepSeek-R1-0528 MXFP4 on MI355X with EAGLE/MTP speculative decoding. -# Mirrors dsr1_fp4_mi355x.sh and adds the speculative-* flags. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -export SGLANG_USE_AITER=1 -export SGLANG_AITER_MLA_PERSIST=1 -export SGLANG_ENABLE_SPEC_V2=1 -export ROCM_QUICK_REDUCE_QUANTIZATION=INT4 - -PREFILL_SIZE=196608 -if [[ "$ISL" == "8192" && "$OSL" == "1024" ]]; then - if [[ "$CONC" -gt "32" ]]; then - PREFILL_SIZE=32768 - fi -fi - -SERVER_LOG=/workspace/server.log - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi -start_gpu_monitor - -set -x -python3 -m sglang.launch_server --model-path=$MODEL --trust-remote-code \ ---host=0.0.0.0 --port=$PORT \ ---tensor-parallel-size=$TP \ ---ep-size $EP_SIZE \ ---chunked-prefill-size=$PREFILL_SIZE \ ---mem-fraction-static=0.8 \ ---disable-radix-cache \ ---num-continuous-decode-steps=4 \ ---max-prefill-tokens=$PREFILL_SIZE \ ---cuda-graph-max-bs=128 \ ---attention-backend aiter \ ---kv-cache-dtype fp8_e4m3 \ ---speculative-algorithm EAGLE \ ---speculative-num-steps 3 \ ---speculative-eagle-topk 1 \ ---speculative-num-draft-tokens 4 \ -$EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --use-chat-template - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/dsr1_fp8_b200.sh b/benchmarks/single_node/fixed_seq_len/dsr1_fp8_b200.sh deleted file mode 100644 index 8a016bb2a..000000000 --- a/benchmarks/single_node/fixed_seq_len/dsr1_fp8_b200.sh +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -export SGL_ENABLE_JIT_DEEPGEMM=false -export SGLANG_ENABLE_FLASHINFER_GEMM=true -SERVER_LOG=/workspace/server.log - -# Default: recv every ~10 requests; if CONC ≥ 16, relax to ~30 requests between scheduler recv polls. -if [[ $TP -eq 8 ]]; then - if [[ $CONC -ge 16 ]]; then - SCHEDULER_RECV_INTERVAL=30 - else - SCHEDULER_RECV_INTERVAL=10 - fi - - # Setting these values (passed in to --cuda-graph-max-bs and --max-running-requests) as the maximum concurrency - # this will help us save memory from being unnecessary used. - MAX_RUNNING_REQUESTS=128 - CUDA_GRAPH_MAX_BATCH_SIZE=128 - - MEM_FRAC_STATIC=0.82 - CHUNKED_PREFILL_SIZE=32768 - MAX_PREFILL_TOKENS=32768 -elif [[ $TP -eq 4 ]]; then - if [[ $ISL -ne 8192 ]] || [[ $OSL -ne 1024 ]]; then - echo "TP=4 not yet supported for ISL=$ISL OSL=$OSL!" - exit 1 - fi - - # Setting these values (passed in to --cuda-graph-max-bs and --max-running-requests) as the maximum concurrency - # this will help us save memory from being unnecessary used. - MAX_RUNNING_REQUESTS=32 - CUDA_GRAPH_MAX_BATCH_SIZE=32 - - MEM_FRAC_STATIC=0.95 - CHUNKED_PREFILL_SIZE=8192 - MAX_PREFILL_TOKENS=8192 - - SCHEDULER_RECV_INTERVAL=10 -else - echo "Unrecognized TP size $TP!" - exit 1 -fi -echo "SCHEDULER_RECV_INTERVAL: $SCHEDULER_RECV_INTERVAL, CONC: $CONC, ISL: $ISL, OSL: $OSL" - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path=$MODEL --host=0.0.0.0 --port=$PORT \ ---tensor-parallel-size=$TP --data-parallel-size=1 \ ---cuda-graph-max-bs $CUDA_GRAPH_MAX_BATCH_SIZE --max-running-requests $MAX_RUNNING_REQUESTS \ ---mem-fraction-static $MEM_FRAC_STATIC --kv-cache-dtype fp8_e4m3 --chunked-prefill-size $CHUNKED_PREFILL_SIZE --max-prefill-tokens $MAX_PREFILL_TOKENS \ ---enable-flashinfer-allreduce-fusion --scheduler-recv-interval $SCHEDULER_RECV_INTERVAL --disable-radix-cache \ ---attention-backend trtllm_mla --stream-interval 30 --ep-size $EP_SIZE --moe-runner-backend flashinfer_trtllm --quantization fp8 $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x \ No newline at end of file diff --git a/benchmarks/single_node/fixed_seq_len/dsr1_fp8_b200_mtp.sh b/benchmarks/single_node/fixed_seq_len/dsr1_fp8_b200_mtp.sh deleted file mode 100755 index 1ad0c9041..000000000 --- a/benchmarks/single_node/fixed_seq_len/dsr1_fp8_b200_mtp.sh +++ /dev/null @@ -1,122 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -export SGLANG_ENABLE_JIT_DEEPGEMM=false - -SERVER_LOG=/workspace/server.log - -# MTP only supports TP=8 for now -if [[ $TP -ne 8 ]]; then - echo "MTP only supports TP=8, got TP=$TP!" - exit 1 -fi - -# Default: recv every ~10 requests; if CONC >= 16, relax to ~30 requests between scheduler recv polls. -if [[ $CONC -ge 16 ]]; then - SCHEDULER_RECV_INTERVAL=30 -else - SCHEDULER_RECV_INTERVAL=10 -fi - -# Setting these values (passed in to --cuda-graph-max-bs and --max-running-requests) as the maximum concurrency -# this will help us save memory from being unnecessary used. -MAX_RUNNING_REQUESTS=512 -CUDA_GRAPH_MAX_BATCH_SIZE=512 - -MEM_FRAC_STATIC=0.82 -CHUNKED_PREFILL_SIZE=16384 -MAX_PREFILL_TOKENS=16384 - -echo "SCHEDULER_RECV_INTERVAL: $SCHEDULER_RECV_INTERVAL, CONC: $CONC, ISL: $ISL, OSL: $OSL" - -# MTP (Multi-Token Prediction) Config - EAGLE speculative decoding -SPECULATIVE_NUM_STEPS=2 -SPECULATIVE_DRAFT_TOKENS=3 -SPECULATIVE_EAGLE_TOPK=1 - -SGLANG_ENABLE_SPEC_V2=1 - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -PYTHONNOUSERSITE=1 python3 -m sglang.launch_server \ - --model-path=$MODEL \ - --host=0.0.0.0 \ - --port=$PORT \ - --tensor-parallel-size=$TP \ - --data-parallel-size=1 \ - --cuda-graph-max-bs $CUDA_GRAPH_MAX_BATCH_SIZE \ - --max-running-requests $MAX_RUNNING_REQUESTS \ - --mem-fraction-static $MEM_FRAC_STATIC \ - --kv-cache-dtype fp8_e4m3 \ - --chunked-prefill-size $CHUNKED_PREFILL_SIZE \ - --max-prefill-tokens $MAX_PREFILL_TOKENS \ - --enable-flashinfer-allreduce-fusion \ - --scheduler-recv-interval $SCHEDULER_RECV_INTERVAL \ - --disable-radix-cache \ - --fp8-gemm-backend=flashinfer_trtllm \ - --attention-backend trtllm_mla \ - --stream-interval 30 \ - --ep-size $EP_SIZE \ - --moe-runner-backend flashinfer_trtllm \ - --quantization fp8 \ - --speculative-algorithm EAGLE \ - --speculative-num-steps $SPECULATIVE_NUM_STEPS \ - --speculative-num-draft-tokens $SPECULATIVE_DRAFT_TOKENS \ - --speculative-eagle-topk $SPECULATIVE_EAGLE_TOPK \ - $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --use-chat-template - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/dsr1_fp8_b200_trt.sh b/benchmarks/single_node/fixed_seq_len/dsr1_fp8_b200_trt.sh deleted file mode 100644 index b0457614e..000000000 --- a/benchmarks/single_node/fixed_seq_len/dsr1_fp8_b200_trt.sh +++ /dev/null @@ -1,152 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - DP_ATTENTION \ - EP_SIZE - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -# temporary, avoids risk of OOM error -export TLLM_OVERRIDE_LAYER_NUM=61 - -# ========= Determine other parameters based on ISL, OSL, CONC ========= -CUDA_GRAPH_MAX_BATCH_SIZE=$CONC -MOE_BACKEND="TRTLLM" -PIECEWISE_CUDA_GRAPHS="false" -DELAY_BATCHING="false" -KV_CACHE_FREE_MEM_FRACTION=0.8 - -if [[ "$ISL" == "1024" && "$OSL" == "1024" ]]; then - if [[ $CONC -ge 64 ]]; then - PIECEWISE_CUDA_GRAPHS="true" - DELAY_BATCHING="true" - fi -elif [[ "$ISL" == "8192" && "$OSL" == "1024" ]]; then - if [[ $CONC -ge 64 ]]; then - PIECEWISE_CUDA_GRAPHS="true" - fi - if [[ "$TP" == "4" ]]; then - KV_CACHE_FREE_MEM_FRACTION=0.75 - fi -fi - -echo "MOE_BACKEND set to '$MOE_BACKEND'" - -SERVER_LOG=/workspace/server.log -EXTRA_CONFIG_FILE="dsr1-fp8.yml" - -cat > $EXTRA_CONFIG_FILE << EOF -cuda_graph_config: - enable_padding: true - max_batch_size: $CUDA_GRAPH_MAX_BATCH_SIZE -enable_attention_dp: $DP_ATTENTION -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: $KV_CACHE_FREE_MEM_FRACTION - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: $MOE_BACKEND -EOF - -if [[ "$DP_ATTENTION" == "true" ]]; then - cat << EOF >> $EXTRA_CONFIG_FILE -attention_dp_config: - batching_wait_iters: 0 - enable_balance: true - timeout_iters: 60 -EOF -fi - -if [[ "$DELAY_BATCHING" == "true" ]]; then - cat << EOF >> $EXTRA_CONFIG_FILE -batch_wait_timeout_iters: 40 -batch_wait_max_tokens_ratio: 0.8 -EOF -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x - -MAX_NUM_TOKENS=$(( ($CONC+$ISL+64+63)/64*64 )) -MAX_MODEL_LEN=$(( MAX_MODEL_LEN > 8192 ? MAX_MODEL_LEN : 8192 )) -MAX_NUM_TOKENS=$(( MAX_NUM_TOKENS > 8192 ? MAX_NUM_TOKENS : 8192 )) - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" - MAX_NUM_TOKENS="$EVAL_MAX_MODEL_LEN" -fi - -if [[ "$PIECEWISE_CUDA_GRAPHS" == "true" ]]; then - # [2^i for i in range(8)] + [i for i in range(256, max_num_tokens, 256)] + [max_num_tokens] - capture_tokens=(1 2 4 8 16 32 64 128) - capture_tokens+=( $(seq 256 256 $MAX_NUM_TOKENS)) - if [ $((MAX_NUM_TOKENS%256)) -ne 0 ]; then - capture_tokens+=($MAX_NUM_TOKENS) - fi - CAPTURE_TOKENS_LIST=$(printf "%s, " "${capture_tokens[@]}") - - cat << EOF >> $EXTRA_CONFIG_FILE -torch_compile_config: - capture_num_tokens: [${CAPTURE_TOKENS_LIST%, }] - enable_piecewise_cuda_graph: true -EOF -fi - -# Launch TRT-LLM server -mpirun -n 1 --oversubscribe --allow-run-as-root \ - trtllm-serve $MODEL --port=$PORT \ - --trust_remote_code \ - --backend=pytorch \ - --max_seq_len=$MAX_MODEL_LEN \ - --max_num_tokens=$MAX_NUM_TOKENS \ - --tp_size=$TP --ep_size=$EP_SIZE \ - --extra_llm_api_options=$EXTRA_CONFIG_FILE \ - > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend openai \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $(( $CONC * 10 )) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x \ No newline at end of file diff --git a/benchmarks/single_node/fixed_seq_len/dsr1_fp8_b200_trt_mtp.sh b/benchmarks/single_node/fixed_seq_len/dsr1_fp8_b200_trt_mtp.sh deleted file mode 100644 index 16f13710e..000000000 --- a/benchmarks/single_node/fixed_seq_len/dsr1_fp8_b200_trt_mtp.sh +++ /dev/null @@ -1,154 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - DP_ATTENTION \ - EP_SIZE - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -# ========= Determine other parameters based on ISL, OSL, CONC ========= -MOE_BACKEND="TRTLLM" -PIECEWISE_CUDA_GRAPHS="true" -MAX_BATCH_SIZE=$CONC -KV_CACHE_FREE_MEM_FRACTION=0.8 -MTP=3 - -# DP ATTENTION requires different optimizations -if [[ "$DP_ATTENTION" == "true" ]]; then - MOE_BACKEND="DEEPGEMM" - PIECEWISE_CUDA_GRAPHS="false" - MAX_BATCH_SIZE=$(( CONC < 8 ? CONC : CONC / 8 )) - KV_CACHE_FREE_MEM_FRACTION=0.7 - # use the new MOE backend from latest trtllm to get better comms - export ENABLE_CONFIGURABLE_MOE=1 - MTP=1 -fi - -# currently narrow CONC cases don't benefit from PW CUDA -if [[ "$ISL" == "1024" && "$OSL" == "1024" ]]; then - if [[ $CONC -le 4 ]]; then - PIECEWISE_CUDA_GRAPHS="false" - fi -elif [[ "$ISL" == "8192" && "$OSL" == "1024" ]]; then - if [[ $CONC -le 16 ]]; then - PIECEWISE_CUDA_GRAPHS="false" - fi -fi - - -echo "MOE_BACKEND='$MOE_BACKEND', MTP='$MTP'" - -SERVER_LOG=/workspace/server.log -EXTRA_CONFIG_FILE="dsr1-fp8-mtp.yml" - -cat > $EXTRA_CONFIG_FILE << EOF -cuda_graph_config: - enable_padding: true - max_batch_size: $MAX_BATCH_SIZE -enable_attention_dp: $DP_ATTENTION -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: $KV_CACHE_FREE_MEM_FRACTION - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: $MOE_BACKEND -speculative_config: - decoding_type: MTP - num_nextn_predict_layers: ${MTP} -EOF - -if [[ "$DP_ATTENTION" == "true" ]]; then - cat << EOF >> $EXTRA_CONFIG_FILE -attention_dp_config: - batching_wait_iters: 0 - enable_balance: true - timeout_iters: 60 -EOF -fi - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi - -MAX_NUM_TOKENS=$(( ((MTP+1)*MAX_BATCH_SIZE+ISL+64+63)/64*64 )) -if [ "${EVAL_ONLY}" = "true" ]; then - MAX_NUM_TOKENS="$EVAL_MAX_MODEL_LEN" -fi - -# prep PW CUDA config per the documentation -if [[ "$PIECEWISE_CUDA_GRAPHS" == "true" ]]; then - # [2^i for i in range(8)] + [i for i in range(256, max_num_tokens, 256)] + [max_num_tokens] - capture_tokens=(1 2 4 8 16 32 64 128) - capture_tokens+=( $(seq 256 256 $MAX_NUM_TOKENS)) - if [ $((MAX_NUM_TOKENS%256)) -ne 0 ]; then - capture_tokens+=($MAX_NUM_TOKENS) - fi - CAPTURE_TOKENS_LIST=$(printf "%s, " "${capture_tokens[@]}") - - cat << EOF >> $EXTRA_CONFIG_FILE -torch_compile_config: - capture_num_tokens: [${CAPTURE_TOKENS_LIST%, }] - enable_piecewise_cuda_graph: true -EOF -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -# Launch TRT-LLM server -mpirun -n 1 --oversubscribe --allow-run-as-root \ - trtllm-serve $MODEL --port=$PORT \ - --trust_remote_code \ - --backend=pytorch \ - --max_batch_size=$MAX_BATCH_SIZE \ - --max_seq_len=$MAX_MODEL_LEN \ - --max_num_tokens=$MAX_NUM_TOKENS \ - --tp_size=$TP --ep_size=$EP_SIZE \ - --extra_llm_api_options=$EXTRA_CONFIG_FILE \ - > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend openai \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $(( $CONC * 10 )) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --use-chat-template - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor \ No newline at end of file diff --git a/benchmarks/single_node/fixed_seq_len/dsr1_fp8_b300_mtp.sh b/benchmarks/single_node/fixed_seq_len/dsr1_fp8_b300_mtp.sh deleted file mode 100755 index 76809ec78..000000000 --- a/benchmarks/single_node/fixed_seq_len/dsr1_fp8_b300_mtp.sh +++ /dev/null @@ -1,137 +0,0 @@ -#!/usr/bin/env bash - -# NOTE: At the time of submission, https://cookbook.sglang.io/autoregressive/DeepSeek/DeepSeek-R1 -# does not have a B300-specific recipe, so this script reuses the existing -# DSR1 FP8 B200 SGLang MTP recipe as-is until B300-specific tuning is available. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - - -export SGLANG_ENABLE_JIT_DEEPGEMM=false - -SERVER_LOG=/workspace/server.log - -# MTP only supports TP=8 for now -if [[ $TP -ne 8 ]]; then - echo "MTP only supports TP=8, got TP=$TP!" - exit 1 -fi - -# Default: recv every ~10 requests; if CONC >= 16, relax to ~30 requests between scheduler recv polls. -if [[ $CONC -ge 16 ]]; then - SCHEDULER_RECV_INTERVAL=30 -else - SCHEDULER_RECV_INTERVAL=10 -fi - -# Setting these values (passed in to --cuda-graph-max-bs and --max-running-requests) as the maximum concurrency -# this will help us save memory from being unnecessary used. -MAX_RUNNING_REQUESTS=512 -CUDA_GRAPH_MAX_BATCH_SIZE=512 - -MEM_FRAC_STATIC=0.82 -CHUNKED_PREFILL_SIZE=16384 -MAX_PREFILL_TOKENS=16384 - -echo "SCHEDULER_RECV_INTERVAL: $SCHEDULER_RECV_INTERVAL, CONC: $CONC, ISL: $ISL, OSL: $OSL" - -# MTP (Multi-Token Prediction) Config - EAGLE speculative decoding -SPECULATIVE_NUM_STEPS=2 -SPECULATIVE_DRAFT_TOKENS=3 -SPECULATIVE_EAGLE_TOPK=1 - -SGLANG_ENABLE_SPEC_V2=1 - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -PYTHONNOUSERSITE=1 python3 -m sglang.launch_server \ - --model-path $MODEL_PATH --served-model-name $MODEL \ - --host 0.0.0.0 \ - --port $PORT \ - --tensor-parallel-size $TP \ - --data-parallel-size 1 \ - --cuda-graph-max-bs $CUDA_GRAPH_MAX_BATCH_SIZE \ - --max-running-requests $MAX_RUNNING_REQUESTS \ - --mem-fraction-static $MEM_FRAC_STATIC \ - --kv-cache-dtype fp8_e4m3 \ - --chunked-prefill-size $CHUNKED_PREFILL_SIZE \ - --max-prefill-tokens $MAX_PREFILL_TOKENS \ - --enable-flashinfer-allreduce-fusion \ - --scheduler-recv-interval $SCHEDULER_RECV_INTERVAL \ - --disable-radix-cache \ - --fp8-gemm-backend flashinfer_trtllm \ - --attention-backend trtllm_mla \ - --stream-interval 30 \ - --ep-size $EP_SIZE \ - --moe-runner-backend flashinfer_trtllm \ - --quantization fp8 \ - --speculative-algorithm EAGLE \ - --speculative-num-steps $SPECULATIVE_NUM_STEPS \ - --speculative-num-draft-tokens $SPECULATIVE_DRAFT_TOKENS \ - --speculative-eagle-topk $SPECULATIVE_EAGLE_TOPK \ - $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --use-chat-template - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/dsr1_fp8_h200.sh b/benchmarks/single_node/fixed_seq_len/dsr1_fp8_h200.sh deleted file mode 100644 index db846b4d2..000000000 --- a/benchmarks/single_node/fixed_seq_len/dsr1_fp8_h200.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -pip3 install --user --break-system-packages sentencepiece - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi -SERVER_LOG=/workspace/server.log - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -export TORCH_CUDA_ARCH_LIST="9.0" - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi - -set -x -if [[ $ISL -eq 1024 && $OSL -eq 1024 ]]; then - PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path $MODEL \ - --host 0.0.0.0 --port $PORT --trust-remote-code \ - --tensor-parallel-size=$TP --data-parallel-size=1 \ - --disable-radix-cache --max-running-requests 512 --cuda-graph-max-bs 512 \ - --chunked-prefill-size 32768 --max-prefill-tokens 32768 --mem-fraction-static 0.82 \ - --attention-backend flashinfer --stream-interval 10 \ - --decode-log-interval 1 \ - $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & -else - PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path $MODEL \ - --host 0.0.0.0 --port $PORT --trust-remote-code \ - --tensor-parallel-size=$TP --data-parallel-size=1 \ - --disable-radix-cache --max-running-requests 256 --cuda-graph-max-bs 256 \ - --chunked-prefill-size 32768 --max-prefill-tokens 32768 --mem-fraction-static 0.82 \ - --attention-backend flashinfer --stream-interval 10 \ - --decode-log-interval 1 \ - $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & -fi - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $(( $CONC * 10 )) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/dsr1_fp8_h200_mtp.sh b/benchmarks/single_node/fixed_seq_len/dsr1_fp8_h200_mtp.sh deleted file mode 100755 index 611f600f6..000000000 --- a/benchmarks/single_node/fixed_seq_len/dsr1_fp8_h200_mtp.sh +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/env bash - -# DeepSeek-R1-0528 FP8 on H200 with EAGLE/MTP speculative decoding. -# Mirrors dsr1_fp8_h200.sh and adds the speculative-* flags from -# dsr1_fp8_b200_mtp.sh (the production sglang MTP template). -# Keeps the H200's flashinfer attention backend (no trtllm_mla path on -# H200 for this image). - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -pip3 install --user --break-system-packages sentencepiece - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -# MTP only supports TP=8 for now (matching dsr1_fp8_b200_mtp.sh) -if [[ $TP -ne 8 ]]; then - echo "MTP only supports TP=8, got TP=$TP!" - exit 1 -fi - -SERVER_LOG=/workspace/server.log - -# MTP (Multi-Token Prediction) Config - EAGLE speculative decoding -SPECULATIVE_NUM_STEPS=2 -SPECULATIVE_DRAFT_TOKENS=3 -SPECULATIVE_EAGLE_TOPK=1 - -export SGLANG_ENABLE_SPEC_V2=1 -export TORCH_CUDA_ARCH_LIST="9.0" - -start_gpu_monitor - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi - -set -x -if [[ $ISL -eq 1024 && $OSL -eq 1024 ]]; then - MAX_RUNNING_REQUESTS=512 - CUDA_GRAPH_MAX_BS=512 -else - MAX_RUNNING_REQUESTS=256 - CUDA_GRAPH_MAX_BS=256 -fi - -PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path $MODEL \ ---host 0.0.0.0 --port $PORT --trust-remote-code \ ---tensor-parallel-size=$TP --data-parallel-size=1 \ ---ep-size $EP_SIZE \ ---disable-radix-cache \ ---max-running-requests $MAX_RUNNING_REQUESTS \ ---cuda-graph-max-bs $CUDA_GRAPH_MAX_BS \ ---chunked-prefill-size 32768 --max-prefill-tokens 32768 --mem-fraction-static 0.82 \ ---attention-backend flashinfer --stream-interval 10 \ ---decode-log-interval 1 \ ---speculative-algorithm EAGLE \ ---speculative-num-steps $SPECULATIVE_NUM_STEPS \ ---speculative-num-draft-tokens $SPECULATIVE_DRAFT_TOKENS \ ---speculative-eagle-topk $SPECULATIVE_EAGLE_TOPK \ -$EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $(( $CONC * 10 )) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --use-chat-template - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/dsr1_fp8_h200_trt.sh b/benchmarks/single_node/fixed_seq_len/dsr1_fp8_h200_trt.sh deleted file mode 100644 index c59eb8625..000000000 --- a/benchmarks/single_node/fixed_seq_len/dsr1_fp8_h200_trt.sh +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - DP_ATTENTION \ - EP_SIZE - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -# ========= Determine DP_ATTENTION, EP_SIZE and MOE_BACKEND based on ISL, OSL, CONC ========= -MOE_BACKEND="CUTLASS" - -echo "MOE_BACKEND set to '$MOE_BACKEND'" - -SERVER_LOG=/workspace/server.log -EXTRA_CONFIG_FILE="dsr1-fp8.yml" - -cat > $EXTRA_CONFIG_FILE << EOF -cuda_graph_config: - enable_padding: true - max_batch_size: 128 -enable_attention_dp: $DP_ATTENTION -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.75 - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: $MOE_BACKEND -EOF - -if [[ "$DP_ATTENTION" == "true" ]]; then - cat << EOF >> $EXTRA_CONFIG_FILE -attention_dp_config: - batching_wait_iters: 0 - enable_balance: true - timeout_iters: 60 -EOF -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x - -MAX_NUM_TOKENS=$(( (CONC + ISL + 64 + 63) / 64 * 64 )) -MAX_MODEL_LEN=$(( MAX_MODEL_LEN > 8192 ? MAX_MODEL_LEN : 8192 )) -MAX_NUM_TOKENS=$(( MAX_NUM_TOKENS > 8192 ? MAX_NUM_TOKENS : 8192 )) - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" - MAX_NUM_TOKENS="$EVAL_MAX_MODEL_LEN" -fi - -# Launch TRT-LLM server -PYTHONNOUSERSITE=1 mpirun -n 1 --oversubscribe --allow-run-as-root \ - trtllm-serve $MODEL --port=$PORT \ - --trust_remote_code \ - --backend=pytorch \ - --max_seq_len=$MAX_MODEL_LEN \ - --max_num_tokens=$MAX_NUM_TOKENS \ - --tp_size=$TP --ep_size=$EP_SIZE \ - --extra_llm_api_options=$EXTRA_CONFIG_FILE \ - > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend openai \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $(( $CONC * 10 )) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/dsr1_fp8_h200_trt_mtp.sh b/benchmarks/single_node/fixed_seq_len/dsr1_fp8_h200_trt_mtp.sh deleted file mode 100644 index c544af6ed..000000000 --- a/benchmarks/single_node/fixed_seq_len/dsr1_fp8_h200_trt_mtp.sh +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - DP_ATTENTION \ - EP_SIZE - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -# ========= Determine MOE_BACKEND and MTP based on DP_ATTENTION ========= -MOE_BACKEND="CUTLASS" - -if [[ "$DP_ATTENTION" == "true" ]]; then - MTP=1 -else - MTP=3 -fi - -echo "MOE_BACKEND='$MOE_BACKEND', MTP='$MTP'" - -SERVER_LOG=/workspace/server.log -EXTRA_CONFIG_FILE="dsr1-fp8-mtp.yml" - -# If ISL=8192 and DP_ATTENTION=true, export PYTORCH_CUDA_ALLOC_CONF=max_split_size_mb:8192 -if [[ "$ISL" == "8192" && "$DP_ATTENTION" == "true" ]]; then - export PYTORCH_CUDA_ALLOC_CONF="max_split_size_mb:8192" -fi - -cat > $EXTRA_CONFIG_FILE << EOF -cuda_graph_config: - enable_padding: true - max_batch_size: 128 -enable_attention_dp: $DP_ATTENTION -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.75 - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: $MOE_BACKEND -speculative_config: - decoding_type: MTP - num_nextn_predict_layers: ${MTP} -EOF - -if [[ "$DP_ATTENTION" == "true" ]]; then - cat << EOF >> $EXTRA_CONFIG_FILE -attention_dp_config: - batching_wait_iters: 0 - enable_balance: true - timeout_iters: 60 -EOF -fi - -if [[ "$DP_ATTENTION" == "true" ]]; then - MAX_BATCH_SIZE=$((CONC/TP)) - if [[ $MAX_BATCH_SIZE -lt 1 ]]; then - MAX_BATCH_SIZE=1 - fi -else - MAX_BATCH_SIZE=$CONC -fi - -MAX_NUM_TOKENS=$(( ((MTP+1)*MAX_BATCH_SIZE+ISL+64+63)/64*64 )) - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" - MAX_NUM_TOKENS="$EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -# Launch TRT-LLM server -PYTHONNOUSERSITE=1 mpirun -n 1 --oversubscribe --allow-run-as-root \ - trtllm-serve $MODEL --port=$PORT \ - --trust_remote_code \ - --backend=pytorch \ - --max_batch_size=$MAX_BATCH_SIZE \ - --max_seq_len=$MAX_MODEL_LEN \ - --max_num_tokens=$MAX_NUM_TOKENS \ - --tp_size=$TP --ep_size=$EP_SIZE \ - --extra_llm_api_options=$EXTRA_CONFIG_FILE \ - > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend openai \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $(( $CONC * 10 )) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --use-chat-template - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor \ No newline at end of file diff --git a/benchmarks/single_node/fixed_seq_len/dsr1_fp8_mi300x.sh b/benchmarks/single_node/fixed_seq_len/dsr1_fp8_mi300x.sh deleted file mode 100644 index da95c0e7a..000000000 --- a/benchmarks/single_node/fixed_seq_len/dsr1_fp8_mi300x.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -# Reference -# https://rocm.docs.amd.com/en/docs-7.0-rc1/preview/benchmark-docker/inference-sglang-deepseek-r1-fp8.html#run-the-inference-benchmark - -# If the machine runs a MEC FW older than 177, RCCL -# cannot reclaim some memory. -# Disable that features to avoid crashes. -# This is related to the changes in the driver at: -# https://rocm.docs.amd.com/en/docs-6.4.3/about/release-notes.html#amdgpu-driver-updates -version=`rocm-smi --showfw | grep MEC | head -n 1 | awk '{print $NF}'` -if [[ "$version" == "" || $version -lt 177 ]]; then - export HSA_NO_SCRATCH_RECLAIM=1 -fi - -export SGLANG_USE_AITER=1 -export SGLANG_AITER_MLA_PERSIST=1 - -SERVER_LOG=/workspace/server.log - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -python3 -m sglang.launch_server \ ---model-path=$MODEL --host=0.0.0.0 --port=$PORT --trust-remote-code \ ---tensor-parallel-size=$TP \ ---mem-fraction-static=0.8 \ ---cuda-graph-max-bs=128 \ ---chunked-prefill-size=131072 \ ---num-continuous-decode-steps=4 \ ---max-prefill-tokens=131072 \ ---kv-cache-dtype fp8_e4m3 \ ---attention-backend aiter \ ---disable-radix-cache $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $(( $CONC * 10 )) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/dsr1_fp8_mi325x.sh b/benchmarks/single_node/fixed_seq_len/dsr1_fp8_mi325x.sh deleted file mode 100644 index 6b1c50265..000000000 --- a/benchmarks/single_node/fixed_seq_len/dsr1_fp8_mi325x.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/bash - -# Source benchmark utilities early -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -SERVER_LOG=/workspace/server.log -PORT=8888 -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -# Reference -# https://rocm.docs.amd.com/en/docs-7.0-rc1/preview/benchmark-docker/inference-sglang-deepseek-r1-fp8.html#run-the-inference-benchmark - -export SGLANG_USE_AITER=1 -export SGLANG_AITER_MLA_PERSIST=1 - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi - -set -x -python3 -m sglang.launch_server \ ---model-path=$MODEL --host=0.0.0.0 --port=$PORT --trust-remote-code \ ---tensor-parallel-size=$TP \ ---mem-fraction-static=0.8 \ ---cuda-graph-max-bs=128 \ ---chunked-prefill-size=131072 \ ---num-continuous-decode-steps=4 \ ---max-prefill-tokens=131072 \ ---kv-cache-dtype fp8_e4m3 \ ---attention-backend aiter \ ---disable-radix-cache \ -$EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $(( $CONC * 10 )) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/dsr1_fp8_mi325x_mtp.sh b/benchmarks/single_node/fixed_seq_len/dsr1_fp8_mi325x_mtp.sh deleted file mode 100755 index 8251c169a..000000000 --- a/benchmarks/single_node/fixed_seq_len/dsr1_fp8_mi325x_mtp.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/bash - -# DeepSeek-R1-0528 FP8 on MI325X with EAGLE/MTP speculative decoding. -# Mirrors dsr1_fp8_mi325x.sh and adds the speculative-* flags. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -SERVER_LOG=/workspace/server.log -PORT=8888 -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -export SGLANG_USE_AITER=1 -export SGLANG_AITER_MLA_PERSIST=1 -export SGLANG_ENABLE_SPEC_V2=1 - -start_gpu_monitor - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi - -set -x -python3 -m sglang.launch_server \ ---model-path=$MODEL --host=0.0.0.0 --port=$PORT --trust-remote-code \ ---tensor-parallel-size=$TP \ ---ep-size $EP_SIZE \ ---mem-fraction-static=0.8 \ ---cuda-graph-max-bs=128 \ ---chunked-prefill-size=131072 \ ---num-continuous-decode-steps=4 \ ---max-prefill-tokens=131072 \ ---kv-cache-dtype fp8_e4m3 \ ---attention-backend aiter \ ---speculative-algorithm EAGLE \ ---speculative-num-steps 3 \ ---speculative-eagle-topk 1 \ ---speculative-num-draft-tokens 4 \ ---disable-radix-cache \ -$EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $(( $CONC * 10 )) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --use-chat-template - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/dsr1_fp8_mi355x.sh b/benchmarks/single_node/fixed_seq_len/dsr1_fp8_mi355x.sh deleted file mode 100644 index d8b596826..000000000 --- a/benchmarks/single_node/fixed_seq_len/dsr1_fp8_mi355x.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -# Reference -# https://rocm.docs.amd.com/en/docs-7.0-docker/benchmark-docker/inference-sglang-deepseek-r1-fp8.html - -export SGLANG_USE_AITER=1 -export RCCL_MSCCL_ENABLE=0 -export ROCM_QUICK_REDUCE_QUANTIZATION=INT4 - -SERVER_LOG=/workspace/server.log - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -python3 -m sglang.launch_server \ - --attention-backend aiter \ - --model-path $MODEL \ - --host=0.0.0.0 \ - --port $PORT \ - --tensor-parallel-size $TP \ - --trust-remote-code \ - --chunked-prefill-size 196608 \ - --mem-fraction-static 0.8 --disable-radix-cache \ - --num-continuous-decode-steps 8 \ - --max-prefill-tokens 196608 \ - --kv-cache-dtype fp8_e4m3 \ - --cuda-graph-max-bs "$CONC" $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/dsr1_fp8_mi355x_atom.sh b/benchmarks/single_node/fixed_seq_len/dsr1_fp8_mi355x_atom.sh deleted file mode 100644 index 6ae8f92ba..000000000 --- a/benchmarks/single_node/fixed_seq_len/dsr1_fp8_mi355x_atom.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE \ - DP_ATTENTION - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -SERVER_LOG=/workspace/server.log - -export OMP_NUM_THREADS=1 - -# Calculate max-model-len based on ISL and OSL -if [ "$ISL" = "1024" ] && [ "$OSL" = "1024" ]; then - CALCULATED_MAX_MODEL_LEN="" -else - CALCULATED_MAX_MODEL_LEN=" --max-model-len 10240 " -fi - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - CALCULATED_MAX_MODEL_LEN=" --max-model-len $EVAL_MAX_MODEL_LEN " -fi - -if [ "$EP_SIZE" -gt 1 ]; then - EP=" --enable-expert-parallel" -else - EP=" " -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x - -BLOCK_SIZE=${BLOCK_SIZE:-16} -python3 -m atom.entrypoints.openai_server \ - --model $MODEL \ - --server-port $PORT \ - -tp $TP \ - --kv_cache_dtype fp8 $CALCULATED_MAX_MODEL_LEN $EP \ - --block-size $BLOCK_SIZE > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x \ No newline at end of file diff --git a/benchmarks/single_node/fixed_seq_len/dsr1_fp8_mi355x_atom_mtp.sh b/benchmarks/single_node/fixed_seq_len/dsr1_fp8_mi355x_atom_mtp.sh deleted file mode 100644 index 2af10d749..000000000 --- a/benchmarks/single_node/fixed_seq_len/dsr1_fp8_mi355x_atom_mtp.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE \ - DP_ATTENTION - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -SERVER_LOG=/workspace/server.log - -export OMP_NUM_THREADS=1 - -CALCULATED_MAX_MODEL_LEN="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - CALCULATED_MAX_MODEL_LEN=" --max-model-len $EVAL_MAX_MODEL_LEN " -fi - -PARALLEL_ARGS=(-tp "$TP") #TP -if [ "$DP_ATTENTION" = "true" ]; then - if [ "$EP_SIZE" -gt 1 ]; then #DP+EP - PARALLEL_ARGS=(-tp "$TP" --enable-expert-parallel --enable-dp-attention ) - else #DP+TP - PARALLEL_ARGS=(-tp "$TP" --enable-dp-attention ) - fi -fi - -SPEC_ARGS=(--method mtp --num-speculative-tokens 3 ) - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x - -python3 -m atom.entrypoints.openai_server \ - --model $MODEL \ - --server-port $PORT \ - "${PARALLEL_ARGS[@]}" \ - "${SPEC_ARGS[@]}" \ - --kv_cache_dtype fp8 $CALCULATED_MAX_MODEL_LEN \ - --no-enable_prefix_caching \ - > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -export PYTHONDONTWRITEBYTECODE=1 -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --use-chat-template - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/dsr1_fp8_mi355x_mtp.sh b/benchmarks/single_node/fixed_seq_len/dsr1_fp8_mi355x_mtp.sh deleted file mode 100755 index d8fc1590b..000000000 --- a/benchmarks/single_node/fixed_seq_len/dsr1_fp8_mi355x_mtp.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env bash - -# DeepSeek-R1-0528 FP8 on MI355X with EAGLE/MTP speculative decoding. -# Mirrors dsr1_fp8_mi355x.sh and adds the speculative-* flags. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -# Reference -# https://rocm.docs.amd.com/en/docs-7.0-docker/benchmark-docker/inference-sglang-deepseek-r1-fp8.html - -export SGLANG_USE_AITER=1 -export SGLANG_AITER_MLA_PERSIST=1 -export SGLANG_ENABLE_SPEC_V2=1 -export RCCL_MSCCL_ENABLE=0 -export ROCM_QUICK_REDUCE_QUANTIZATION=INT4 - -SERVER_LOG=/workspace/server.log - -# Keep server-side speculative decoding capacity aligned with the matrix row. -MAX_RUNNING_REQUESTS="${MAX_RUNNING_REQUESTS:-$CONC}" -CUDA_GRAPH_MAX_BATCH_SIZE="${CUDA_GRAPH_MAX_BATCH_SIZE:-$CONC}" - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi -start_gpu_monitor - -python3 -m sglang.launch_server \ - --attention-backend aiter \ - --model-path $MODEL \ - --host=0.0.0.0 \ - --port $PORT \ - --tensor-parallel-size $TP \ - --ep-size $EP_SIZE \ - --trust-remote-code \ - --chunked-prefill-size 196608 \ - --mem-fraction-static 0.8 --disable-radix-cache \ - --num-continuous-decode-steps 8 \ - --max-prefill-tokens 196608 \ - --kv-cache-dtype fp8_e4m3 \ - --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BATCH_SIZE" \ - --max-running-requests "$MAX_RUNNING_REQUESTS" \ - --speculative-algorithm EAGLE \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 \ - $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --use-chat-template - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b200.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b200.sh deleted file mode 100755 index 898402776..000000000 --- a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b200.sh +++ /dev/null @@ -1,148 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -nvidia-smi - -# Common SGLANG env vars (apply to every config). -export SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT=1 - -# TODO(Cam): the lmsysorg/sglang:deepseek-v4-blackwell image installs sglang -# editable at /workspace/sglang/python; prior sglang tags used /sgl-workspace/sglang. -# The runner mounts our repo at a non-/workspace path for this image so the editable -# install stays visible. Paths in this script are $PWD-relative for that reason. -# Drop the runner conditional once lmsys moves sglang back out of /workspace. - -SERVER_LOG="$PWD/server.log" -PORT=${PORT:-8888} - -echo "TP: $TP, DP_ATTENTION: $DP_ATTENTION, CONC: $CONC, ISL: $ISL, OSL: $OSL" - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi - -start_gpu_monitor --output "$PWD/gpu_metrics.csv" - -# 1k inputs need more SWA cache headroom than 8k inputs do. -if [[ "$ISL" == "1024" ]]; then - SWA_FULL_TOKENS_RATIO=0.5 -else - SWA_FULL_TOKENS_RATIO=0.1 -fi - -# Pick the parallelism + MoE backend based on DP_ATTENTION. DP-attention turns on -# EP-MoE (megamoe) + the mega_moe / mixed-chunk optimizations; single-instance -# uses flashinfer_mxfp4. -if [ "${DP_ATTENTION}" = "true" ]; then - export SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION=8 - export SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN=1 - export SGLANG_OPT_USE_FAST_MASK_EP=1 - export SGLANG_OPT_FIX_MEGA_MOE_MEMORY=1 - export SGLANG_OPT_FIX_NEXTN_MEGA_MOE=1 - export NVSHMEM_DISABLE_IB=1 - export SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW=1 - export SGLANG_OPT_USE_ONLINE_COMPRESS=1 - export SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=2048 - export SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS=1 - export SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND=1 - export SGLANG_EXPERIMENTAL_ENABLE_PIECEWISE_CUDA_GRAPH_MOE_A2A=1 - export NCCL_MNNVL_ENABLE=1 - export NCCL_CUMEM_ENABLE=1 - export MC_FORCE_MNNVL=1 - export SGLANG_MOONCAKE_CUSTOM_MEM_POOL=True - - MEM_FRACTION_STATIC=0.835 - MAX_RUNNING_REQUESTS=4352 - SWA_FULL_TOKENS_RATIO=0.12 - - PARALLEL_ARGS=( - --dp-size "$TP" - --enable-dp-attention - --moe-a2a-backend megamoe - --cuda-graph-max-bs 544 - --enable-mixed-chunk - --chunked-prefill-size 16384 - --max-prefill-tokens 16384 - --tokenizer-worker-num 8 - --stream-interval 30 - --enable-prefill-delayer - ) -else - MEM_FRACTION_STATIC=0.90 - MAX_RUNNING_REQUESTS=512 - PARALLEL_ARGS=( - --moe-runner-backend flashinfer_mxfp4 - --chunked-prefill-size 8192 - --disable-flashinfer-autotune - --cuda-graph-max-bs 512 - --tokenizer-worker-num 8 - --stream-interval 30 - --enable-prefill-delayer - ) -fi - -# Print all SGLANG_* env vars to both the CI step log and server.log so the -# launch config is auditable from the result artifact alone. -{ - echo "=== SGLANG_* env vars at launch ===" - env | grep -E '^SGLANG_' | sort - echo "===================================" -} | tee "$SERVER_LOG" - -set -x -PYTHONNOUSERSITE=1 sglang serve \ - --model-path $MODEL \ - --host 0.0.0.0 \ - --port $PORT \ - --trust-remote-code \ - --tp $TP \ - --disable-radix-cache \ - --max-running-requests "$MAX_RUNNING_REQUESTS" \ - --mem-fraction-static "$MEM_FRACTION_STATIC" \ - --swa-full-tokens-ratio "$SWA_FULL_TOKENS_RATIO" \ - "${PARALLEL_ARGS[@]}" $EVAL_CONTEXT_ARGS >> $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $((CONC * 10)) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir "$PWD/" - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b200_trt.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b200_trt.sh deleted file mode 100644 index ce567c908..000000000 --- a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b200_trt.sh +++ /dev/null @@ -1,170 +0,0 @@ -#!/usr/bin/env bash - -# DeepSeek-V4-Pro single-node TRTLLM recipe for B200. The configured image -# already contains a TensorRT-LLM DeepSeek-V4 build; do not build TRTLLM at -# runtime from this benchmark path. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - DP_ATTENTION \ - EP_SIZE - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -export TRTLLM_DSV4_USE_MPIRUN="${TRTLLM_DSV4_USE_MPIRUN:-1}" -export TRTLLM_DSV4_SANITIZE_SLURM_MPI_ENV="${TRTLLM_DSV4_SANITIZE_SLURM_MPI_ENV:-1}" - -sanitize_slurm_mpi_env_for_trtllm() { - if [[ "${TRTLLM_DSV4_SANITIZE_SLURM_MPI_ENV:-0}" != "1" ]]; then - return 0 - fi - - echo "Sanitizing Slurm/PMI environment for TensorRT-LLM launch" - while IFS='=' read -r name _; do - case "$name" in - SLURM_*|PMIX*|PMI*|OMPI_*|ORTE_*) - unset "$name" - ;; - esac - done < <(env) -} - -sanitize_slurm_mpi_env_for_trtllm - -export NCCL_NVLS_ENABLE="${NCCL_NVLS_ENABLE:-0}" -echo "NCCL_NVLS_ENABLE: $NCCL_NVLS_ENABLE" - -export TRTLLM_SERVER_DISABLE_GC="${TRTLLM_SERVER_DISABLE_GC:-1}" -export TRTLLM_WORKER_DISABLE_GC="${TRTLLM_WORKER_DISABLE_GC:-1}" -export NCCL_GRAPH_MIXING_SUPPORT="${NCCL_GRAPH_MIXING_SUPPORT:-0}" -export MIMALLOC_PURGE_DELAY="${MIMALLOC_PURGE_DELAY:-0}" -export PYTORCH_CUDA_ALLOC_CONF="${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True}" - -if [[ "$MODEL" != /* ]]; then - hf download "$MODEL" -fi - -nvidia-smi - -SERVER_LOG="$PWD/server.log" -EXTRA_CONFIG_FILE="dsv4-fp4-trt.yml" - -# MoE backend: TRTLLM at low/mid concurrency; switch to MEGAMOE_DEEPGEMM at the -# top concurrency for short ISL (1k). -if [[ "$ISL" -le 1024 && "$CONC" -ge 2048 ]]; then - MOE_BACKEND="${MOE_BACKEND:-MEGAMOE_DEEPGEMM}" -else - MOE_BACKEND="${MOE_BACKEND:-TRTLLM}" -fi -MAX_BATCH_SIZE=$(( CONC > 16 ? CONC : 16 )) -CUDA_GRAPH_MAX_BATCH_SIZE="$MAX_BATCH_SIZE" -if [[ "$DP_ATTENTION" == "true" ]]; then - KV_CACHE_FREE_MEM_FRACTION="${KV_CACHE_FREE_MEM_FRACTION:-0.7}" -else - KV_CACHE_FREE_MEM_FRACTION="${KV_CACHE_FREE_MEM_FRACTION:-0.9}" -fi - -ATTENTION_DP_CONFIG="" -if [[ "$DP_ATTENTION" == "true" ]]; then - ATTENTION_DP_CONFIG=" -attention_dp_config: - batching_wait_iters: 30 - enable_balance: true" -fi - -cat > "$EXTRA_CONFIG_FILE" << EOF -cuda_graph_config: - enable_padding: true - max_batch_size: $CUDA_GRAPH_MAX_BATCH_SIZE -enable_attention_dp: $DP_ATTENTION$ATTENTION_DP_CONFIG -print_iter_log: true -kv_cache_config: - tokens_per_block: 128 - dtype: fp8 - free_gpu_memory_fraction: $KV_CACHE_FREE_MEM_FRACTION - enable_block_reuse: false -stream_interval: 100 -num_postprocess_workers: 4 -moe_config: - backend: $MOE_BACKEND - use_low_precision_moe_combine: true -EOF - -echo "Generated config file contents:" -cat "$EXTRA_CONFIG_FILE" - -MAX_MODEL_LEN=$(( MAX_MODEL_LEN > 8192 ? MAX_MODEL_LEN : 8192 )) -MAX_NUM_TOKENS=$(( ISL + 256 )) -MAX_NUM_TOKENS=$(( MAX_NUM_TOKENS > 8192 ? MAX_NUM_TOKENS : 8192 )) - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" - MAX_NUM_TOKENS="$EVAL_MAX_MODEL_LEN" -fi - -start_gpu_monitor --output "$PWD/gpu_metrics.csv" - -set -x -SERVE_CMD=( - trtllm-serve "$MODEL" \ - --host 0.0.0.0 \ - --port "$PORT" \ - --trust_remote_code \ - --backend pytorch \ - --max_batch_size "$MAX_BATCH_SIZE" \ - --max_seq_len "$MAX_MODEL_LEN" \ - --max_num_tokens "$MAX_NUM_TOKENS" \ - --tp_size "$TP" \ - --ep_size "$EP_SIZE" \ - --custom_tokenizer deepseek_v4 \ - --config "$EXTRA_CONFIG_FILE" -) - -if [[ "${TRTLLM_DSV4_USE_MPIRUN:-1}" == "0" ]]; then - "${SERVE_CMD[@]}" > "$SERVER_LOG" 2>&1 & -else - mpirun -n 1 --oversubscribe --allow-run-as-root \ - "${SERVE_CMD[@]}" \ - > "$SERVER_LOG" 2>&1 & -fi - -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend openai-chat \ - --endpoint /v1/chat/completions \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$(( CONC * 10 ))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir "$PWD/" \ - --trust-remote-code \ - --server-pid "$SERVER_PID" - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b200_trt_mtp.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b200_trt_mtp.sh deleted file mode 100644 index 0c7f32363..000000000 --- a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b200_trt_mtp.sh +++ /dev/null @@ -1,181 +0,0 @@ -#!/usr/bin/env bash - -# DeepSeek-V4-Pro B200 TensorRT-LLM MTP variant. The configured image already -# contains the DeepSeek-V4 TRTLLM build; this path only toggles speculative MTP. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - DP_ATTENTION \ - EP_SIZE - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -export TRTLLM_DSV4_USE_MPIRUN="${TRTLLM_DSV4_USE_MPIRUN:-1}" -export TRTLLM_DSV4_SANITIZE_SLURM_MPI_ENV="${TRTLLM_DSV4_SANITIZE_SLURM_MPI_ENV:-1}" - -sanitize_slurm_mpi_env_for_trtllm() { - if [[ "${TRTLLM_DSV4_SANITIZE_SLURM_MPI_ENV:-0}" != "1" ]]; then - return 0 - fi - - echo "Sanitizing Slurm/PMI environment for TensorRT-LLM launch" - while IFS='=' read -r name _; do - case "$name" in - SLURM_*|PMIX*|PMI*|OMPI_*|ORTE_*) - unset "$name" - ;; - esac - done < <(env) -} - -sanitize_slurm_mpi_env_for_trtllm - -export NCCL_NVLS_ENABLE="${NCCL_NVLS_ENABLE:-0}" -echo "NCCL_NVLS_ENABLE: $NCCL_NVLS_ENABLE" - -export TRTLLM_SERVER_DISABLE_GC="${TRTLLM_SERVER_DISABLE_GC:-1}" -export TRTLLM_WORKER_DISABLE_GC="${TRTLLM_WORKER_DISABLE_GC:-1}" -export NCCL_GRAPH_MIXING_SUPPORT="${NCCL_GRAPH_MIXING_SUPPORT:-0}" -export MIMALLOC_PURGE_DELAY="${MIMALLOC_PURGE_DELAY:-0}" -export PYTORCH_CUDA_ALLOC_CONF="${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True}" - -if [[ "$MODEL" != /* ]]; then - hf download "$MODEL" -fi - -nvidia-smi - -SERVER_LOG="$PWD/server.log" -EXTRA_CONFIG_FILE="dsv4-fp4-trt-mtp.yml" - -# MoE backend: TRTLLM at low/mid concurrency; switch to MEGAMOE_DEEPGEMM at high -# concurrency for short ISL (1k). -if [[ "$ISL" -le 1024 && "$CONC" -ge 512 ]]; then - MOE_BACKEND="${MOE_BACKEND:-MEGAMOE_DEEPGEMM}" -else - MOE_BACKEND="${MOE_BACKEND:-TRTLLM}" -fi -# MTP draft length: 3 at low/mid concurrency; steps down to 2 at high concurrency -# for long ISL (8k). -if [[ "$ISL" -ge 4096 && "$CONC" -ge 128 ]]; then - MTP="${TRTLLM_DSV4_MTP_NUM_NEXTN_LAYERS:-2}" -else - MTP="${TRTLLM_DSV4_MTP_NUM_NEXTN_LAYERS:-3}" -fi -MAX_BATCH_SIZE=$(( CONC > 16 ? CONC : 16 )) -CUDA_GRAPH_MAX_BATCH_SIZE="$MAX_BATCH_SIZE" -if [[ "$DP_ATTENTION" == "true" ]]; then - KV_CACHE_FREE_MEM_FRACTION="${KV_CACHE_FREE_MEM_FRACTION:-0.6}" -else - KV_CACHE_FREE_MEM_FRACTION="${KV_CACHE_FREE_MEM_FRACTION:-0.9}" -fi - -ATTENTION_DP_CONFIG="" -if [[ "$DP_ATTENTION" == "true" ]]; then - ATTENTION_DP_CONFIG=" -attention_dp_config: - batching_wait_iters: 30 - enable_balance: true -enable_lm_head_tp_in_adp: true" -fi - -cat > "$EXTRA_CONFIG_FILE" << EOF -cuda_graph_config: - enable_padding: true - max_batch_size: $CUDA_GRAPH_MAX_BATCH_SIZE -enable_attention_dp: $DP_ATTENTION$ATTENTION_DP_CONFIG -print_iter_log: true -kv_cache_config: - tokens_per_block: 128 - dtype: fp8 - free_gpu_memory_fraction: $KV_CACHE_FREE_MEM_FRACTION - enable_block_reuse: false -stream_interval: 100 -num_postprocess_workers: 4 -moe_config: - backend: $MOE_BACKEND - use_low_precision_moe_combine: true -speculative_config: - decoding_type: MTP - max_draft_len: $MTP -EOF - -echo "Generated config file contents:" -cat "$EXTRA_CONFIG_FILE" - -MAX_MODEL_LEN=$(( MAX_MODEL_LEN > 8192 ? MAX_MODEL_LEN : 8192 )) -MAX_NUM_TOKENS=$(( ISL + (MTP + 1) * MAX_BATCH_SIZE + 256 )) -MAX_NUM_TOKENS=$(( MAX_NUM_TOKENS > 8192 ? MAX_NUM_TOKENS : 8192 )) - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" - MAX_NUM_TOKENS="$EVAL_MAX_MODEL_LEN" -fi - -start_gpu_monitor --output "$PWD/gpu_metrics.csv" - -set -x -SERVE_CMD=( - trtllm-serve "$MODEL" \ - --host 0.0.0.0 \ - --port "$PORT" \ - --trust_remote_code \ - --backend pytorch \ - --max_batch_size "$MAX_BATCH_SIZE" \ - --max_seq_len "$MAX_MODEL_LEN" \ - --max_num_tokens "$MAX_NUM_TOKENS" \ - --tp_size "$TP" \ - --ep_size "$EP_SIZE" \ - --custom_tokenizer deepseek_v4 \ - --config "$EXTRA_CONFIG_FILE" -) - -if [[ "${TRTLLM_DSV4_USE_MPIRUN:-1}" == "0" ]]; then - "${SERVE_CMD[@]}" > "$SERVER_LOG" 2>&1 & -else - mpirun -n 1 --oversubscribe --allow-run-as-root \ - "${SERVE_CMD[@]}" \ - > "$SERVER_LOG" 2>&1 & -fi - -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend openai \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$(( CONC * 10 ))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir "$PWD/" \ - --trust-remote-code \ - --use-chat-template \ - --dsv4 \ - --server-pid "$SERVER_PID" - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b200_vllm.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b200_vllm.sh deleted file mode 100755 index 2cafd4b32..000000000 --- a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b200_vllm.sh +++ /dev/null @@ -1,122 +0,0 @@ -#!/usr/bin/env bash - -# DeepSeek-V4-Pro B200 single-node vLLM recipe derived from the B200 pareto -# sweep. TP mode (dp-attn=false) runs without expert parallel; DP mode -# (dp-attn=true) enables expert parallel (EP_SIZE=TP value = DP size). - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -SERVER_LOG=/workspace/server.log - -# DeepSeek-V4-Pro weights are large; engine startup can exceed the default -# 600s. Give it an hour to load. -export VLLM_ENGINE_READY_TIMEOUT_S=3600 - -PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") -fi - -EP_ARGS=() -if [ "${EP_SIZE:-1}" -gt 1 ]; then - EP_ARGS=(--enable-expert-parallel) -fi - -GMU_ARGS=() -MOE_ARGS=() -EPLB_ARGS=() -if [ "${DP_ATTENTION}" = "true" ]; then - MOE_ARGS=(--moe-backend deep_gemm_mega_moe) - EPLB_ARGS=(--enable-eplb --eplb-config '{"communicator":"torch_nccl", "use_async": false}') -fi - -if [ "${ISL}" -eq 8192 ] && [ "${CONC}" -le 128 ]; then - MAX_NUM_BATCHED_TOKENS=${ISL} -else - MAX_NUM_BATCHED_TOKENS=2048 -fi - -MAX_CUDAGRAPH_CAPTURE_SIZE=2048 - -BENCHMARK_MAX_MODEL_LEN="$MAX_MODEL_LEN" - -if [ "${EVAL_ONLY}" = "true" ]; then - EVAL_MAX_MODEL_LEN=$(compute_eval_context_length "$MODEL" "$BENCHMARK_MAX_MODEL_LEN") - export EVAL_MAX_MODEL_LEN - SERVE_MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -else - SERVE_MAX_MODEL_LEN="$BENCHMARK_MAX_MODEL_LEN" -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve "$MODEL" --host 0.0.0.0 --port "$PORT" \ - --trust-remote-code \ - --kv-cache-dtype fp8 \ - --block-size 256 \ - --no-enable-prefix-caching \ - "${PARALLEL_ARGS[@]}" \ - "${EP_ARGS[@]}" \ - "${GMU_ARGS[@]}" \ - "${MOE_ARGS[@]}" \ - "${EPLB_ARGS[@]}" \ - --compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' \ - --attention_config.use_fp4_indexer_cache=True \ - --tokenizer-mode deepseek_v4 \ - --tool-call-parser deepseek_v4 \ - --enable-auto-tool-choice \ - --reasoning-parser deepseek_v4 \ - --max-cudagraph-capture-size "$MAX_CUDAGRAPH_CAPTURE_SIZE" \ - --max-model-len "$SERVE_MAX_MODEL_LEN" \ - --max-num-batched-tokens "$MAX_NUM_BATCHED_TOKENS" > "$SERVER_LOG" 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b200_vllm_mtp.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b200_vllm_mtp.sh deleted file mode 100755 index 6846223e8..000000000 --- a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b200_vllm_mtp.sh +++ /dev/null @@ -1,124 +0,0 @@ -#!/usr/bin/env bash - -# DeepSeek-V4-Pro B200 single-node vLLM MTP variant of dsv4_fp4_b200_vllm.sh. -# Adds --speculative-config '{"method":"mtp","num_speculative_tokens":2}' and -# routes prompts through chat-formatted encoding via --dsv4 (required for -# meaningful MTP acceptance numbers). - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -SERVER_LOG=/workspace/server.log - -# DeepSeek-V4-Pro weights are large; engine startup can exceed the default -# 600s. Give it an hour to load. -export VLLM_ENGINE_READY_TIMEOUT_S=3600 - -PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") -fi - -EP_ARGS=() -if [ "${EP_SIZE:-1}" -gt 1 ]; then - EP_ARGS=(--enable-expert-parallel) -fi - -# Mega-MoE backend and the lower GMU only kick in on the DP-attn path, -# per the vLLM v0.20.0 DeepSeek-V4-Pro recipe. All configs share the -# FULL_AND_PIECEWISE compilation config. -GMU_ARGS=() -MOE_ARGS=() -if [ "${DP_ATTENTION}" = "true" ]; then - GMU_ARGS=(--gpu-memory-utilization 0.95) - MOE_ARGS=(--moe-backend deep_gemm_mega_moe) -fi - -MAX_NUM_BATCHED_TOKENS=$(( ISL * 2 )) -BENCHMARK_MAX_MODEL_LEN="$MAX_MODEL_LEN" - -if [ "${EVAL_ONLY}" = "true" ]; then - EVAL_MAX_MODEL_LEN=$(compute_eval_context_length "$MODEL" "$BENCHMARK_MAX_MODEL_LEN") - export EVAL_MAX_MODEL_LEN - SERVE_MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -else - SERVE_MAX_MODEL_LEN="$BENCHMARK_MAX_MODEL_LEN" -fi - -# use 2 speculative tokens for all configs for now -NUM_SPEC_TOKENS=2 - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve "$MODEL" --host 0.0.0.0 --port "$PORT" \ - --trust-remote-code \ - --kv-cache-dtype fp8 \ - --block-size 256 \ - --no-enable-prefix-caching \ - "${PARALLEL_ARGS[@]}" \ - "${EP_ARGS[@]}" \ - "${GMU_ARGS[@]}" \ - "${MOE_ARGS[@]}" \ - --compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' \ - --attention_config.use_fp4_indexer_cache=True \ - --tokenizer-mode deepseek_v4 \ - --tool-call-parser deepseek_v4 \ - --enable-auto-tool-choice \ - --reasoning-parser deepseek_v4 \ - --max-cudagraph-capture-size 2048 \ - --speculative-config "{\"method\": \"mtp\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS}" \ - --max-model-len "$SERVE_MAX_MODEL_LEN" \ - --max-num-batched-tokens "$MAX_NUM_BATCHED_TOKENS" > "$SERVER_LOG" 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -# MTP acceptance rate degrades on raw random tokens; --dsv4 routes prompts -# through chat-formatted encoding as required for speculative decoding benchmarks. -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code \ - --dsv4 - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b300_sglang.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b300_sglang.sh deleted file mode 100755 index b451dee0d..000000000 --- a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b300_sglang.sh +++ /dev/null @@ -1,191 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -# ─── Common env vars (all profiles) ─────────────────────────────────────────── -export SGLANG_JIT_DEEPGEMM_PRECOMPILE=0 -export SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT=1 - -SERVER_LOG="$PWD/server.log" - -echo "TP: $TP, DP_ATTENTION: $DP_ATTENTION, CONC: $CONC, ISL: $ISL, OSL: $OSL" - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi - -start_gpu_monitor --output "$PWD/gpu_metrics.csv" - -# ─── Per-concurrency launch profile ────────────────────────────────────────── -# Each block sets: PARALLEL_ARGS, MEM_FRACTION_STATIC, SWA_FULL_TOKENS_RATIO, -# and optionally MAX_RUNNING_REQUESTS plus profile-specific env vars. -# -# SWA ratio: 1k inputs need more SWA cache headroom than 8k inputs; 0.5 was -# tuned empirically for the 1k1k recipe, while 0.1 is the cookbook default. - -if [ "$CONC" = "1" ] || [ "$CONC" = "32" ]; then - # TP-only, no DP attention - MEM_FRACTION_STATIC=0.90 - SWA_FULL_TOKENS_RATIO=$([[ "$ISL" == "1024" ]] && echo 0.5 || echo 0.1) - PARALLEL_ARGS=( - --moe-runner-backend flashinfer_mxfp4 - --chunked-prefill-size 8192 - --disable-flashinfer-autotune - ) - -elif [ "$CONC" = "512" ]; then - # DP attention, flashinfer_mxfp4 - export SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN=1 - MEM_FRACTION_STATIC=0.94 - SWA_FULL_TOKENS_RATIO=$([[ "$ISL" == "1024" ]] && echo 0.5 || echo 0.1) - PARALLEL_ARGS=( - --dp-size "$TP" - --enable-dp-attention - --moe-runner-backend flashinfer_mxfp4 - --disable-flashinfer-autotune - --chunked-prefill-size 16384 - --enable-prefill-delayer - ) - -elif [ "$CONC" = "2048" ]; then - # DP attention, megamoe - export SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN=1 - export NVSHMEM_DISABLE_IB=1 - export SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW=1 - export SGLANG_LOG_FORWARD_ITERS=1 - export SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=8320 - MEM_FRACTION_STATIC=0.87 - SWA_FULL_TOKENS_RATIO=0.06 - MAX_RUNNING_REQUESTS=2560 - PARALLEL_ARGS=( - --dp-size "$TP" - --enable-dp-attention - --moe-a2a-backend megamoe - --cuda-graph-max-bs 288 - --chunked-prefill-size 65536 - --tokenizer-worker-num 4 - --enable-prefill-delayer - ) - -elif [ "$CONC" = "4096" ]; then - # DP attention, megamoe - export SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN=1 - export NVSHMEM_DISABLE_IB=1 - export SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW=1 - export SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=8320 - MEM_FRACTION_STATIC=0.835 - SWA_FULL_TOKENS_RATIO=0.075 - MAX_RUNNING_REQUESTS=4352 - PARALLEL_ARGS=( - --dp-size "$TP" - --enable-dp-attention - --moe-a2a-backend megamoe - --cuda-graph-max-bs 544 - --chunked-prefill-size 65536 - --tokenizer-worker-num 8 - --enable-prefill-delayer - --decode-log-interval 5 - ) - -elif [ "$CONC" = "8192" ]; then - # DP attention, megamoe - export SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN=1 - export NVSHMEM_DISABLE_IB=1 - export SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW=1 - export SGLANG_OPT_USE_ONLINE_COMPRESS=1 - export SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=8256 - MEM_FRACTION_STATIC=0.80 - SWA_FULL_TOKENS_RATIO=0.3 - MAX_RUNNING_REQUESTS=8192 - PARALLEL_ARGS=( - --dp-size "$TP" - --enable-dp-attention - --moe-a2a-backend megamoe - --cuda-graph-max-bs 1088 - --chunked-prefill-size 65536 - --tokenizer-worker-num 16 - --enable-prefill-delayer - --stream-interval 30 - ) - -else - echo "ERROR: unsupported CONC=$CONC" >&2 - exit 1 -fi - -# Print all SGLANG_* env vars to both the CI step log and server.log so the -# launch config is auditable from the result artifact alone. -{ - echo "=== SGLANG_* env vars at launch ===" - env | grep -E '^SGLANG_' | sort - echo "===================================" -} | tee "$SERVER_LOG" - -set -x -PYTHONNOUSERSITE=1 sglang serve \ - --model-path $MODEL_PATH --served-model-name $MODEL \ - --host 0.0.0.0 \ - --port $PORT \ - --trust-remote-code \ - --tp $TP \ - --max-running-requests "${MAX_RUNNING_REQUESTS:-$(( CONC * 3 / 2 > 8 ? CONC * 3 / 2 : 8 ))}" \ - --mem-fraction-static "$MEM_FRACTION_STATIC" \ - --swa-full-tokens-ratio "$SWA_FULL_TOKENS_RATIO" \ - "${PARALLEL_ARGS[@]}" $EVAL_CONTEXT_ARGS >> $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas -pip install -q --upgrade transformers - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $((CONC * 10)) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir "$PWD/" - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b300_sglang_mtp.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b300_sglang_mtp.sh deleted file mode 100755 index 9c8bab961..000000000 --- a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b300_sglang_mtp.sh +++ /dev/null @@ -1,172 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -# Tuning inputs from the matrix (all required): -# TP -- tensor parallel size -> --tp -# EP_SIZE -- expert parallel size -> --ep-size -# DP_ATTENTION -- "true" enables --enable-dp-attention --dp-size $TP -# Also selects MoE backend / chunked-prefill / EAGLE chain -# / mem-fraction-static / max-running-requests: -# true -> flashinfer_mxfp4 + DP-attn + chunked-prefill 32768 -# + EAGLE (1,1,2) + mem-fraction 0.92 + max-running 256 -# false -> flashinfer_mxfp4 (TP-only) + chunked-prefill 8192 -# + EAGLE (3,1,4) + mem-fraction 0.90 + max-running CONC*3/2 -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -# Common SGLANG env vars. -export SGLANG_JIT_DEEPGEMM_FAST_WARMUP=1 -export SGLANG_RADIX_FORCE_MISS=1 -export SGLANG_DEFAULT_THINKING=1 -export SGLANG_DSV4_REASONING_EFFORT=max -export SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT=1 - -# TODO(Cam): the deepseek-v4 sglang images install sglang editable at -# /workspace/sglang/python; prior sglang tags used /sgl-workspace/sglang. -# The runner mounts our repo at a non-/workspace path for these images so the -# editable install stays visible. Paths in this script are $PWD-relative for -# that reason. Drop the runner conditional once lmsys moves sglang back out of -# /workspace. - -SERVER_LOG="$PWD/server.log" - -echo "TP: $TP, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION, CONC: $CONC, ISL: $ISL, OSL: $OSL" - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi - -start_gpu_monitor --output "$PWD/gpu_metrics.csv" - -# Recipe path is selected by DP_ATTENTION; MoE backend, chunked-prefill, EAGLE -# chain, mem-fraction, and max-running all follow. -DEEPEP_CONFIG='{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}' - -if [ "${DP_ATTENTION}" = "true" ]; then - # DP-attn path: flashinfer_mxfp4 + DP-attn (covers conc 16-256). - export SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN=1 - export SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW=1 - export SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_FP4_ACTS=1 - export SGLANG_OPT_DEEPGEMM_MEGA_MOE_USE_MXF4_KIND=1 - export SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=8192 - export SGLANG_REQUEST_STATE_WAIT_TIMEOUT=60 - SPEC_FLAGS=( - --speculative-algorithm EAGLE - --speculative-num-steps 3 - --speculative-eagle-topk 1 - --speculative-num-draft-tokens 4 - ) - PARALLEL_ARGS=( - --dp-size "$TP" - --enable-dp-attention - --moe-runner-backend flashinfer_mxfp4 - --disable-flashinfer-autotune - --deepep-config "$DEEPEP_CONFIG" - --cuda-graph-max-bs 256 - --enable-deepseek-v4-fp4-indexer - ) - CHUNKED_PREFILL_SIZE=32768 - MEM_FRACTION_STATIC=0.92 - MAX_RUNNING_REQUESTS=256 -else - # TP-only fallback for low-conc: flashinfer_mxfp4 + EAGLE (3,1,4). - SPEC_FLAGS=( - --speculative-algorithm EAGLE - --speculative-num-steps 3 - --speculative-eagle-topk 1 - --speculative-num-draft-tokens 4 - ) - PARALLEL_ARGS=( - --moe-runner-backend flashinfer_mxfp4 - --disable-flashinfer-autotune - --enable-deepseek-v4-fp4-indexer - ) - CHUNKED_PREFILL_SIZE=8192 - MEM_FRACTION_STATIC=0.90 - MAX_RUNNING_REQUESTS="$(( CONC * 3 / 2 > 8 ? CONC * 3 / 2 : 8 ))" -fi - -# Print all SGLANG_* env vars to both the CI step log and server.log so the -# launch config is auditable from the result artifact alone. -{ - echo "=== SGLANG_* env vars at launch ===" - env | grep -E '^SGLANG_' | sort - echo "===================================" -} | tee "$SERVER_LOG" - -set -x -PYTHONNOUSERSITE=1 sglang serve \ - --model-path $MODEL_PATH --served-model-name $MODEL \ - --host 0.0.0.0 \ - --port $PORT \ - --trust-remote-code \ - --tp $TP \ - --ep-size $EP_SIZE \ - --chunked-prefill-size "$CHUNKED_PREFILL_SIZE" \ - --max-running-requests "$MAX_RUNNING_REQUESTS" \ - --mem-fraction-static "$MEM_FRACTION_STATIC" \ - --swa-full-tokens-ratio 0.1 \ - "${SPEC_FLAGS[@]}" \ - "${PARALLEL_ARGS[@]}" $EVAL_CONTEXT_ARGS >> $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -# --dsv4 routes prompts through encoding_dsv4.py (PR #1153), which emits the -# ... framing DeepSeek-V4-Pro expects. The DSv4-Pro -# tokenizer ships without a jinja chat_template, so plain --use-chat-template -# would crash; --dsv4 sidesteps that and satisfies the AGENTS.md rule that all -# MTP scripts must benchmark against chat-formatted inputs (EAGLE acceptance -# silently regresses on raw random tokens). -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $((CONC * 10)) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir "$PWD/" \ - --dsv4 - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b300_trt.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b300_trt.sh deleted file mode 100644 index bcd1fbf6a..000000000 --- a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b300_trt.sh +++ /dev/null @@ -1,187 +0,0 @@ -#!/usr/bin/env bash - -# DeepSeek-V4-Pro single-node TRTLLM recipe for B300. The configured image -# already contains a TensorRT-LLM DeepSeek-V4 build; do not build TRTLLM at -# runtime from this benchmark path. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - DP_ATTENTION \ - EP_SIZE - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -export TRTLLM_DSV4_USE_MPIRUN="${TRTLLM_DSV4_USE_MPIRUN:-1}" -export TRTLLM_DSV4_SANITIZE_SLURM_MPI_ENV="${TRTLLM_DSV4_SANITIZE_SLURM_MPI_ENV:-1}" - -sanitize_slurm_mpi_env_for_trtllm() { - if [[ "${TRTLLM_DSV4_SANITIZE_SLURM_MPI_ENV:-0}" != "1" ]]; then - return 0 - fi - - echo "Sanitizing Slurm/PMI environment for TensorRT-LLM launch" - while IFS='=' read -r name _; do - case "$name" in - SLURM_*|PMIX*|PMI*|OMPI_*|ORTE_*) - unset "$name" - ;; - esac - done < <(env) -} - -sanitize_slurm_mpi_env_for_trtllm - -export NCCL_NVLS_ENABLE="${NCCL_NVLS_ENABLE:-0}" -echo "NCCL_NVLS_ENABLE: $NCCL_NVLS_ENABLE" - -export TRTLLM_SERVER_DISABLE_GC="${TRTLLM_SERVER_DISABLE_GC:-1}" -export TRTLLM_WORKER_DISABLE_GC="${TRTLLM_WORKER_DISABLE_GC:-1}" -export NCCL_GRAPH_MIXING_SUPPORT="${NCCL_GRAPH_MIXING_SUPPORT:-0}" -export MIMALLOC_PURGE_DELAY="${MIMALLOC_PURGE_DELAY:-0}" -export PYTORCH_CUDA_ALLOC_CONF="${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True}" - -nvidia-smi - -SERVER_LOG="$PWD/server.log" -EXTRA_CONFIG_FILE="dsv4-fp4-trt.yml" - -# MoE backend: TRTLLM at low/mid concurrency; switch to MEGAMOE_DEEPGEMM at the -# top concurrency for short ISL (1k). -if [[ "$ISL" -le 1024 && "$CONC" -ge 2048 ]]; then - MOE_BACKEND="${MOE_BACKEND:-MEGAMOE_DEEPGEMM}" -else - MOE_BACKEND="${MOE_BACKEND:-TRTLLM}" -fi -MAX_BATCH_SIZE=$(( CONC > 16 ? CONC : 16 )) -# Cap CUDA-graph capture at batch 1024. TRTLLM_MLA_EXTRA_OVERLAP hands MLA -# prologue tensors across streams without record_stream(), so graph warmup at -# decode batch >1024 (repros at 1088, e.g. tp8/ep8 dp-attn conc-2048 on B300) -# hits a use-after-free -> CUDA_ERROR_ILLEGAL_ADDRESS. Fixed upstream in -# NVIDIA/TensorRT-LLM#15265; cap until that fix ships in the image. Runtime -# --max_batch_size stays = CONC, so batches >1024 just run eager. -CUDA_GRAPH_MAX_BATCH_SIZE=$(( MAX_BATCH_SIZE < 1024 ? MAX_BATCH_SIZE : 1024 )) -if [[ "$DP_ATTENTION" == "true" ]]; then - KV_CACHE_FREE_MEM_FRACTION="${KV_CACHE_FREE_MEM_FRACTION:-0.7}" -else - KV_CACHE_FREE_MEM_FRACTION="${KV_CACHE_FREE_MEM_FRACTION:-0.9}" -fi - -ATTENTION_DP_CONFIG="" -if [[ "$DP_ATTENTION" == "true" ]]; then - ATTENTION_DP_CONFIG=" -attention_dp_config: - batching_wait_iters: 30 - enable_balance: true" -fi - -cat > "$EXTRA_CONFIG_FILE" << EOF -cuda_graph_config: - enable_padding: true - max_batch_size: $CUDA_GRAPH_MAX_BATCH_SIZE -enable_attention_dp: $DP_ATTENTION$ATTENTION_DP_CONFIG -print_iter_log: true -kv_cache_config: - tokens_per_block: 128 - dtype: fp8 - free_gpu_memory_fraction: $KV_CACHE_FREE_MEM_FRACTION - enable_block_reuse: false -stream_interval: 100 -num_postprocess_workers: 4 -moe_config: - backend: $MOE_BACKEND - use_low_precision_moe_combine: true -EOF - -echo "Generated config file contents:" -cat "$EXTRA_CONFIG_FILE" - -MAX_MODEL_LEN=$(( MAX_MODEL_LEN > 8192 ? MAX_MODEL_LEN : 8192 )) -MAX_NUM_TOKENS=$(( ISL + 256 )) -MAX_NUM_TOKENS=$(( MAX_NUM_TOKENS > 8192 ? MAX_NUM_TOKENS : 8192 )) - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" - MAX_NUM_TOKENS="$EVAL_MAX_MODEL_LEN" -fi - -export TRTLLM_MHC_ENABLE_FUSED_HC="${TRTLLM_MHC_ENABLE_FUSED_HC:-1}" -echo "TRTLLM_MHC_ENABLE_FUSED_HC: $TRTLLM_MHC_ENABLE_FUSED_HC" - -start_gpu_monitor --output "$PWD/gpu_metrics.csv" - -set -x -SERVE_CMD=( - trtllm-serve "$MODEL_PATH" \ - --host 0.0.0.0 \ - --port "$PORT" \ - --trust_remote_code \ - --backend pytorch \ - --max_batch_size "$MAX_BATCH_SIZE" \ - --max_seq_len "$MAX_MODEL_LEN" \ - --max_num_tokens "$MAX_NUM_TOKENS" \ - --tp_size "$TP" \ - --ep_size "$EP_SIZE" \ - --custom_tokenizer deepseek_v4 \ - --config "$EXTRA_CONFIG_FILE" -) - -if [[ "${TRTLLM_DSV4_USE_MPIRUN:-1}" == "0" ]]; then - "${SERVE_CMD[@]}" > "$SERVER_LOG" 2>&1 & -else - mpirun -n 1 --oversubscribe --allow-run-as-root \ - "${SERVE_CMD[@]}" \ - > "$SERVER_LOG" 2>&1 & -fi - -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend openai-chat \ - --endpoint /v1/chat/completions \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$(( CONC * 10 ))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir "$PWD/" \ - --trust-remote-code \ - --server-pid "$SERVER_PID" - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b300_trt_mtp.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b300_trt_mtp.sh deleted file mode 100644 index bb0362c25..000000000 --- a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b300_trt_mtp.sh +++ /dev/null @@ -1,198 +0,0 @@ -#!/usr/bin/env bash - -# DeepSeek-V4-Pro B300 TensorRT-LLM MTP variant. The configured image already -# contains the DeepSeek-V4 TRTLLM build; this path only toggles speculative MTP. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - DP_ATTENTION \ - EP_SIZE - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -export TRTLLM_DSV4_USE_MPIRUN="${TRTLLM_DSV4_USE_MPIRUN:-1}" -export TRTLLM_DSV4_SANITIZE_SLURM_MPI_ENV="${TRTLLM_DSV4_SANITIZE_SLURM_MPI_ENV:-1}" - -sanitize_slurm_mpi_env_for_trtllm() { - if [[ "${TRTLLM_DSV4_SANITIZE_SLURM_MPI_ENV:-0}" != "1" ]]; then - return 0 - fi - - echo "Sanitizing Slurm/PMI environment for TensorRT-LLM launch" - while IFS='=' read -r name _; do - case "$name" in - SLURM_*|PMIX*|PMI*|OMPI_*|ORTE_*) - unset "$name" - ;; - esac - done < <(env) -} - -sanitize_slurm_mpi_env_for_trtllm - -export NCCL_NVLS_ENABLE="${NCCL_NVLS_ENABLE:-0}" -echo "NCCL_NVLS_ENABLE: $NCCL_NVLS_ENABLE" - -export TRTLLM_SERVER_DISABLE_GC="${TRTLLM_SERVER_DISABLE_GC:-1}" -export TRTLLM_WORKER_DISABLE_GC="${TRTLLM_WORKER_DISABLE_GC:-1}" -export NCCL_GRAPH_MIXING_SUPPORT="${NCCL_GRAPH_MIXING_SUPPORT:-0}" -export MIMALLOC_PURGE_DELAY="${MIMALLOC_PURGE_DELAY:-0}" -export PYTORCH_CUDA_ALLOC_CONF="${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True}" - -nvidia-smi - -SERVER_LOG="$PWD/server.log" -EXTRA_CONFIG_FILE="dsv4-fp4-trt-mtp.yml" - -# MoE backend: TRTLLM at low/mid concurrency; switch to MEGAMOE_DEEPGEMM at high -# concurrency for short ISL (1k). -if [[ "$ISL" -le 1024 && "$CONC" -ge 512 ]]; then - MOE_BACKEND="${MOE_BACKEND:-MEGAMOE_DEEPGEMM}" -else - MOE_BACKEND="${MOE_BACKEND:-TRTLLM}" -fi -# MTP draft length: 3 at low/mid concurrency; steps down to 2 at high concurrency -# for long ISL (8k). -if [[ "$ISL" -ge 4096 && "$CONC" -ge 128 ]]; then - MTP="${TRTLLM_DSV4_MTP_NUM_NEXTN_LAYERS:-2}" -else - MTP="${TRTLLM_DSV4_MTP_NUM_NEXTN_LAYERS:-3}" -fi -MAX_BATCH_SIZE=$(( CONC > 16 ? CONC : 16 )) -# Cap CUDA-graph capture at batch 1024. TRTLLM_MLA_EXTRA_OVERLAP hands MLA -# prologue tensors across streams without record_stream(), so graph warmup at -# decode batch >1024 (repros at 1088, e.g. tp8/ep8 dp-attn conc-2048 on B300) -# hits a use-after-free -> CUDA_ERROR_ILLEGAL_ADDRESS. Fixed upstream in -# NVIDIA/TensorRT-LLM#15265; cap until that fix ships in the image. Runtime -# --max_batch_size stays = CONC, so batches >1024 just run eager. -CUDA_GRAPH_MAX_BATCH_SIZE=$(( MAX_BATCH_SIZE < 1024 ? MAX_BATCH_SIZE : 1024 )) -if [[ "$DP_ATTENTION" == "true" ]]; then - KV_CACHE_FREE_MEM_FRACTION="${KV_CACHE_FREE_MEM_FRACTION:-0.6}" -else - KV_CACHE_FREE_MEM_FRACTION="${KV_CACHE_FREE_MEM_FRACTION:-0.9}" -fi - -ATTENTION_DP_CONFIG="" -if [[ "$DP_ATTENTION" == "true" ]]; then - ATTENTION_DP_CONFIG=" -attention_dp_config: - batching_wait_iters: 30 - enable_balance: true -enable_lm_head_tp_in_adp: true" -fi - -cat > "$EXTRA_CONFIG_FILE" << EOF -cuda_graph_config: - enable_padding: true - max_batch_size: $CUDA_GRAPH_MAX_BATCH_SIZE -enable_attention_dp: $DP_ATTENTION$ATTENTION_DP_CONFIG -print_iter_log: true -kv_cache_config: - tokens_per_block: 128 - dtype: fp8 - free_gpu_memory_fraction: $KV_CACHE_FREE_MEM_FRACTION - enable_block_reuse: false -stream_interval: 100 -num_postprocess_workers: 4 -moe_config: - backend: $MOE_BACKEND - use_low_precision_moe_combine: true -speculative_config: - decoding_type: MTP - max_draft_len: $MTP -EOF - -echo "Generated config file contents:" -cat "$EXTRA_CONFIG_FILE" - -MAX_MODEL_LEN=$(( MAX_MODEL_LEN > 8192 ? MAX_MODEL_LEN : 8192 )) -MAX_NUM_TOKENS=$(( ISL + (MTP + 1) * MAX_BATCH_SIZE + 256 )) -MAX_NUM_TOKENS=$(( MAX_NUM_TOKENS > 8192 ? MAX_NUM_TOKENS : 8192 )) - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" - MAX_NUM_TOKENS="$EVAL_MAX_MODEL_LEN" -fi - -export TRTLLM_MHC_ENABLE_FUSED_HC="${TRTLLM_MHC_ENABLE_FUSED_HC:-1}" -echo "TRTLLM_MHC_ENABLE_FUSED_HC: $TRTLLM_MHC_ENABLE_FUSED_HC" - -start_gpu_monitor --output "$PWD/gpu_metrics.csv" - -set -x -SERVE_CMD=( - trtllm-serve "$MODEL_PATH" \ - --host 0.0.0.0 \ - --port "$PORT" \ - --trust_remote_code \ - --backend pytorch \ - --max_batch_size "$MAX_BATCH_SIZE" \ - --max_seq_len "$MAX_MODEL_LEN" \ - --max_num_tokens "$MAX_NUM_TOKENS" \ - --tp_size "$TP" \ - --ep_size "$EP_SIZE" \ - --custom_tokenizer deepseek_v4 \ - --config "$EXTRA_CONFIG_FILE" -) - -if [[ "${TRTLLM_DSV4_USE_MPIRUN:-1}" == "0" ]]; then - "${SERVE_CMD[@]}" > "$SERVER_LOG" 2>&1 & -else - mpirun -n 1 --oversubscribe --allow-run-as-root \ - "${SERVE_CMD[@]}" \ - > "$SERVER_LOG" 2>&1 & -fi - -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend openai \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$(( CONC * 10 ))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir "$PWD/" \ - --trust-remote-code \ - --use-chat-template \ - --dsv4 \ - --server-pid "$SERVER_PID" - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b300_vllm.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b300_vllm.sh deleted file mode 100755 index 8aef70a8e..000000000 --- a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b300_vllm.sh +++ /dev/null @@ -1,127 +0,0 @@ -#!/usr/bin/env bash - -# DeepSeek-V4-Pro B300 single-node aggregate recipe from the submitted B300 -# pareto sweep. TP mode (dp-attn=false) runs without expert parallel; DP mode -# (dp-attn=true) enables expert parallel (EP_SIZE=TP value = DP size). - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - - -SERVER_LOG=/workspace/server.log - -# DeepSeek-V4-Pro weights are large; engine startup can exceed the default -# 600s. Give it an hour to load. -export VLLM_ENGINE_READY_TIMEOUT_S=3600 - -PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") -fi - -EP_ARGS=() -if [ "${EP_SIZE:-1}" -gt 1 ]; then - EP_ARGS=(--enable-expert-parallel) -fi - -MOE_ARGS=() -if [ "${DP_ATTENTION}" = "true" ]; then - MOE_ARGS=(--moe-backend deep_gemm_mega_moe) -fi - -if [ "${DP_ATTENTION}" = "true" ]; then - MAX_NUM_BATCHED_TOKENS=2048 -else - MAX_NUM_BATCHED_TOKENS=$(( ISL * 2 )) -fi - -BENCHMARK_MAX_MODEL_LEN="$MAX_MODEL_LEN" - -if [ "${EVAL_ONLY}" = "true" ]; then - EVAL_MAX_MODEL_LEN=$(compute_eval_context_length "$MODEL" "$BENCHMARK_MAX_MODEL_LEN") - export EVAL_MAX_MODEL_LEN - SERVE_MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -else - SERVE_MAX_MODEL_LEN="$BENCHMARK_MAX_MODEL_LEN" -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve "$MODEL_PATH" --served-model-name "$MODEL" --host 0.0.0.0 --port "$PORT" \ - "${PARALLEL_ARGS[@]}" \ - --pipeline-parallel-size 1 \ - --kv-cache-dtype fp8 \ - --trust-remote-code \ - --block-size 256 \ - --no-enable-prefix-caching \ - "${EP_ARGS[@]}" \ - "${MOE_ARGS[@]}" \ - --compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' \ - --attention_config.use_fp4_indexer_cache True \ - --tokenizer-mode deepseek_v4 \ - --tool-call-parser deepseek_v4 \ - --enable-auto-tool-choice \ - --reasoning-parser deepseek_v4 \ - --max-cudagraph-capture-size 2048 \ - --max-model-len "$SERVE_MAX_MODEL_LEN" \ - --max-num-batched-tokens "$MAX_NUM_BATCHED_TOKENS" > "$SERVER_LOG" 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b300_vllm_mtp.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b300_vllm_mtp.sh deleted file mode 100755 index a5e7dd28c..000000000 --- a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_b300_vllm_mtp.sh +++ /dev/null @@ -1,121 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - - -SERVER_LOG=/workspace/server.log - -export VLLM_ENGINE_READY_TIMEOUT_S=3600 - -PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") -fi - -EP_ARGS=() -if [ "${EP_SIZE:-1}" -gt 1 ]; then - EP_ARGS=(--enable-expert-parallel) -fi - -MOE_ARGS=() -if [ "${DP_ATTENTION}" = "true" ]; then - MOE_ARGS=(--moe-backend deep_gemm_mega_moe) - MAX_NUM_BATCHED_TOKENS=2048 -else - MAX_NUM_BATCHED_TOKENS=$(( ISL * 2 )) -fi - -BENCHMARK_MAX_MODEL_LEN=$MAX_MODEL_LEN - -if [ "${EVAL_ONLY}" = "true" ]; then - EVAL_MAX_MODEL_LEN=$(compute_eval_context_length "$MODEL" "$BENCHMARK_MAX_MODEL_LEN") - export EVAL_MAX_MODEL_LEN - SERVE_MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -else - SERVE_MAX_MODEL_LEN="$BENCHMARK_MAX_MODEL_LEN" -fi - -# use 2 speculative tokens for all configs for now -NUM_SPEC_TOKENS=2 - -start_gpu_monitor - -set -x -vllm serve "$MODEL_PATH" --served-model-name "$MODEL" --host 0.0.0.0 --port "$PORT" \ - "${PARALLEL_ARGS[@]}" \ - --pipeline-parallel-size 1 \ - --kv-cache-dtype fp8 \ - --trust-remote-code \ - --block-size 256 \ - --no-enable-prefix-caching \ - "${EP_ARGS[@]}" \ - "${MOE_ARGS[@]}" \ - --compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' \ - --attention_config.use_fp4_indexer_cache True \ - --tokenizer-mode deepseek_v4 \ - --tool-call-parser deepseek_v4 \ - --enable-auto-tool-choice \ - --reasoning-parser deepseek_v4 \ - --max-cudagraph-capture-size 2048 \ - --speculative-config "{\"method\": \"mtp\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS}" \ - --max-model-len "$SERVE_MAX_MODEL_LEN" \ - --max-num-batched-tokens "$MAX_NUM_BATCHED_TOKENS" > "$SERVER_LOG" 2>&1 & - -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -# MTP acceptance rate degrades on raw random tokens; --dsv4 routes prompts -# through chat-formatted encoding as required for speculative decoding benchmarks. -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code \ - --dsv4 - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_atom.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_atom.sh deleted file mode 100644 index 66be04a52..000000000 --- a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_atom.sh +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE \ - DP_ATTENTION - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -SERVER_LOG=/workspace/server.log - -PARALLEL_ARGS=(-tp "$TP") #TP -CUDAGRAPH_SIZES='[1, 2, 4, 8, 16, 32, 48, 64, 128, 256, 512]' -if [ "$DP_ATTENTION" = "true" ]; then - if [ "$EP_SIZE" -gt 1 ]; then #DP+EP - PARALLEL_ARGS=(-tp "$TP" --enable-expert-parallel --enable-dp-attention ) - else #DPA+TP - #DPA+TP+TBO - if [ "$ISL" -eq 1024 ] && [ "$OSL" -eq 1024 ] && [ "$CONC" -ge 1024 ]; then - PARALLEL_ARGS=(-tp "$TP" --enable-dp-attention --enable-tbo) - export GPU_MAX_HW_QUEUES=5 - elif [ "$ISL" -eq 8192 ] && [ "$OSL" -eq 1024 ] && [ "$CONC" -ge 256 ]; then - PARALLEL_ARGS=(-tp "$TP" --enable-dp-attention --enable-tbo) - export GPU_MAX_HW_QUEUES=5 - else - PARALLEL_ARGS=(-tp "$TP" --enable-dp-attention ) - fi - fi -fi - -BENCHMARK_MAX_MODEL_LEN="$MAX_MODEL_LEN" - -if [ "${EVAL_ONLY}" = "true" ]; then - EVAL_MAX_MODEL_LEN=$(compute_eval_context_length "$MODEL" "$BENCHMARK_MAX_MODEL_LEN") - export EVAL_MAX_MODEL_LEN -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -export ATOM_DISABLE_MMAP=true -export AITER_BF16_FP8_MOE_BOUND=0 -export ATOM_MOE_GU_ITLV=1 -MEM_FRAC_STATIC=0.9 - -python3 -m atom.entrypoints.openai_server \ - --model $MODEL \ - --server-port $PORT \ - "${PARALLEL_ARGS[@]}" \ - --kv_cache_dtype fp8 \ - --trust-remote-code \ - --gpu-memory-utilization $MEM_FRAC_STATIC \ - --no-enable_prefix_caching \ - --cudagraph-capture-sizes "${CUDAGRAPH_SIZES}" \ - > "$SERVER_LOG" 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_atom_mtp.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_atom_mtp.sh deleted file mode 100755 index e46c19126..000000000 --- a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_atom_mtp.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE \ - DP_ATTENTION - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -SERVER_LOG=/workspace/server.log -PORT=${PORT:-8888} - -PARALLEL_ARGS=(-tp "$TP") #TP -if [ "$DP_ATTENTION" = "true" ]; then - if [ "$EP_SIZE" -gt 1 ]; then #DP+EP - PARALLEL_ARGS=(-tp "$TP" --enable-expert-parallel --enable-dp-attention ) - else #DP+TP - PARALLEL_ARGS=(-tp "$TP" --enable-dp-attention ) - fi -fi - -SPEC_ARGS=(--method mtp --num-speculative-tokens 3 ) - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -export ATOM_DISABLE_MMAP=true -export AITER_BF16_FP8_MOE_BOUND=0 -export ATOM_MOE_GU_ITLV=1 -python3 -m atom.entrypoints.openai_server \ - --model $MODEL \ - --server-port $PORT \ - "${PARALLEL_ARGS[@]}" \ - "${SPEC_ARGS[@]}" \ - --kv_cache_dtype fp8 \ - --trust-remote-code \ - > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -# --dsv4 routes prompts through encoding_dsv4.py (PR #1153), which emits the -# ... framing DeepSeek-V4-Pro expects. The DSv4-Pro -# tokenizer ships without a jinja chat_template, so plain --use-chat-template -# would crash; --dsv4 sidesteps that and satisfies the AGENTS.md rule that all -# MTP scripts must benchmark against chat-formatted inputs (EAGLE acceptance -# silently regresses on raw random tokens). -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code \ - --dsv4 - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_sglang.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_sglang.sh deleted file mode 100755 index 2ced86836..000000000 --- a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_sglang.sh +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - DP_ATTENTION \ - EP_SIZE \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - MAX_MODEL_LEN - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -# sglang ships in the image at the SHA encoded in the image tag (built -# from the amd/deepseek_v4 branch in sgl-project/sglang). To bump sglang, -# bump the image tag in configs/amd-master.yaml. - -export SGLANG_DEFAULT_THINKING=1 -export SGLANG_DSV4_REASONING_EFFORT=max -export SGLANG_OPT_DEEPGEMM_HC_PRENORM=false -export SGLANG_USE_AITER=1 -export SGLANG_USE_ROCM700A=0 -export SGLANG_DP_USE_GATHERV=1 -export SGLANG_OPT_USE_FUSED_COMPRESS=true -export SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton -export SGLANG_OPT_FP8_WO_A_GEMM=false -export SGLANG_OPT_USE_JIT_INDEXER_METADATA=false -export SGLANG_OPT_USE_TOPK_V2=false -export SGLANG_OPT_USE_AITER_INDEXER=true -export SGLANG_OPT_USE_TILELANG_INDEXER=false -export SGLANG_OPT_USE_TILELANG_MHC_PRE=false -export SGLANG_OPT_USE_TILELANG_MHC_POST=false -export SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1 -export SGLANG_OPT_USE_FUSED_COMPRESS_TRITON=true -export AITER_BF16_FP8_MOE_BOUND=0 -export SGLANG_EAGER_INPUT_NO_COPY=true - -# multi-stream -export SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=false -export SGLANG_ROCM_USE_MULTI_STREAM=false - -SERVER_LOG=/workspace/server.log - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -PARALLEL_ARGS=( - --tensor-parallel-size "$TP" -) -CHUNKED_PREFILL_SIZE=$ISL -if [ "${DP_ATTENTION}" = "true" ]; then - CHUNKED_PREFILL_SIZE=$((ISL * TP)) - PARALLEL_ARGS+=( - --dp "$TP" - --enable-dp-attention - --enable-prefill-delayer - --prefill-delayer-max-delay-ms 5000 - ) -fi -if [ "${EP_SIZE:-1}" -gt 1 ]; then - PARALLEL_ARGS+=(--ep-size "$EP_SIZE") -fi - -sglang serve \ - --model-path $MODEL \ - --host=0.0.0.0 \ - --port $PORT \ - "${PARALLEL_ARGS[@]}" \ - --trust-remote-code \ - --disable-radix-cache \ - --attention-backend dsv4 \ - --max-running-requests ${CONC} \ - --mem-fraction-static 0.90 \ - --swa-full-tokens-ratio 0.15 \ - --page-size 256 \ - --context-length $MAX_MODEL_LEN \ - --chunked-prefill-size $CHUNKED_PREFILL_SIZE \ - --disable-shared-experts-fusion \ - --tool-call-parser deepseekv4 \ - --reasoning-parser deepseek-v4 \ - --chat-template "$(dirname "$0")/../chat_templates/deepseek_v4_thinking.jinja" \ - --watchdog-timeout 1800 $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_sglang_mtp.sh deleted file mode 100755 index c4c59c92e..000000000 --- a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_sglang_mtp.sh +++ /dev/null @@ -1,240 +0,0 @@ -#!/usr/bin/env bash - -# DeepSeek-V4-Pro on MI355X via SGLang — MTP variant of dsv4_fp4_mi355x_sglang.sh. -# Adds EAGLE/MTP speculative decoding per sgl-project/sglang#26383 -# ([AMD][DSV4] DSV4 MTP graph + sparse triton attn optimizations, merged -# 2026-05-27, commit deaba74), which fixes the ROCm HIP-radix backend's -# per-step draft out_cache_loc slicing under CUDA graph (the bug behind the -# false-EOS / truncated-generation symptom in sgl issue #20404) and validates -# GSM8K 0.950 with MTP on. The EAGLE chain follows that PR's accuracy config -# for the DP-attention path (steps=2, topk=1, draft=3); the TP-only -# low-concurrency path uses the (3,1,4) chain shared with dsr1_fp4_mi355x_mtp.sh. -# -# Image: #26383 is on sglang `main`, so this runs on the mainline ROCm nightly -# (lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-*), NOT a rocm/sgl-dev:*-DSv4 -# build. The -DSv4 images are cut from the amd/deepseek_v4 branch, which has not -# merged #26383 (latest da28108 = f96ac98 + build fixes + an unrelated MLA-decode -# refactor; it still crashes at MTP graph capture, run 26723126211). Mainline -# carries #26383 but omits deep_gemm, which DSv4-Pro's default fp8 wo_a path -# imports. AMD doesn't need deep_gemm (it uses aiter/tilelang/torch), and every -# deep_gemm use on the DSv4 path is behind an env-flag fallback, so the block -# below detects deep_gemm's absence and routes around it: SGLANG_OPT_FP8_WO_A_GEMM=0 -# (dequant fp8 wo_a -> bf16 + torch.einsum; also skips the weight-load -# transform_sf_into_required_layout that crashed run 26727984372) and -# SGLANG_TOPK_TRANSFORM_512_TORCH=1 (torch topk). The indexer already routes to -# tilelang + torch paged-MQA-logits and MHC to aiter via flags set below. On a -# -DSv4 image that carries #26383, bump amd-master.yaml and the detect restores -# the deep_gemm perf path. RUN_EVAL on the high-conc points gates accuracy. - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - DP_ATTENTION \ - EP_SIZE \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - MAX_MODEL_LEN - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -# sglang ships in the image at the SHA encoded in the image tag (built -# from the amd/deepseek_v4 branch in sgl-project/sglang). To bump sglang, -# bump the image tag in configs/amd-master.yaml. - -# Transformers in the container doesn't recognize the `deepseek_v4` model_type. -# PR #23608's fallback in hf_transformers_utils.get_config tries to handle this -# by writing a patched config to /tmp, but in practice isn't catching the error -# in this image. Patch the cached config.json directly instead: set model_type -# to `deepseek_v3` so AutoConfig.from_pretrained succeeds, and keep -# architectures=['DeepseekV4ForCausalLM'] so SGLang dispatches to its native -# DSv4 model class (python/sglang/srt/models/deepseek_v4.py). -python3 << PYEOF -import json -from huggingface_hub import hf_hub_download -path = hf_hub_download(repo_id="$MODEL", filename="config.json") -with open(path) as f: - config = json.load(f) -if config.get("model_type") == "deepseek_v4": - config["model_type"] = "deepseek_v3" - with open(path, "w") as f: - json.dump(config, f, indent=2) - print(f"Patched {path}: model_type deepseek_v4 -> deepseek_v3") -else: - print(f"No patch needed: model_type is {config.get('model_type')!r}") -PYEOF - -# DSv4 FP4-experts path. Tracks the env block in python/run_dsv4.sh on the -# amd/deepseek_v4 branch (HEAD's active block is FP8; we override the two -# FP4-specific flags below): -# SGLANG_DSV4_FP4_EXPERTS=True -> route experts through the FP4 kernels -# SGLANG_FORCE_TRITON_MOE_FP8=0 -> dispatch MoE through aiter and apply -# the swiglu_limit clamp in the triton -# MoE fallback path. -export SGLANG_REASONING_EFFORT=max -export SGLANG_OPT_USE_FUSED_COMPRESS=true -export SGLANG_OPT_USE_OLD_COMPRESSOR=false -export SGLANG_OPT_USE_TILELANG_SWA_PREPARE=false -export SGLANG_OPT_USE_JIT_KERNEL_FUSED_TOPK=false -export SGLANG_OPT_USE_FUSED_HASH_TOPK=true -export SGLANG_OPT_DEEPGEMM_HC_PRENORM=false -export SGLANG_OPT_USE_TILELANG_MHC_PRE=false -export SGLANG_OPT_USE_TILELANG_MHC_POST=false -export SGLANG_OPT_USE_AITER_MHC_PRE=true -export SGLANG_OPT_USE_AITER_MHC_POST=true -export SGLANG_ENABLE_THINKING=1 -export SGLANG_USE_AITER=1 -export SGLANG_USE_ROCM700A=1 -export SGLANG_TOPK_TRANSFORM_512_TORCH=0 -export SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1 -export SGLANG_DSV4_FP4_EXPERTS=True -export SGLANG_OPT_DPSK_V4_RADIX=1 -export SGLANG_OPT_USE_OVERLAP_STORE_CACHE=false -export SGLANG_OPT_USE_FUSED_STORE_CACHE=true -export SGLANG_FORCE_TRITON_MOE_FP8=0 -export SGLANG_HACK_FLASHMLA_BACKEND=triton -export SGLANG_OPT_USE_TILELANG_INDEXER=true -export SGLANG_OPT_USE_TRITON_SWA_PREPARE=true -export AITER_BF16_FP8_MOE_BOUND=0 -export SGLANG_OPT_FUSE_WQA_WKV=true -export SGLANG_OPT_USE_FUSED_PAGED_COMPRESS=true -export SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=0 - -# MTP-specific knobs landed alongside the graph fix in sgl#26383: -# SGLANG_OPT_USE_TRITON_FUSED_MHC -> fused Triton mhc_post_pre for low conc -# (defaults True in post-#26383 images; -# set explicitly so the recipe is auditable) -# SGLANG_OPT_C4_SPARSE_TOPK -> sparse-attention top-k used in the PR's -# DSv4 MTP accuracy run -export SGLANG_OPT_USE_TRITON_FUSED_MHC=1 -export SGLANG_OPT_C4_SPARSE_TOPK=512 - -# Mainline ROCm nightlies carry #26383 but omit deep_gemm (only rocm/sgl-dev:*-DSv4 -# builds bundle it). DSv4-Pro's default fp8 wo_a path imports deep_gemm at weight -# load; detect its absence and route the deep_gemm-touching paths to their torch -# fallbacks. No-op on a deep_gemm-bearing image, so this recipe works on both. -# SGLANG_OPT_FP8_WO_A_GEMM=0 -> wo_a fp8 weights dequantized to bf16 at load -# (_dequant_fp8_wo_a) + o-proj via torch.einsum; -# also skips the post-load deep_gemm -# transform_sf_into_required_layout that crashed. -# SGLANG_TOPK_TRANSFORM_512_TORCH=1 -> torch topk-transform instead of the kernel. -# SGLANG_OPT_USE_TOPK_V2=0 -> skip plan_topk_v2 in the indexer metadata; -# its jit kernel is CUDA-only (topk/ptx.cuh -# #includes ) and won't build for -# gfx950. topk_metadata is unused on the torch -# topk path, so empty is fine. -# SGLANG_ENABLE_JIT_DEEPGEMM=0 -> global off; nothing to JIT without the module. -if python3 -c "import deep_gemm" >/dev/null 2>&1; then - echo "deep_gemm present -> using fp8 wo_a / deep_gemm perf path" -else - echo "deep_gemm absent -> routing DSv4 fp8 wo_a / topk around it (mainline nightly)" - export SGLANG_OPT_FP8_WO_A_GEMM=0 - export SGLANG_TOPK_TRANSFORM_512_TORCH=1 - export SGLANG_OPT_USE_TOPK_V2=0 - export SGLANG_ENABLE_JIT_DEEPGEMM=0 -fi - -SERVER_LOG=/workspace/server.log -PORT=${PORT:-8888} - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -PARALLEL_ARGS=( - --tensor-parallel-size "$TP" -) -# EAGLE chain is selected by DP_ATTENTION. The DP-attention path mirrors the -# sgl#26383 DSv4 ROCm accuracy config (steps=2, topk=1, draft=3); the TP-only -# low-concurrency fallback uses the longer (3,1,4) chain that low batch sizes -# benefit from, matching dsr1_fp4_mi355x_mtp.sh. -SPEC_FLAGS=( - --speculative-algorithm EAGLE - --speculative-num-steps 3 - --speculative-eagle-topk 1 - --speculative-num-draft-tokens 4 -) -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS+=( - --dp "$TP" - --enable-dp-attention - --enable-prefill-delayer - ) - SPEC_FLAGS=( - --speculative-algorithm EAGLE - --speculative-num-steps 2 - --speculative-eagle-topk 1 - --speculative-num-draft-tokens 3 - ) -fi -if [ "${EP_SIZE:-1}" -gt 1 ]; then - PARALLEL_ARGS+=(--ep-size "$EP_SIZE") -fi - -set -x -python3 -m sglang.launch_server \ - --model-path $MODEL \ - --host=0.0.0.0 \ - --port $PORT \ - "${PARALLEL_ARGS[@]}" \ - "${SPEC_FLAGS[@]}" \ - --trust-remote-code \ - --disable-radix-cache \ - --attention-backend compressed \ - --max-running-requests ${CONC} \ - --mem-fraction-static 0.90 \ - --swa-full-tokens-ratio 0.15 \ - --page-size 256 \ - --context-length $MAX_MODEL_LEN \ - --chunked-prefill-size 8192 \ - --disable-shared-experts-fusion \ - --tool-call-parser deepseekv4 \ - --reasoning-parser deepseek-v4 \ - --chat-template "$(dirname "$0")/chat_templates/deepseek_v4_thinking.jinja" \ - --watchdog-timeout 1800 $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -# --dsv4 routes prompts through encoding_dsv4.py, emitting the -# ... framing DeepSeek-V4-Pro expects. EAGLE/MTP -# acceptance silently regresses on raw random tokens, so MTP benchmarks must -# use chat-formatted inputs (AGENTS.md). The DSv4-Pro tokenizer ships without a -# jinja chat_template, so plain --use-chat-template would crash; --dsv4 handles -# the framing directly. -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --dsv4 - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_vllm.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_vllm.sh deleted file mode 100755 index e6f02db07..000000000 --- a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_vllm.sh +++ /dev/null @@ -1,107 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail - -# DeepSeek-V4-Pro on MI355X via vLLM. -# The DeepSeek-V4-Pro checkpoint is mixed-precision FP4+FP8 (FP4 MoE -# expert weights dominate the ~960 GB footprint, FP8 on attention/norm/ -# router, FP8 KV cache at runtime). InferenceX classifies this as the -# fp4 variant. -# -# Serving flags follow the validated MI355X recipe from -# vllm-project/recipes#433 (DeepSeek-V4-Pro, TP=8). DEP probes reuse the -# same ROCm recipe while switching parallelism to vLLM's DP+EP form. -# Image-pin details live in amd-master.yaml. -# -# Use the AITER MoE backend (VLLM_ROCM_USE_AITER_MOE=1 + --moe-backend aiter) -# for the FP4 MoE expert weights of deepseek-ai/DeepSeek-V4-Pro. The AITER -# MXFP4 path registers the FP4 scale parameters (w13_weight_scale / -# w2_weight_scale), so safetensors loads correctly and decode runs on the -# fused AITER experts instead of triton_unfused. -# -# --compilation-config mode=3 with FULL_AND_PIECEWISE cudagraph mode -# enables full CUDA graph capture for improved throughput on MI355X. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -if [ -n "$ROCR_VISIBLE_DEVICES" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -export VLLM_ROCM_USE_AITER=1 -export VLLM_ROCM_USE_AITER_MOE=1 - -SERVER_LOG=/workspace/server.log - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi - -start_gpu_monitor - -PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") -fi - -EP_ARGS=() -if [ "${EP_SIZE:-1}" -gt 1 ]; then - EP_ARGS=(--enable-expert-parallel) -fi - -set -x -vllm serve $MODEL --port $PORT \ - "${PARALLEL_ARGS[@]}" \ - "${EP_ARGS[@]}" \ - --async-scheduling \ - --no-enable-prefix-caching \ - --distributed-executor-backend mp \ - --gpu-memory-utilization 0.8 \ - --kv-cache-dtype fp8 \ - --trust-remote-code \ - --moe-backend aiter \ - --tokenizer-mode deepseek_v4 \ - --reasoning-parser deepseek_v4 \ - --compilation-config '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE"}' > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_vllm_mtp.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_vllm_mtp.sh deleted file mode 100755 index ce51f8c5d..000000000 --- a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_vllm_mtp.sh +++ /dev/null @@ -1,110 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail - -# DeepSeek-V4-Pro on MI355X via vLLM — MTP variant of dsv4_fp4_mi355x_vllm.sh. -# Adds MTP speculative decoding per vllm-project/vllm#43385 (ROCm DeepSeek-V4 -# MTP support, merged 2026-05-24, present in v0.22.0 tagged 2026-05-29): -# --speculative-config '{"method":"mtp","num_speculative_tokens":2}'. -# -# Benchmark prompts are routed through DeepSeek-V4 chat encoding via --dsv4 -# (which auto-enables --use-chat-template). EAGLE/MTP-style spec decoding is -# trained against chat-formatted inputs; benchmarking against raw random -# prompts silently regresses the acceptance rate. -# -# All other serving flags mirror the non-MTP MI355X recipe (TP=8, -# VLLM_ROCM_USE_AITER=1, AITER MoE, FP8 KV cache, mp executor, async -# scheduling, mode=3 FULL_AND_PIECEWISE compilation). See -# dsv4_fp4_mi355x_vllm.sh for per-flag rationale. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -if [ -n "$ROCR_VISIBLE_DEVICES" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -export VLLM_ROCM_USE_AITER=1 -export VLLM_ROCM_USE_AITER_MOE=1 - -SERVER_LOG=/workspace/server.log -PORT=${PORT:-8888} - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi - -start_gpu_monitor - -PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") -fi - -EP_ARGS=() -if [ "${EP_SIZE:-1}" -gt 1 ]; then - EP_ARGS=(--enable-expert-parallel) -fi - -# use 2 speculative tokens for all configs for now -NUM_SPEC_TOKENS=2 - -set -x -vllm serve $MODEL --port $PORT \ - "${PARALLEL_ARGS[@]}" \ - "${EP_ARGS[@]}" \ - --async-scheduling \ - --no-enable-prefix-caching \ - --distributed-executor-backend mp \ - --gpu-memory-utilization 0.8 \ - --kv-cache-dtype fp8 \ - --trust-remote-code \ - --moe-backend aiter \ - --tokenizer-mode deepseek_v4 \ - --reasoning-parser deepseek_v4 \ - --speculative-config "{\"method\": \"mtp\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS}" \ - --compilation-config '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE"}' > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -# --dsv4 routes prompts through DeepSeek-V4 chat encoding (auto-enables -# --use-chat-template); required for meaningful MTP acceptance numbers. -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code \ - --dsv4 - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp8_h200.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp8_h200.sh deleted file mode 100644 index 274dee995..000000000 --- a/benchmarks/single_node/fixed_seq_len/dsv4_fp8_h200.sh +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env bash - -# Per https://vllm.ai/blog/deepseek-v4 the DeepSeek-V4-Pro H200 recipe uses -# the cu129 image and omits the FP4 indexer cache flag (H200 has no FP4 -# path). Max-model-len is pinned at 800k per the recipe. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -SERVER_LOG=/workspace/server.log - -# DeepSeek-V4-Pro weights are large; engine startup can exceed the default -# 600s. Give it an hour to load. -export VLLM_ENGINE_READY_TIMEOUT_S=3600 - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN_ARG="--max-model-len $EVAL_MAX_MODEL_LEN" -else - MAX_MODEL_LEN_ARG="--max-model-len $MAX_MODEL_LEN" -fi - -# DP_ATTENTION=true runs DP-attention with expert parallel (DP size = TP); -# DP_ATTENTION=false runs pure tensor parallel. -PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") -fi - -EP_ARGS=() -if [ "${EP_SIZE:-1}" -gt 1 ]; then - EP_ARGS=(--enable-expert-parallel) -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve $MODEL --host 0.0.0.0 --port $PORT \ ---trust-remote-code \ ---kv-cache-dtype fp8 \ ---block-size 256 \ ---no-enable-prefix-caching \ -"${PARALLEL_ARGS[@]}" \ -"${EP_ARGS[@]}" \ -$MAX_MODEL_LEN_ARG \ ---quantization deepseek_v4_fp8 \ ---gpu-memory-utilization 0.95 \ ---max-num-seqs 512 \ ---max-num-batched-tokens 512 \ ---no-enable-flashinfer-autotune \ ---compilation-config '{"mode":0,"cudagraph_mode":"FULL_DECODE_ONLY"}' \ ---tokenizer-mode deepseek_v4 \ ---tool-call-parser deepseek_v4 \ ---enable-auto-tool-choice \ ---reasoning-parser deepseek_v4 > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp8_h200_mtp.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp8_h200_mtp.sh deleted file mode 100755 index bf37eb2d0..000000000 --- a/benchmarks/single_node/fixed_seq_len/dsv4_fp8_h200_mtp.sh +++ /dev/null @@ -1,115 +0,0 @@ -#!/usr/bin/env bash - -# DeepSeek-V4-Pro H200 vLLM MTP variant of the recipe at -# https://vllm.ai/blog/deepseek-v4. Mirrors dsv4_fp8_h200.sh but adds -# --speculative-config '{"method":"mtp","num_speculative_tokens":2}' and -# routes prompts through chat-formatted encoding via --dsv4 (required for -# meaningful MTP acceptance numbers per AGENTS.md). - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -SERVER_LOG=/workspace/server.log - -# DeepSeek-V4-Pro weights are large; engine startup can exceed the default -# 600s. Give it an hour to load. -export VLLM_ENGINE_READY_TIMEOUT_S=3600 - -# Skip the cudagraph-memory estimator during the worker memory profiling -# phase — it overestimates and pushes us over the GPU memory budget on -# H200 + MTP, even though the actual cudagraph capture works fine. -export VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0 - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN_ARG="--max-model-len $EVAL_MAX_MODEL_LEN" -else - MAX_MODEL_LEN_ARG="--max-model-len $MAX_MODEL_LEN" -fi - -# DP_ATTENTION=true runs DP-attention with expert parallel (DP size = TP); -# DP_ATTENTION=false runs pure tensor parallel. -PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") -fi - -EP_ARGS=() -if [ "${EP_SIZE:-1}" -gt 1 ]; then - EP_ARGS=(--enable-expert-parallel) -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve $MODEL --host 0.0.0.0 --port $PORT \ ---trust-remote-code \ ---kv-cache-dtype fp8 \ ---block-size 256 \ ---no-enable-prefix-caching \ -"${PARALLEL_ARGS[@]}" \ -"${EP_ARGS[@]}" \ -$MAX_MODEL_LEN_ARG \ ---quantization deepseek_v4_fp8 \ ---gpu-memory-utilization 0.95 \ ---max-num-seqs 512 \ ---max-num-batched-tokens 512 \ ---no-enable-flashinfer-autotune \ ---compilation-config '{"mode":0,"cudagraph_mode":"FULL_DECODE_ONLY"}' \ ---speculative-config '{"method":"mtp","num_speculative_tokens":2}' \ ---tokenizer-mode deepseek_v4 \ ---tool-call-parser deepseek_v4 \ ---enable-auto-tool-choice \ ---reasoning-parser deepseek_v4 > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -# MTP acceptance rate degrades on raw random tokens; --dsv4 routes prompts -# through chat-formatted encoding as required for speculative decoding benchmarks. -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code \ - --dsv4 - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp8_h200_sglang.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp8_h200_sglang.sh deleted file mode 100644 index 3e7132ebe..000000000 --- a/benchmarks/single_node/fixed_seq_len/dsv4_fp8_h200_sglang.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -nvidia-smi - -SERVER_LOG="$PWD/server.log" - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL" - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi - -start_gpu_monitor --output "$PWD/gpu_metrics.csv" - -set -x -PYTHONNOUSERSITE=1 sglang serve \ - --model-path $MODEL \ - --host 0.0.0.0 \ - --port $PORT \ - --trust-remote-code \ - --tp $TP \ - --moe-runner-backend marlin \ - --chunked-prefill-size 4096 \ - --disable-flashinfer-autotune \ - --disable-radix-cache \ - --mem-fraction-static 0.88 \ - --max-running-requests "$(( CONC * 3 / 2 > 8 ? CONC * 3 / 2 : 8 ))" \ - $EVAL_CONTEXT_ARGS >> $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $((CONC * 10)) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir "$PWD/" - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp8_h200_sglang_mtp.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp8_h200_sglang_mtp.sh deleted file mode 100644 index 788eff5b8..000000000 --- a/benchmarks/single_node/fixed_seq_len/dsv4_fp8_h200_sglang_mtp.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -nvidia-smi - -SERVER_LOG="$PWD/server.log" - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL" - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi - -start_gpu_monitor --output "$PWD/gpu_metrics.csv" - -set -x -PYTHONNOUSERSITE=1 sglang serve \ - --model-path $MODEL \ - --host 0.0.0.0 \ - --port $PORT \ - --trust-remote-code \ - --tp $TP \ - --moe-runner-backend marlin \ - --chunked-prefill-size 4096 \ - --disable-flashinfer-autotune \ - --disable-radix-cache \ - --mem-fraction-static 0.88 \ - --max-running-requests "$(( CONC * 3 / 2 > 8 ? CONC * 3 / 2 : 8 ))" \ - --speculative-algorithm EAGLE \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 \ - $EVAL_CONTEXT_ARGS >> $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -# --dsv4 routes prompts through encoding_dsv4.py (PR #1153), which emits the -# ... framing DeepSeek-V4-Pro expects. The DSv4-Pro -# tokenizer ships without a jinja chat_template, so plain --use-chat-template -# would crash; --dsv4 sidesteps that and satisfies the AGENTS.md rule that all -# MTP scripts must benchmark against chat-formatted inputs (EAGLE acceptance -# silently regresses on raw random tokens). -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $((CONC * 10)) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir "$PWD/" \ - --dsv4 - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/glm5.1_fp4_mi355x.sh b/benchmarks/single_node/fixed_seq_len/glm5.1_fp4_mi355x.sh deleted file mode 100644 index 4e0d507c6..000000000 --- a/benchmarks/single_node/fixed_seq_len/glm5.1_fp4_mi355x.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env bash -set -x - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -# ROCm / SGLang performance tuning for MI355X -export SGLANG_ROCM_FUSED_DECODE_MLA=0 -export ROCM_QUICK_REDUCE_QUANTIZATION=INT4 -export SAFETENSORS_FAST_GPU=1 - -SERVER_LOG=/workspace/server.log -CONTEXT_LENGTH=$((ISL + OSL + 32)) - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -pip install -U transformers - -python3 -m sglang.launch_server \ - --model-path $MODEL \ - --host=0.0.0.0 \ - --port $PORT \ - --tensor-parallel-size $TP \ - --trust-remote-code \ - --cuda-graph-max-bs $CONC \ - --context-length $CONTEXT_LENGTH \ - --mem-fraction-static 0.85 \ - --tool-call-parser glm47 \ - --reasoning-parser glm45 \ - --model-loader-extra-config '{"enable_multithread_load": true, "num_threads": 8}' \ - --nsa-prefill-backend tilelang \ - --nsa-decode-backend tilelang $EVAL_CONTEXT_ARGS \ - --kv-cache-dtype fp8_e4m3 \ - --tokenizer-worker-num $((TP*2)) \ - --enable-aiter-allreduce-fusion \ - --disable-radix-cache> $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/glm5.1_fp4_mi355x_atom.sh b/benchmarks/single_node/fixed_seq_len/glm5.1_fp4_mi355x_atom.sh deleted file mode 100644 index b1d1b61c8..000000000 --- a/benchmarks/single_node/fixed_seq_len/glm5.1_fp4_mi355x_atom.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE \ - DP_ATTENTION - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -SERVER_LOG=/workspace/server.log - -export OMP_NUM_THREADS=1 - -# Calculate max-model-len based on ISL and OSL -if [ "$ISL" = "1024" ] && [ "$OSL" = "1024" ]; then - CALCULATED_MAX_MODEL_LEN="" -else - CALCULATED_MAX_MODEL_LEN=" --max-model-len 10240 " -fi - -if [ "$EP_SIZE" -gt 1 ]; then - EP=" --enable-expert-parallel" -else - EP=" " -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor -MEM_FRAC_STATIC=0.9 - -set -x -pip install -U transformers -python3 -m atom.entrypoints.openai_server \ - --model $MODEL \ - --server-port $PORT \ - -tp $TP \ - --kv_cache_dtype fp8 $CALCULATED_MAX_MODEL_LEN $EP \ - --gpu-memory-utilization $MEM_FRAC_STATIC \ - --default-chat-template-kwargs '{"enable_thinking": false}' \ - --trust-remote-code \ - > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -export PYTHONDONTWRITEBYTECODE=1 -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/glm5_fp4_b200.sh b/benchmarks/single_node/fixed_seq_len/glm5_fp4_b200.sh deleted file mode 100755 index a1ae27021..000000000 --- a/benchmarks/single_node/fixed_seq_len/glm5_fp4_b200.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -SERVER_LOG=/workspace/server.log - -echo "EP_SIZE: $EP_SIZE, CONC: $CONC, ISL: $ISL, OSL: $OSL" - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path=$MODEL --host=0.0.0.0 --port=$PORT \ ---trust-remote-code \ ---tensor-parallel-size=$TP \ ---data-parallel-size 1 --expert-parallel-size $EP_SIZE \ ---disable-radix-cache \ ---quantization modelopt_fp4 \ ---kv-cache-dtype fp8_e4m3 \ ---nsa-decode-backend trtllm \ ---nsa-prefill-backend trtllm \ ---moe-runner-backend flashinfer_trtllm \ ---enable-flashinfer-allreduce-fusion \ ---cuda-graph-max-bs 256 \ ---max-prefill-tokens 32768 \ ---chunked-prefill-size 32768 \ ---mem-fraction-static 0.9 \ ---stream-interval 30 \ ---scheduler-recv-interval 10 \ ---tokenizer-worker-num 6 \ ---tokenizer-path $MODEL $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/glm5_fp4_b200_mtp.sh b/benchmarks/single_node/fixed_seq_len/glm5_fp4_b200_mtp.sh deleted file mode 100755 index b42dfbefa..000000000 --- a/benchmarks/single_node/fixed_seq_len/glm5_fp4_b200_mtp.sh +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - - -export SGLANG_ENABLE_JIT_DEEPGEMM=1 -export SGLANG_ENABLE_SPEC_V2=1 - -SERVER_LOG=/workspace/server.log - - -echo "CONC: $CONC, ISL: $ISL, OSL: $OSL" - -MEM_FRACTION_STATIC=0.85 -if [[ "$CONC" -gt 128 ]]; then - MEM_FRACTION_STATIC=0.8 -fi -echo "MEM_FRACTION_STATIC: $MEM_FRACTION_STATIC" - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path=$MODEL --host=0.0.0.0 --port=$PORT \ ---trust-remote-code \ ---tensor-parallel-size=$TP \ ---data-parallel-size 1 --expert-parallel-size 1 \ ---tool-call-parser glm47 \ ---reasoning-parser glm45 \ ---kv-cache-dtype fp8_e4m3 --quantization fp8 \ ---attention-backend nsa \ ---nsa-decode-backend trtllm --nsa-prefill-backend trtllm \ ---moe-runner-backend flashinfer_trtllm \ ---cuda-graph-max-bs $CONC --max-running-requests $CONC \ ---mem-fraction-static $MEM_FRACTION_STATIC \ ---chunked-prefill-size 32768 --max-prefill-tokens 32768 \ ---enable-flashinfer-allreduce-fusion --disable-radix-cache \ ---stream-interval 30 \ ---speculative-algorithm EAGLE \ ---speculative-num-steps 3 \ ---speculative-eagle-topk 1 \ ---speculative-num-draft-tokens 4 \ ---model-loader-extra-config '{"enable_multithread_load": true}' $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --use-chat-template - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/glm5_fp4_b300.sh b/benchmarks/single_node/fixed_seq_len/glm5_fp4_b300.sh deleted file mode 100755 index b85db6b8b..000000000 --- a/benchmarks/single_node/fixed_seq_len/glm5_fp4_b300.sh +++ /dev/null @@ -1,98 +0,0 @@ -#!/usr/bin/env bash - -# NOTE: At the time of submission, https://cookbook.sglang.io/autoregressive/GLM/GLM-5 -# does not have a B300-specific recipe, so this script reuses the existing -# GLM-5 FP4 B200 SGLang recipe as-is until B300-specific tuning is available. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - - -SERVER_LOG=/workspace/server.log - -echo "EP_SIZE: $EP_SIZE, CONC: $CONC, ISL: $ISL, OSL: $OSL" - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path $MODEL_PATH --served-model-name $MODEL --host 0.0.0.0 --port $PORT \ ---trust-remote-code \ ---tensor-parallel-size $TP \ ---data-parallel-size 1 --expert-parallel-size $EP_SIZE \ ---disable-radix-cache \ ---quantization modelopt_fp4 \ ---kv-cache-dtype fp8_e4m3 \ ---nsa-decode-backend trtllm \ ---nsa-prefill-backend trtllm \ ---moe-runner-backend flashinfer_trtllm \ ---enable-flashinfer-allreduce-fusion \ ---cuda-graph-max-bs 256 \ ---max-prefill-tokens 32768 \ ---chunked-prefill-size 32768 \ ---mem-fraction-static 0.9 \ ---stream-interval 30 \ ---scheduler-recv-interval 10 \ ---tokenizer-worker-num 6 \ ---tokenizer-path $MODEL_PATH $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/glm5_fp4_b300_mtp.sh b/benchmarks/single_node/fixed_seq_len/glm5_fp4_b300_mtp.sh deleted file mode 100755 index 7a859fc54..000000000 --- a/benchmarks/single_node/fixed_seq_len/glm5_fp4_b300_mtp.sh +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env bash - -# NOTE: At the time of submission, https://cookbook.sglang.io/autoregressive/GLM/GLM-5.1 -# does not have a B300-specific recipe, so this script reuses the existing -# GLM5 FP8 B200 SGLang recipe as-is until B300-specific tuning is available. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - - - -export SGLANG_ENABLE_JIT_DEEPGEMM=1 -export SGLANG_ENABLE_SPEC_V2=1 - -SERVER_LOG=/workspace/server.log - - -echo "CONC: $CONC, ISL: $ISL, OSL: $OSL" - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path $MODEL_PATH --served-model-name $MODEL --host 0.0.0.0 --port $PORT \ ---trust-remote-code \ ---tensor-parallel-size $TP \ ---data-parallel-size 1 --expert-parallel-size 1 \ ---tool-call-parser glm47 \ ---reasoning-parser glm45 \ ---kv-cache-dtype fp8_e4m3 --quantization fp8 \ ---attention-backend nsa \ ---nsa-decode-backend trtllm --nsa-prefill-backend trtllm \ ---moe-runner-backend flashinfer_trtllm \ ---cuda-graph-max-bs $CONC --max-running-requests $CONC \ ---mem-fraction-static 0.85 \ ---chunked-prefill-size 32768 --max-prefill-tokens 32768 \ ---enable-flashinfer-allreduce-fusion --disable-radix-cache \ ---stream-interval 30 \ ---speculative-algorithm EAGLE \ ---speculative-num-steps 3 \ ---speculative-eagle-topk 1 \ ---speculative-num-draft-tokens 4 \ ---model-loader-extra-config '{"enable_multithread_load": true}' $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --use-chat-template - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/glm5_fp8_b200.sh b/benchmarks/single_node/fixed_seq_len/glm5_fp8_b200.sh deleted file mode 100755 index ebcecc5e5..000000000 --- a/benchmarks/single_node/fixed_seq_len/glm5_fp8_b200.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -export SGLANG_ENABLE_JIT_DEEPGEMM=1 - -SERVER_LOG=/workspace/server.log - - -echo "CONC: $CONC, ISL: $ISL, OSL: $OSL" - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path=$MODEL --host=0.0.0.0 --port=$PORT \ ---trust-remote-code \ ---tensor-parallel-size=$TP \ ---data-parallel-size 1 --expert-parallel-size 1 \ ---tool-call-parser glm47 \ ---reasoning-parser glm45 \ ---kv-cache-dtype fp8_e4m3 --quantization fp8 \ ---attention-backend nsa \ ---nsa-decode-backend trtllm --nsa-prefill-backend trtllm \ ---moe-runner-backend flashinfer_trtllm \ ---cuda-graph-max-bs $CONC --max-running-requests $CONC \ ---mem-fraction-static 0.85 \ ---chunked-prefill-size 32768 --max-prefill-tokens 32768 \ ---enable-flashinfer-allreduce-fusion --disable-radix-cache \ ---stream-interval 30 \ ---model-loader-extra-config '{"enable_multithread_load": true}' $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/glm5_fp8_b200_mtp.sh b/benchmarks/single_node/fixed_seq_len/glm5_fp8_b200_mtp.sh deleted file mode 100755 index a7c627f46..000000000 --- a/benchmarks/single_node/fixed_seq_len/glm5_fp8_b200_mtp.sh +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -export SGLANG_ENABLE_JIT_DEEPGEMM=1 -export SGLANG_ENABLE_SPEC_V2=1 - -SERVER_LOG=/workspace/server.log - - -echo "CONC: $CONC, ISL: $ISL, OSL: $OSL" - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path=$MODEL --host=0.0.0.0 --port=$PORT \ ---trust-remote-code \ ---tensor-parallel-size=$TP \ ---data-parallel-size 1 --expert-parallel-size 1 \ ---tool-call-parser glm47 \ ---reasoning-parser glm45 \ ---kv-cache-dtype fp8_e4m3 --quantization fp8 \ ---attention-backend nsa \ ---nsa-decode-backend trtllm --nsa-prefill-backend trtllm \ ---moe-runner-backend flashinfer_trtllm \ ---cuda-graph-max-bs $CONC --max-running-requests $CONC \ ---mem-fraction-static 0.85 \ ---chunked-prefill-size 32768 --max-prefill-tokens 32768 \ ---enable-flashinfer-allreduce-fusion --disable-radix-cache \ ---stream-interval 30 \ ---speculative-algorithm EAGLE \ ---speculative-num-steps 3 \ ---speculative-eagle-topk 1 \ ---speculative-num-draft-tokens 4 \ ---model-loader-extra-config '{"enable_multithread_load": true}' $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --use-chat-template - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/glm5_fp8_b300.sh b/benchmarks/single_node/fixed_seq_len/glm5_fp8_b300.sh deleted file mode 100644 index e5a0b2c29..000000000 --- a/benchmarks/single_node/fixed_seq_len/glm5_fp8_b300.sh +++ /dev/null @@ -1,96 +0,0 @@ -#!/usr/bin/env bash - -# NOTE: At the time of submission, https://cookbook.sglang.io/autoregressive/GLM/GLM-5.1 -# does not have a B300-specific recipe, so this script reuses the existing -# GLM5 FP8 B200 SGLang recipe as-is until B300-specific tuning is available. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -export SGLANG_ENABLE_JIT_DEEPGEMM=1 - -SERVER_LOG=/workspace/server.log - - -echo "CONC: $CONC, ISL: $ISL, OSL: $OSL" - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path $MODEL_PATH --served-model-name $MODEL --host 0.0.0.0 --port $PORT \ ---trust-remote-code \ ---tensor-parallel-size $TP \ ---data-parallel-size 1 --expert-parallel-size 1 \ ---tool-call-parser glm47 \ ---reasoning-parser glm45 \ ---kv-cache-dtype fp8_e4m3 --quantization fp8 \ ---attention-backend nsa \ ---nsa-decode-backend trtllm --nsa-prefill-backend trtllm \ ---moe-runner-backend flashinfer_trtllm \ ---cuda-graph-max-bs $CONC --max-running-requests $CONC \ ---mem-fraction-static 0.85 \ ---chunked-prefill-size 32768 --max-prefill-tokens 32768 \ ---enable-flashinfer-allreduce-fusion --disable-radix-cache \ ---stream-interval 30 \ ---model-loader-extra-config '{"enable_multithread_load": true}' $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/glm5_fp8_b300_mtp.sh b/benchmarks/single_node/fixed_seq_len/glm5_fp8_b300_mtp.sh deleted file mode 100755 index 7a859fc54..000000000 --- a/benchmarks/single_node/fixed_seq_len/glm5_fp8_b300_mtp.sh +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env bash - -# NOTE: At the time of submission, https://cookbook.sglang.io/autoregressive/GLM/GLM-5.1 -# does not have a B300-specific recipe, so this script reuses the existing -# GLM5 FP8 B200 SGLang recipe as-is until B300-specific tuning is available. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - - - -export SGLANG_ENABLE_JIT_DEEPGEMM=1 -export SGLANG_ENABLE_SPEC_V2=1 - -SERVER_LOG=/workspace/server.log - - -echo "CONC: $CONC, ISL: $ISL, OSL: $OSL" - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path $MODEL_PATH --served-model-name $MODEL --host 0.0.0.0 --port $PORT \ ---trust-remote-code \ ---tensor-parallel-size $TP \ ---data-parallel-size 1 --expert-parallel-size 1 \ ---tool-call-parser glm47 \ ---reasoning-parser glm45 \ ---kv-cache-dtype fp8_e4m3 --quantization fp8 \ ---attention-backend nsa \ ---nsa-decode-backend trtllm --nsa-prefill-backend trtllm \ ---moe-runner-backend flashinfer_trtllm \ ---cuda-graph-max-bs $CONC --max-running-requests $CONC \ ---mem-fraction-static 0.85 \ ---chunked-prefill-size 32768 --max-prefill-tokens 32768 \ ---enable-flashinfer-allreduce-fusion --disable-radix-cache \ ---stream-interval 30 \ ---speculative-algorithm EAGLE \ ---speculative-num-steps 3 \ ---speculative-eagle-topk 1 \ ---speculative-num-draft-tokens 4 \ ---model-loader-extra-config '{"enable_multithread_load": true}' $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --use-chat-template - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/glm5_fp8_h200.sh b/benchmarks/single_node/fixed_seq_len/glm5_fp8_h200.sh deleted file mode 100644 index 266587de9..000000000 --- a/benchmarks/single_node/fixed_seq_len/glm5_fp8_h200.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -SERVER_LOG=/workspace/server.log - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -python3 -m sglang.launch_server \ - --model-path "$MODEL" \ - --host 0.0.0.0 \ - --port "$PORT" \ - --tp-size "$TP" \ - --tool-call-parser glm47 \ - --reasoning-parser glm45 \ - --mem-fraction-static 0.85 \ - --served-model-name glm-5-fp8 \ - --trust-remote-code \ - --enable-flashinfer-allreduce-fusion \ - $EVAL_CONTEXT_ARGS > "$SERVER_LOG" 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $(( CONC * 10 )) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -# Server accepts glm-5-fp8 (--served-model-name); lm-eval must use that model name -if [ "${RUN_EVAL}" = "true" ]; then - export MODEL_NAME=glm-5-fp8 - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/glm5_fp8_h200_mtp.sh b/benchmarks/single_node/fixed_seq_len/glm5_fp8_h200_mtp.sh deleted file mode 100755 index 133d757dc..000000000 --- a/benchmarks/single_node/fixed_seq_len/glm5_fp8_h200_mtp.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env bash - -# GLM-5 FP8 on H200 (Hopper) with EAGLE / MTP speculative decoding. -# Mirrors glm5_fp8_h200.sh but adds the speculative-* flags. We keep the -# server-arg shape from the non-MTP H200 recipe (sglang defaults — no -# nsa/trtllm-mha) since those backends are Blackwell-specific and not -# applicable to Hopper. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -SERVER_LOG=/workspace/server.log - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi - -start_gpu_monitor - -set -x -python3 -m sglang.launch_server \ - --model-path "$MODEL" \ - --host 0.0.0.0 \ - --port "$PORT" \ - --tp-size "$TP" \ - --tool-call-parser glm47 \ - --reasoning-parser glm45 \ - --mem-fraction-static 0.85 \ - --served-model-name glm-5-fp8 \ - --trust-remote-code \ - --speculative-algorithm EAGLE \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 \ - $EVAL_CONTEXT_ARGS > "$SERVER_LOG" 2>&1 & - -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $(( CONC * 10 )) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code \ - --use-chat-template - -if [ "${RUN_EVAL}" = "true" ]; then - export MODEL_NAME=glm-5-fp8 - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/glm5_fp8_mi325x.sh b/benchmarks/single_node/fixed_seq_len/glm5_fp8_mi325x.sh deleted file mode 100755 index 0564ef8d8..000000000 --- a/benchmarks/single_node/fixed_seq_len/glm5_fp8_mi325x.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -SERVER_LOG=/workspace/server.log -CONTEXT_LENGTH=$((ISL + OSL + 20)) -MAX_PREFILL_TOKENS=32768 - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -else EVAL_CONTEXT_ARGS="--context-length $CONTEXT_LENGTH" -fi - -start_gpu_monitor - -# Launch args follow sglang issue #25672 comment 4485916205: -# tilelang NSA backends + fp8_e4m3 KV cache + multithread model load. -python3 -m sglang.launch_server \ - --model-path $MODEL \ - --host=0.0.0.0 \ - --port $PORT \ - --tensor-parallel-size $TP \ - --data-parallel-size 1 \ - --trust-remote-code \ - --tool-call-parser glm47 \ - --reasoning-parser glm45 \ - --tokenizer-worker-num 6 \ - --cuda-graph-max-bs $CONC \ - --disable-radix-cache \ - --max-prefill-tokens $MAX_PREFILL_TOKENS \ - --scheduler-recv-interval 30 \ - --mem-fraction-static 0.80 \ - --model-loader-extra-config '{"enable_multithread_load": true, "num_threads": 8}' \ - --nsa-prefill-backend tilelang \ - --nsa-decode-backend tilelang \ - --kv-cache-dtype fp8_e4m3 \ - $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/glm5_fp8_mi325x_mtp.sh b/benchmarks/single_node/fixed_seq_len/glm5_fp8_mi325x_mtp.sh deleted file mode 100755 index fb77d84c2..000000000 --- a/benchmarks/single_node/fixed_seq_len/glm5_fp8_mi325x_mtp.sh +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env bash - -# GLM-5 FP8 on MI325X with EAGLE / MTP speculative decoding. -# Mirrors glm5_fp8_mi325x.sh and adds the speculative-* flags. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -SERVER_LOG=/workspace/server.log -CONTEXT_LENGTH=$((ISL + OSL + 20)) -MAX_PREFILL_TOKENS=32768 - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -else EVAL_CONTEXT_ARGS="--context-length $CONTEXT_LENGTH" -fi - -start_gpu_monitor - -# Launch args follow sglang issue #25672 comment 4485916205: -# tilelang NSA backends + fp8_e4m3 KV cache + multithread model load, -# plus EAGLE/MTP speculative decoding. -python3 -m sglang.launch_server \ - --model-path $MODEL \ - --host=0.0.0.0 \ - --port $PORT \ - --tensor-parallel-size $TP \ - --ep-size $EP_SIZE \ - --trust-remote-code \ - --tool-call-parser glm47 \ - --reasoning-parser glm45 \ - --tokenizer-worker-num 6 \ - --cuda-graph-max-bs $CONC \ - --disable-radix-cache \ - --max-prefill-tokens $MAX_PREFILL_TOKENS \ - --scheduler-recv-interval 30 \ - --mem-fraction-static 0.80 \ - --model-loader-extra-config '{"enable_multithread_load": true, "num_threads": 8}' \ - --nsa-prefill-backend tilelang \ - --nsa-decode-backend tilelang \ - --kv-cache-dtype fp8_e4m3 \ - --speculative-algorithm EAGLE \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 \ - $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --use-chat-template - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/glm5_fp8_mi355x.sh b/benchmarks/single_node/fixed_seq_len/glm5_fp8_mi355x.sh deleted file mode 100755 index 21defe90c..000000000 --- a/benchmarks/single_node/fixed_seq_len/glm5_fp8_mi355x.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -# ROCm / SGLang performance tuning for MI355X -export SGLANG_ROCM_FUSED_DECODE_MLA=0 -export ROCM_QUICK_REDUCE_QUANTIZATION=INT4 -export SAFETENSORS_FAST_GPU=1 - -SERVER_LOG=/workspace/server.log - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -python3 -m sglang.launch_server \ - --model-path $MODEL \ - --host=0.0.0.0 \ - --port $PORT \ - --tensor-parallel-size $TP \ - --trust-remote-code \ - --tool-call-parser glm47 \ - --reasoning-parser glm45 \ - --mem-fraction-static 0.85 \ - --model-loader-extra-config '{"enable_multithread_load": true, "num_threads": 8}' \ - --nsa-prefill-backend tilelang \ - --nsa-decode-backend tilelang $EVAL_CONTEXT_ARGS \ - --kv-cache-dtype fp8_e4m3 \ - --max-running-requests ${CONC} \ - --cuda-graph-max-bs ${CONC} \ - --disable-radix-cache> $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/glm5_fp8_mi355x_atom.sh b/benchmarks/single_node/fixed_seq_len/glm5_fp8_mi355x_atom.sh deleted file mode 100644 index b1d1b61c8..000000000 --- a/benchmarks/single_node/fixed_seq_len/glm5_fp8_mi355x_atom.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE \ - DP_ATTENTION - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -SERVER_LOG=/workspace/server.log - -export OMP_NUM_THREADS=1 - -# Calculate max-model-len based on ISL and OSL -if [ "$ISL" = "1024" ] && [ "$OSL" = "1024" ]; then - CALCULATED_MAX_MODEL_LEN="" -else - CALCULATED_MAX_MODEL_LEN=" --max-model-len 10240 " -fi - -if [ "$EP_SIZE" -gt 1 ]; then - EP=" --enable-expert-parallel" -else - EP=" " -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor -MEM_FRAC_STATIC=0.9 - -set -x -pip install -U transformers -python3 -m atom.entrypoints.openai_server \ - --model $MODEL \ - --server-port $PORT \ - -tp $TP \ - --kv_cache_dtype fp8 $CALCULATED_MAX_MODEL_LEN $EP \ - --gpu-memory-utilization $MEM_FRAC_STATIC \ - --default-chat-template-kwargs '{"enable_thinking": false}' \ - --trust-remote-code \ - > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -export PYTHONDONTWRITEBYTECODE=1 -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/glm5_fp8_mi355x_mtp.sh b/benchmarks/single_node/fixed_seq_len/glm5_fp8_mi355x_mtp.sh deleted file mode 100755 index 90fa04f5d..000000000 --- a/benchmarks/single_node/fixed_seq_len/glm5_fp8_mi355x_mtp.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env bash -set -x - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -# ROCm / SGLang performance tuning for MI355X -export SGLANG_ROCM_FUSED_DECODE_MLA=0 -export ROCM_QUICK_REDUCE_QUANTIZATION=INT4 -export SAFETENSORS_FAST_GPU=1 -export SGLANG_ENABLE_SPEC_V2=1 - -SERVER_LOG=/workspace/server.log -CONTEXT_LENGTH=$((ISL + OSL + 32)) - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -python3 -m sglang.launch_server \ - --model-path $MODEL \ - --host=0.0.0.0 \ - --port $PORT \ - --tensor-parallel-size $TP \ - --trust-remote-code \ - --cuda-graph-max-bs $CONC \ - --context-length $CONTEXT_LENGTH \ - --mem-fraction-static 0.85 \ - --tool-call-parser glm47 \ - --reasoning-parser glm45 \ - --model-loader-extra-config '{"enable_multithread_load": true, "num_threads": 8}' \ - --nsa-prefill-backend tilelang \ - --nsa-decode-backend tilelang $EVAL_CONTEXT_ARGS \ - --kv-cache-dtype fp8_e4m3 \ - --speculative-algorithm EAGLE \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 \ - --tokenizer-worker-num $((TP*2)) \ - --disable-radix-cache> $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --use-chat-template - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/gptoss_fp4_b200.sh b/benchmarks/single_node/fixed_seq_len/gptoss_fp4_b200.sh deleted file mode 100644 index 743974df3..000000000 --- a/benchmarks/single_node/fixed_seq_len/gptoss_fp4_b200.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -nvidia-smi - -# Calculate max-model-len based on ISL and OSL -if [ "$ISL" = "1024" ] && [ "$OSL" = "1024" ]; then - CALCULATED_MAX_MODEL_LEN=$((ISL + OSL + 20)) -elif [ "$ISL" = "8192" ] || [ "$OSL" = "8192" ]; then - CALCULATED_MAX_MODEL_LEN=$((ISL + OSL + 256)) -else - CALCULATED_MAX_MODEL_LEN=${MAX_MODEL_LEN:-10240} -fi - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - CALCULATED_MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi - -cat > config.yaml << EOF -kv-cache-dtype: fp8 -compilation-config: '{"pass_config":{"fuse_allreduce_rms":true,"eliminate_noops":true}}' -no-enable-prefix-caching: true -max-cudagraph-capture-size: 2048 -max-num-batched-tokens: 8192 -max-model-len: $CALCULATED_MAX_MODEL_LEN -EOF - -export TORCH_CUDA_ARCH_LIST="10.0" -export PYTHONNOUSERSITE=1 -export VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8=1 - -SERVER_LOG=/workspace/server.log - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve $MODEL --host 0.0.0.0 --port $PORT \ ---config config.yaml \ ---gpu-memory-utilization 0.9 \ ---tensor-parallel-size $TP \ ---max-num-seqs 512 > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $(( CONC * 10 )) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/gptoss_fp4_b200_trt.sh b/benchmarks/single_node/fixed_seq_len/gptoss_fp4_b200_trt.sh deleted file mode 100644 index ced9162f9..000000000 --- a/benchmarks/single_node/fixed_seq_len/gptoss_fp4_b200_trt.sh +++ /dev/null @@ -1,124 +0,0 @@ -#!/usr/bin/env bash - -# Source benchmark utilities early -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - DP_ATTENTION \ - EP_SIZE - -# GPTOSS TRTLLM Deployment Guide: -# https://github.com/NVIDIA/TensorRT-LLM/blob/main/docs/source/deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.md - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi -SERVER_LOG=/workspace/server.log - -# ========= Determine DP_ATTENTION, EP_SIZE and MOE_BACKEND based on ISL, OSL, CONC ========= -MOE_BACKEND="TRTLLM" - -echo "MOE_BACKEND set to '$MOE_BACKEND'" - -EXTRA_CONFIG_FILE="gptoss-fp4.yml" -export TRTLLM_ENABLE_PDL=1 - -cat > $EXTRA_CONFIG_FILE << EOF -cuda_graph_config: - enable_padding: true - max_batch_size: $CONC -enable_attention_dp: $DP_ATTENTION -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: $MOE_BACKEND -EOF - -if [[ "$DP_ATTENTION" == "true" ]]; then - # DISABLE All2All for MoE TP - if [[ "$EP_SIZE" -eq 1 ]]; then - # DTP Alltoall Environment variables for EP_SIZE == 1 - export TRTLLM_FORCE_ALLTOALL_METHOD="NotEnabled" - elif [[ "$EP_SIZE" -gt 1 ]]; then - # DEP - export TRTLLM_MOE_ALLTOALL_BACKEND="mnnvlthroughput" - export TRTLLM_FORCE_ALLTOALL_METHOD="MNNVL" - export TRTLLM_MOE_A2A_WORKSPACE_MB="2048" - fi - cat << EOF >> $EXTRA_CONFIG_FILE -attention_dp_config: - enable_balance: true -EOF -fi - -echo "Generated config file contents:" -cat $EXTRA_CONFIG_FILE - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x - -MAX_NUM_TOKENS=20000 - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" - MAX_NUM_TOKENS="$EVAL_MAX_MODEL_LEN" -fi - -# Launch TRT-LLM server -mpirun -n 1 --oversubscribe --allow-run-as-root \ - trtllm-serve $MODEL --port=$PORT \ - --trust_remote_code \ - --backend=pytorch \ - --max_batch_size 512 \ - --max_seq_len=$MAX_MODEL_LEN \ - --max_num_tokens=$MAX_NUM_TOKENS \ - --tp_size=$TP --ep_size=$EP_SIZE \ - --extra_llm_api_options=$EXTRA_CONFIG_FILE \ - > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend openai \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $(( $CONC * 10 )) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/gptoss_fp4_h100.sh b/benchmarks/single_node/fixed_seq_len/gptoss_fp4_h100.sh deleted file mode 100644 index dfd842a88..000000000 --- a/benchmarks/single_node/fixed_seq_len/gptoss_fp4_h100.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -MAX_MODEL_LEN=10240 - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi - -cat > config.yaml << EOF -no-enable-prefix-caching: true -max-cudagraph-capture-size: 2048 -max-num-batched-tokens: 8192 -max-model-len: $MAX_MODEL_LEN -EOF - -export PYTHONNOUSERSITE=1 -export VLLM_MXFP4_USE_MARLIN=1 -SERVER_LOG=/workspace/server.log - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve $MODEL --host=0.0.0.0 --port=$PORT \ ---config config.yaml \ ---gpu-memory-utilization=0.9 \ ---tensor-parallel-size=$TP \ ---max-num-seqs=$CONC > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $(( $CONC * 10 )) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/gptoss_fp4_h200.sh b/benchmarks/single_node/fixed_seq_len/gptoss_fp4_h200.sh deleted file mode 100644 index b65c86782..000000000 --- a/benchmarks/single_node/fixed_seq_len/gptoss_fp4_h200.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -pip install datasets pandas - -# Calculate max-model-len based on ISL and OSL -if [ "$ISL" = "1024" ] && [ "$OSL" = "1024" ]; then - CALCULATED_MAX_MODEL_LEN=$((ISL + OSL + 20)) -elif [ "$ISL" = "8192" ] || [ "$OSL" = "8192" ]; then - CALCULATED_MAX_MODEL_LEN=$((ISL + OSL + 256)) -else - CALCULATED_MAX_MODEL_LEN=${MAX_MODEL_LEN:-10240} -fi - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - CALCULATED_MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi - -# Create config.yaml -cat > config.yaml << EOF -no-enable-prefix-caching: true -max-cudagraph-capture-size: 2048 -max-num-batched-tokens: 8192 -max-model-len: $CALCULATED_MAX_MODEL_LEN -EOF - -SERVER_LOG=/workspace/server.log -export TORCH_CUDA_ARCH_LIST="9.0" - -export VLLM_MXFP4_USE_MARLIN=1 - -PYTHONNOUSERSITE=1 vllm serve $MODEL --host 0.0.0.0 --port $PORT \ - --config config.yaml \ - --gpu-memory-utilization 0.9 \ - --tensor-parallel-size $TP \ - --max-num-seqs $CONC > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $(( $CONC * 10 )) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/gptoss_fp4_h200_trt.sh b/benchmarks/single_node/fixed_seq_len/gptoss_fp4_h200_trt.sh deleted file mode 100644 index 02dd05bc9..000000000 --- a/benchmarks/single_node/fixed_seq_len/gptoss_fp4_h200_trt.sh +++ /dev/null @@ -1,96 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - DP_ATTENTION \ - EP_SIZE - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi -SERVER_LOG=/workspace/server.log - -set +x - -export TRTLLM_ENABLE_PDL=1 - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -cat > gptoss-config.yml << EOF -cuda_graph_config: - enable_padding: true - max_batch_size: $CONC -enable_attention_dp: $DP_ATTENTION -kv_cache_config: - dtype: auto - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -EOF - -MAX_NUM_TOKENS=20000 - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" - MAX_NUM_TOKENS="$EVAL_MAX_MODEL_LEN" -fi - -PYTHONNOUSERSITE=1 mpirun -n 1 --oversubscribe --allow-run-as-root \ -trtllm-serve $MODEL \ ---max_batch_size $CONC \ ---max_num_tokens $MAX_NUM_TOKENS \ ---max_seq_len=$MAX_MODEL_LEN \ ---backend pytorch \ ---extra_llm_api_options gptoss-config.yml \ ---ep_size=$EP_SIZE \ ---trust_remote_code \ ---gpus_per_node 8 \ ---host 0.0.0.0 \ ---port $PORT \ ---tp_size=$TP \ ---pp_size=1 \ -> $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend openai \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $(( $CONC * 10 )) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x \ No newline at end of file diff --git a/benchmarks/single_node/fixed_seq_len/gptoss_fp4_mi300x.sh b/benchmarks/single_node/fixed_seq_len/gptoss_fp4_mi300x.sh deleted file mode 100644 index c18a5a3ee..000000000 --- a/benchmarks/single_node/fixed_seq_len/gptoss_fp4_mi300x.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -# If the machine runs a MEC FW older than 177, RCCL -# cannot reclaim some memory. -# Disable that features to avoid crashes. -# This is related to the changes in the driver at: -# https://rocm.docs.amd.com/en/docs-6.4.3/about/release-notes.html#amdgpu-driver-updates -version=`rocm-smi --showfw | grep MEC | head -n 1 | awk '{print $NF}'` -if [[ "$version" == "" || $version -lt 177 ]]; then - export HSA_NO_SCRATCH_RECLAIM=1 -fi - -# Set HIP_VISIBLE_DEVICES to match ROCR_VISIBLE_DEVICES for Ray compatibility in vLLM 0.14+ -if [ -n "$ROCR_VISIBLE_DEVICES" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -export AMDGCN_USE_BUFFER_OPS=0 -export VLLM_ROCM_USE_AITER=1 -export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 -ATTN_BACKEND="--attention-backend ROCM_AITER_UNIFIED_ATTN" -FUSE_ROPE_KVCACHE="-cc.pass_config.fuse_rope_kvcache=True -cc.use_inductor_graph_partition=True" - -SERVER_LOG=/workspace/server.log - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve $MODEL --port $PORT \ - $ATTN_BACKEND $FUSE_ROPE_KVCACHE \ - --tensor-parallel-size=$TP \ - --gpu-memory-utilization 0.95 \ - --max-model-len $MAX_MODEL_LEN \ - --block-size=64 \ - --no-enable-prefix-caching > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $(( $CONC * 10 )) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/gptoss_fp4_mi325x.sh b/benchmarks/single_node/fixed_seq_len/gptoss_fp4_mi325x.sh deleted file mode 100644 index c18a5a3ee..000000000 --- a/benchmarks/single_node/fixed_seq_len/gptoss_fp4_mi325x.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -# If the machine runs a MEC FW older than 177, RCCL -# cannot reclaim some memory. -# Disable that features to avoid crashes. -# This is related to the changes in the driver at: -# https://rocm.docs.amd.com/en/docs-6.4.3/about/release-notes.html#amdgpu-driver-updates -version=`rocm-smi --showfw | grep MEC | head -n 1 | awk '{print $NF}'` -if [[ "$version" == "" || $version -lt 177 ]]; then - export HSA_NO_SCRATCH_RECLAIM=1 -fi - -# Set HIP_VISIBLE_DEVICES to match ROCR_VISIBLE_DEVICES for Ray compatibility in vLLM 0.14+ -if [ -n "$ROCR_VISIBLE_DEVICES" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -export AMDGCN_USE_BUFFER_OPS=0 -export VLLM_ROCM_USE_AITER=1 -export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 -ATTN_BACKEND="--attention-backend ROCM_AITER_UNIFIED_ATTN" -FUSE_ROPE_KVCACHE="-cc.pass_config.fuse_rope_kvcache=True -cc.use_inductor_graph_partition=True" - -SERVER_LOG=/workspace/server.log - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve $MODEL --port $PORT \ - $ATTN_BACKEND $FUSE_ROPE_KVCACHE \ - --tensor-parallel-size=$TP \ - --gpu-memory-utilization 0.95 \ - --max-model-len $MAX_MODEL_LEN \ - --block-size=64 \ - --no-enable-prefix-caching > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $(( $CONC * 10 )) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/gptoss_fp4_mi355x.sh b/benchmarks/single_node/fixed_seq_len/gptoss_fp4_mi355x.sh deleted file mode 100644 index 14dedb141..000000000 --- a/benchmarks/single_node/fixed_seq_len/gptoss_fp4_mi355x.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -# If the machine runs a MEC FW older than 177, RCCL -# cannot reclaim some memory. -# Disable that features to avoid crashes. -# This is related to the changes in the driver at: -# https://rocm.docs.amd.com/en/docs-6.4.3/about/release-notes.html#amdgpu-driver-updates -version=`rocm-smi --showfw | grep MEC | head -n 1 | awk '{print $NF}'` -if [[ "$version" == "" || $version -lt 177 ]]; then - export HSA_NO_SCRATCH_RECLAIM=1 -fi - -# Set HIP_VISIBLE_DEVICES to match ROCR_VISIBLE_DEVICES for Ray compatibility in vLLM 0.14+ -if [ -n "$ROCR_VISIBLE_DEVICES" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -export AMDGCN_USE_BUFFER_OPS=0 -export VLLM_ROCM_USE_AITER=1 -export VLLM_ROCM_USE_AITER_TRITON_ROPE=1 -export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 -ATTN_BACKEND="--attention-backend ROCM_AITER_UNIFIED_ATTN" -FUSE_ROPE_KVCACHE="-cc.pass_config.fuse_rope_kvcache=True -cc.use_inductor_graph_partition=True" - -SERVER_LOG=/workspace/server.log - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve $MODEL --port $PORT \ - $ATTN_BACKEND $FUSE_ROPE_KVCACHE \ - --tensor-parallel-size=$TP \ - --gpu-memory-utilization 0.95 \ - --max-model-len $MAX_MODEL_LEN \ - --block-size=64 \ - --no-enable-prefix-caching > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/gptoss_fp4_mi355x_atom.sh b/benchmarks/single_node/fixed_seq_len/gptoss_fp4_mi355x_atom.sh deleted file mode 100644 index d3a8a66a1..000000000 --- a/benchmarks/single_node/fixed_seq_len/gptoss_fp4_mi355x_atom.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE \ - DP_ATTENTION - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -SERVER_LOG=/workspace/server.log - -export OMP_NUM_THREADS=1 - -# Calculate max-model-len based on ISL and OSL -if [ "$ISL" = "1024" ] && [ "$OSL" = "1024" ]; then - CALCULATED_MAX_MODEL_LEN="" -else - CALCULATED_MAX_MODEL_LEN=" --max-model-len 10240 " -fi - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - CALCULATED_MAX_MODEL_LEN=" --max-model-len $EVAL_MAX_MODEL_LEN " -fi - -if [ "$EP_SIZE" -gt 1 ]; then - EP=" --enable-expert-parallel" -else - EP=" " -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor -MEM_FRAC_STATIC=0.9 - -set -x - -BLOCK_SIZE=${BLOCK_SIZE:-16} -export ATOM_GPT_OSS_MODEL=1 #TODO remove this -python3 -m atom.entrypoints.openai_server \ - --model $MODEL \ - --server-port $PORT \ - -tp $TP \ - --kv_cache_dtype fp8 $CALCULATED_MAX_MODEL_LEN $EP \ - --gpu-memory-utilization $MEM_FRAC_STATIC \ - --block-size $BLOCK_SIZE > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x \ No newline at end of file diff --git a/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b200.sh b/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b200.sh deleted file mode 100644 index 59b55c90c..000000000 --- a/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b200.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -nvidia-smi - -export TORCH_CUDA_ARCH_LIST="10.0" -export PYTHONNOUSERSITE=1 - -SERVER_LOG=/workspace/server.log - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -# vLLM v0.20.2+'s CUDA-graph memory profiler pre-reserves ~57 GB/GPU upfront -# (~32% of total), which collides with --gpu-memory-utilization=0.90 and -# leaves negative space for the KV cache. Disable the profiler — our 0.90 -# already leaves ~18 GB/GPU as safety net (same pattern as -# benchmarks/single_node/agentic/kimik2.5_fp4_b200.sh). -export VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0 - -set -x -vllm serve $MODEL --host 0.0.0.0 --port $PORT \ ---tensor-parallel-size=$TP \ ---gpu-memory-utilization 0.90 \ ---max-model-len $MAX_MODEL_LEN \ ---max-num-seqs $CONC \ ---reasoning-parser kimi_k2 \ ---tool-call-parser kimi_k2 \ ---compilation_config.pass_config.fuse_allreduce_rms true \ ---kv-cache-dtype fp8 \ ---max-cudagraph-capture-size 2048 \ ---max-num-batched-tokens "$((ISL * 2 ))" \ ---stream-interval 20 --no-enable-prefix-caching \ ---trust-remote-code > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $(( CONC * 10 )) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x \ No newline at end of file diff --git a/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b300.sh b/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b300.sh deleted file mode 100755 index db6d3fb0d..000000000 --- a/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b300.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env bash - -# NOTE: At the time of submission, https://docs.vllm.ai/projects/recipes/en/latest/moonshotai/Kimi-K2.5.html -# does not have a B300-specific recipe, so this script reuses the existing -# Kimi-K2.5 FP4 B200 vLLM recipe as-is until B300-specific tuning is available. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - - -nvidia-smi - -export TORCH_CUDA_ARCH_LIST="10.0" -export PYTHONNOUSERSITE=1 - -SERVER_LOG=/workspace/server.log - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve $MODEL_PATH --served-model-name $MODEL --host 0.0.0.0 --port $PORT \ ---tensor-parallel-size $TP \ ---gpu-memory-utilization 0.90 \ ---max-model-len $MAX_MODEL_LEN \ ---max-num-seqs $CONC \ ---reasoning-parser kimi_k2 \ ---tool-call-parser kimi_k2 \ ---compilation_config.pass_config.fuse_allreduce_rms true \ ---no-enable-prefix-caching \ ---trust-remote-code > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $(( CONC * 10 )) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_mi355x.sh b/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_mi355x.sh deleted file mode 100755 index d4616143a..000000000 --- a/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_mi355x.sh +++ /dev/null @@ -1,106 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -# Install amd-quark for MXFP4 quantization support -# need to manually install due to ROCm vLLM bug -# https://github.com/vllm-project/vllm/issues/35633 -pip install amd-quark - -# Set HIP_VISIBLE_DEVICES to match ROCR_VISIBLE_DEVICES for Ray compatibility in vLLM 0.14+ -if [ -n "$ROCR_VISIBLE_DEVICES" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -SERVER_LOG=/workspace/server.log - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi - -# If the machine runs a MEC FW older than 177, RCCL -# cannot reclaim some memory. -# Disable that features to avoid crashes. -# This is related to the changes in the driver at: -# https://rocm.docs.amd.com/en/docs-6.4.3/about/release-notes.html#amdgpu-driver-updates -version=`rocm-smi --showfw | grep MEC | head -n 1 | awk '{print $NF}'` -if [[ "$version" == "" || $version -lt 177 ]]; then - export HSA_NO_SCRATCH_RECLAIM=1 -fi - -export VLLM_ROCM_USE_AITER=1 -export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 - -# Disable AITER RMSNorm for TP < 8 due to accuracy issues -if [ "${TP}" -lt 8 ]; then - export VLLM_ROCM_USE_AITER_RMSNORM=0 -fi - -if [ "${EP_SIZE:-0}" -gt 1 ]; then - EP=" --enable-expert-parallel" -else - EP=" " -fi - -# following AMD andy luo's recipe -# https://x.com/linluo77/status/2017024513595301985 - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve $MODEL --port $PORT \ ---tensor-parallel-size=$TP \ -$EP \ ---gpu-memory-utilization 0.90 \ ---max-model-len $MAX_MODEL_LEN \ ---block-size=1 \ ---no-enable-prefix-caching \ ---trust-remote-code \ ---no-enable-prefix-caching \ ---mm-encoder-tp-mode data > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_mi355x_atom.sh b/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_mi355x_atom.sh deleted file mode 100755 index 6730aded2..000000000 --- a/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_mi355x_atom.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE \ - DP_ATTENTION - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -SERVER_LOG=/workspace/server.log - -export OMP_NUM_THREADS=1 - -# Calculate max-model-len based on ISL and OSL -if [ "$ISL" = "1024" ] && [ "$OSL" = "1024" ]; then - CALCULATED_MAX_MODEL_LEN="" -else - CALCULATED_MAX_MODEL_LEN=" --max-model-len 10240 " -fi - -if [ "$EP_SIZE" -gt 1 ]; then - EP=" --enable-expert-parallel" -else - EP=" " -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x - -python3 -m atom.entrypoints.openai_server \ - --model $MODEL \ - --server-port $PORT \ - -tp $TP \ - --kv_cache_dtype fp8 $CALCULATED_MAX_MODEL_LEN $EP \ - --trust-remote-code \ - > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -export PYTHONDONTWRITEBYTECODE=1 -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/kimik2.5_int4_b200.sh b/benchmarks/single_node/fixed_seq_len/kimik2.5_int4_b200.sh deleted file mode 100755 index cbef22d67..000000000 --- a/benchmarks/single_node/fixed_seq_len/kimik2.5_int4_b200.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -nvidia-smi - -export PYTHONNOUSERSITE=1 -export VLLM_USE_FLASHINFER_MOE_INT4=1 - -SERVER_LOG=/workspace/server.log - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve $MODEL --host 0.0.0.0 --port $PORT \ ---gpu-memory-utilization 0.95 \ ---tensor-parallel-size $TP \ ---max-model-len $MAX_MODEL_LEN \ ---max-num-seqs $CONC \ ---reasoning-parser kimi_k2 \ ---tool-call-parser kimi_k2 \ ---compilation_config.pass_config.fuse_allreduce_rms true \ ---trust-remote-code \ ---no-enable-prefix-caching > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $(( CONC * 10 )) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/kimik2.5_int4_b300.sh b/benchmarks/single_node/fixed_seq_len/kimik2.5_int4_b300.sh deleted file mode 100755 index e66140141..000000000 --- a/benchmarks/single_node/fixed_seq_len/kimik2.5_int4_b300.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env bash - -# NOTE: At the time of submission, https://docs.vllm.ai/projects/recipes/en/latest/moonshotai/Kimi-K2.5.html -# does not have a B300-specific recipe, so this script reuses the existing -# Kimi-K2.5 INT4 B200 vLLM recipe as-is until B300-specific tuning is available. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - - -nvidia-smi - -export PYTHONNOUSERSITE=1 -export VLLM_USE_FLASHINFER_MOE_INT4=1 - -SERVER_LOG=/workspace/server.log - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve $MODEL_PATH --served-model-name $MODEL --host 0.0.0.0 --port $PORT \ ---gpu-memory-utilization 0.95 \ ---tensor-parallel-size $TP \ ---max-model-len $MAX_MODEL_LEN \ ---max-num-seqs $CONC \ ---reasoning-parser kimi_k2 \ ---tool-call-parser kimi_k2 \ ---compilation_config.pass_config.fuse_allreduce_rms true \ ---trust-remote-code \ ---no-enable-prefix-caching > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $(( CONC * 10 )) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/kimik2.5_int4_h200.sh b/benchmarks/single_node/fixed_seq_len/kimik2.5_int4_h200.sh deleted file mode 100755 index 1f18032ff..000000000 --- a/benchmarks/single_node/fixed_seq_len/kimik2.5_int4_h200.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -nvidia-smi - -export PYTHONNOUSERSITE=1 - -SERVER_LOG=/workspace/server.log - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi - -# following https://docs.vllm.ai/projects/recipes/en/latest/moonshotai/Kimi-K2.5.html recipe - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve $MODEL --host 0.0.0.0 --port $PORT \ ---gpu-memory-utilization 0.95 \ ---tensor-parallel-size $TP \ ---max-model-len $MAX_MODEL_LEN \ ---max-num-seqs $CONC \ ---reasoning-parser kimi_k2 \ ---tool-call-parser kimi_k2 \ ---compilation_config.pass_config.fuse_allreduce_rms true \ ---trust-remote-code \ ---no-enable-prefix-caching > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts $(( CONC * 10 )) \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/kimik2.5_int4_mi300x.sh b/benchmarks/single_node/fixed_seq_len/kimik2.5_int4_mi300x.sh deleted file mode 100755 index bb5145a66..000000000 --- a/benchmarks/single_node/fixed_seq_len/kimik2.5_int4_mi300x.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -# Set HIP_VISIBLE_DEVICES to match ROCR_VISIBLE_DEVICES for Ray compatibility in vLLM 0.14+ -if [ -n "$ROCR_VISIBLE_DEVICES" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -SERVER_LOG=/workspace/server.log - -# following AMD andy luo's recipe -# https://x.com/linluo77/status/2017024513595301985 -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -export VLLM_ROCM_USE_AITER=1 -vllm serve $MODEL --port $PORT \ ---tensor-parallel-size=$TP \ ---gpu-memory-utilization 0.95 \ ---max-model-len $MAX_MODEL_LEN \ ---block-size=64 \ ---trust-remote-code \ ---no-enable-prefix-caching \ ---max-num-seqs 256 \ ---mm-encoder-tp-mode data > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/kimik2.5_int4_mi325x.sh b/benchmarks/single_node/fixed_seq_len/kimik2.5_int4_mi325x.sh deleted file mode 100755 index bb5145a66..000000000 --- a/benchmarks/single_node/fixed_seq_len/kimik2.5_int4_mi325x.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -# Set HIP_VISIBLE_DEVICES to match ROCR_VISIBLE_DEVICES for Ray compatibility in vLLM 0.14+ -if [ -n "$ROCR_VISIBLE_DEVICES" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -SERVER_LOG=/workspace/server.log - -# following AMD andy luo's recipe -# https://x.com/linluo77/status/2017024513595301985 -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -export VLLM_ROCM_USE_AITER=1 -vllm serve $MODEL --port $PORT \ ---tensor-parallel-size=$TP \ ---gpu-memory-utilization 0.95 \ ---max-model-len $MAX_MODEL_LEN \ ---block-size=64 \ ---trust-remote-code \ ---no-enable-prefix-caching \ ---max-num-seqs 256 \ ---mm-encoder-tp-mode data > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/kimik2.5_int4_mi355x.sh b/benchmarks/single_node/fixed_seq_len/kimik2.5_int4_mi355x.sh deleted file mode 100755 index dc16f1e53..000000000 --- a/benchmarks/single_node/fixed_seq_len/kimik2.5_int4_mi355x.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -# Set HIP_VISIBLE_DEVICES to match ROCR_VISIBLE_DEVICES for Ray compatibility in vLLM 0.14+ -if [ -n "$ROCR_VISIBLE_DEVICES" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -SERVER_LOG=/workspace/server.log - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -export VLLM_ROCM_USE_AITER=1 -vllm serve $MODEL --port $PORT \ ---tensor-parallel-size=$TP \ ---gpu-memory-utilization 0.95 \ ---max-model-len $MAX_MODEL_LEN \ ---block-size=64 \ ---trust-remote-code \ ---no-enable-prefix-caching \ ---max-num-seqs 256 \ ---moe-backend flydsl \ ---compilation-config '{"pass_config": {"fuse_allreduce_rms": false}}' \ ---mm-encoder-tp-mode data > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b200.sh b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b200.sh deleted file mode 100755 index 966448b0c..000000000 --- a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b200.sh +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env bash - -# MiniMax-M3 NVFP4 B200 single-node vLLM recipe. -# Same shape as minimaxm3_fp8_b200.sh but uses the nvidia/MiniMax-M3-NVFP4 -# checkpoint. MiniMax-M3 modelopt NVFP4 support (vllm-project/vllm PR #46380) is -# baked into the perf container image, so no runtime patch is needed. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -# launch_b200-dgxc.sh rewrites MODEL to the pre-downloaded path; only download -# when handed a bare HF id (b200-cw / b200-nb runners). -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -SERVER_LOG=/workspace/server.log - -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export VLLM_FLOAT32_MATMUL_PRECISION=high -export VLLM_FLASHINFER_ALLREDUCE_BACKEND=trtllm - -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS="--tensor-parallel-size=1 --data-parallel-size=$TP --enable-expert-parallel" -elif [ "$EP_SIZE" -gt 1 ]; then - PARALLEL_ARGS="--tensor-parallel-size=$TP --enable-expert-parallel" -else - PARALLEL_ARGS="--tensor-parallel-size=$TP" -fi - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi -start_gpu_monitor - -set -x -vllm serve $MODEL --port $PORT \ -$PARALLEL_ARGS \ ---gpu-memory-utilization 0.95 \ ---max-model-len $MAX_MODEL_LEN \ ---kv-cache-dtype fp8 \ ---block-size 128 \ ---language-model-only \ ---max-cudagraph-capture-size 2048 \ ---max-num-batched-tokens "$((ISL * 2 ))" \ ---stream-interval 20 --no-enable-prefix-caching \ ---trust-remote-code > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b200_mtp.sh b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b200_mtp.sh deleted file mode 100755 index 94ee39083..000000000 --- a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b200_mtp.sh +++ /dev/null @@ -1,107 +0,0 @@ -#!/usr/bin/env bash - -# MiniMax-M3 NVFP4 B200 single-node vLLM recipe with EAGLE3 speculative -# decoding — same shape as minimaxm3_fp8_b200_mtp.sh but uses the -# nvidia/MiniMax-M3-NVFP4 checkpoint. MiniMax-M3 modelopt NVFP4 support -# (vllm-project/vllm PR #46380) is baked into the perf container image, so no -# runtime patch is needed. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3" - -# launch_b200-dgxc.sh rewrites MODEL to the pre-downloaded path; only download -# the target when handed a bare HF id (b200-cw / b200-nb runners). The EAGLE3 -# draft is never pre-staged, so fetch it either way: next to the target weights -# when MODEL is a local path, into the HF cache otherwise. -if [[ "$MODEL" != /* ]]; then - hf download "$MODEL" - hf download "$DRAFT_MODEL" - DRAFT_MODEL_PATH="$DRAFT_MODEL" -else - DRAFT_MODEL_PATH="$(dirname "$MODEL")/${DRAFT_MODEL##*/}" - if [[ ! -d "$DRAFT_MODEL_PATH" || -z "$(ls -A "$DRAFT_MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$DRAFT_MODEL" --local-dir "$DRAFT_MODEL_PATH" - fi -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -SERVER_LOG=/workspace/server.log - -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export VLLM_FLOAT32_MATMUL_PRECISION=high - -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS="--tensor-parallel-size=1 --data-parallel-size=$TP --enable-expert-parallel" -elif [ "$EP_SIZE" -gt 1 ]; then - PARALLEL_ARGS="--tensor-parallel-size=$TP --enable-expert-parallel" -else - PARALLEL_ARGS="--tensor-parallel-size=$TP" -fi - -# use 3 speculative tokens for all configs for now -NUM_SPEC_TOKENS=3 - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi -start_gpu_monitor - -set -x -vllm serve $MODEL --port $PORT \ -$PARALLEL_ARGS \ ---gpu-memory-utilization 0.90 \ ---max-model-len $MAX_MODEL_LEN \ ---block-size 128 \ ---language-model-only \ ---max-cudagraph-capture-size 2048 \ ---max-num-batched-tokens "$((ISL * 2 ))" \ ---speculative-config "{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL_PATH\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"attention_backend\": \"FLASH_ATTN\"}" \ ---stream-interval 20 --no-enable-prefix-caching \ ---trust-remote-code > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code \ - --use-chat-template - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b300.sh b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b300.sh deleted file mode 100755 index f91419edb..000000000 --- a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b300.sh +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env bash - -# MiniMax-M3 NVFP4 B300 single-node vLLM recipe. -# Same shape as minimaxm3_fp8_b300.sh but uses the nvidia/MiniMax-M3-NVFP4 -# checkpoint. MiniMax-M3 modelopt NVFP4 support (vllm-project/vllm PR #46380) is -# baked into the perf container image, so no runtime patch is needed. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -SERVER_LOG=/workspace/server.log - -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export VLLM_FLOAT32_MATMUL_PRECISION=high -export VLLM_FLASHINFER_ALLREDUCE_BACKEND=trtllm - -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS="--tensor-parallel-size=1 --data-parallel-size=$TP --enable-expert-parallel" -elif [ "$EP_SIZE" -gt 1 ]; then - PARALLEL_ARGS="--tensor-parallel-size=$TP --enable-expert-parallel" -else - PARALLEL_ARGS="--tensor-parallel-size=$TP" -fi - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi -start_gpu_monitor - -set -x -vllm serve "$MODEL_PATH" --served-model-name "$MODEL" --host 0.0.0.0 --port $PORT \ -$PARALLEL_ARGS \ ---gpu-memory-utilization 0.95 \ ---max-model-len $MAX_MODEL_LEN \ ---kv-cache-dtype fp8 \ ---block-size 128 \ ---language-model-only \ ---max-cudagraph-capture-size 2048 \ ---max-num-batched-tokens "$((ISL * 2 ))" \ ---stream-interval 20 --no-enable-prefix-caching \ ---trust-remote-code > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b300_mtp.sh b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b300_mtp.sh deleted file mode 100755 index 74cbcd020..000000000 --- a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_b300_mtp.sh +++ /dev/null @@ -1,112 +0,0 @@ -#!/usr/bin/env bash - -# MiniMax-M3 NVFP4 B300 single-node vLLM recipe with EAGLE3 speculative -# decoding — same shape as minimaxm3_fp8_b300_mtp.sh but uses the -# nvidia/MiniMax-M3-NVFP4 checkpoint. MiniMax-M3 modelopt NVFP4 support -# (vllm-project/vllm PR #46380) is baked into the perf container image, so no -# runtime patch is needed. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3" - -# The target weights are launched from MODEL_PATH (the b300 launcher points it -# at the pre-staged read-only /scratch/models/MiniMax-M3-NVFP4). The EAGLE3 -# draft is not pre-staged and must be downloaded, so it cannot live next to the -# read-only target — fetch it into the writable models dir (/data/models) -# instead. When MODEL_PATH is unset (stand-alone runs) fall back to the HF cache. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi - DRAFT_MODEL_PATH="/data/models/${DRAFT_MODEL##*/}" - if [[ ! -d "$DRAFT_MODEL_PATH" || -z "$(ls -A "$DRAFT_MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$DRAFT_MODEL" --local-dir "$DRAFT_MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" - hf download "$DRAFT_MODEL" - DRAFT_MODEL_PATH="$DRAFT_MODEL" -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -SERVER_LOG=/workspace/server.log - -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export VLLM_FLOAT32_MATMUL_PRECISION=high - -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS="--tensor-parallel-size=1 --data-parallel-size=$TP --enable-expert-parallel" -elif [ "$EP_SIZE" -gt 1 ]; then - PARALLEL_ARGS="--tensor-parallel-size=$TP --enable-expert-parallel" -else - PARALLEL_ARGS="--tensor-parallel-size=$TP" -fi - -# use 3 speculative tokens for all configs for now -NUM_SPEC_TOKENS=3 - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi -start_gpu_monitor - -set -x -vllm serve "$MODEL_PATH" --served-model-name "$MODEL" --host 0.0.0.0 --port $PORT \ -$PARALLEL_ARGS \ ---gpu-memory-utilization 0.90 \ ---max-model-len $MAX_MODEL_LEN \ ---block-size 128 \ ---language-model-only \ ---max-cudagraph-capture-size 2048 \ ---max-num-batched-tokens "$((ISL * 2 ))" \ ---speculative-config "{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL_PATH\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"attention_backend\": \"FLASH_ATTN\"}" \ ---stream-interval 20 --no-enable-prefix-caching \ ---trust-remote-code > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code \ - --use-chat-template - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_mi355x_atom.sh b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_mi355x_atom.sh deleted file mode 100644 index dfb1e7a9f..000000000 --- a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_mi355x_atom.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE \ - DP_ATTENTION - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -SERVER_LOG=/workspace/server.log - -PARALLEL_ARGS=(-tp "$TP") #TP -if [ "$DP_ATTENTION" = "true" ]; then - if [ "$EP_SIZE" -gt 1 ]; then #DP+EP - PARALLEL_ARGS=(-tp "$TP" --enable-expert-parallel --enable-dp-attention ) - else #DP+TP - PARALLEL_ARGS=(-tp "$TP" --enable-dp-attention ) - fi -fi - -SPEC_ARGS=() -OPT_ARGS=(--online_quant_config '{"global_quant_config": "ptpc_fp8", "exclude_layer": ["lm_head", "model.embed_tokens", "vision_tower", "multi_modal_projector", "patch_merge_mlp", "*block_sparse_moe"]}') - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor -MEM_FRAC_STATIC=0.8 - -set -x -export AITER_QUICK_REDUCE_QUANTIZATION=INT4 -export ATOM_FORCE_ATTN_TRITON=1 -export MAX_MODEL_LEN=32768 -export MAX_NUM_BATCHED_TOKENS=32768 -export MAX_NUM_SEQS=256 -python3 -m atom.entrypoints.openai_server \ - --model $MODEL \ - --server-port $PORT \ - "${PARALLEL_ARGS[@]}" \ - "${SPEC_ARGS[@]}" \ - "${OPT_ARGS[@]}" \ - --block-size 128 \ - --gpu-memory-utilization $MEM_FRAC_STATIC \ - --max-model-len $MAX_MODEL_LEN \ - --max-num-batched-tokens $MAX_NUM_BATCHED_TOKENS \ - --max-num-seqs $MAX_NUM_SEQS \ - --kv_cache_dtype fp8 \ - --trust-remote-code \ - --no-enable_prefix_caching \ - > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -export PYTHONDONTWRITEBYTECODE=1 -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code $( [[ ${#SPEC_ARGS[@]} -gt 0 ]] && echo "--use-chat-template" ) - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x \ No newline at end of file diff --git a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_mi355x_atom_mtp.sh b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_mi355x_atom_mtp.sh deleted file mode 100644 index 4ef60e71e..000000000 --- a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_mi355x_atom_mtp.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE \ - DP_ATTENTION - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -SERVER_LOG=/workspace/server.log - -PARALLEL_ARGS=(-tp "$TP") #TP -if [ "$DP_ATTENTION" = "true" ]; then - if [ "$EP_SIZE" -gt 1 ]; then #DP+EP - PARALLEL_ARGS=(-tp "$TP" --enable-expert-parallel --enable-dp-attention ) - else #DP+TP - PARALLEL_ARGS=(-tp "$TP" --enable-dp-attention ) - fi -fi - -SPEC_ARGS=(--method eagle3 --draft-model Inferact/MiniMax-M3-EAGLE3 --num-speculative-tokens 3 ) -OPT_ARGS=(--online_quant_config '{"global_quant_config": "ptpc_fp8", "exclude_layer": ["lm_head", "model.embed_tokens", "vision_tower", "multi_modal_projector", "patch_merge_mlp", "*block_sparse_moe"]}') - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor -MEM_FRAC_STATIC=0.8 - -set -x -export AITER_QUICK_REDUCE_QUANTIZATION=INT4 -export ATOM_FORCE_ATTN_TRITON=1 -export MAX_MODEL_LEN=32768 -export MAX_NUM_BATCHED_TOKENS=32768 -export MAX_NUM_SEQS=256 -python3 -m atom.entrypoints.openai_server \ - --model $MODEL \ - --server-port $PORT \ - "${PARALLEL_ARGS[@]}" \ - "${SPEC_ARGS[@]}" \ - "${OPT_ARGS[@]}" \ - --block-size 128 \ - --gpu-memory-utilization $MEM_FRAC_STATIC \ - --max-model-len $MAX_MODEL_LEN \ - --max-num-batched-tokens $MAX_NUM_BATCHED_TOKENS \ - --max-num-seqs $MAX_NUM_SEQS \ - --kv_cache_dtype fp8 \ - --trust-remote-code \ - --no-enable_prefix_caching \ - > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -export PYTHONDONTWRITEBYTECODE=1 -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code $( [[ ${#SPEC_ARGS[@]} -gt 0 ]] && echo "--use-chat-template" ) - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x \ No newline at end of file diff --git a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_mi355x_vllm.sh b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_mi355x_vllm.sh deleted file mode 100755 index 4be977a80..000000000 --- a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_mi355x_vllm.sh +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/env bash - -# MiniMax-M3 MXFP4 MI355X (gfx950) single-node vLLM recipe. -# https://huggingface.co/amd/MiniMax-M3-MXFP4#reproduction -# Block size 128 is mandatory for MSA. This fixed-sequence benchmark uses the -# text-only language-model path with AITER MoE (vllm-project/vllm#46419). - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -if [ -n "$ROCR_VISIBLE_DEVICES" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -SERVER_LOG=/workspace/server.log -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export VLLM_USE_BREAKABLE_CUDAGRAPH=0 -export VLLM_ROCM_USE_AITER=1 -export VLLM_ROCM_USE_AITER_MOE=1 -export VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context -fi - -PARALLEL_ARGS=(--tensor-parallel-size "$TP") -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS=( - --tensor-parallel-size 1 - --data-parallel-size "$TP" - --enable-expert-parallel - ) -elif [ "$EP_SIZE" -gt 1 ]; then - PARALLEL_ARGS+=(--enable-expert-parallel) -fi - -start_gpu_monitor - -set -x -vllm serve "$MODEL" --port "$PORT" \ - "${PARALLEL_ARGS[@]}" \ - --trust-remote-code \ - --block-size 128 \ - --no-enable-prefix-caching \ - --language-model-only \ - --max-model-len "$MAX_MODEL_LEN" \ - --attention-backend TRITON_ATTN \ - --moe-backend aiter \ - --tool-call-parser minimax_m3 \ - --enable-auto-tool-choice \ - --reasoning-parser minimax_m3 > "$SERVER_LOG" 2>&1 & - -SERVER_PID=$! -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_mi355x_vllm_mtp.sh b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_mi355x_vllm_mtp.sh deleted file mode 100755 index 8a15b8c89..000000000 --- a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_mi355x_vllm_mtp.sh +++ /dev/null @@ -1,111 +0,0 @@ -#!/usr/bin/env bash - -# MiniMax-M3 MXFP4 MI355X (gfx950) single-node vLLM recipe with EAGLE3 -# speculative decoding. This is the spec-decoding=mtp variant of -# minimaxm3_fp4_mi355x_vllm.sh and uses three speculative tokens from -# Inferact/MiniMax-M3-EAGLE3. The pinned nightly includes upstream AMD -# MiniMax-M3 SupportsEagle3 support, so no runtime model patch is needed. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3" - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi -hf download "$DRAFT_MODEL" - -if [ -n "$ROCR_VISIBLE_DEVICES" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -SERVER_LOG=/workspace/server.log -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export VLLM_USE_BREAKABLE_CUDAGRAPH=0 -# Use AITER MoE for the MXFP4 experts, matching minimaxm3_fp4_mi355x_vllm.sh. -# This is required for ALL configs including expert parallelism: with EP enabled -# and moe_backend=auto, the AITER MXFP4 backend is skipped and selection falls -# back to Mxfp4MoeBackend.EMULATION, which triggers a first-time build of the -# Quark hw-emulation C++ kernel (kernel_ext, 9 ROCm arches) on every worker at -# warmup. Concurrent EP workers deadlock on the shared torch_extensions build -# lock, hanging engine-core for hours. Forcing --moe-backend aiter selects the -# AITER_MXFP4_MXFP4 backend instead (verified working under TP4+EP4 with EAGLE3 -# spec decoding), avoiding the emulation build entirely. -export VLLM_ROCM_USE_AITER=1 -export VLLM_ROCM_USE_AITER_MOE=1 -export VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context -fi - -PARALLEL_ARGS=(--tensor-parallel-size "$TP") -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS=( - --tensor-parallel-size 1 - --data-parallel-size "$TP" - --enable-expert-parallel - ) -elif [ "$EP_SIZE" -gt 1 ]; then - PARALLEL_ARGS+=(--enable-expert-parallel) -fi - -NUM_SPEC_TOKENS=3 - -start_gpu_monitor - -set -x -vllm serve "$MODEL" --port "$PORT" \ - "${PARALLEL_ARGS[@]}" \ - --trust-remote-code \ - --block-size 128 \ - --no-enable-prefix-caching \ - --language-model-only \ - --max-model-len "$MAX_MODEL_LEN" \ - --attention-backend TRITON_ATTN \ - --moe-backend aiter \ - --speculative-config "{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS}" \ - --tool-call-parser minimax_m3 \ - --enable-auto-tool-choice \ - --reasoning-parser minimax_m3 > "$SERVER_LOG" 2>&1 & - -SERVER_PID=$! -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code \ - --use-chat-template - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_b200.sh b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_b200.sh deleted file mode 100755 index 7ac314a09..000000000 --- a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_b200.sh +++ /dev/null @@ -1,132 +0,0 @@ -#!/usr/bin/env bash - -# MiniMax-M3 MXFP8 B200 single-node vLLM recipe -# (https://recipes.vllm.ai/MiniMaxAI/MiniMax-M3). 427B/26B-active MoE with MSA -# sparse attention. --block-size 128 is mandatory (MSA sparse_block_size is -# 128; the default 16 misaligns sparse indexing). The benchmark is text-only, -# so --language-model-only skips the vision encoder and frees VRAM for KV. -# dp-attn=true maps to DP×EP (DEP) per the recipe's "DP8 + Expert Parallel" -# layout; ep>1 maps to TP+EP (TEP). - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -# The 0618 image keeps MiniMax M3 top-k indices in a persistent -# [head_kv, max_tokens, topK] buffer for CUDA graphs. Slicing that buffer to -# the actual prefill length is non-contiguous when TP leaves multiple local KV -# heads, and the MSA CSR builder rejects it. Materialize the slice until the -# image includes this fix. -python3 - <<'PYEOF' || { echo "MiniMax M3 MSA contiguity patch failed" >&2; exit 1; } -import importlib.util -import pathlib - -spec = importlib.util.find_spec("vllm") -if spec is None or not spec.submodule_search_locations: - raise RuntimeError("Could not locate the installed vllm package") - -target = ( - pathlib.Path(next(iter(spec.submodule_search_locations))) - / "models" - / "minimax_m3" - / "nvidia" - / "sparse_attention_msa.py" -) -src = target.read_text() -old = " prefill_topk = topk[:, nd:num_tokens, :]\n" -new = " prefill_topk = topk[:, nd:num_tokens, :].contiguous()\n" - -if new in src: - print(f"[minimax-m3-msa-patch] already applied: {target}") -elif src.count(old) == 1: - target.write_text(src.replace(old, new, 1)) - print(f"[minimax-m3-msa-patch] patched: {target}") -else: - raise RuntimeError(f"Expected exactly one patch anchor in {target}") -PYEOF - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -# launch_b200-dgxc.sh rewrites MODEL to the pre-downloaded -# /lustre/fsw/gharunners/models/MiniMax-M3-MXFP8 path; only download when -# handed a bare HF id (b200-cw / b200-nb runners). -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -SERVER_LOG=/workspace/server.log - -# 444 GB of MXFP8 weights off shared FS; engine startup can exceed the -# default 600s readiness window. -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export VLLM_FLOAT32_MATMUL_PRECISION=high - -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS="--tensor-parallel-size=1 --data-parallel-size=$TP --enable-expert-parallel" -elif [ "$EP_SIZE" -gt 1 ]; then - PARALLEL_ARGS="--tensor-parallel-size=$TP --enable-expert-parallel" -else - PARALLEL_ARGS="--tensor-parallel-size=$TP" -fi - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve $MODEL --port $PORT \ -$PARALLEL_ARGS \ ---gpu-memory-utilization 0.90 \ ---max-model-len $MAX_MODEL_LEN \ ---block-size 128 \ ---attention-config '{"backend": "FLASHINFER", "use_trtllm_attention": true}' \ ---attention-config.indexer_kv_dtype "fp8" \ ---kv-cache-dtype fp8 \ ---language-model-only \ ---max-cudagraph-capture-size 2048 \ ---max-num-batched-tokens "$((ISL * 2 ))" \ ---stream-interval 20 --no-enable-prefix-caching \ ---trust-remote-code > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_b200_mtp.sh b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_b200_mtp.sh deleted file mode 100644 index 51147129d..000000000 --- a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_b200_mtp.sh +++ /dev/null @@ -1,160 +0,0 @@ -#!/usr/bin/env bash - -# MiniMax-M3 MXFP8 B200 single-node vLLM recipe with EAGLE3 speculative -# decoding — the repo's spec-decoding=mtp variant of minimaxm3_fp8_b200.sh -# (https://recipes.vllm.ai/MiniMaxAI/MiniMax-M3). Adds the -# Inferact/MiniMax-M3-EAGLE3 draft head via --speculative-config with 3 -# speculative tokens. Everything else keeps the non-MTP serve shape: -# --block-size 128 is mandatory (MSA sparse_block_size is 128; the default 16 -# misaligns sparse indexing), and --language-model-only skips the vision -# encoder for the text-only benchmark. dp-attn=true maps to DP×EP (DEP); -# ep>1 maps to TP+EP (TEP). -# -# The target uses the FlashInfer TRT-LLM attention path. The EAGLE3 drafter is -# pinned separately to TRITON_ATTN. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -# The 0618 image keeps MiniMax M3 top-k indices in a persistent -# [head_kv, max_tokens, topK] buffer for CUDA graphs. Slicing that buffer to -# the actual prefill length is non-contiguous when TP leaves multiple local KV -# heads, and the MSA CSR builder rejects it. Materialize the slice until the -# image includes this fix. -python3 - <<'PYEOF' || { echo "MiniMax M3 MSA contiguity patch failed" >&2; exit 1; } -import importlib.util -import pathlib - -spec = importlib.util.find_spec("vllm") -if spec is None or not spec.submodule_search_locations: - raise RuntimeError("Could not locate the installed vllm package") - -target = ( - pathlib.Path(next(iter(spec.submodule_search_locations))) - / "models" - / "minimax_m3" - / "nvidia" - / "sparse_attention_msa.py" -) -src = target.read_text() -old = " prefill_topk = topk[:, nd:num_tokens, :]\n" -new = " prefill_topk = topk[:, nd:num_tokens, :].contiguous()\n" - -if new in src: - print(f"[minimax-m3-msa-patch] already applied: {target}") -elif src.count(old) == 1: - target.write_text(src.replace(old, new, 1)) - print(f"[minimax-m3-msa-patch] patched: {target}") -else: - raise RuntimeError(f"Expected exactly one patch anchor in {target}") -PYEOF - -DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3" - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -# launch_b200-dgxc.sh rewrites MODEL to the pre-downloaded -# /lustre/fsw/gharunners/models/MiniMax-M3-MXFP8 path; only download the target -# when handed a bare HF id (b200-cw / b200-nb runners). The EAGLE3 draft is -# never pre-staged, so fetch it either way: next to the target weights when -# MODEL is a local path (the gharunners tree is writable), into the HF cache -# otherwise. -if [[ "$MODEL" != /* ]]; then - hf download "$MODEL" - hf download "$DRAFT_MODEL" - DRAFT_MODEL_PATH="$DRAFT_MODEL" -else - DRAFT_MODEL_PATH="$(dirname "$MODEL")/${DRAFT_MODEL##*/}" - if [[ ! -d "$DRAFT_MODEL_PATH" || -z "$(ls -A "$DRAFT_MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$DRAFT_MODEL" --local-dir "$DRAFT_MODEL_PATH" - fi -fi - -SERVER_LOG=/workspace/server.log - -# 444 GB of MXFP8 weights off shared FS; engine startup can exceed the -# default 600s readiness window. -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export VLLM_FLOAT32_MATMUL_PRECISION=high - -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS="--tensor-parallel-size=1 --data-parallel-size=$TP --enable-expert-parallel" -elif [ "$EP_SIZE" -gt 1 ]; then - PARALLEL_ARGS="--tensor-parallel-size=$TP --enable-expert-parallel" -else - PARALLEL_ARGS="--tensor-parallel-size=$TP" -fi - -# use 3 speculative tokens for all configs for now -NUM_SPEC_TOKENS=3 - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve $MODEL --port $PORT \ -$PARALLEL_ARGS \ ---gpu-memory-utilization 0.90 \ ---max-model-len $MAX_MODEL_LEN \ ---block-size 128 \ ---attention-config '{"backend": "FLASHINFER", "use_trtllm_attention": true}' \ ---attention-config.indexer_kv_dtype "fp8" \ ---kv-cache-dtype fp8 \ ---language-model-only \ ---max-cudagraph-capture-size 2048 \ ---max-num-batched-tokens "$((ISL * 2 ))" \ ---speculative-config "{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL_PATH\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"attention_backend\": \"TRITON_ATTN\"}" \ ---stream-interval 20 --no-enable-prefix-caching \ ---trust-remote-code > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -# Spec-decode acceptance rate degrades on raw random tokens; route prompts -# through the chat template as the other MTP recipes do. -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code \ - --use-chat-template - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_b300.sh b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_b300.sh deleted file mode 100755 index c0ee15f1f..000000000 --- a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_b300.sh +++ /dev/null @@ -1,140 +0,0 @@ -#!/usr/bin/env bash - -# MiniMax-M3 MXFP8 B300 single-node vLLM recipe -# (https://recipes.vllm.ai/MiniMaxAI/MiniMax-M3). Same shape as the B200 -# script, but follows the b300 launcher's MODEL/MODEL_PATH split: -# launch_b300-nv.sh keeps MODEL as the HF id and points MODEL_PATH at -# /data/models/ (writable NFS) for models not in the SRE-staged -# /scratch/models list — MiniMax-M3 is not staged. --block-size 128 is -# mandatory (MSA sparse/index cache); the benchmark is text-only, so -# --language-model-only frees the vision encoder's VRAM. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -# The 0618 image keeps MiniMax M3 top-k indices in a persistent -# [head_kv, max_tokens, topK] buffer for CUDA graphs. Slicing that buffer to -# the actual prefill length is non-contiguous when TP leaves multiple local KV -# heads, and the MSA CSR builder rejects it. Materialize the slice until the -# image includes this fix. -python3 - <<'PYEOF' || { echo "MiniMax M3 MSA contiguity patch failed" >&2; exit 1; } -import importlib.util -import pathlib - -spec = importlib.util.find_spec("vllm") -if spec is None or not spec.submodule_search_locations: - raise RuntimeError("Could not locate the installed vllm package") - -target = ( - pathlib.Path(next(iter(spec.submodule_search_locations))) - / "models" - / "minimax_m3" - / "nvidia" - / "sparse_attention_msa.py" -) -src = target.read_text() -old = " prefill_topk = topk[:, nd:num_tokens, :]\n" -new = " prefill_topk = topk[:, nd:num_tokens, :].contiguous()\n" - -if new in src: - print(f"[minimax-m3-msa-patch] already applied: {target}") -elif src.count(old) == 1: - target.write_text(src.replace(old, new, 1)) - print(f"[minimax-m3-msa-patch] patched: {target}") -else: - raise RuntimeError(f"Expected exactly one patch anchor in {target}") -PYEOF - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE. -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -SERVER_LOG=/workspace/server.log - -# 444 GB of MXFP8 weights off shared FS; engine startup can exceed the -# default 600s readiness window. -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export VLLM_FLOAT32_MATMUL_PRECISION=high - -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS="--tensor-parallel-size=1 --data-parallel-size=$TP --enable-expert-parallel" -elif [ "$EP_SIZE" -gt 1 ]; then - PARALLEL_ARGS="--tensor-parallel-size=$TP --enable-expert-parallel" -else - PARALLEL_ARGS="--tensor-parallel-size=$TP" -fi - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve "$MODEL_PATH" --served-model-name "$MODEL" --host 0.0.0.0 --port $PORT \ -$PARALLEL_ARGS \ ---gpu-memory-utilization 0.90 \ ---max-model-len $MAX_MODEL_LEN \ ---block-size 128 \ ---attention-config '{"backend": "FLASHINFER", "use_trtllm_attention": true}' \ ---attention-config.indexer_kv_dtype "fp8" \ ---kv-cache-dtype fp8 \ ---language-model-only \ ---max-cudagraph-capture-size 2048 \ ---max-num-batched-tokens "$((ISL * 2 ))" \ ---stream-interval 20 --no-enable-prefix-caching \ ---trust-remote-code > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_b300_mtp.sh b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_b300_mtp.sh deleted file mode 100644 index 01bf23eb6..000000000 --- a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_b300_mtp.sh +++ /dev/null @@ -1,162 +0,0 @@ -#!/usr/bin/env bash - -# MiniMax-M3 MXFP8 B300 single-node vLLM recipe with EAGLE3 speculative -# decoding — the repo's spec-decoding=mtp variant of minimaxm3_fp8_b300.sh -# (https://recipes.vllm.ai/MiniMaxAI/MiniMax-M3). Adds the -# Inferact/MiniMax-M3-EAGLE3 draft head via --speculative-config with 3 -# speculative tokens. Everything else keeps the non-MTP serve shape: -# --block-size 128 is mandatory (MSA sparse/index cache); the benchmark is -# text-only, so --language-model-only frees the vision encoder's VRAM. -# -# The target uses the FlashInfer TRT-LLM attention path. The EAGLE3 drafter is -# pinned separately to TRITON_ATTN. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -# The 0618 image keeps MiniMax M3 top-k indices in a persistent -# [head_kv, max_tokens, topK] buffer for CUDA graphs. Slicing that buffer to -# the actual prefill length is non-contiguous when TP leaves multiple local KV -# heads, and the MSA CSR builder rejects it. Materialize the slice until the -# image includes this fix. -python3 - <<'PYEOF' || { echo "MiniMax M3 MSA contiguity patch failed" >&2; exit 1; } -import importlib.util -import pathlib - -spec = importlib.util.find_spec("vllm") -if spec is None or not spec.submodule_search_locations: - raise RuntimeError("Could not locate the installed vllm package") - -target = ( - pathlib.Path(next(iter(spec.submodule_search_locations))) - / "models" - / "minimax_m3" - / "nvidia" - / "sparse_attention_msa.py" -) -src = target.read_text() -old = " prefill_topk = topk[:, nd:num_tokens, :]\n" -new = " prefill_topk = topk[:, nd:num_tokens, :].contiguous()\n" - -if new in src: - print(f"[minimax-m3-msa-patch] already applied: {target}") -elif src.count(old) == 1: - target.write_text(src.replace(old, new, 1)) - print(f"[minimax-m3-msa-patch] patched: {target}") -else: - raise RuntimeError(f"Expected exactly one patch anchor in {target}") -PYEOF - -DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3" - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE. -# Either way, MODEL_PATH is what the server is launched with. The EAGLE3 -# draft follows the same split: it lands next to the target weights (writable -# /data/models on b300 via the launcher's MODEL/MODEL_PATH split) when -# MODEL_PATH is set, in the HF cache otherwise. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi - DRAFT_MODEL_PATH="$(dirname "$MODEL_PATH")/${DRAFT_MODEL##*/}" - if [[ ! -d "$DRAFT_MODEL_PATH" || -z "$(ls -A "$DRAFT_MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$DRAFT_MODEL" --local-dir "$DRAFT_MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" - hf download "$DRAFT_MODEL" - DRAFT_MODEL_PATH="$DRAFT_MODEL" -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -SERVER_LOG=/workspace/server.log - -# 444 GB of MXFP8 weights off shared FS; engine startup can exceed the -# default 600s readiness window. -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export VLLM_FLOAT32_MATMUL_PRECISION=high - -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS="--tensor-parallel-size=1 --data-parallel-size=$TP --enable-expert-parallel" -elif [ "$EP_SIZE" -gt 1 ]; then - PARALLEL_ARGS="--tensor-parallel-size=$TP --enable-expert-parallel" -else - PARALLEL_ARGS="--tensor-parallel-size=$TP" -fi - -# use 3 speculative tokens for all configs for now -NUM_SPEC_TOKENS=3 - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -vllm serve "$MODEL_PATH" --served-model-name "$MODEL" --host 0.0.0.0 --port $PORT \ -$PARALLEL_ARGS \ ---gpu-memory-utilization 0.90 \ ---max-model-len $MAX_MODEL_LEN \ ---block-size 128 \ ---attention-config '{"backend": "FLASHINFER", "use_trtllm_attention": true}' \ ---attention-config.indexer_kv_dtype "fp8" \ ---kv-cache-dtype fp8 \ ---language-model-only \ ---max-cudagraph-capture-size 2048 \ ---max-num-batched-tokens "$((ISL * 2 ))" \ ---speculative-config "{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL_PATH\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"attention_backend\": \"TRITON_ATTN\"}" \ ---stream-interval 20 --no-enable-prefix-caching \ ---trust-remote-code > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -# Spec-decode acceptance rate degrades on raw random tokens; route prompts -# through the chat template as the other MTP recipes do. -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code \ - --use-chat-template - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_h100.sh b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_h100.sh deleted file mode 100755 index d8a795987..000000000 --- a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_h100.sh +++ /dev/null @@ -1,131 +0,0 @@ -#!/usr/bin/env bash - -# MiniMax-M3 MXFP8 H100 single-node vLLM recipe -# (https://recipes.vllm.ai/MiniMaxAI/MiniMax-M3). 427B/26B-active MoE with MSA -# sparse attention. --block-size 128 is mandatory (MSA sparse_block_size is -# 128; the default 16 misaligns sparse indexing). The benchmark is text-only, -# so --language-model-only skips the vision encoder and frees VRAM for KV. -# dp-attn=true maps to DP×EP (DEP) per the recipe's "DP8 + Expert Parallel" -# layout; ep>1 maps to TP+EP (TEP). Hopper has no native MX tensor cores, so -# the MXFP8 MoE runs through vLLM's Hopper-compatible backends (Marlin / -# DeepGEMM) selected by the mxfp8 oracle in the minimax-m3 image. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -# The shared HF cache lives on a network FS; concurrent day-zero downloads of -# the same ~444 GB checkpoint from sibling nodes hit huggingface_hub's -# WeakFileLock "[Errno 116] Stale file handle" race. Retry the download (it -# resumes), then serve with HF_HUB_OFFLINE=1 so vllm's snapshot_download does -# a lock-free local-cache read instead of re-contending the lock files. -SERVE_OFFLINE=() -if [[ "$MODEL" != /* ]]; then - for attempt in 1 2 3 4 5; do - hf download "$MODEL" && break - if [ "$attempt" = 5 ]; then echo "hf download failed after $attempt attempts" >&2; exit 1; fi - echo "hf download attempt $attempt failed; retrying in 60s" >&2 - sleep 60 - done - SERVE_OFFLINE=(env HF_HUB_OFFLINE=1) -fi - -SERVER_LOG=/workspace/server.log - -export PYTHONNOUSERSITE=1 -# ~444 GB of MXFP8 weights off shared FS; engine startup can exceed the -# default 600s readiness window. -export VLLM_ENGINE_READY_TIMEOUT_S=3600 - -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS="--tensor-parallel-size=1 --data-parallel-size=$TP --enable-expert-parallel" -elif [ "$EP_SIZE" -gt 1 ]; then - PARALLEL_ARGS="--tensor-parallel-size=$TP --enable-expert-parallel" -else - PARALLEL_ARGS="--tensor-parallel-size=$TP" -fi - -# Fixed-seq-len runs don't need graphs past the request concurrency: capture -# up to the next power of two >= CONC (per-DP-rank batch is CONC/DP but ragged -# arrival makes the full CONC bound safer), capped at vLLM's 2048 ceiling. -CAPTURE_SIZE=4 -while (( CAPTURE_SIZE < CONC )); do CAPTURE_SIZE=$((CAPTURE_SIZE * 2)); done -(( CAPTURE_SIZE > 2048 )) && CAPTURE_SIZE=2048 - -# H100 DEP is weights-bound: every DP rank replicates the ~20 GB -# BF16-dequantized attention/dense/embedding weights next to its ~52 GB -# expert shard, and at gmu 0.90 KV-cache init fails outright at high conc -# (sweep 27441767143, conc 256: "No available memory for the cache blocks"). -# Claw back headroom: higher gpu-memory-utilization, and decode graphs -# capped at 2x the per-rank batch share instead of the full CONC bound. -GMU=0.90 -if [ "${DP_ATTENTION}" = "true" ]; then - GMU=0.94 - PER_RANK_BOUND=$(( 2 * ((CONC + TP - 1) / TP) )) - CAPTURE_SIZE=4 - while (( CAPTURE_SIZE < PER_RANK_BOUND )); do CAPTURE_SIZE=$((CAPTURE_SIZE * 2)); done - (( CAPTURE_SIZE > 2048 )) && CAPTURE_SIZE=2048 -fi - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -"${SERVE_OFFLINE[@]}" vllm serve $MODEL --port $PORT \ -$PARALLEL_ARGS \ ---gpu-memory-utilization $GMU \ ---max-model-len $MAX_MODEL_LEN \ ---block-size 128 \ ---language-model-only \ ---max-cudagraph-capture-size $CAPTURE_SIZE \ ---max-num-batched-tokens "$((ISL * 2 ))" \ ---stream-interval 20 --no-enable-prefix-caching \ ---trust-remote-code > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_h100_mtp.sh b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_h100_mtp.sh deleted file mode 100644 index 8d9f6333f..000000000 --- a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_h100_mtp.sh +++ /dev/null @@ -1,157 +0,0 @@ -#!/usr/bin/env bash - -# MiniMax-M3 MXFP8 H100 single-node vLLM recipe with EAGLE3 speculative -# decoding — the repo's spec-decoding=mtp variant of minimaxm3_fp8_h100.sh -# (https://recipes.vllm.ai/MiniMaxAI/MiniMax-M3). Adds the -# Inferact/MiniMax-M3-EAGLE3 draft head via --speculative-config with 3 -# speculative tokens. Everything else keeps the non-MTP serve shape: -# --block-size 128 is mandatory (MSA sparse_block_size is 128), the benchmark -# is text-only so --language-model-only frees the vision encoder's VRAM, and -# the MXFP8 MoE runs through vLLM's Hopper-compatible backends. H100 is -# TP8-only (~56 GB of weights per 80 GB GPU below TP8 leaves no room). -# -# The drafter is pinned to FLASH_ATTN: the EAGLE3 head is MHA, and FlashInfer -# only supports page size 128 through its trtllm-gen kernel, which requires -# GQA/MQA — engine init dies in FlashInferMetadataBuilder otherwise (hit on -# the B300 MTP canary). FLASH_ATTN takes any multiple-of-16 block size, so -# the mandatory 128 is fine for the draft. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3" - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -# The shared HF cache lives on a network FS; concurrent day-zero downloads of -# the same ~444 GB checkpoint from sibling nodes hit huggingface_hub's -# WeakFileLock "[Errno 116] Stale file handle" race. Retry the download (it -# resumes), then serve with HF_HUB_OFFLINE=1 so vllm's snapshot_download does -# a lock-free local-cache read instead of re-contending the lock files. The -# EAGLE3 draft is fetched the same way so the offline serve finds it cached. -SERVE_OFFLINE=() -if [[ "$MODEL" != /* ]]; then - for attempt in 1 2 3 4 5; do - hf download "$MODEL" && break - if [ "$attempt" = 5 ]; then echo "hf download failed after $attempt attempts" >&2; exit 1; fi - echo "hf download attempt $attempt failed; retrying in 60s" >&2 - sleep 60 - done - for attempt in 1 2 3 4 5; do - hf download "$DRAFT_MODEL" && break - if [ "$attempt" = 5 ]; then echo "hf download failed after $attempt attempts" >&2; exit 1; fi - echo "hf download attempt $attempt failed; retrying in 60s" >&2 - sleep 60 - done - SERVE_OFFLINE=(env HF_HUB_OFFLINE=1) -fi - -SERVER_LOG=/workspace/server.log - -export PYTHONNOUSERSITE=1 -# ~444 GB of MXFP8 weights off shared FS; engine startup can exceed the -# default 600s readiness window. -export VLLM_ENGINE_READY_TIMEOUT_S=3600 - -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS="--tensor-parallel-size=1 --data-parallel-size=$TP --enable-expert-parallel" -elif [ "$EP_SIZE" -gt 1 ]; then - PARALLEL_ARGS="--tensor-parallel-size=$TP --enable-expert-parallel" -else - PARALLEL_ARGS="--tensor-parallel-size=$TP" -fi - -# use 3 speculative tokens for all configs for now -NUM_SPEC_TOKENS=3 - -# Fixed-seq-len runs don't need graphs past the decode step's token count: -# with spec decoding every running request contributes 1 + NUM_SPEC_TOKENS -# tokens per step, so capture up to the next power of two >= -# CONC * (1 + NUM_SPEC_TOKENS), capped at vLLM's 2048 ceiling. -CAPTURE_SIZE=4 -while (( CAPTURE_SIZE < CONC * (1 + NUM_SPEC_TOKENS) )); do CAPTURE_SIZE=$((CAPTURE_SIZE * 2)); done -(( CAPTURE_SIZE > 2048 )) && CAPTURE_SIZE=2048 - -# H100 DEP is weights-bound: every DP rank replicates the ~20 GB -# BF16-dequantized attention/dense/embedding weights next to its ~52 GB -# expert shard, and at gmu 0.90 KV-cache init fails outright at high conc -# (sweep 27441767143, conc 256: "No available memory for the cache blocks"). -# Claw back headroom: higher gpu-memory-utilization, and decode graphs -# capped at 2x the per-rank batch share (spec-token-scaled) instead of the -# full CONC bound. -GMU=0.90 -if [ "${DP_ATTENTION}" = "true" ]; then - GMU=0.94 - PER_RANK_BOUND=$(( 2 * ((CONC + TP - 1) / TP) * (1 + NUM_SPEC_TOKENS) )) - CAPTURE_SIZE=4 - while (( CAPTURE_SIZE < PER_RANK_BOUND )); do CAPTURE_SIZE=$((CAPTURE_SIZE * 2)); done - (( CAPTURE_SIZE > 2048 )) && CAPTURE_SIZE=2048 -fi - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -"${SERVE_OFFLINE[@]}" vllm serve $MODEL --port $PORT \ -$PARALLEL_ARGS \ ---gpu-memory-utilization $GMU \ ---max-model-len $MAX_MODEL_LEN \ ---block-size 128 \ ---language-model-only \ ---max-cudagraph-capture-size $CAPTURE_SIZE \ ---max-num-batched-tokens "$((ISL * 2 ))" \ ---speculative-config "{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"attention_backend\": \"FLASH_ATTN\"}" \ ---stream-interval 20 --no-enable-prefix-caching \ ---trust-remote-code > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -# Spec-decode acceptance rate degrades on raw random tokens; route prompts -# through the chat template as the other MTP recipes do. -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code \ - --use-chat-template - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_h200.sh b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_h200.sh deleted file mode 100755 index 057c0c230..000000000 --- a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_h200.sh +++ /dev/null @@ -1,116 +0,0 @@ -#!/usr/bin/env bash - -# MiniMax-M3 MXFP8 H200 single-node vLLM recipe -# (https://recipes.vllm.ai/MiniMaxAI/MiniMax-M3). 427B/26B-active MoE with MSA -# sparse attention. --block-size 128 is mandatory (MSA sparse_block_size is -# 128; the default 16 misaligns sparse indexing). The benchmark is text-only, -# so --language-model-only skips the vision encoder and frees VRAM for KV. -# dp-attn=true maps to DP×EP (DEP) per the recipe's "DP8 + Expert Parallel" -# layout; ep>1 maps to TP+EP (TEP). Hopper has no native MX tensor cores, so -# the MXFP8 MoE runs through vLLM's Hopper-compatible backends (Marlin / -# DeepGEMM) selected by the mxfp8 oracle in the minimax-m3 image. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -# The shared HF cache lives on a network FS; concurrent day-zero downloads of -# the same ~444 GB checkpoint from sibling nodes hit huggingface_hub's -# WeakFileLock "[Errno 116] Stale file handle" race. Retry the download (it -# resumes), then serve with HF_HUB_OFFLINE=1 so vllm's snapshot_download does -# a lock-free local-cache read instead of re-contending the lock files. -SERVE_OFFLINE=() -if [[ "$MODEL" != /* ]]; then - for attempt in 1 2 3 4 5; do - hf download "$MODEL" && break - if [ "$attempt" = 5 ]; then echo "hf download failed after $attempt attempts" >&2; exit 1; fi - echo "hf download attempt $attempt failed; retrying in 60s" >&2 - sleep 60 - done - SERVE_OFFLINE=(env HF_HUB_OFFLINE=1) -fi - -SERVER_LOG=/workspace/server.log - -export PYTHONNOUSERSITE=1 -# ~444 GB of MXFP8 weights off shared FS; engine startup can exceed the -# default 600s readiness window. -export VLLM_ENGINE_READY_TIMEOUT_S=3600 - -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS="--tensor-parallel-size=1 --data-parallel-size=$TP --enable-expert-parallel" -elif [ "$EP_SIZE" -gt 1 ]; then - PARALLEL_ARGS="--tensor-parallel-size=$TP --enable-expert-parallel" -else - PARALLEL_ARGS="--tensor-parallel-size=$TP" -fi - -# Fixed-seq-len runs don't need graphs past the request concurrency: capture -# up to the next power of two >= CONC (per-DP-rank batch is CONC/DP but ragged -# arrival makes the full CONC bound safer), capped at vLLM's 2048 ceiling. -CAPTURE_SIZE=4 -while (( CAPTURE_SIZE < CONC )); do CAPTURE_SIZE=$((CAPTURE_SIZE * 2)); done -(( CAPTURE_SIZE > 2048 )) && CAPTURE_SIZE=2048 - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -"${SERVE_OFFLINE[@]}" vllm serve $MODEL --port $PORT \ -$PARALLEL_ARGS \ ---gpu-memory-utilization 0.90 \ ---max-model-len $MAX_MODEL_LEN \ ---block-size 128 \ ---language-model-only \ ---max-cudagraph-capture-size $CAPTURE_SIZE \ ---max-num-batched-tokens "$((ISL * 2 ))" \ ---stream-interval 20 --no-enable-prefix-caching \ ---trust-remote-code > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_h200_mtp.sh b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_h200_mtp.sh deleted file mode 100644 index 2bc98d4b8..000000000 --- a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_h200_mtp.sh +++ /dev/null @@ -1,140 +0,0 @@ -#!/usr/bin/env bash - -# MiniMax-M3 MXFP8 H200 single-node vLLM recipe with EAGLE3 speculative -# decoding — the repo's spec-decoding=mtp variant of minimaxm3_fp8_h200.sh -# (https://recipes.vllm.ai/MiniMaxAI/MiniMax-M3). Adds the -# Inferact/MiniMax-M3-EAGLE3 draft head via --speculative-config with 3 -# speculative tokens. Everything else keeps the non-MTP serve shape: -# --block-size 128 is mandatory (MSA sparse_block_size is 128), the benchmark -# is text-only so --language-model-only frees the vision encoder's VRAM, and -# the MXFP8 MoE runs through vLLM's Hopper-compatible backends. -# -# The drafter is pinned to FLASH_ATTN: the EAGLE3 head is MHA, and FlashInfer -# only supports page size 128 through its trtllm-gen kernel, which requires -# GQA/MQA — engine init dies in FlashInferMetadataBuilder otherwise (hit on -# the B300 MTP canary). FLASH_ATTN takes any multiple-of-16 block size, so -# the mandatory 128 is fine for the draft. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3" - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -# The shared HF cache lives on a network FS; concurrent day-zero downloads of -# the same ~444 GB checkpoint from sibling nodes hit huggingface_hub's -# WeakFileLock "[Errno 116] Stale file handle" race. Retry the download (it -# resumes), then serve with HF_HUB_OFFLINE=1 so vllm's snapshot_download does -# a lock-free local-cache read instead of re-contending the lock files. The -# EAGLE3 draft is fetched the same way so the offline serve finds it cached. -SERVE_OFFLINE=() -if [[ "$MODEL" != /* ]]; then - for attempt in 1 2 3 4 5; do - hf download "$MODEL" && break - if [ "$attempt" = 5 ]; then echo "hf download failed after $attempt attempts" >&2; exit 1; fi - echo "hf download attempt $attempt failed; retrying in 60s" >&2 - sleep 60 - done - for attempt in 1 2 3 4 5; do - hf download "$DRAFT_MODEL" && break - if [ "$attempt" = 5 ]; then echo "hf download failed after $attempt attempts" >&2; exit 1; fi - echo "hf download attempt $attempt failed; retrying in 60s" >&2 - sleep 60 - done - SERVE_OFFLINE=(env HF_HUB_OFFLINE=1) -fi - -SERVER_LOG=/workspace/server.log - -export PYTHONNOUSERSITE=1 -# ~444 GB of MXFP8 weights off shared FS; engine startup can exceed the -# default 600s readiness window. -export VLLM_ENGINE_READY_TIMEOUT_S=3600 - -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS="--tensor-parallel-size=1 --data-parallel-size=$TP --enable-expert-parallel" -elif [ "$EP_SIZE" -gt 1 ]; then - PARALLEL_ARGS="--tensor-parallel-size=$TP --enable-expert-parallel" -else - PARALLEL_ARGS="--tensor-parallel-size=$TP" -fi - -# use 3 speculative tokens for all configs for now -NUM_SPEC_TOKENS=3 - -# Fixed-seq-len runs don't need graphs past the decode step's token count: -# with spec decoding every running request contributes 1 + NUM_SPEC_TOKENS -# tokens per step, so capture up to the next power of two >= -# CONC * (1 + NUM_SPEC_TOKENS), capped at vLLM's 2048 ceiling. -CAPTURE_SIZE=4 -while (( CAPTURE_SIZE < CONC * (1 + NUM_SPEC_TOKENS) )); do CAPTURE_SIZE=$((CAPTURE_SIZE * 2)); done -(( CAPTURE_SIZE > 2048 )) && CAPTURE_SIZE=2048 - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -"${SERVE_OFFLINE[@]}" vllm serve $MODEL --port $PORT \ -$PARALLEL_ARGS \ ---gpu-memory-utilization 0.90 \ ---max-model-len $MAX_MODEL_LEN \ ---block-size 128 \ ---language-model-only \ ---max-cudagraph-capture-size $CAPTURE_SIZE \ ---max-num-batched-tokens "$((ISL * 2 ))" \ ---speculative-config "{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"attention_backend\": \"FLASH_ATTN\"}" \ ---stream-interval 20 --no-enable-prefix-caching \ ---trust-remote-code > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -# Spec-decode acceptance rate degrades on raw random tokens; route prompts -# through the chat template as the other MTP recipes do. -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code \ - --use-chat-template - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_mi300x.sh b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_mi300x.sh deleted file mode 100755 index f2cdaf284..000000000 --- a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_mi300x.sh +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env bash - -# MiniMax-M3 MXFP8 MI300X (gfx942) single-node vLLM recipe. -# Reuses the dedicated ROCm image and the MI355X serving shape. Block size 128 -# is mandatory for MSA sparse attention. Keep the default BF16 KV cache on -# gfx942: the checkpoint has no calibrated q/prob scales for ROCm FP8 -# attention, and vLLM's fallback scale of 1.0 corrupts model accuracy. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -if [ -n "$ROCR_VISIBLE_DEVICES" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -SERVER_LOG=/workspace/server.log -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export VLLM_USE_BREAKABLE_CUDAGRAPH=0 - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context -fi - -PARALLEL_ARGS=(--tensor-parallel-size "$TP") -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS=( - --tensor-parallel-size 1 - --data-parallel-size "$TP" - --enable-expert-parallel - ) -elif [ "$EP_SIZE" -gt 1 ]; then - PARALLEL_ARGS+=(--enable-expert-parallel) -fi - -start_gpu_monitor - -set -x -vllm serve "$MODEL" --port "$PORT" \ - "${PARALLEL_ARGS[@]}" \ - --block-size 128 \ - --no-enable-prefix-caching \ - --language-model-only \ - --max-model-len "$MAX_MODEL_LEN" \ - --attention-backend TRITON_ATTN \ - --tool-call-parser minimax_m3 \ - --reasoning-parser minimax_m3 \ - --enable-auto-tool-choice > "$SERVER_LOG" 2>&1 & - -SERVER_PID=$! -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_mi300x_mtp.sh b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_mi300x_mtp.sh deleted file mode 100644 index 64f4b1fa1..000000000 --- a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_mi300x_mtp.sh +++ /dev/null @@ -1,115 +0,0 @@ -#!/usr/bin/env bash - -# MiniMax-M3 MXFP8 MI300X (gfx942) single-node vLLM recipe with EAGLE3 -# speculative decoding — the spec-decoding=mtp variant of -# minimaxm3_fp8_mi300x.sh. Adds the Inferact/MiniMax-M3-EAGLE3 draft head via -# --speculative-config with 3 speculative tokens. Everything else mirrors the -# non-MTP MI300X recipe: mandatory --block-size 128, --language-model-only for -# the text-only benchmark, --attention-backend TRITON_ATTN, and -# --no-enable-prefix-caching. Runs with CUDA graphs (no --enforce-eager); -# VLLM_USE_BREAKABLE_CUDAGRAPH=0 avoids the M3-decode breakable-cudagraph path. -# FP8 KV cache reduces memory pressure and increases concurrency headroom. -# -# Unlike the CUDA recipes, the drafter needs no attention_backend override: -# the FlashInfer "page size 128 requires GQA/MQA" limitation that forced -# FLASH_ATTN for the EAGLE3 MHA head on Blackwell is FlashInfer/CUDA-specific. -# Here the whole server runs on TRITON_ATTN (set globally below), which serves -# the MHA draft fine. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3" - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -# MODEL is a bare HF id on the mi300x single-node runner (a fast cache hit when -# pre-staged). The EAGLE3 draft is not staged; fetch it into the same cache. -if [[ "$MODEL" != /* ]]; then - hf download "$MODEL" - hf download "$DRAFT_MODEL" -fi - -if [ -n "$ROCR_VISIBLE_DEVICES" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -SERVER_LOG=/workspace/server.log -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export VLLM_USE_BREAKABLE_CUDAGRAPH=0 - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context -fi - -PARALLEL_ARGS=(--tensor-parallel-size "$TP") -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS=( - --tensor-parallel-size 1 - --data-parallel-size "$TP" - --enable-expert-parallel - ) -elif [ "$EP_SIZE" -gt 1 ]; then - PARALLEL_ARGS+=(--enable-expert-parallel) -fi - -# use 3 speculative tokens for all configs for now -NUM_SPEC_TOKENS=3 - -start_gpu_monitor - -set -x -vllm serve "$MODEL" --port "$PORT" \ - "${PARALLEL_ARGS[@]}" \ - --block-size 128 \ - --kv-cache-dtype fp8 \ - --no-enable-prefix-caching \ - --language-model-only \ - --max-model-len "$MAX_MODEL_LEN" \ - --attention-backend TRITON_ATTN \ - --speculative-config "{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS}" \ - --tool-call-parser minimax_m3 \ - --reasoning-parser minimax_m3 \ - --enable-auto-tool-choice > "$SERVER_LOG" 2>&1 & - -SERVER_PID=$! -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -# Spec-decode acceptance rate degrades on raw random tokens; route prompts -# through the chat template as the other MTP recipes do. -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code \ - --use-chat-template - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_mi325x.sh b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_mi325x.sh deleted file mode 100755 index 6290722fe..000000000 --- a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_mi325x.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env bash - -# MiniMax-M3 MXFP8 MI325X (gfx942) single-node vLLM recipe. -# https://recipes.vllm.ai/MiniMaxAI/MiniMax-M3?hardware=mi325x&variant=mxfp8 -# MXFP8 runs from TP=4 on gfx942; block size 128 is mandatory for MSA. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -if [ -n "$ROCR_VISIBLE_DEVICES" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -SERVER_LOG=/workspace/server.log -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export VLLM_USE_BREAKABLE_CUDAGRAPH=0 - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context -fi - -PARALLEL_ARGS=(--tensor-parallel-size "$TP") -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS=( - --tensor-parallel-size 1 - --data-parallel-size "$TP" - --enable-expert-parallel - ) -elif [ "$EP_SIZE" -gt 1 ]; then - PARALLEL_ARGS+=(--enable-expert-parallel) -fi - -start_gpu_monitor - -set -x -vllm serve "$MODEL" --port "$PORT" \ - "${PARALLEL_ARGS[@]}" \ - --block-size 128 \ - --language-model-only \ - --max-model-len "$MAX_MODEL_LEN" \ - --attention-backend TRITON_ATTN \ - --no-enable-prefix-caching \ - --tool-call-parser minimax_m3 \ - --reasoning-parser minimax_m3 \ - --enable-auto-tool-choice > "$SERVER_LOG" 2>&1 & - -SERVER_PID=$! -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_mi325x_mtp.sh b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_mi325x_mtp.sh deleted file mode 100644 index 4ba15e761..000000000 --- a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_mi325x_mtp.sh +++ /dev/null @@ -1,214 +0,0 @@ -#!/usr/bin/env bash - -# MiniMax-M3 MXFP8 MI325X (gfx942) single-node vLLM recipe with EAGLE3 -# speculative decoding — the spec-decoding=mtp variant of -# minimaxm3_fp8_mi325x.sh. Adds the Inferact/MiniMax-M3-EAGLE3 draft head via -# --speculative-config with 3 speculative tokens. Everything else mirrors the -# non-MTP MI325X recipe: mandatory --block-size 128, --language-model-only for -# the text-only benchmark, --attention-backend TRITON_ATTN, and -# --no-enable-prefix-caching. Runs with CUDA graphs (no --enforce-eager); -# VLLM_USE_BREAKABLE_CUDAGRAPH=0 avoids the M3-decode breakable-cudagraph path. -# The default BF16 KV cache is retained (unlike the MI355X recipe's FP8 KV -# cache): gfx942 has no calibrated q/prob scales for ROCm FP8 attention and -# vLLM's fallback scale of 1.0 corrupts accuracy. -# -# Unlike the CUDA recipes, the drafter needs no attention_backend override: -# the FlashInfer "page size 128 requires GQA/MQA" limitation that forced -# FLASH_ATTN for the EAGLE3 MHA head on Blackwell is FlashInfer/CUDA-specific. -# Here the whole server runs on TRITON_ATTN (set globally below), which serves -# the MHA draft fine. -# -# [AI generated draft test] The shipped vllm/vllm-openai-rocm:minimax-m3 image -# does NOT implement SupportsEagle3 on the AMD MiniMax-M3 model, so EAGLE3 -# engine init fails with "Model does not support EAGLE3 interface but -# aux_hidden_state_outputs was requested". This recipe applies that fix -# (functionstackx/vllm#1 — ported from nvidia/model.py, upstreamed as -# vllm-project/vllm#45546) in-place to the installed vllm before serving, so we -# can validate EAGLE3 on real MI325X hardware ahead of an image rebuild. The -# same patch is validated green on MI355X. It is idempotent and fails the job -# loudly if the installed amd/model.py has drifted from the expected base. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3" - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -# MODEL is a bare HF id on the mi325x single-node runner (a fast cache hit when -# pre-staged). The EAGLE3 draft is not staged; fetch it into the same cache. -if [[ "$MODEL" != /* ]]; then - hf download "$MODEL" - hf download "$DRAFT_MODEL" -fi - -if [ -n "$ROCR_VISIBLE_DEVICES" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -SERVER_LOG=/workspace/server.log -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export VLLM_USE_BREAKABLE_CUDAGRAPH=0 - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context -fi - -PARALLEL_ARGS=(--tensor-parallel-size "$TP") -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS=( - --tensor-parallel-size 1 - --data-parallel-size "$TP" - --enable-expert-parallel - ) -elif [ "$EP_SIZE" -gt 1 ]; then - PARALLEL_ARGS+=(--enable-expert-parallel) -fi - -# use 3 speculative tokens for all configs for now -NUM_SPEC_TOKENS=3 - -# [AI generated draft test] Patch the installed AMD MiniMax-M3 model to add the -# SupportsEagle3 interface (functionstackx/vllm#1, upstream vllm-project/vllm#45546). -# Mirrors nvidia/model.py: adds EagleModelMixin to the inner model + -# aux-hidden-state emission, and SupportsEagle3 to the two outer classes. -# Idempotent; hard-fails if the installed file has drifted from the expected -# base (so we never silently run unpatched and mislabel the result). -python3 - <<'PYEOF' || { echo "EAGLE3 in-place patch failed" >&2; exit 1; } -import ast, importlib.util, pathlib, sys - -spec = importlib.util.find_spec("vllm") -root = pathlib.Path(spec.submodule_search_locations[0]) -target = root / "models" / "minimax_m3" / "amd" / "model.py" -src = target.read_text() - -if "EagleModelMixin" in src and "class MiniMaxM3Model(nn.Module, EagleModelMixin):" in src: - print(f"[eagle3-patch] already applied: {target}") - sys.exit(0) - -edits = [ - ( - "from vllm.model_executor.models.interfaces import (\n" - " MultiModalEmbeddings,\n" - " SupportsMultiModal,\n" - ")", - "from vllm.model_executor.models.interfaces import (\n" - " EagleModelMixin,\n" - " MultiModalEmbeddings,\n" - " SupportsEagle3,\n" - " SupportsMultiModal,\n" - ")", - ), - ( - "class MiniMaxM3Model(nn.Module):", - "class MiniMaxM3Model(nn.Module, EagleModelMixin):", - ), - ( - " inputs_embeds: torch.Tensor | None = None,\n" - " ) -> torch.Tensor:\n" - " if inputs_embeds is not None:", - " inputs_embeds: torch.Tensor | None = None,\n" - " ) -> torch.Tensor | tuple[torch.Tensor, list[torch.Tensor]]:\n" - " if inputs_embeds is not None:", - ), - ( - " residual = None\n\n" - " for layer in self.layers[self.start_layer : self.end_layer]:\n" - " hidden_states, residual = layer(positions, hidden_states, residual)\n\n" - " hidden_states, _ = self.norm(hidden_states, residual)\n" - " return hidden_states", - " residual = None\n\n" - " # EAGLE3 is not yet compatible with pipeline parallel\n" - " aux_hidden_states = self._maybe_add_hidden_state([], 0, hidden_states, residual)\n" - " for idx, layer in enumerate(self.layers[self.start_layer : self.end_layer]):\n" - " hidden_states, residual = layer(positions, hidden_states, residual)\n" - " self._maybe_add_hidden_state(\n" - " aux_hidden_states, idx + 1, hidden_states, residual\n" - " )\n\n" - " hidden_states, _ = self.norm(hidden_states, residual)\n\n" - " if len(aux_hidden_states) > 0:\n" - " return hidden_states, aux_hidden_states\n" - " return hidden_states", - ), - ( - "class MiniMaxM3SparseForCausalLM(nn.Module):", - "class MiniMaxM3SparseForCausalLM(nn.Module, SupportsEagle3):", - ), - ( - "class MiniMaxM3SparseForConditionalGeneration(nn.Module, SupportsMultiModal):", - "class MiniMaxM3SparseForConditionalGeneration(\n" - " nn.Module, SupportsMultiModal, SupportsEagle3\n" - "):", - ), -] - -for old, new in edits: - count = src.count(old) - if count != 1: - sys.exit( - f"[eagle3-patch] anchor matched {count} times (expected 1); " - f"installed {target} has drifted from the expected base — aborting" - ) - src = src.replace(old, new) - -ast.parse(src) -target.write_text(src) -print(f"[eagle3-patch] applied EAGLE3 support to {target}") -PYEOF - -start_gpu_monitor - -set -x -vllm serve "$MODEL" --port "$PORT" \ - "${PARALLEL_ARGS[@]}" \ - --block-size 128 \ - --no-enable-prefix-caching \ - --language-model-only \ - --max-model-len "$MAX_MODEL_LEN" \ - --attention-backend TRITON_ATTN \ - --speculative-config "{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS}" \ - --tool-call-parser minimax_m3 \ - --reasoning-parser minimax_m3 \ - --enable-auto-tool-choice > "$SERVER_LOG" 2>&1 & - -SERVER_PID=$! -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -# Spec-decode acceptance rate degrades on raw random tokens; route prompts -# through the chat template as the other MTP recipes do. -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code \ - --use-chat-template - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_mi355x.sh b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_mi355x.sh deleted file mode 100755 index 89c136c27..000000000 --- a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_mi355x.sh +++ /dev/null @@ -1,106 +0,0 @@ -#!/usr/bin/env bash - -# MiniMax-M3 MXFP8 MI355X (gfx950) single-node vLLM recipe. -# https://github.com/vllm-project/recipes/commit/2a3728ed9892debfd767a72a58ebc90b33f186e5 -# The recipe recommends MXFP8 from TP=4 on gfx950 and requires block size 128. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -if [ -n "$ROCR_VISIBLE_DEVICES" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -SERVER_LOG=/workspace/server.log -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export VLLM_USE_BREAKABLE_CUDAGRAPH=0 -# MI355X mxfp8 recipe (vllm-project/recipes#581): INT6 quick all-reduce plus -# the router-append shared-experts MoE fusion (vllm-project/vllm#46545). The -# fusion checks this env directly and runs on both the aiter and native MXFP8 -# MoE paths (it is independent of the AITER master switch, and self-disables -# under expert parallelism inside the model), so enable it unconditionally. -# (The AITER master switch itself is set below, gated on expert parallelism.) -export VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 -export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT6 - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context -fi - -PARALLEL_ARGS=(--tensor-parallel-size "$TP") -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS=( - --tensor-parallel-size 1 - --data-parallel-size "$TP" - --enable-expert-parallel - ) -elif [ "$EP_SIZE" -gt 1 ]; then - PARALLEL_ARGS+=(--enable-expert-parallel) -fi - -# Gate the AITER master switch on expert parallelism. With EP, the aiter fused -# MoE path is the auto-selected backend (no --moe-backend override). With EP -# disabled (TP-only) the AITER master switch produces degenerate MiniMax-M3 -# output, so leave it off and fall back to the native MXFP8 path (the -# shared-experts fusion set above still applies — it is master-independent). -if printf '%s\n' "${PARALLEL_ARGS[@]}" | grep -qxF -- '--enable-expert-parallel'; then - export VLLM_ROCM_USE_AITER=1 -else - export VLLM_ROCM_USE_AITER=0 -fi - -start_gpu_monitor - -set -x -vllm serve "$MODEL" --port "$PORT" \ - "${PARALLEL_ARGS[@]}" \ - --block-size 128 \ - --no-enable-prefix-caching \ - --language-model-only \ - --max-model-len "$MAX_MODEL_LEN" \ - --kv-cache-dtype fp8 \ - --attention-backend TRITON_ATTN \ - --tool-call-parser minimax_m3 \ - --reasoning-parser minimax_m3 \ - --enable-auto-tool-choice > "$SERVER_LOG" 2>&1 & - -SERVER_PID=$! -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_mi355x_atom.sh b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_mi355x_atom.sh deleted file mode 100644 index c5ce072fc..000000000 --- a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_mi355x_atom.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE \ - DP_ATTENTION - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -SERVER_LOG=/workspace/server.log - -PARALLEL_ARGS=(-tp "$TP") #TP -if [ "$DP_ATTENTION" = "true" ]; then - if [ "$EP_SIZE" -gt 1 ]; then #DP+EP - PARALLEL_ARGS=(-tp "$TP" --enable-expert-parallel --enable-dp-attention ) - else #DP+TP - PARALLEL_ARGS=(-tp "$TP" --enable-dp-attention ) - fi -fi - -SPEC_ARGS=() -OPT_ARGS=(--online_quant_config '{"global_quant_config": "ptpc_fp8", "exclude_layer": ["lm_head", "model.embed_tokens", "vision_tower", "multi_modal_projector", "patch_merge_mlp", "*block_sparse_moe"]}') - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor -MEM_FRAC_STATIC=0.8 - -set -x -export AITER_QUICK_REDUCE_QUANTIZATION=INT4 -export ATOM_FORCE_ATTN_TRITON=1 -export MAX_MODEL_LEN=32768 -export MAX_NUM_BATCHED_TOKENS=32768 -export MAX_NUM_SEQS=256 -python3 -m atom.entrypoints.openai_server \ - --model $MODEL \ - --server-port $PORT \ - "${PARALLEL_ARGS[@]}" \ - "${SPEC_ARGS[@]}" \ - "${OPT_ARGS[@]}" \ - --block-size 128 \ - --gpu-memory-utilization $MEM_FRAC_STATIC \ - --max-model-len $MAX_MODEL_LEN \ - --max-num-batched-tokens $MAX_NUM_BATCHED_TOKENS \ - --max-num-seqs $MAX_NUM_SEQS \ - --kv_cache_dtype fp8 \ - --trust-remote-code \ - --no-enable_prefix_caching \ - > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -export PYTHONDONTWRITEBYTECODE=1 -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code $( [[ ${#SPEC_ARGS[@]} -gt 0 ]] && echo "--use-chat-template" ) - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_mi355x_atom_mtp.sh b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_mi355x_atom_mtp.sh deleted file mode 100644 index 66320d03d..000000000 --- a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_mi355x_atom_mtp.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE \ - DP_ATTENTION - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -SERVER_LOG=/workspace/server.log - -PARALLEL_ARGS=(-tp "$TP") #TP -if [ "$DP_ATTENTION" = "true" ]; then - if [ "$EP_SIZE" -gt 1 ]; then #DP+EP - PARALLEL_ARGS=(-tp "$TP" --enable-expert-parallel --enable-dp-attention ) - else #DP+TP - PARALLEL_ARGS=(-tp "$TP" --enable-dp-attention ) - fi -fi - -SPEC_ARGS=(--method eagle3 --draft-model Inferact/MiniMax-M3-EAGLE3 --num-speculative-tokens 3 ) -OPT_ARGS=(--online_quant_config '{"global_quant_config": "ptpc_fp8", "exclude_layer": ["lm_head", "model.embed_tokens", "vision_tower", "multi_modal_projector", "patch_merge_mlp", "*block_sparse_moe"]}') - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor -MEM_FRAC_STATIC=0.8 - -set -x -export AITER_QUICK_REDUCE_QUANTIZATION=INT4 -export ATOM_FORCE_ATTN_TRITON=1 -export MAX_MODEL_LEN=32768 -export MAX_NUM_BATCHED_TOKENS=32768 -export MAX_NUM_SEQS=256 -python3 -m atom.entrypoints.openai_server \ - --model $MODEL \ - --server-port $PORT \ - "${PARALLEL_ARGS[@]}" \ - "${SPEC_ARGS[@]}" \ - "${OPT_ARGS[@]}" \ - --block-size 128 \ - --gpu-memory-utilization $MEM_FRAC_STATIC \ - --max-model-len $MAX_MODEL_LEN \ - --max-num-batched-tokens $MAX_NUM_BATCHED_TOKENS \ - --max-num-seqs $MAX_NUM_SEQS \ - --kv_cache_dtype fp8 \ - --trust-remote-code \ - --no-enable_prefix_caching \ - > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -export PYTHONDONTWRITEBYTECODE=1 -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code $( [[ ${#SPEC_ARGS[@]} -gt 0 ]] && echo "--use-chat-template" ) - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_mi355x_mtp.sh b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_mi355x_mtp.sh deleted file mode 100644 index 50a7d6d9f..000000000 --- a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp8_mi355x_mtp.sh +++ /dev/null @@ -1,231 +0,0 @@ -#!/usr/bin/env bash - -# MiniMax-M3 MXFP8 MI355X (gfx950) single-node vLLM recipe with EAGLE3 -# speculative decoding — the spec-decoding=mtp variant of -# minimaxm3_fp8_mi355x.sh. Adds the Inferact/MiniMax-M3-EAGLE3 draft head via -# --speculative-config with 3 speculative tokens. Everything else mirrors the -# non-MTP recipe: MXFP8 from TP=4 on gfx950, mandatory --block-size 128, -# --language-model-only for the text-only benchmark, FP8 KV cache, and -# --attention-backend TRITON_ATTN. Runs with CUDA graphs (no --enforce-eager); -# VLLM_USE_BREAKABLE_CUDAGRAPH=0 avoids the M3-decode breakable-cudagraph path. -# -# Unlike the CUDA recipes, the drafter needs no attention_backend override: -# the FlashInfer "page size 128 requires GQA/MQA" limitation that forced -# FLASH_ATTN for the EAGLE3 MHA head on Blackwell is FlashInfer/CUDA-specific. -# Here the whole server runs on TRITON_ATTN (set globally below), which serves -# the MHA draft fine. -# -# [AI generated draft test] The shipped vllm/vllm-openai-rocm:minimax-m3 image -# does NOT implement SupportsEagle3 on the AMD MiniMax-M3 model, so EAGLE3 -# engine init fails with "Model does not support EAGLE3 interface but -# aux_hidden_state_outputs was requested". This recipe applies that fix -# (functionstackx/vllm#1 — ported from nvidia/model.py) in-place to the -# installed vllm before serving, so we can validate EAGLE3 on real MI355X -# hardware ahead of an image rebuild. The patch is idempotent and fails the -# job loudly if the installed amd/model.py has drifted from the expected base. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - EP_SIZE \ - DP_ATTENTION \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3" - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -# MODEL stays a bare HF id on the mi355x single-node runner (weights are -# pre-staged in the mounted NFS HF cache, so this is a fast cache hit). The -# EAGLE3 draft is not staged; fetch it into the same cache. -if [[ "$MODEL" != /* ]]; then - hf download "$MODEL" - hf download "$DRAFT_MODEL" -fi - -if [ -n "$ROCR_VISIBLE_DEVICES" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -SERVER_LOG=/workspace/server.log -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -# Run with CUDA graphs (no --enforce-eager): VLLM_USE_BREAKABLE_CUDAGRAPH=0 -# avoids the M3-decode breakable-cudagraph path that previously forced eager. -export VLLM_USE_BREAKABLE_CUDAGRAPH=0 -# MI355X mxfp8 recipe (vllm-project/recipes#581): INT6 quick all-reduce plus -# the router-append shared-experts MoE fusion (vllm-project/vllm#46545). The -# fusion checks this env directly and runs on both the aiter and native MXFP8 -# MoE paths (it is independent of the AITER master switch, and self-disables -# under expert parallelism inside the model), so enable it unconditionally. -# (The AITER master switch itself is set below, gated on expert parallelism.) -export VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 -export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT6 - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context -fi - -PARALLEL_ARGS=(--tensor-parallel-size "$TP") -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS=( - --tensor-parallel-size 1 - --data-parallel-size "$TP" - --enable-expert-parallel - ) -elif [ "$EP_SIZE" -gt 1 ]; then - PARALLEL_ARGS+=(--enable-expert-parallel) -fi - -# Gate the AITER master switch on expert parallelism. With EP, the aiter fused -# MoE path is the auto-selected backend (no --moe-backend override). With EP -# disabled (TP-only) the AITER master switch produces degenerate MiniMax-M3 -# output, so leave it off and fall back to the native MXFP8 path (the -# shared-experts fusion set above still applies — it is master-independent). -if printf '%s\n' "${PARALLEL_ARGS[@]}" | grep -qxF -- '--enable-expert-parallel'; then - export VLLM_ROCM_USE_AITER=1 -else - export VLLM_ROCM_USE_AITER=0 -fi - -# use 3 speculative tokens for all configs for now -NUM_SPEC_TOKENS=3 - -# [AI generated draft test] Patch the installed AMD MiniMax-M3 model to add the -# SupportsEagle3 interface (functionstackx/vllm#1). Mirrors nvidia/model.py: -# adds EagleModelMixin to the inner model + aux-hidden-state emission, and -# SupportsEagle3 to the two outer classes. Idempotent; hard-fails if the -# installed file has drifted from the expected base (so we never silently run -# unpatched and mislabel the result). -python3 - <<'PYEOF' || { echo "EAGLE3 in-place patch failed" >&2; exit 1; } -import ast, importlib.util, pathlib, sys - -spec = importlib.util.find_spec("vllm") -root = pathlib.Path(spec.submodule_search_locations[0]) -target = root / "models" / "minimax_m3" / "amd" / "model.py" -src = target.read_text() - -if "EagleModelMixin" in src and "class MiniMaxM3Model(nn.Module, EagleModelMixin):" in src: - print(f"[eagle3-patch] already applied: {target}") - sys.exit(0) - -edits = [ - ( - "from vllm.model_executor.models.interfaces import (\n" - " MultiModalEmbeddings,\n" - " SupportsMultiModal,\n" - ")", - "from vllm.model_executor.models.interfaces import (\n" - " EagleModelMixin,\n" - " MultiModalEmbeddings,\n" - " SupportsEagle3,\n" - " SupportsMultiModal,\n" - ")", - ), - ( - "class MiniMaxM3Model(nn.Module):", - "class MiniMaxM3Model(nn.Module, EagleModelMixin):", - ), - ( - " inputs_embeds: torch.Tensor | None = None,\n" - " ) -> torch.Tensor:\n" - " if inputs_embeds is not None:", - " inputs_embeds: torch.Tensor | None = None,\n" - " ) -> torch.Tensor | tuple[torch.Tensor, list[torch.Tensor]]:\n" - " if inputs_embeds is not None:", - ), - ( - " residual = None\n\n" - " for layer in self.layers[self.start_layer : self.end_layer]:\n" - " hidden_states, residual = layer(positions, hidden_states, residual)\n\n" - " hidden_states, _ = self.norm(hidden_states, residual)\n" - " return hidden_states", - " residual = None\n\n" - " # EAGLE3 is not yet compatible with pipeline parallel\n" - " aux_hidden_states = self._maybe_add_hidden_state([], 0, hidden_states, residual)\n" - " for idx, layer in enumerate(self.layers[self.start_layer : self.end_layer]):\n" - " hidden_states, residual = layer(positions, hidden_states, residual)\n" - " self._maybe_add_hidden_state(\n" - " aux_hidden_states, idx + 1, hidden_states, residual\n" - " )\n\n" - " hidden_states, _ = self.norm(hidden_states, residual)\n\n" - " if len(aux_hidden_states) > 0:\n" - " return hidden_states, aux_hidden_states\n" - " return hidden_states", - ), - ( - "class MiniMaxM3SparseForCausalLM(nn.Module):", - "class MiniMaxM3SparseForCausalLM(nn.Module, SupportsEagle3):", - ), - ( - "class MiniMaxM3SparseForConditionalGeneration(nn.Module, SupportsMultiModal):", - "class MiniMaxM3SparseForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsEagle3):", - ), -] - -for old, new in edits: - count = src.count(old) - if count != 1: - sys.exit( - f"[eagle3-patch] anchor matched {count} times (expected 1); " - f"installed {target} has drifted from the expected base — aborting" - ) - src = src.replace(old, new) - -ast.parse(src) -target.write_text(src) -print(f"[eagle3-patch] applied EAGLE3 support to {target}") -PYEOF - -start_gpu_monitor - -set -x -vllm serve "$MODEL" --port "$PORT" \ - "${PARALLEL_ARGS[@]}" \ - --block-size 128 \ - --no-enable-prefix-caching \ - --language-model-only \ - --max-model-len "$MAX_MODEL_LEN" \ - --kv-cache-dtype fp8 \ - --attention-backend TRITON_ATTN \ - --speculative-config "{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS}" \ - --tool-call-parser minimax_m3 \ - --reasoning-parser minimax_m3 \ - --enable-auto-tool-choice > "$SERVER_LOG" 2>&1 & - -SERVER_PID=$! -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -# Spec-decode acceptance rate degrades on raw random tokens; route prompts -# through the chat template as the other MTP recipes do. -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code \ - --use-chat-template - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/qwen3.5_bf16_b200.sh b/benchmarks/single_node/fixed_seq_len/qwen3.5_bf16_b200.sh deleted file mode 100755 index 3f7c6a314..000000000 --- a/benchmarks/single_node/fixed_seq_len/qwen3.5_bf16_b200.sh +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -export NCCL_NVLS_ENABLE=1 -export SGL_ENABLE_JIT_DEEPGEMM=false -export SGLANG_ENABLE_FLASHINFER_GEMM=true -export PYTHONUNBUFFERED=1 - -SERVER_LOG=/workspace/server.log - -# Default: recv every ~10 requests; if CONC ≥ 16, relax to ~30 requests between scheduler recv polls. -if [[ $CONC -ge 16 ]]; then - SCHEDULER_RECV_INTERVAL=30 -else - SCHEDULER_RECV_INTERVAL=10 -fi - -MEM_FRAC_STATIC=0.82 -CHUNKED_PREFILL_SIZE=32768 -MAX_PREFILL_TOKENS=32768 -CUDA_GRAPH_MAX_BATCH_SIZE=$CONC -MAX_RUNNING_REQUESTS=128 -CONTEXT_LENGTH=$((ISL + OSL + 20)) -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - CONTEXT_LENGTH="$EVAL_MAX_MODEL_LEN" -fi - -echo "SCHEDULER_RECV_INTERVAL: $SCHEDULER_RECV_INTERVAL, CONC: $CONC, ISL: $ISL, OSL: $OSL" - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path=$MODEL --host=0.0.0.0 --port=$PORT \ ---served-model-name "Qwen/Qwen3.5-397B-A17B" --trust-remote-code \ ---tensor-parallel-size=$TP --data-parallel-size=1 --ep-size $EP_SIZE \ ---cuda-graph-max-bs $CUDA_GRAPH_MAX_BATCH_SIZE --max-running-requests $MAX_RUNNING_REQUESTS \ ---mem-fraction-static $MEM_FRAC_STATIC --chunked-prefill-size $CHUNKED_PREFILL_SIZE --max-prefill-tokens $MAX_PREFILL_TOKENS \ ---context-length $CONTEXT_LENGTH --disable-radix-cache \ ---attention-backend trtllm_mha --moe-runner-backend flashinfer_trtllm \ ---enable-flashinfer-allreduce-fusion --scheduler-recv-interval $SCHEDULER_RECV_INTERVAL \ ---tokenizer-worker-num 6 --stream-interval 30 > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/qwen3.5_bf16_b200_mtp.sh b/benchmarks/single_node/fixed_seq_len/qwen3.5_bf16_b200_mtp.sh deleted file mode 100755 index be314c872..000000000 --- a/benchmarks/single_node/fixed_seq_len/qwen3.5_bf16_b200_mtp.sh +++ /dev/null @@ -1,97 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -export NCCL_NVLS_ENABLE=1 -export SGL_ENABLE_JIT_DEEPGEMM=false -export SGLANG_ENABLE_FLASHINFER_GEMM=true -export PYTHONUNBUFFERED=1 - -SERVER_LOG=/workspace/server.log - -# Default: recv every ~10 requests; if CONC ≥ 16, relax to ~30 requests between scheduler recv polls. -if [[ $CONC -ge 16 ]]; then - SCHEDULER_RECV_INTERVAL=30 -else - SCHEDULER_RECV_INTERVAL=10 -fi - -MEM_FRAC_STATIC=0.82 -CHUNKED_PREFILL_SIZE=32768 -MAX_PREFILL_TOKENS=32768 -CUDA_GRAPH_MAX_BATCH_SIZE=$CONC -MAX_RUNNING_REQUESTS=128 -CONTEXT_LENGTH=$((ISL + OSL + 20)) -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - CONTEXT_LENGTH="$EVAL_MAX_MODEL_LEN" -fi - -echo "SCHEDULER_RECV_INTERVAL: $SCHEDULER_RECV_INTERVAL, CONC: $CONC, ISL: $ISL, OSL: $OSL" - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path=$MODEL --host=0.0.0.0 --port=$PORT \ ---served-model-name "Qwen/Qwen3.5-397B-A17B" --trust-remote-code \ ---tensor-parallel-size=$TP --data-parallel-size=1 --ep-size $EP_SIZE \ ---cuda-graph-max-bs $CUDA_GRAPH_MAX_BATCH_SIZE --max-running-requests $MAX_RUNNING_REQUESTS \ ---mem-fraction-static $MEM_FRAC_STATIC --chunked-prefill-size $CHUNKED_PREFILL_SIZE --max-prefill-tokens $MAX_PREFILL_TOKENS \ ---context-length $CONTEXT_LENGTH --disable-radix-cache \ ---attention-backend trtllm_mha --moe-runner-backend flashinfer_trtllm \ ---enable-flashinfer-allreduce-fusion --scheduler-recv-interval $SCHEDULER_RECV_INTERVAL \ ---tokenizer-worker-num 6 --stream-interval 30 \ ---speculative-algorithm EAGLE \ ---speculative-num-steps 3 \ ---speculative-eagle-topk 1 \ ---speculative-num-draft-tokens 4 \ -> $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --use-chat-template - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/qwen3.5_bf16_b300.sh b/benchmarks/single_node/fixed_seq_len/qwen3.5_bf16_b300.sh deleted file mode 100755 index 32e4197cb..000000000 --- a/benchmarks/single_node/fixed_seq_len/qwen3.5_bf16_b300.sh +++ /dev/null @@ -1,102 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - - -export NCCL_NVLS_ENABLE=1 -export SGL_ENABLE_JIT_DEEPGEMM=false -export SGLANG_ENABLE_FLASHINFER_GEMM=true -export PYTHONUNBUFFERED=1 - -SERVER_LOG=/workspace/server.log - -# Default: recv every ~10 requests; if CONC ≥ 16, relax to ~30 requests between scheduler recv polls. -if [[ $CONC -ge 16 ]]; then - SCHEDULER_RECV_INTERVAL=30 -else - SCHEDULER_RECV_INTERVAL=10 -fi - -MEM_FRAC_STATIC=0.82 -CHUNKED_PREFILL_SIZE=32768 -MAX_PREFILL_TOKENS=32768 -CUDA_GRAPH_MAX_BATCH_SIZE=$CONC -MAX_RUNNING_REQUESTS=128 -CONTEXT_LENGTH=$((ISL + OSL + 20)) -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - CONTEXT_LENGTH="$EVAL_MAX_MODEL_LEN" -fi - -echo "SCHEDULER_RECV_INTERVAL: $SCHEDULER_RECV_INTERVAL, CONC: $CONC, ISL: $ISL, OSL: $OSL" - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path $MODEL_PATH --host 0.0.0.0 --port $PORT \ ---served-model-name $MODEL --trust-remote-code \ ---tensor-parallel-size $TP --data-parallel-size 1 --ep-size $EP_SIZE \ ---cuda-graph-max-bs $CUDA_GRAPH_MAX_BATCH_SIZE --max-running-requests $MAX_RUNNING_REQUESTS \ ---mem-fraction-static $MEM_FRAC_STATIC --chunked-prefill-size $CHUNKED_PREFILL_SIZE --max-prefill-tokens $MAX_PREFILL_TOKENS \ ---context-length $CONTEXT_LENGTH --disable-radix-cache \ ---attention-backend trtllm_mha --mm-attention-backend triton_attn --moe-runner-backend flashinfer_trtllm \ ---enable-flashinfer-allreduce-fusion --scheduler-recv-interval $SCHEDULER_RECV_INTERVAL \ ---tokenizer-worker-num 6 --stream-interval 30 > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/qwen3.5_bf16_b300_mtp.sh b/benchmarks/single_node/fixed_seq_len/qwen3.5_bf16_b300_mtp.sh deleted file mode 100755 index 4e12d1284..000000000 --- a/benchmarks/single_node/fixed_seq_len/qwen3.5_bf16_b300_mtp.sh +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - - -export NCCL_NVLS_ENABLE=1 -export SGL_ENABLE_JIT_DEEPGEMM=false -export SGLANG_ENABLE_FLASHINFER_GEMM=true -export PYTHONUNBUFFERED=1 - -SERVER_LOG=/workspace/server.log - -# Default: recv every ~10 requests; if CONC ≥ 16, relax to ~30 requests between scheduler recv polls. -if [[ $CONC -ge 16 ]]; then - SCHEDULER_RECV_INTERVAL=30 -else - SCHEDULER_RECV_INTERVAL=10 -fi - -MEM_FRAC_STATIC=0.82 -CHUNKED_PREFILL_SIZE=32768 -MAX_PREFILL_TOKENS=32768 -CUDA_GRAPH_MAX_BATCH_SIZE=$CONC -MAX_RUNNING_REQUESTS=128 -CONTEXT_LENGTH=$((ISL + OSL + 20)) -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - CONTEXT_LENGTH="$EVAL_MAX_MODEL_LEN" -fi - -echo "SCHEDULER_RECV_INTERVAL: $SCHEDULER_RECV_INTERVAL, CONC: $CONC, ISL: $ISL, OSL: $OSL" - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path $MODEL_PATH --host 0.0.0.0 --port $PORT \ ---served-model-name $MODEL --trust-remote-code \ ---tensor-parallel-size $TP --data-parallel-size 1 --ep-size $EP_SIZE \ ---cuda-graph-max-bs $CUDA_GRAPH_MAX_BATCH_SIZE --max-running-requests $MAX_RUNNING_REQUESTS \ ---mem-fraction-static $MEM_FRAC_STATIC --chunked-prefill-size $CHUNKED_PREFILL_SIZE --max-prefill-tokens $MAX_PREFILL_TOKENS \ ---context-length $CONTEXT_LENGTH --disable-radix-cache \ ---attention-backend trtllm_mha --mm-attention-backend triton_attn --moe-runner-backend flashinfer_trtllm \ ---enable-flashinfer-allreduce-fusion --scheduler-recv-interval $SCHEDULER_RECV_INTERVAL \ ---tokenizer-worker-num 6 --stream-interval 30 \ ---speculative-algorithm EAGLE \ ---speculative-num-steps 3 \ ---speculative-eagle-topk 1 \ ---speculative-num-draft-tokens 4 \ -> $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --use-chat-template - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/qwen3.5_bf16_mi300x.sh b/benchmarks/single_node/fixed_seq_len/qwen3.5_bf16_mi300x.sh deleted file mode 100755 index 32fe60a73..000000000 --- a/benchmarks/single_node/fixed_seq_len/qwen3.5_bf16_mi300x.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -SERVER_LOG=/workspace/server.log -CONTEXT_LENGTH=$((ISL + OSL + 20)) -MAX_PREFILL_TOKENS=32768 - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -else EVAL_CONTEXT_ARGS="--context-length $CONTEXT_LENGTH" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -# following Andy Luo linkedin's recipe https://www.linkedin.com/feed/update/urn:li:activity:7429203734389280768/ -python3 -m sglang.launch_server \ - --attention-backend aiter \ - --model-path $MODEL \ - --host=0.0.0.0 \ - --port $PORT \ - --tensor-parallel-size $TP \ - --data-parallel-size 1 \ - --trust-remote-code \ - --tokenizer-worker-num 6 \ - --enable-aiter-allreduce-fusion \ - --cuda-graph-max-bs $CONC \ - --disable-radix-cache \ - --max-prefill-tokens $MAX_PREFILL_TOKENS \ - --scheduler-recv-interval 30 \ - --mem-fraction-static 0.75 $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/qwen3.5_bf16_mi325x.sh b/benchmarks/single_node/fixed_seq_len/qwen3.5_bf16_mi325x.sh deleted file mode 100644 index 32fe60a73..000000000 --- a/benchmarks/single_node/fixed_seq_len/qwen3.5_bf16_mi325x.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -SERVER_LOG=/workspace/server.log -CONTEXT_LENGTH=$((ISL + OSL + 20)) -MAX_PREFILL_TOKENS=32768 - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -else EVAL_CONTEXT_ARGS="--context-length $CONTEXT_LENGTH" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -# following Andy Luo linkedin's recipe https://www.linkedin.com/feed/update/urn:li:activity:7429203734389280768/ -python3 -m sglang.launch_server \ - --attention-backend aiter \ - --model-path $MODEL \ - --host=0.0.0.0 \ - --port $PORT \ - --tensor-parallel-size $TP \ - --data-parallel-size 1 \ - --trust-remote-code \ - --tokenizer-worker-num 6 \ - --enable-aiter-allreduce-fusion \ - --cuda-graph-max-bs $CONC \ - --disable-radix-cache \ - --max-prefill-tokens $MAX_PREFILL_TOKENS \ - --scheduler-recv-interval 30 \ - --mem-fraction-static 0.75 $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/qwen3.5_bf16_mi325x_mtp.sh b/benchmarks/single_node/fixed_seq_len/qwen3.5_bf16_mi325x_mtp.sh deleted file mode 100755 index e9df93c7d..000000000 --- a/benchmarks/single_node/fixed_seq_len/qwen3.5_bf16_mi325x_mtp.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - EP_SIZE \ - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -SERVER_LOG=/workspace/server.log -CONTEXT_LENGTH=$((ISL + OSL + 20)) -MAX_PREFILL_TOKENS=32768 - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -else EVAL_CONTEXT_ARGS="--context-length $CONTEXT_LENGTH" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -# following Andy Luo linkedin's recipe https://www.linkedin.com/feed/update/urn:li:activity:7429203734389280768/ -python3 -m sglang.launch_server \ - --attention-backend aiter \ - --model-path $MODEL \ - --host=0.0.0.0 \ - --port $PORT \ - --tensor-parallel-size $TP \ - --ep-size $EP_SIZE \ - --trust-remote-code \ - --tokenizer-worker-num 6 \ - --enable-aiter-allreduce-fusion \ - --cuda-graph-max-bs $CONC \ - --disable-radix-cache \ - --max-prefill-tokens $MAX_PREFILL_TOKENS \ - --scheduler-recv-interval 30 \ - --speculative-algorithm EAGLE \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 \ - --mem-fraction-static 0.75 $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - EP_SIZE \ - --result-dir /workspace/ \ - --use-chat-template - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/qwen3.5_bf16_mi355x.sh b/benchmarks/single_node/fixed_seq_len/qwen3.5_bf16_mi355x.sh deleted file mode 100755 index 1661df465..000000000 --- a/benchmarks/single_node/fixed_seq_len/qwen3.5_bf16_mi355x.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -SERVER_LOG=/workspace/server.log -CONTEXT_LENGTH=$((ISL + OSL + 20)) -MAX_PREFILL_TOKENS=32768 - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -else EVAL_CONTEXT_ARGS="--context-length $CONTEXT_LENGTH" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -python3 -m sglang.launch_server \ - --attention-backend triton \ - --model-path $MODEL \ - --host=0.0.0.0 \ - --port $PORT \ - --tensor-parallel-size $TP \ - --ep-size $EP_SIZE \ - --trust-remote-code \ - --tokenizer-worker-num 6 \ - --enable-aiter-allreduce-fusion \ - --cuda-graph-max-bs $CONC \ - --disable-radix-cache \ - --max-prefill-tokens $MAX_PREFILL_TOKENS \ - --scheduler-recv-interval 30 \ - --mem-fraction-static 0.8 $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/qwen3.5_bf16_mi355x_mtp.sh b/benchmarks/single_node/fixed_seq_len/qwen3.5_bf16_mi355x_mtp.sh deleted file mode 100755 index 38230cc88..000000000 --- a/benchmarks/single_node/fixed_seq_len/qwen3.5_bf16_mi355x_mtp.sh +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -SERVER_LOG=/workspace/server.log -CONTEXT_LENGTH=$((ISL + OSL + 20)) -MAX_PREFILL_TOKENS=32768 - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -else EVAL_CONTEXT_ARGS="--context-length $CONTEXT_LENGTH" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -python3 -m sglang.launch_server \ - --attention-backend triton \ - --model-path $MODEL \ - --host=0.0.0.0 \ - --port $PORT \ - --tensor-parallel-size $TP \ - --ep-size $EP_SIZE \ - --trust-remote-code \ - --tokenizer-worker-num 6 \ - --enable-aiter-allreduce-fusion \ - --cuda-graph-max-bs $CONC \ - --disable-radix-cache \ - --max-prefill-tokens $MAX_PREFILL_TOKENS \ - --scheduler-recv-interval 30 \ - --mem-fraction-static 0.8 \ - --speculative-algorithm EAGLE \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 \ - $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --use-chat-template - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_b200.sh b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_b200.sh deleted file mode 100755 index 638bc85ec..000000000 --- a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_b200.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -SERVER_LOG=/workspace/server.log - -CONTEXT_LENGTH=$((ISL + OSL + 20)) -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - CONTEXT_LENGTH="$EVAL_MAX_MODEL_LEN" -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path=$MODEL --host=0.0.0.0 --port=$PORT \ ---trust-remote-code \ ---tensor-parallel-size=$TP --data-parallel-size=1 --expert-parallel-size=$EP_SIZE \ ---enable-symm-mem \ ---disable-radix-cache \ ---quantization modelopt_fp4 \ ---kv-cache-dtype fp8_e4m3 \ ---mamba-ssm-dtype bfloat16 \ ---attention-backend trtllm_mha \ ---moe-runner-backend flashinfer_trtllm \ ---cuda-graph-max-bs $CONC \ ---max-prefill-tokens 16384 \ ---chunked-prefill-size 16384 \ ---mem-fraction-static 0.8 \ ---stream-interval 50 \ ---scheduler-recv-interval $( [[ $CONC -gt 4 ]] && echo 30 || echo 10 ) \ ---tokenizer-worker-num 6 \ ---tokenizer-path $MODEL \ ---context-length $CONTEXT_LENGTH > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_b200_mtp.sh b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_b200_mtp.sh deleted file mode 100755 index 5da51d974..000000000 --- a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_b200_mtp.sh +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -SERVER_LOG=/workspace/server.log - -CONTEXT_LENGTH=$((ISL + OSL + 20)) -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - CONTEXT_LENGTH="$EVAL_MAX_MODEL_LEN" -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -SGLANG_ENABLE_SPEC_V2=1 PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path=$MODEL --host=0.0.0.0 --port=$PORT \ ---trust-remote-code \ ---tensor-parallel-size=$TP --data-parallel-size=1 --expert-parallel-size=$EP_SIZE \ ---enable-symm-mem \ ---disable-radix-cache \ ---quantization modelopt_fp4 \ ---kv-cache-dtype fp8_e4m3 \ ---mamba-ssm-dtype bfloat16 \ ---attention-backend trtllm_mha \ ---moe-runner-backend flashinfer_trtllm \ ---cuda-graph-max-bs $CONC \ ---max-running-requests $CONC \ ---max-prefill-tokens 16384 \ ---chunked-prefill-size 16384 \ ---mem-fraction-static 0.8 \ ---stream-interval 50 \ ---scheduler-recv-interval $( [[ $CONC -gt 4 ]] && echo 30 || echo 10 ) \ ---tokenizer-worker-num 6 \ ---tokenizer-path $MODEL \ ---speculative-algorithm EAGLE \ ---speculative-num-steps 3 \ ---speculative-eagle-topk 1 \ ---speculative-num-draft-tokens 4 \ ---context-length $CONTEXT_LENGTH > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --use-chat-template - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_b200_trt.sh b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_b200_trt.sh deleted file mode 100644 index 3cdc282ed..000000000 --- a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_b200_trt.sh +++ /dev/null @@ -1,146 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - DP_ATTENTION \ - EP_SIZE - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -nvidia-smi - -SERVER_LOG=/workspace/server.log -EXTRA_CONFIG_FILE="qwen3.5-fp4-trt.yml" - -if [[ "$DP_ATTENTION" == "true" ]]; then - case "$TP" in - 4) MAX_BATCH_SIZE=256 ;; # tp4 / ep4 with attention DP - 8) MAX_BATCH_SIZE=128 ;; # tp8 / ep8 with attention DP - *) MAX_BATCH_SIZE=$(( CONC > 16 ? CONC : 16 )) ;; - esac -elif [[ "$TP" == "2" ]]; then - if [[ "$EP_SIZE" == "2" && "$CONC" -ge 32 ]]; then - MAX_BATCH_SIZE=32 # tp2 / ep2 at high concurrency - else - MAX_BATCH_SIZE=256 # tp2 / ep1, or tp2 / ep2 at low concurrency - fi -elif [[ "$TP" -ge 4 ]]; then - MAX_BATCH_SIZE=512 # tp>=4 without attention DP -else - MAX_BATCH_SIZE=$(( CONC > 16 ? CONC : 16 )) -fi - -if [[ "$DP_ATTENTION" == "true" ]]; then - MOE_BACKEND="CUTEDSL" - MODE_CONFIG="attention_dp_config: - enable_balance: true - batching_wait_iters: 10 - timeout_iters: 500" -else - MOE_BACKEND="TRTLLM" - MODE_CONFIG="batch_wait_timeout_iters: 50 -batch_wait_max_tokens_ratio: 0.45" -fi - -cat > "$EXTRA_CONFIG_FILE" << EOF -backend: pytorch -print_iter_log: true -enable_layerwise_nvtx_marker: false -disable_overlap_scheduler: false -enable_iter_perf_stats: true -enable_chunked_prefill: false -stream_interval: 20 -num_postprocess_workers: 4 -enable_attention_dp: $DP_ATTENTION -scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - context_chunking_policy: FIRST_COME_FIRST_SERVED -kv_cache_config: - free_gpu_memory_fraction: 0.9 - enable_block_reuse: false - dtype: fp8 -cuda_graph_config: - enable_padding: true - max_batch_size: $MAX_BATCH_SIZE -moe_config: - backend: $MOE_BACKEND - use_low_precision_moe_combine: true -$MODE_CONFIG -EOF - -echo "Generated config file contents:" -cat "$EXTRA_CONFIG_FILE" - -MAX_MODEL_LEN=$(( MAX_MODEL_LEN > 8192 ? MAX_MODEL_LEN : 8192 )) - -case "${ISL}_${OSL}" in - 8192_1024) MAX_NUM_TOKENS=32768 ;; - 1024_1024) MAX_NUM_TOKENS=16384 ;; - *) - MAX_NUM_TOKENS=$(( ISL + OSL + 256 )) - MAX_NUM_TOKENS=$(( MAX_NUM_TOKENS > 8192 ? MAX_NUM_TOKENS : 8192 )) - ;; -esac - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" - MAX_NUM_TOKENS="$EVAL_MAX_MODEL_LEN" -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -mpirun -n 1 --oversubscribe --allow-run-as-root \ - trtllm-serve "$MODEL" --port="$PORT" \ - --trust_remote_code \ - --backend=pytorch \ - --max_batch_size "$MAX_BATCH_SIZE" \ - --max_seq_len="$MAX_MODEL_LEN" \ - --max_num_tokens="$MAX_NUM_TOKENS" \ - --tp_size="$TP" --ep_size="$EP_SIZE" \ - --extra_llm_api_options="$EXTRA_CONFIG_FILE" \ - > "$SERVER_LOG" 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend openai \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$(( CONC * 10 ))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_b200_trt_mtp.sh b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_b200_trt_mtp.sh deleted file mode 100644 index e0e706e78..000000000 --- a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_b200_trt_mtp.sh +++ /dev/null @@ -1,170 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - MAX_MODEL_LEN \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - DP_ATTENTION \ - EP_SIZE - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -# MTP (multi-token prediction) speculative decode requires the FlashInfer GDN -# prefill path to be disabled. -export TLLM_USE_FLASHINFER_GDN_PREFILL="0" - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -nvidia-smi - -SERVER_LOG=/workspace/server.log -EXTRA_CONFIG_FILE="qwen3.5-fp4-trt-mtp.yml" -NUM_NEXTN_PREDICT_LAYERS=3 - -# Attention-DP layouts run CUTEDSL MoE; everything else runs the TRTLLM backend. -# With MTP the served batch is much smaller than raw concurrency: attention-DP -# runs at CONC/8, everything else at CONC. The KV-cache memory fraction is tuned -# per layout (there is no single derivable rule). -if [[ "$DP_ATTENTION" == "true" ]]; then - MAX_BATCH_SIZE=$(( CONC / 8 )) - MOE_BACKEND="CUTEDSL" - # attention-DP: 0.9 up to conc 512, backed off to 0.8 at conc 1024. - if (( CONC >= 1024 )); then KV_MEMORY_FRACTION=0.8; else KV_MEMORY_FRACTION=0.9; fi - MODE_CONFIG="enable_attention_dp: true -attention_dp_config: - enable_balance: true - batching_wait_iters: 10 - timeout_iters: 500" -else - MAX_BATCH_SIZE="$CONC" - MOE_BACKEND="TRTLLM" - # non-attention-DP fraction, tuned per (ISL, TP, EP) layout. - case "${ISL}_tp${TP}_ep${EP_SIZE}" in - 1024_tp2_ep1) KV_MEMORY_FRACTION=0.6 ;; - 1024_tp2_ep2) KV_MEMORY_FRACTION=0.75 ;; - 1024_tp8_ep8) KV_MEMORY_FRACTION=0.8 ;; - 8192_tp2_ep1) KV_MEMORY_FRACTION=0.7 ;; - 8192_tp2_ep2) KV_MEMORY_FRACTION=0.6 ;; - 8192_tp4_ep4) KV_MEMORY_FRACTION=0.75 ;; - 8192_tp8_ep8) KV_MEMORY_FRACTION=0.8 ;; - *) KV_MEMORY_FRACTION=0.8 ;; - esac - # Short-context runs hold less in flight, so they wait on a tighter token - # ratio before flushing a batch. - case "$ISL" in - 1024) BATCH_WAIT_MAX_TOKENS_RATIO=0.0625 ;; - *) BATCH_WAIT_MAX_TOKENS_RATIO=0.45 ;; - esac - MODE_CONFIG="batch_wait_timeout_iters: 50 -batch_wait_max_tokens_ratio: $BATCH_WAIT_MAX_TOKENS_RATIO" -fi - -cat > "$EXTRA_CONFIG_FILE" << EOF -backend: pytorch -print_iter_log: true -enable_layerwise_nvtx_marker: false -disable_overlap_scheduler: false -enable_iter_perf_stats: true -enable_chunked_prefill: false -stream_interval: 20 -num_postprocess_workers: 4 -scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION - context_chunking_policy: FIRST_COME_FIRST_SERVED -kv_cache_config: - free_gpu_memory_fraction: $KV_MEMORY_FRACTION - enable_block_reuse: false - dtype: fp8 -cuda_graph_config: - enable_padding: true - batch_sizes: - - 1 - - 2 - - 4 - - 8 - - 16 - - 32 - - 64 - - 128 -moe_config: - backend: $MOE_BACKEND - use_low_precision_moe_combine: true -speculative_config: - decoding_type: MTP - num_nextn_predict_layers: $NUM_NEXTN_PREDICT_LAYERS -$MODE_CONFIG -EOF - -echo "Generated config file contents:" -cat "$EXTRA_CONFIG_FILE" - -MAX_MODEL_LEN=$(( MAX_MODEL_LEN > 8192 ? MAX_MODEL_LEN : 8192 )) - -case "${ISL}_${OSL}" in - 8192_1024) MAX_NUM_TOKENS=32768 ;; - 1024_1024) MAX_NUM_TOKENS=16384 ;; - *) - MAX_NUM_TOKENS=$(( ISL + OSL + 256 )) - MAX_NUM_TOKENS=$(( MAX_NUM_TOKENS > 8192 ? MAX_NUM_TOKENS : 8192 )) - ;; -esac - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" - MAX_NUM_TOKENS="$EVAL_MAX_MODEL_LEN" -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -mpirun -n 1 --oversubscribe --allow-run-as-root \ - trtllm-serve "$MODEL" --port="$PORT" \ - --trust_remote_code \ - --backend=pytorch \ - --max_batch_size "$MAX_BATCH_SIZE" \ - --max_seq_len="$MAX_MODEL_LEN" \ - --max_num_tokens="$MAX_NUM_TOKENS" \ - --tp_size="$TP" --ep_size="$EP_SIZE" \ - --extra_llm_api_options="$EXTRA_CONFIG_FILE" \ - > "$SERVER_LOG" 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend openai \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$(( CONC * 10 ))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --use-chat-template - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_b300.sh b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_b300.sh deleted file mode 100755 index a9e219097..000000000 --- a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_b300.sh +++ /dev/null @@ -1,117 +0,0 @@ -#!/usr/bin/env bash - -# Follows the SGLang cookbook recipe at -# https://cookbook.sglang.io/autoregressive/Qwen/Qwen3.5 as of 2026-04-17. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - - -export NCCL_NVLS_ENABLE=1 -export SGL_ENABLE_JIT_DEEPGEMM=false -export SGLANG_ENABLE_FLASHINFER_GEMM=true -export PYTHONUNBUFFERED=1 - -SERVER_LOG=/workspace/server.log - -# Default: recv every ~10 requests; if CONC >= 16, relax to ~30 requests between scheduler recv polls. -if [[ $CONC -ge 16 ]]; then - SCHEDULER_RECV_INTERVAL=30 -else - SCHEDULER_RECV_INTERVAL=10 -fi - -MEM_FRAC_STATIC=0.8 -CHUNKED_PREFILL_SIZE=32768 -MAX_PREFILL_TOKENS=32768 -CUDA_GRAPH_MAX_BATCH_SIZE=$CONC -MAX_RUNNING_REQUESTS=128 -CONTEXT_LENGTH=$((ISL + OSL + 20)) -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - CONTEXT_LENGTH="$EVAL_MAX_MODEL_LEN" -fi - -if [[ $TP -eq 8 ]]; then - EXTRA_ARGS="--enable-flashinfer-allreduce-fusion" -else - EXTRA_ARGS="" -fi - -echo "SCHEDULER_RECV_INTERVAL: $SCHEDULER_RECV_INTERVAL, CONC: $CONC, ISL: $ISL, OSL: $OSL" - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path $MODEL_PATH --served-model-name $MODEL --host 0.0.0.0 --port $PORT \ ---trust-remote-code \ ---tensor-parallel-size $TP --data-parallel-size 1 --ep-size $EP_SIZE \ ---reasoning-parser qwen3 \ ---tool-call-parser qwen3_coder \ ---mamba-scheduler-strategy no_buffer \ ---quantization modelopt_fp4 --fp4-gemm-backend flashinfer_cutlass \ ---kv-cache-dtype fp8_e4m3 \ ---mamba-ssm-dtype bfloat16 \ ---cuda-graph-max-bs $CUDA_GRAPH_MAX_BATCH_SIZE --max-running-requests $MAX_RUNNING_REQUESTS \ ---mem-fraction-static $MEM_FRAC_STATIC --chunked-prefill-size $CHUNKED_PREFILL_SIZE --max-prefill-tokens $MAX_PREFILL_TOKENS \ ---context-length $CONTEXT_LENGTH --disable-radix-cache \ ---attention-backend trtllm_mha --mm-attention-backend triton_attn --moe-runner-backend flashinfer_trtllm \ -$EXTRA_ARGS --scheduler-recv-interval $SCHEDULER_RECV_INTERVAL \ ---tokenizer-worker-num 6 --stream-interval 30 > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_b300_mtp.sh b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_b300_mtp.sh deleted file mode 100755 index ba7b77b19..000000000 --- a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_b300_mtp.sh +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env bash - -# Follows the SGLang cookbook recipe at -# https://cookbook.sglang.io/autoregressive/Qwen/Qwen3.5 as of 2026-04-17. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - - -export NCCL_NVLS_ENABLE=1 -export SGL_ENABLE_JIT_DEEPGEMM=false -export SGLANG_ENABLE_FLASHINFER_GEMM=true -export PYTHONUNBUFFERED=1 - -SERVER_LOG=/workspace/server.log - -# Default: recv every ~10 requests; if CONC >= 16, relax to ~30 requests between scheduler recv polls. -if [[ $CONC -ge 16 ]]; then - SCHEDULER_RECV_INTERVAL=30 -else - SCHEDULER_RECV_INTERVAL=10 -fi - -MEM_FRAC_STATIC=0.8 -CHUNKED_PREFILL_SIZE=32768 -MAX_PREFILL_TOKENS=32768 -CUDA_GRAPH_MAX_BATCH_SIZE=$CONC -MAX_RUNNING_REQUESTS=128 -CONTEXT_LENGTH=$((ISL + OSL + 20)) -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - CONTEXT_LENGTH="$EVAL_MAX_MODEL_LEN" -fi - -if [[ $TP -eq 8 ]]; then - EXTRA_ARGS="--enable-flashinfer-allreduce-fusion" -else - EXTRA_ARGS="" -fi - -echo "SCHEDULER_RECV_INTERVAL: $SCHEDULER_RECV_INTERVAL, CONC: $CONC, ISL: $ISL, OSL: $OSL" - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path $MODEL_PATH --served-model-name $MODEL --host 0.0.0.0 --port $PORT \ ---trust-remote-code \ ---tensor-parallel-size $TP --data-parallel-size 1 --ep-size $EP_SIZE \ ---reasoning-parser qwen3 \ ---tool-call-parser qwen3_coder \ ---mamba-scheduler-strategy no_buffer \ ---quantization modelopt_fp4 --fp4-gemm-backend flashinfer_cutlass \ ---kv-cache-dtype fp8_e4m3 \ ---mamba-ssm-dtype bfloat16 \ ---cuda-graph-max-bs $CUDA_GRAPH_MAX_BATCH_SIZE --max-running-requests $MAX_RUNNING_REQUESTS \ ---mem-fraction-static $MEM_FRAC_STATIC --chunked-prefill-size $CHUNKED_PREFILL_SIZE --max-prefill-tokens $MAX_PREFILL_TOKENS \ ---context-length $CONTEXT_LENGTH --disable-radix-cache \ ---attention-backend trtllm_mha --mm-attention-backend triton_attn --moe-runner-backend flashinfer_trtllm \ -$EXTRA_ARGS --scheduler-recv-interval $SCHEDULER_RECV_INTERVAL \ ---tokenizer-worker-num 6 --stream-interval 30 \ ---speculative-algorithm EAGLE \ ---speculative-num-steps 3 \ ---speculative-eagle-topk 1 \ ---speculative-num-draft-tokens 4 \ -> $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --use-chat-template - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_mi355x.sh b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_mi355x.sh deleted file mode 100644 index b036e060e..000000000 --- a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_mi355x.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -export SGLANG_USE_AITER=1 -export SGLANG_USE_AITER_UNIFIED_ATTN=1 -export AITER_FLYDSL_FORCE=1 - -SERVER_LOG=/workspace/server.log -MEM_FRAC_STATIC=${MEM_FRAC_STATIC:-0.8} - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -python3 -m sglang.launch_server --model-path=$MODEL --trust-remote-code \ ---host=0.0.0.0 --port=$PORT \ ---tensor-parallel-size=$TP \ ---attention-backend aiter \ ---mem-fraction-static $MEM_FRAC_STATIC \ ---model-loader-extra-config '{"enable_multithread_load": true}' \ ---watchdog-timeout 1200 \ ---disable-radix-cache \ ---enable-aiter-allreduce-fusion --max-running-requests $CONC \ ---page-size 16 \ -> $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" --sleep-interval 60 - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_mi355x_atom.sh b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_mi355x_atom.sh deleted file mode 100644 index 325c97726..000000000 --- a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_mi355x_atom.sh +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE \ - DP_ATTENTION - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -SERVER_LOG=/workspace/server.log - -export OMP_NUM_THREADS=1 - -# Calculate max-model-len based on ISL and OSL -if [ "$ISL" = "1024" ] && [ "$OSL" = "1024" ]; then - CALCULATED_MAX_MODEL_LEN="" -else - CALCULATED_MAX_MODEL_LEN=" --max-model-len 10240 " -fi - -if [ "$EP_SIZE" -gt 1 ]; then - EP=" --enable-expert-parallel" -else - EP=" " -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor -MEM_FRAC_STATIC=0.9 - -set -x - -python3 -m atom.entrypoints.openai_server \ - --model $MODEL \ - --server-port $PORT \ - -tp $TP \ - --kv_cache_dtype fp8 $CALCULATED_MAX_MODEL_LEN $EP \ - --gpu-memory-utilization $MEM_FRAC_STATIC \ - --trust-remote-code \ - > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -export PYTHONDONTWRITEBYTECODE=1 -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_mi355x_mtp.sh b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_mi355x_mtp.sh deleted file mode 100755 index 8081b824e..000000000 --- a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_mi355x_mtp.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -hf download "$MODEL" - -export SGLANG_USE_AITER=1 -export SGLANG_USE_AITER_UNIFIED_ATTN=1 -export AITER_FLYDSL_FORCE=1 - -SERVER_LOG=/workspace/server.log -MEM_FRAC_STATIC=${MEM_FRAC_STATIC:-0.8} - -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -python3 -m sglang.launch_server --model-path=$MODEL --trust-remote-code \ ---host=0.0.0.0 --port=$PORT \ ---tensor-parallel-size=$TP \ ---attention-backend aiter \ ---mem-fraction-static $MEM_FRAC_STATIC \ ---model-loader-extra-config '{"enable_multithread_load": true}' \ ---watchdog-timeout 1200 \ ---disable-radix-cache \ ---enable-aiter-allreduce-fusion --max-running-requests $CONC \ ---page-size 16 \ ---speculative-algorithm EAGLE \ ---speculative-num-steps 3 \ ---speculative-eagle-topk 1 \ ---speculative-num-draft-tokens 4 \ -> $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" --sleep-interval 60 - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --use-chat-template - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp8_b200.sh b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp8_b200.sh deleted file mode 100755 index 4b9005eb8..000000000 --- a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp8_b200.sh +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -SERVER_LOG=/workspace/server.log - -CONTEXT_LENGTH=$((ISL + OSL + 20)) -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - CONTEXT_LENGTH="$EVAL_MAX_MODEL_LEN" -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path=$MODEL --host=0.0.0.0 --port=$PORT \ ---trust-remote-code \ ---tensor-parallel-size=$TP --data-parallel-size=1 --expert-parallel-size=$EP_SIZE \ ---enable-symm-mem \ ---disable-radix-cache \ ---quantization fp8 \ ---kv-cache-dtype fp8_e4m3 \ ---mamba-ssm-dtype bfloat16 \ ---attention-backend trtllm_mha \ ---moe-runner-backend flashinfer_trtllm \ ---cuda-graph-max-bs $CONC \ ---max-prefill-tokens 16384 \ ---chunked-prefill-size 16384 \ ---mem-fraction-static 0.8 \ ---stream-interval 50 \ ---scheduler-recv-interval $( [[ $CONC -gt 4 ]] && echo 30 || echo 10 ) \ ---tokenizer-worker-num 6 \ ---context-length $CONTEXT_LENGTH > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp8_b200_mtp.sh b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp8_b200_mtp.sh deleted file mode 100755 index a7093d4b8..000000000 --- a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp8_b200_mtp.sh +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -SERVER_LOG=/workspace/server.log - -CONTEXT_LENGTH=$((ISL + OSL + 20)) -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - CONTEXT_LENGTH="$EVAL_MAX_MODEL_LEN" -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -SGLANG_ENABLE_SPEC_V2=1 PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path=$MODEL --host=0.0.0.0 --port=$PORT \ ---trust-remote-code \ ---tensor-parallel-size=$TP --data-parallel-size=1 --expert-parallel-size=$EP_SIZE \ ---enable-symm-mem \ ---disable-radix-cache \ ---quantization fp8 \ ---kv-cache-dtype fp8_e4m3 \ ---mamba-ssm-dtype bfloat16 \ ---attention-backend trtllm_mha \ ---moe-runner-backend flashinfer_trtllm \ ---cuda-graph-max-bs $CONC \ ---max-running-requests $CONC \ ---max-prefill-tokens 16384 \ ---chunked-prefill-size 16384 \ ---mem-fraction-static 0.8 \ ---stream-interval 50 \ ---scheduler-recv-interval $( [[ $CONC -gt 4 ]] && echo 30 || echo 10 ) \ ---tokenizer-worker-num 6 \ ---tokenizer-path $MODEL \ ---speculative-algorithm EAGLE \ ---speculative-num-steps 3 \ ---speculative-eagle-topk 1 \ ---speculative-num-draft-tokens 4 \ ---context-length $CONTEXT_LENGTH > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --use-chat-template - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp8_b300.sh b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp8_b300.sh deleted file mode 100644 index b07acebca..000000000 --- a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp8_b300.sh +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -SERVER_LOG=/workspace/server.log - -CONTEXT_LENGTH=$((ISL + OSL + 20)) -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - CONTEXT_LENGTH="$EVAL_MAX_MODEL_LEN" -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path $MODEL_PATH --served-model-name $MODEL --host 0.0.0.0 --port $PORT \ ---trust-remote-code \ ---tensor-parallel-size $TP --data-parallel-size 1 --expert-parallel-size $EP_SIZE \ ---enable-symm-mem \ ---disable-radix-cache \ ---quantization fp8 \ ---kv-cache-dtype fp8_e4m3 \ ---mamba-ssm-dtype bfloat16 \ ---attention-backend trtllm_mha \ ---mm-attention-backend triton_attn \ ---moe-runner-backend flashinfer_trtllm \ ---cuda-graph-max-bs $CONC \ ---max-running-requests $CONC \ ---max-prefill-tokens 16384 \ ---chunked-prefill-size 16384 \ ---mem-fraction-static 0.8 \ ---stream-interval 50 \ ---scheduler-recv-interval 10 \ ---tokenizer-worker-num 6 \ ---tokenizer-path $MODEL_PATH \ ---context-length $CONTEXT_LENGTH > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --use-chat-template - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp8_b300_mtp.sh b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp8_b300_mtp.sh deleted file mode 100644 index 6a14ff236..000000000 --- a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp8_b300_mtp.sh +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -SERVER_LOG=/workspace/server.log - -CONTEXT_LENGTH=$((ISL + OSL + 20)) -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - CONTEXT_LENGTH="$EVAL_MAX_MODEL_LEN" -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -SGLANG_ENABLE_SPEC_V2=1 PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path $MODEL_PATH --served-model-name $MODEL --host 0.0.0.0 --port $PORT \ ---trust-remote-code \ ---tensor-parallel-size $TP --data-parallel-size 1 --expert-parallel-size $EP_SIZE \ ---enable-symm-mem \ ---disable-radix-cache \ ---quantization fp8 \ ---kv-cache-dtype fp8_e4m3 \ ---mamba-ssm-dtype bfloat16 \ ---attention-backend trtllm_mha \ ---mm-attention-backend triton_attn \ ---moe-runner-backend flashinfer_trtllm \ ---cuda-graph-max-bs $CONC \ ---max-running-requests $CONC \ ---max-prefill-tokens 16384 \ ---chunked-prefill-size 16384 \ ---mem-fraction-static 0.8 \ ---stream-interval 50 \ ---scheduler-recv-interval 10 \ ---tokenizer-worker-num 6 \ ---tokenizer-path $MODEL_PATH \ ---speculative-algorithm EAGLE \ ---speculative-num-steps 3 \ ---speculative-eagle-topk 1 \ ---speculative-num-draft-tokens 4 \ ---context-length $CONTEXT_LENGTH > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --use-chat-template - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp8_h100.sh b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp8_h100.sh deleted file mode 100755 index ceff22da8..000000000 --- a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp8_h100.sh +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/env bash - -# Qwen-3.5-397B-A17B FP8 on H100 via sglang. -# Uses TP8/EP1 at conc 1-8 and TP8/EP8 at conc 16-256. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -SERVER_LOG=/workspace/server.log -MAX_SEQ_LEN=$((ISL + OSL + 20)) -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_SEQ_LEN="$EVAL_MAX_MODEL_LEN" -fi - -PARALLEL_ARGS=(--tp "$TP") -if [ "${EP_SIZE}" -gt 1 ]; then - PARALLEL_ARGS+=(--expert-parallel-size "$EP_SIZE") -fi - -SCHEDULER_RECV_INTERVAL= -case "$CONC" in - 1|2|4) - SCHEDULER_RECV_INTERVAL=2 - ;; - 8) - SCHEDULER_RECV_INTERVAL=60 - ;; - 16) - SCHEDULER_RECV_INTERVAL=30 - ;; - 32) - SCHEDULER_RECV_INTERVAL=1200 - ;; - 64) - SCHEDULER_RECV_INTERVAL=600 - ;; - 128|256) - SCHEDULER_RECV_INTERVAL=1920 - ;; - *) - echo "Unsupported CONC=$CONC for qwen3.5 FP8 H100 SGLang recipe" >&2 - exit 1 - ;; -esac - -SCHEDULER_ARGS=() -if [ -n "$SCHEDULER_RECV_INTERVAL" ]; then - SCHEDULER_ARGS=(--scheduler-recv-interval "$SCHEDULER_RECV_INTERVAL") -fi - -echo "TP: $TP, EP_SIZE: $EP_SIZE, CONC: $CONC, ISL: $ISL, OSL: $OSL, MAX_SEQ_LEN: $MAX_SEQ_LEN" -echo "SCHEDULER_RECV_INTERVAL: ${SCHEDULER_RECV_INTERVAL:-none}" -echo "SCHEDULER_ARGS: ${SCHEDULER_ARGS[*]}" - -start_gpu_monitor - -set -x -python3 -m sglang.launch_server \ - --model "$MODEL" \ - --host 0.0.0.0 \ - --port "$PORT" \ - "${PARALLEL_ARGS[@]}" \ - --reasoning-parser qwen3 \ - --tool-call-parser qwen3_coder \ - --enable-flashinfer-allreduce-fusion \ - --max-running-requests 256 \ - --chunked-prefill-size 16384 \ - --decode-log-interval 1 \ - --mem-fraction-static 0.8 \ - --cuda-graph-max-bs "$CONC" \ - --context-length "$MAX_SEQ_LEN" \ - --kv-cache-dtype fp8_e4m3 \ - --quantization fp8 \ - --attention-backend flashinfer \ - --stream-interval 50 \ - --tokenizer-worker-num 6 \ - --mamba-ssm-dtype bfloat16 \ - --disable-radix-cache \ - --enable-symm-mem \ - --trust-remote-code \ - "${SCHEDULER_ARGS[@]}" \ - > "$SERVER_LOG" 2>&1 & - -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp8_h100_mtp.sh b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp8_h100_mtp.sh deleted file mode 100755 index faa666f8b..000000000 --- a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp8_h100_mtp.sh +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/bin/env bash - -# Qwen-3.5-397B-A17B FP8 on H100 with EAGLE / MTP speculative decoding. -# Mirrors qwen3.5_fp8_h100.sh; adds the speculative-* flags + SGLANG_ENABLE_SPEC_V2=1 -# and passes --use-chat-template per AGENTS.md. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -export SGLANG_ENABLE_SPEC_V2=1 - -SERVER_LOG=/workspace/server.log -MAX_SEQ_LEN=$((ISL + OSL + 20)) -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_SEQ_LEN="$EVAL_MAX_MODEL_LEN" -fi - -echo "CONC: $CONC, ISL: $ISL, OSL: $OSL, MAX_SEQ_LEN: $MAX_SEQ_LEN" - -start_gpu_monitor - -set -x -python3 -m sglang.launch_server \ - --model "$MODEL" \ - --host 0.0.0.0 \ - --port "$PORT" \ - --tp "$TP" \ - --expert-parallel-size "$EP_SIZE" \ - --reasoning-parser qwen3 \ - --tool-call-parser qwen3_coder \ - --enable-flashinfer-allreduce-fusion \ - --max-running-requests 64 \ - --chunked-prefill-size 8192 \ - --decode-log-interval 1 \ - --mem-fraction-static 0.75 \ - --cuda-graph-max-bs "$CONC" \ - --context-length "$MAX_SEQ_LEN" \ - --kv-cache-dtype fp8_e4m3 \ - --quantization fp8 \ - --attention-backend flashinfer \ - --stream-interval 50 \ - --tokenizer-worker-num 6 \ - --mamba-ssm-dtype bfloat16 \ - --disable-radix-cache \ - --trust-remote-code \ - --speculative-algorithm EAGLE \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 \ - > "$SERVER_LOG" 2>&1 & - -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --use-chat-template - -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp8_h200.sh b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp8_h200.sh deleted file mode 100644 index 07ce08a58..000000000 --- a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp8_h200.sh +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -SERVER_LOG=/workspace/server.log -MAX_SEQ_LEN=$((ISL + OSL + 20)) -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - MAX_SEQ_LEN="$EVAL_MAX_MODEL_LEN" -fi - -echo "CONC: $CONC, ISL: $ISL, OSL: $OSL, MAX_SEQ_LEN: $MAX_SEQ_LEN" - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -python3 -m sglang.launch_server \ - --model "$MODEL" \ - --host 0.0.0.0 \ - --port "$PORT" \ - --tp "$TP" \ - --expert-parallel-size "$EP_SIZE" \ - --reasoning-parser qwen3 \ - --tool-call-parser qwen3_coder \ - --enable-flashinfer-allreduce-fusion \ - --max-running-requests 128 \ - --chunked-prefill-size 16384 \ - --decode-log-interval 1 \ - --mem-fraction-static 0.8 \ - --cuda-graph-max-bs "$CONC" \ - --context-length "$MAX_SEQ_LEN" \ - --kv-cache-dtype fp8_e4m3 \ - --quantization fp8 \ - --attention-backend flashinfer \ - --stream-interval 50 \ - --tokenizer-worker-num 6 \ - --mamba-ssm-dtype bfloat16 \ - --disable-radix-cache \ - --trust-remote-code \ - > "$SERVER_LOG" 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp8_h200_mtp.sh b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp8_h200_mtp.sh deleted file mode 100644 index 98c1ec9db..000000000 --- a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp8_h200_mtp.sh +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE \ - MAX_MODEL_LEN - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -nvidia-smi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -SERVER_LOG=/workspace/server.log - -# MTP (Multi-Token Prediction) Config - EAGLE speculative decoding -SPECULATIVE_NUM_STEPS=3 -SPECULATIVE_DRAFT_TOKENS=4 -SPECULATIVE_EAGLE_TOPK=1 - -echo "CONC: $CONC, ISL: $ISL, OSL: $OSL, MAX_MODEL_LEN: $MAX_MODEL_LEN" - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -set -x -SGLANG_ENABLE_SPEC_V2=1 python3 -m sglang.launch_server \ - --model "$MODEL" \ - --host 0.0.0.0 \ - --port "$PORT" \ - --tp "$TP" \ - --expert-parallel-size "$EP_SIZE" \ - --reasoning-parser qwen3 \ - --tool-call-parser qwen3_coder \ - --enable-flashinfer-allreduce-fusion \ - --max-running-requests 128 \ - --chunked-prefill-size 16384 \ - --mem-fraction-static 0.8 \ - --cuda-graph-max-bs "$CONC" \ - --context-length "$MAX_MODEL_LEN" \ - --kv-cache-dtype fp8_e4m3 \ - --quantization fp8 \ - --attention-backend flashinfer \ - --stream-interval 50 \ - --tokenizer-worker-num 6 \ - --mamba-ssm-dtype bfloat16 \ - --disable-radix-cache \ - --trust-remote-code \ - --speculative-algorithm EAGLE \ - --speculative-num-steps "$SPECULATIVE_NUM_STEPS" \ - --speculative-num-draft-tokens "$SPECULATIVE_DRAFT_TOKENS" \ - --speculative-eagle-topk "$SPECULATIVE_EAGLE_TOPK" \ - > "$SERVER_LOG" 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -pip install -q datasets pandas - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --use-chat-template \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - export EVAL_CONCURRENT_REQUESTS="${EVAL_CONCURRENT_REQUESTS:-$CONC}" - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp8_mi300x.sh b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp8_mi300x.sh deleted file mode 100755 index e1607860d..000000000 --- a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp8_mi300x.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -SERVER_LOG=/workspace/server.log -CONTEXT_LENGTH=$((ISL + OSL + 20)) -MAX_PREFILL_TOKENS=32768 - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -else EVAL_CONTEXT_ARGS="--context-length $CONTEXT_LENGTH" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -# following AMD Andy linkedin's recipe -# https://www.linkedin.com/feed/update/urn:li:activity:7429203734389280768/ -python3 -m sglang.launch_server \ - --attention-backend aiter \ - --model-path $MODEL \ - --host=0.0.0.0 \ - --port $PORT \ - --tensor-parallel-size $TP \ - --data-parallel-size 1 \ - --trust-remote-code \ - --tokenizer-worker-num 6 \ - --enable-aiter-allreduce-fusion \ - --cuda-graph-max-bs $CONC \ - --disable-radix-cache \ - --max-prefill-tokens $MAX_PREFILL_TOKENS \ - --scheduler-recv-interval 30 \ - --mem-fraction-static 0.75 $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp8_mi325x.sh b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp8_mi325x.sh deleted file mode 100755 index e1607860d..000000000 --- a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp8_mi325x.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -SERVER_LOG=/workspace/server.log -CONTEXT_LENGTH=$((ISL + OSL + 20)) -MAX_PREFILL_TOKENS=32768 - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -else EVAL_CONTEXT_ARGS="--context-length $CONTEXT_LENGTH" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -# following AMD Andy linkedin's recipe -# https://www.linkedin.com/feed/update/urn:li:activity:7429203734389280768/ -python3 -m sglang.launch_server \ - --attention-backend aiter \ - --model-path $MODEL \ - --host=0.0.0.0 \ - --port $PORT \ - --tensor-parallel-size $TP \ - --data-parallel-size 1 \ - --trust-remote-code \ - --tokenizer-worker-num 6 \ - --enable-aiter-allreduce-fusion \ - --cuda-graph-max-bs $CONC \ - --disable-radix-cache \ - --max-prefill-tokens $MAX_PREFILL_TOKENS \ - --scheduler-recv-interval 30 \ - --mem-fraction-static 0.75 $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp8_mi325x_mtp.sh b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp8_mi325x_mtp.sh deleted file mode 100755 index a8e04064b..000000000 --- a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp8_mi325x_mtp.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME - EP_SIZE \ - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -SERVER_LOG=/workspace/server.log -CONTEXT_LENGTH=$((ISL + OSL + 20)) -MAX_PREFILL_TOKENS=32768 - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -else EVAL_CONTEXT_ARGS="--context-length $CONTEXT_LENGTH" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -# following AMD Andy linkedin's recipe -# https://www.linkedin.com/feed/update/urn:li:activity:7429203734389280768/ -python3 -m sglang.launch_server \ - --attention-backend aiter \ - --model-path $MODEL \ - --host=0.0.0.0 \ - --port $PORT \ - --tensor-parallel-size $TP \ - --ep-size $EP_SIZE \ - --trust-remote-code \ - --tokenizer-worker-num 6 \ - --enable-aiter-allreduce-fusion \ - --cuda-graph-max-bs $CONC \ - --disable-radix-cache \ - --max-prefill-tokens $MAX_PREFILL_TOKENS \ - --scheduler-recv-interval 30 \ - --speculative-algorithm EAGLE \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 \ - --mem-fraction-static 0.75 $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - EP_SIZE \ - --result-dir /workspace/ \ - --use-chat-template - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp8_mi355x.sh b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp8_mi355x.sh deleted file mode 100644 index 7ff150630..000000000 --- a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp8_mi355x.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -export SGLANG_USE_AITER_UNIFIED_ATTN=1 -export SGLANG_USE_AITER=1 - -SERVER_LOG=/workspace/server.log -CONTEXT_LENGTH=$((ISL + OSL + 20)) - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -else EVAL_CONTEXT_ARGS="--context-length $CONTEXT_LENGTH" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -python3 -m sglang.launch_server \ - --attention-backend aiter \ - --model-path $MODEL \ - --host=0.0.0.0 \ - --port $PORT \ - --tensor-parallel-size $TP \ - --ep-size $EP_SIZE \ - --trust-remote-code \ - --tokenizer-worker-num 6 \ - --enable-aiter-allreduce-fusion \ - --max-running-requests $CONC \ - --cuda-graph-max-bs $CONC \ - --disable-radix-cache \ - --chunked-prefill-size 32768 \ - --scheduler-recv-interval 30 \ - --mem-fraction-static 0.8 \ - --model-loader-extra-config '{"enable_multithread_load": true}' \ - --page-size 16 $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp8_mi355x_atom.sh b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp8_mi355x_atom.sh deleted file mode 100644 index 325c97726..000000000 --- a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp8_mi355x_atom.sh +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE \ - DP_ATTENTION - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -SERVER_LOG=/workspace/server.log - -export OMP_NUM_THREADS=1 - -# Calculate max-model-len based on ISL and OSL -if [ "$ISL" = "1024" ] && [ "$OSL" = "1024" ]; then - CALCULATED_MAX_MODEL_LEN="" -else - CALCULATED_MAX_MODEL_LEN=" --max-model-len 10240 " -fi - -if [ "$EP_SIZE" -gt 1 ]; then - EP=" --enable-expert-parallel" -else - EP=" " -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor -MEM_FRAC_STATIC=0.9 - -set -x - -python3 -m atom.entrypoints.openai_server \ - --model $MODEL \ - --server-port $PORT \ - -tp $TP \ - --kv_cache_dtype fp8 $CALCULATED_MAX_MODEL_LEN $EP \ - --gpu-memory-utilization $MEM_FRAC_STATIC \ - --trust-remote-code \ - > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -export PYTHONDONTWRITEBYTECODE=1 -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp8_mi355x_atom_mtp.sh b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp8_mi355x_atom_mtp.sh deleted file mode 100644 index 29351cf33..000000000 --- a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp8_mi355x_atom_mtp.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE \ - DP_ATTENTION - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" - -SERVER_LOG=/workspace/server.log - -export OMP_NUM_THREADS=1 - -# Calculate max-model-len based on ISL and OSL -if [ "$ISL" = "1024" ] && [ "$OSL" = "1024" ]; then - CALCULATED_MAX_MODEL_LEN="" -else - CALCULATED_MAX_MODEL_LEN=" --max-model-len 10240 " -fi - -if [ "$EP_SIZE" -gt 1 ]; then - EP=" --enable-expert-parallel" -else - EP=" " -fi - -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor -MEM_FRAC_STATIC=0.9 - -set -x - -python3 -m atom.entrypoints.openai_server \ - --model $MODEL \ - --server-port $PORT \ - -tp $TP \ - --kv_cache_dtype fp8 $CALCULATED_MAX_MODEL_LEN $EP \ - --gpu-memory-utilization $MEM_FRAC_STATIC \ - --method mtp \ - --num-speculative-tokens 3 \ - --trust-remote-code \ - > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -export PYTHONDONTWRITEBYTECODE=1 -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --trust-remote-code \ - --use-chat-template - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp8_mi355x_mtp.sh b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp8_mi355x_mtp.sh deleted file mode 100755 index 50130482a..000000000 --- a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp8_mi355x_mtp.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL \ - TP \ - CONC \ - ISL \ - OSL \ - RANDOM_RANGE_RATIO \ - RESULT_FILENAME \ - EP_SIZE - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi - -export SGLANG_USE_AITER_UNIFIED_ATTN=1 -export SGLANG_USE_AITER=1 - -SERVER_LOG=/workspace/server.log -CONTEXT_LENGTH=$((ISL + OSL + 20)) - -EVAL_CONTEXT_ARGS="" -if [ "${EVAL_ONLY}" = "true" ]; then - setup_eval_context - EVAL_CONTEXT_ARGS="--context-length $EVAL_MAX_MODEL_LEN" -else EVAL_CONTEXT_ARGS="--context-length $CONTEXT_LENGTH" -fi -# Start GPU monitoring (power, temperature, clocks every second) -start_gpu_monitor - -python3 -m sglang.launch_server \ - --attention-backend aiter \ - --model-path $MODEL \ - --host=0.0.0.0 \ - --port $PORT \ - --tensor-parallel-size $TP \ - --ep-size $EP_SIZE \ - --trust-remote-code \ - --tokenizer-worker-num 6 \ - --enable-aiter-allreduce-fusion \ - --max-running-requests $CONC \ - --cuda-graph-max-bs $CONC \ - --disable-radix-cache \ - --chunked-prefill-size 32768 \ - --scheduler-recv-interval 30 \ - --mem-fraction-static 0.8 \ - --model-loader-extra-config '{"enable_multithread_load": true}' \ - --page-size 16 \ - --speculative-algorithm EAGLE \ - --speculative-num-steps 3 \ - --speculative-eagle-topk 1 \ - --speculative-num-draft-tokens 4 \ - $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & - -SERVER_PID=$! - -# Wait for server to be ready -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -run_benchmark_serving \ - --model "$MODEL" \ - --port "$PORT" \ - --backend vllm \ - --input-len "$ISL" \ - --output-len "$OSL" \ - --random-range-ratio "$RANDOM_RANGE_RATIO" \ - --num-prompts "$((CONC * 10))" \ - --max-concurrency "$CONC" \ - --result-filename "$RESULT_FILENAME" \ - --result-dir /workspace/ \ - --use-chat-template - -# After throughput, run evaluation only if RUN_EVAL is true -if [ "${RUN_EVAL}" = "true" ]; then - run_eval --framework lm-eval --port "$PORT" - append_lm_eval_summary -fi - -# Stop GPU monitoring -stop_gpu_monitor -set +x diff --git a/benchmarks/single_node/speedbench/dsr1_fp4_b300_vllm.sh b/benchmarks/single_node/speedbench/dsr1_fp4_b300_vllm.sh deleted file mode 100755 index d0357c6b4..000000000 --- a/benchmarks/single_node/speedbench/dsr1_fp4_b300_vllm.sh +++ /dev/null @@ -1,243 +0,0 @@ -#!/usr/bin/env bash - -# DeepSeek-R1 B300 vLLM SPEED-Bench AL matrix collector. -# -# Produces the golden acceptance-length (AL) reference matrix consumed by the -# synthetic-acceptance framework: for each MTP level (num_speculative_tokens), -# measure the REAL AL on a single SPEED-Bench category (default: coding) and emit -# a YAML matrix identical in shape to benchmarks/speedbench-reference-al.yaml. -# This measures real MTP acceptance; the synthetic value is injected downstream -# by the throughput recipe, not here. -# -# Adapted from speedbench/dsv4_fp4_b300_vllm.sh. DeepSeek-R1 is DeepSeek-V3 -# architecture (MLA dense attention), NOT V4 (DSA / Lightning Indexer), so vs the -# DSV4 collector: -# - NO --tokenizer-mode deepseek_v4 / --reasoning-parser deepseek_v4 / -# --tool-call-parser deepseek_v4 (all V4-specific; the official vLLM R1 -# serve command is bare). reasoning-parser is irrelevant here anyway: AL is -# read from /metrics, not from parsed output. -# - NO --attention_config.use_fp4_indexer_cache (that knob is dsv32/MLA-indexer -# only; R1 is plain MLA and never reads it). -# - NO --block-size / --compilation-config (the official R1 recipe omits them; -# defaults apply). --kv-cache-dtype fp8 IS kept, to match the dsv4/qwen/glm -# collectors so all golden AL values share one kv-cache numeric regime. -# - FP4 on Blackwell needs FlashInfer MoE: export VLLM_USE_FLASHINFER_MOE_FP4=1. -# - THINKING: R1 is a pure reasoning model and always emits (its chat -# template has no enable_thinking toggle). There is no thinking-off mode, so -# this collector measures thinking_on only and needs no --chat-template-kwargs -# shim (the default client-side template render already enables thinking). -# -# Checkpoint (B300 / Blackwell): NVFP4 build nvidia/DeepSeek-R1-0528-NVFP4-v2, -# basename dsr1-fp4 on the runner (resolved by launch_b300-nv.sh). -# -# Usage (inside the vLLM container, on a B300 node): -# export MODEL=/data/models/dsr1-fp4 -# bash benchmarks/single_node/speedbench/dsr1_fp4_b300_vllm.sh -# -# Tunables (env): -# MTP_LIST space-separated MTP levels (default "1 2 3 4 5 6 7 8") -# THINKING_MODES space-separated: on (default "on"; R1 has no off) -# CATEGORY SPEED-Bench category (default coding) -# SPEEDBENCH_OUTPUT_LEN per-request output len (default 4096) -# OUT_YAML output matrix path (default $RESULTS_DIR/speedbench-reference-al.yaml) - -set -uo pipefail -source "$(dirname "$0")/../../benchmark_lib.sh" - -MODEL="${MODEL:?MODEL env var required (e.g. /data/models/dsr1-fp4)}" -SERVE_MODEL="${MODEL_PATH:-$MODEL}" -TP="${TP:-8}" -DP_ATTENTION="${DP_ATTENTION:-false}" -PORT="${PORT:-8888}" - -MTP_LIST="${MTP_LIST:-1 2 3 4 5 6 7 8}" -THINKING_MODES="${THINKING_MODES:-on}" -CATEGORY="${CATEGORY:-coding}" -MODEL_KEY="${MODEL_KEY:-$(basename "$SERVE_MODEL" | tr '[:upper:]' '[:lower:]')}" -SPEEDBENCH_OUTPUT_LEN="${SPEEDBENCH_OUTPUT_LEN:-4096}" -CONCURRENCY="${CONCURRENCY:-1}" -# Provider-recommended sampling from the DeepSeek-R1 checkpoint generation_config -# (temperature 0.6, top_p 0.95; no top_k). vLLM's own default top_p is 1.0, so it -# MUST be passed explicitly or the measured AL is taken at the wrong settings. -TEMPERATURE="${TEMPERATURE:-0.6}" -TOP_P="${TOP_P:-0.95}" - -SPEEDBENCH_DIR="${SPEEDBENCH_DIR:-/workspace/speed_bench_data}" -# Flat results dir to match the speedbench-al.yml artifact glob -# (speedbench_results/server_*.log) and its pre-run `rm -rf speedbench_results`. -RESULTS_DIR="${RESULTS_DIR:-/workspace/speedbench_results}" -OUT_YAML="${OUT_YAML:-$RESULTS_DIR/speedbench-reference-al.yaml}" - -# Blackwell FP4 MoE path (DeepSeek-R1 FP4 on B-series): required per vLLM R1 docs. -export VLLM_USE_FLASHINFER_MOE_FP4="${VLLM_USE_FLASHINFER_MOE_FP4:-1}" -export VLLM_ENGINE_READY_TIMEOUT_S=3600 - -mkdir -p "$RESULTS_DIR" -nvidia-smi -if [[ "$SERVE_MODEL" != /* ]]; then hf download "$SERVE_MODEL"; fi - -# ---- Download SPEED-Bench dataset ---- -echo "=== Downloading SPEED-Bench dataset ===" -pip install -q datasets tiktoken -curl -LsSf https://raw.githubusercontent.com/NVIDIA-NeMo/Skills/refs/heads/main/nemo_skills/dataset/speed-bench/prepare.py \ - | python3 - --config qualitative --output_dir "$SPEEDBENCH_DIR" - -if [[ ! -f "$SPEEDBENCH_DIR/qualitative.jsonl" ]]; then - echo "CRITICAL: SPEED-Bench download failed — $SPEEDBENCH_DIR/qualitative.jsonl not found" - exit 1 -fi - -PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") -fi - -fetch_metric() { - local port="$1" name="$2" - curl -s "http://localhost:${port}/metrics" \ - | grep -oP "${name}\\{[^}]*\\} \\K[0-9.]+" || echo "0" -} - -SERVER_PID="" -_descendants() { - local pid="$1" child - for child in $(pgrep -P "$pid" 2>/dev/null || true); do - echo "$child" - _descendants "$child" - done -} -cleanup_server() { - if [[ -n "$SERVER_PID" ]]; then - local descendants - descendants=$(_descendants "$SERVER_PID") - kill "$SERVER_PID" 2>/dev/null || true - wait "$SERVER_PID" 2>/dev/null || true - local pid - for pid in $descendants; do - kill -9 "$pid" 2>/dev/null || true - done - local waited=0 - while [[ $waited -lt 120 ]]; do - local used - used=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits 2>/dev/null | sort -rn | head -1) - if [[ -z "$used" || "$used" -lt 2000 ]]; then break; fi - sleep 3; waited=$((waited + 3)) - done - SERVER_PID="" - fi -} -trap 'cleanup_server' EXIT - -start_gpu_monitor - -declare -A AL_RESULT - -run_cell() { - local mode="$1" mtp="$2" - - echo "" - echo "==========================================" - echo " Cell: thinking=$mode MTP=$mtp category=$CATEGORY" - echo "==========================================" - - local serve_args=( - --host 0.0.0.0 --port "$PORT" - "${PARALLEL_ARGS[@]}" - --pipeline-parallel-size 1 - --trust-remote-code - --enable-expert-parallel - --kv-cache-dtype fp8 - --no-enable-prefix-caching - --max-model-len 16384 - --speculative-config "{\"method\": \"mtp\", \"num_speculative_tokens\": $mtp}" - ) - - local server_log="$RESULTS_DIR/server_${mode}_mtp${mtp}.log" - vllm serve "$SERVE_MODEL" "${serve_args[@]}" > "$server_log" 2>&1 & - SERVER_PID=$! - - if ! wait_for_server_ready --port "$PORT" --server-log "$server_log" --server-pid "$SERVER_PID"; then - echo " -> server failed to start (thinking=$mode mtp=$mtp), recording N/A" - AL_RESULT["${mode}_${mtp}"]="N/A" - cleanup_server - return - fi - - local acc_before drf_before acc_after drf_after - acc_before=$(fetch_metric "$PORT" "vllm:spec_decode_num_accepted_tokens_total") - drf_before=$(fetch_metric "$PORT" "vllm:spec_decode_num_drafts_total") - - vllm bench serve \ - --model "$SERVE_MODEL" \ - --port "$PORT" \ - --dataset-name speed_bench \ - --dataset-path "$SPEEDBENCH_DIR" \ - --speed-bench-category "$CATEGORY" \ - --speed-bench-output-len "$SPEEDBENCH_OUTPUT_LEN" \ - --num-prompts -1 \ - --max-concurrency "$CONCURRENCY" \ - --save-result \ - --save-detailed \ - --result-dir "$RESULTS_DIR" \ - --result-filename "speedbench_${mode}_mtp${mtp}" \ - --trust-remote-code \ - --temperature "$TEMPERATURE" \ - --top-p "$TOP_P" - - acc_after=$(fetch_metric "$PORT" "vllm:spec_decode_num_accepted_tokens_total") - drf_after=$(fetch_metric "$PORT" "vllm:spec_decode_num_drafts_total") - - local delta_acc delta_drf al - delta_acc=$(awk "BEGIN {printf \"%d\", $acc_after - $acc_before}") - delta_drf=$(awk "BEGIN {printf \"%d\", $drf_after - $drf_before}") - if [[ "$delta_drf" -gt 0 ]]; then - al=$(awk "BEGIN {printf \"%.2f\", 1 + ($delta_acc / $delta_drf)}") - else - al="N/A" - fi - echo " -> thinking=$mode MTP=$mtp AL=$al (accepted=$delta_acc drafts=$delta_drf)" - AL_RESULT["${mode}_${mtp}"]="$al" - - cleanup_server -} - -for mode in $THINKING_MODES; do - for mtp in $MTP_LIST; do - run_cell "$mode" "$mtp" - done -done - -stop_gpu_monitor - -# ---- Emit the YAML matrix ---- -emit_mode_block() { - local mode="$1" - for mtp in $MTP_LIST; do - echo " $mtp: ${AL_RESULT[${mode}_${mtp}]:-N/A}" - done -} - -{ - echo "# Acceptance Length (AL) reference values measured with SPEED-Bench." - echo "# dataset: $CATEGORY | temperature: $TEMPERATURE | top_p: $TOP_P | output_len: $SPEEDBENCH_OUTPUT_LEN" - echo "# DeepSeek-R1 always reasons (no thinking-off mode), so only thinking_on is emitted." - echo "# Measured on $MODEL_KEY (B300, vLLM MTP), per num_speculative_tokens." - echo "# Auto-generated by benchmarks/single_node/speedbench/dsr1_fp4_b300_vllm.sh (speedbench-al.yml)." - echo "#" - echo "# key = num_speculative_tokens (MTP level); value = golden AL" - echo "${MODEL_KEY}:" - if [[ " $THINKING_MODES " == *" on "* ]]; then - echo " thinking_on:" - emit_mode_block on - fi - if [[ " $THINKING_MODES " == *" off "* ]]; then - echo " thinking_off:" - emit_mode_block off - fi -} > "$OUT_YAML" - -echo "" -echo "==========================================" -echo " SPEED-Bench AL matrix written to: $OUT_YAML" -echo "==========================================" -cat "$OUT_YAML" diff --git a/benchmarks/single_node/speedbench/dsv4_fp4_b300_vllm.sh b/benchmarks/single_node/speedbench/dsv4_fp4_b300_vllm.sh deleted file mode 100755 index b8550a350..000000000 --- a/benchmarks/single_node/speedbench/dsv4_fp4_b300_vllm.sh +++ /dev/null @@ -1,342 +0,0 @@ -#!/usr/bin/env bash - -# DSV4-Pro B300 vLLM SPEED-Bench AL matrix collector. -# -# Produces the golden acceptance-length (AL) reference matrix consumed by the -# synthetic-acceptance framework: for each thinking mode (on/off) and each MTP -# level (num_speculative_tokens), measure the AL on a single SPEED-Bench -# category (default: coding) and emit a YAML matrix identical in shape to -# benchmarks/speedbench-reference-al.yaml. -# -# This is the "AL distribution collection" script wired into the -# speedbench-al.yml GitHub Action (workflow_dispatch / push-button). -# -# Usage (inside the vLLM container, on a B300 node): -# export MODEL=/data/models/dsv4-pro -# bash benchmarks/single_node/speedbench/dsv4_fp4_b300_vllm.sh -# -# Tunables (env): -# MTP_LIST space-separated MTP levels (default "1 2 3 4 5 6 7 8") -# THINKING_MODES space-separated: off|on (default "off on") -# CATEGORY SPEED-Bench category (default coding) -# SPEEDBENCH_OUTPUT_LEN per-request output len (default 4096) -# OUT_YAML output matrix path (default $RESULTS_DIR/speedbench-reference-al.yaml) - -set -uo pipefail -source "$(dirname "$0")/../../benchmark_lib.sh" - -MODEL="${MODEL:?MODEL env var required (e.g. /data/models/dsv4-pro)}" -# Serve from the local weights dir resolved by the launcher (MODEL_PATH points -# at the pre-staged copy, e.g. /scratch/models/DeepSeek-V4-Pro). Falls back to -# MODEL for a standalone local run where MODEL is itself a path. A leading "/" -# makes the download guard below a no-op. -SERVE_MODEL="${MODEL_PATH:-$MODEL}" -TP="${TP:-8}" -DP_ATTENTION="${DP_ATTENTION:-false}" -EP_SIZE="${EP_SIZE:-1}" -PORT="${PORT:-8888}" - -MTP_LIST="${MTP_LIST:-1 2 3 4 5 6 7 8}" -THINKING_MODES="${THINKING_MODES:-off on}" -CATEGORY="${CATEGORY:-coding}" -# Top-level key in the emitted YAML matrix. Derived from the model by the -# workflow (e.g. deepseek-v4-pro); falls back to the model basename, lowercased. -MODEL_KEY="${MODEL_KEY:-$(basename "$SERVE_MODEL" | tr '[:upper:]' '[:lower:]')}" -SPEEDBENCH_OUTPUT_LEN="${SPEEDBENCH_OUTPUT_LEN:-4096}" -CONCURRENCY="${CONCURRENCY:-1}" -TEMPERATURE="${TEMPERATURE:-1.0}" -# thinking-on chat_template_kwargs. MUST match the production/golden config: -# the reference matrix (benchmarks/speedbench-reference-al.yaml) was measured -# with reasoning_effort=high. -DEFAULT_CHAT_TEMPLATE_KWARGS_ON='{"thinking": true, "reasoning_effort": "high"}' -CHAT_TEMPLATE_KWARGS_ON="${CHAT_TEMPLATE_KWARGS_ON:-$DEFAULT_CHAT_TEMPLATE_KWARGS_ON}" - -SPEEDBENCH_DIR="${SPEEDBENCH_DIR:-/workspace/speed_bench_data}" -RESULTS_DIR="${RESULTS_DIR:-/workspace/speedbench_results}" -OUT_YAML="${OUT_YAML:-$RESULTS_DIR/speedbench-reference-al.yaml}" - -export VLLM_ENGINE_READY_TIMEOUT_S=3600 - -mkdir -p "$RESULTS_DIR" -nvidia-smi -if [[ "$SERVE_MODEL" != /* ]]; then hf download "$SERVE_MODEL"; fi - -# ---- Download SPEED-Bench dataset ---- -echo "=== Downloading SPEED-Bench dataset ===" -pip install -q datasets tiktoken -curl -LsSf https://raw.githubusercontent.com/NVIDIA-NeMo/Skills/refs/heads/main/nemo_skills/dataset/speed-bench/prepare.py \ - | python3 - --config qualitative --output_dir "$SPEEDBENCH_DIR" - -if [[ ! -f "$SPEEDBENCH_DIR/qualitative.jsonl" ]]; then - echo "CRITICAL: SPEED-Bench download failed — $SPEEDBENCH_DIR/qualitative.jsonl not found" - exit 1 -fi - -# ---- Temporary shim: add a real --chat-template-kwargs CLI option ---- -# Upstream gap (until vllm-project/vllm#44244 lands): speed_bench/CustomDataset -# pre-renders the chat template client-side WITHOUT chat_template_kwargs and -# posts to /v1/completions, so thinking mode cannot be enabled via --extra-body -# or --default-chat-template-kwargs. This wires a proper --chat-template-kwargs -# option through get_samples into CustomDataset.sample's apply_chat_template. -# TODO: delete this whole block once #44244 is released in the benchmark image; -# the patch is idempotent (marker check) so it is safe to leave until then. -apply_chat_template_kwargs_shim() { - echo "=== Patching vLLM benchmark to add --chat-template-kwargs (temporary shim) ===" - python3 - <<'PYEOF' -import vllm.benchmarks.serve as S -import vllm.benchmarks.datasets.datasets as D - -def patch(mod, edits, marker): - f = mod.__file__ - src = open(f).read() - if marker in src: - print("already patched:", f) - return - for old, new in edits: - n = src.count(old) - assert n == 1, f"anchor matched {n} times in {f}, aborting:\n{old[:80]}..." - src = src.replace(old, new, 1) - open(f, "w").write(src) - print("patched OK ->", f) - -# Edit 1: serve.py -- declare the --chat-template-kwargs argument before --extra-body -serve_old = ''' parser.add_argument( - "--extra-body",''' -serve_new = ''' parser.add_argument( - "--chat-template-kwargs", - type=json.loads, - default=None, - help="JSON dict forwarded to apply_chat_template during " - "client-side prompt rendering, e.g. to enable reasoning mode.", - ) - parser.add_argument( - "--extra-body",''' -patch(S, [(serve_old, serve_new)], marker='"--chat-template-kwargs"') - -# Edit 2: datasets.py -- forward args.chat_template_kwargs into the speed_bench .sample() call -disp_old = ''' output_len=args.speed_bench_output_len, - enable_multimodal_chat=args.enable_multimodal_chat,''' -disp_new = ''' output_len=args.speed_bench_output_len, - chat_template_kwargs=args.chat_template_kwargs, - enable_multimodal_chat=args.enable_multimodal_chat,''' - -# Edit 3: datasets.py -- forward chat_template_kwargs into CustomDataset.sample's template call -samp_old = ''' # apply template - if not skip_chat_template: - prompt = tokenizer.apply_chat_template( - [{"role": "user", "content": prompt}], - add_generation_prompt=True, - tokenize=False, - ) - - prompt_len = len(tokenizer(prompt).input_ids)''' -samp_new = ''' # apply template - if not skip_chat_template: - _ctk = kwargs.get("chat_template_kwargs") or {} - prompt = tokenizer.apply_chat_template( - [{"role": "user", "content": prompt}], - add_generation_prompt=True, - tokenize=False, - **_ctk, - ) - - prompt_len = len(tokenizer(prompt).input_ids)''' -patch(D, [(disp_old, disp_new), (samp_old, samp_new)], - marker="chat_template_kwargs=args.chat_template_kwargs") -PYEOF -} - -# Apply the shim once if any thinking-on cell is requested. -if [[ " $THINKING_MODES " == *" on "* ]]; then - if ! apply_chat_template_kwargs_shim; then - echo "CRITICAL: --chat-template-kwargs shim failed — aborting" - exit 1 - fi -fi - -PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") -fi -EP_ARGS=() -if [ "${EP_SIZE:-1}" -gt 1 ]; then - EP_ARGS=(--enable-expert-parallel) -fi -MOE_ARGS=() -if [ "${DP_ATTENTION}" = "true" ]; then - MOE_ARGS=(--moe-backend deep_gemm_mega_moe) -fi - -fetch_metric() { - local port="$1" name="$2" - curl -s "http://localhost:${port}/metrics" \ - | grep -oP "${name}\\{[^}]*\\} \\K[0-9.]+" || echo "0" -} - -SERVER_PID="" -# List all descendant PIDs of $1 recursively, matched by PARENT pid. This can -# never include this script (the script is an ancestor of the server, not a -# descendant), so it avoids the self-kill a name-based `pkill -f vllm` caused -# (the script filename contains "vllm"). -_descendants() { - local pid="$1" child - for child in $(pgrep -P "$pid" 2>/dev/null || true); do - echo "$child" - _descendants "$child" - done -} -cleanup_server() { - if [[ -n "$SERVER_PID" ]]; then - # Snapshot the server's worker/EngineCore subprocesses BEFORE killing the - # parent: once the parent dies the children reparent to init and the tree - # link is lost. Killing the captured PIDs guarantees no orphaned worker - # survives to hold GPU memory and OOM the next server start. - local descendants - descendants=$(_descendants "$SERVER_PID") - kill "$SERVER_PID" 2>/dev/null || true - wait "$SERVER_PID" 2>/dev/null || true - local pid - for pid in $descendants; do - kill -9 "$pid" 2>/dev/null || true - done - # Wait for GPU memory to actually free before the next server starts. - local waited=0 - while [[ $waited -lt 120 ]]; do - local used - used=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits 2>/dev/null | sort -rn | head -1) - if [[ -z "$used" || "$used" -lt 2000 ]]; then break; fi - sleep 3; waited=$((waited + 3)) - done - SERVER_PID="" - fi -} -trap 'cleanup_server' EXIT - -start_gpu_monitor - -# Per-cell AL is collected into associative arrays keyed by "mode_mtp". -declare -A AL_RESULT - -run_cell() { - local mode="$1" mtp="$2" - local think_args=() - if [[ "$mode" == "on" ]]; then - think_args=(--chat-template-kwargs "$CHAT_TEMPLATE_KWARGS_ON") - fi - - echo "" - echo "==========================================" - echo " Cell: thinking=$mode MTP=$mtp category=$CATEGORY" - echo "==========================================" - - local serve_args=( - --host 0.0.0.0 --port "$PORT" - "${PARALLEL_ARGS[@]}" - --pipeline-parallel-size 1 - --kv-cache-dtype fp8 - --trust-remote-code - --block-size 256 - --no-enable-prefix-caching - "${EP_ARGS[@]}" - "${MOE_ARGS[@]}" - --compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' - --attention_config.use_fp4_indexer_cache True - --tokenizer-mode deepseek_v4 - --tool-call-parser deepseek_v4 - --enable-auto-tool-choice - --reasoning-parser deepseek_v4 - --max-cudagraph-capture-size 2048 - --max-model-len 16384 - --speculative-config "{\"method\": \"mtp\", \"num_speculative_tokens\": $mtp}" - ) - - local server_log="$RESULTS_DIR/server_${mode}_mtp${mtp}.log" - vllm serve "$SERVE_MODEL" "${serve_args[@]}" > "$server_log" 2>&1 & - SERVER_PID=$! - - if ! wait_for_server_ready --port "$PORT" --server-log "$server_log" --server-pid "$SERVER_PID"; then - echo " -> server failed to start (thinking=$mode mtp=$mtp), recording N/A" - AL_RESULT["${mode}_${mtp}"]="N/A" - cleanup_server - return - fi - - local acc_before drf_before acc_after drf_after - acc_before=$(fetch_metric "$PORT" "vllm:spec_decode_num_accepted_tokens_total") - drf_before=$(fetch_metric "$PORT" "vllm:spec_decode_num_drafts_total") - - vllm bench serve \ - --model "$SERVE_MODEL" \ - --port "$PORT" \ - --dataset-name speed_bench \ - --dataset-path "$SPEEDBENCH_DIR" \ - --speed-bench-category "$CATEGORY" \ - --speed-bench-output-len "$SPEEDBENCH_OUTPUT_LEN" \ - --num-prompts -1 \ - --max-concurrency "$CONCURRENCY" \ - --save-result \ - --save-detailed \ - --result-dir "$RESULTS_DIR" \ - --result-filename "speedbench_${mode}_mtp${mtp}" \ - --trust-remote-code \ - --tokenizer-mode deepseek_v4 \ - --temperature "$TEMPERATURE" \ - "${think_args[@]}" - - acc_after=$(fetch_metric "$PORT" "vllm:spec_decode_num_accepted_tokens_total") - drf_after=$(fetch_metric "$PORT" "vllm:spec_decode_num_drafts_total") - - local delta_acc delta_drf al - delta_acc=$(awk "BEGIN {printf \"%d\", $acc_after - $acc_before}") - delta_drf=$(awk "BEGIN {printf \"%d\", $drf_after - $drf_before}") - if [[ "$delta_drf" -gt 0 ]]; then - al=$(awk "BEGIN {printf \"%.2f\", 1 + ($delta_acc / $delta_drf)}") - else - al="N/A" - fi - echo " -> thinking=$mode MTP=$mtp AL=$al (accepted=$delta_acc drafts=$delta_drf)" - AL_RESULT["${mode}_${mtp}"]="$al" - - cleanup_server -} - -for mode in $THINKING_MODES; do - for mtp in $MTP_LIST; do - run_cell "$mode" "$mtp" - done -done - -stop_gpu_monitor - -# ---- Emit the YAML matrix ---- -emit_mode_block() { - local mode="$1" - for mtp in $MTP_LIST; do - echo " $mtp: ${AL_RESULT[${mode}_${mtp}]:-N/A}" - done -} - -{ - echo "# Acceptance Length (AL) reference values measured with SPEED-Bench." - echo "# dataset: $CATEGORY | temperature: $TEMPERATURE | output_len: $SPEEDBENCH_OUTPUT_LEN" - echo "# thinking_on chat_template_kwargs: $CHAT_TEMPLATE_KWARGS_ON" - echo "# Measured on $MODEL_KEY (B300, vLLM MTP), per num_speculative_tokens." - echo "# Auto-generated by benchmarks/single_node/speedbench/dsv4_fp4_b300_vllm.sh (speedbench-al.yml)." - echo "#" - echo "# key = num_speculative_tokens (MTP level); value = golden AL" - echo "${MODEL_KEY}:" - if [[ " $THINKING_MODES " == *" on "* ]]; then - echo " thinking_on:" - emit_mode_block on - fi - if [[ " $THINKING_MODES " == *" off "* ]]; then - echo " thinking_off:" - emit_mode_block off - fi -} > "$OUT_YAML" - -echo "" -echo "==========================================" -echo " SPEED-Bench AL matrix written to: $OUT_YAML" -echo "==========================================" -cat "$OUT_YAML" diff --git a/benchmarks/single_node/speedbench/glm52_fp4_b300_vllm.sh b/benchmarks/single_node/speedbench/glm52_fp4_b300_vllm.sh deleted file mode 100644 index f7290fe6b..000000000 --- a/benchmarks/single_node/speedbench/glm52_fp4_b300_vllm.sh +++ /dev/null @@ -1,251 +0,0 @@ -#!/usr/bin/env bash - -# GLM-5.2 B300 vLLM SPEED-Bench AL matrix collector. -# -# Identical to glm5_fp4_b300_vllm.sh (same GLM DSA architecture, same MTP, -# same serve flags) but with a proper download guard: if MODEL_PATH points to -# an empty directory (model not pre-staged), the script downloads weights from -# HuggingFace before starting the server. The GLM-5 collector skips the -# download when MODEL_PATH is already set (assumes pre-staged); this variant -# handles the not-yet-staged case for GLM-5.2. -# -# Serve parameters, sampling, thinking kwargs, and the chat-template-kwargs -# shim are all inherited from the GLM-5 collector unchanged — GLM-5.2 shares -# the same architecture (glm_moe_dsa), MTP head, and chat template. -# -# Usage (inside the vLLM container, on a B300 node): -# export MODEL=zai-org/GLM-5.2-FP8 -# bash benchmarks/single_node/speedbench/glm52_fp4_b300_vllm.sh -# -# Tunables (env): same as glm5_fp4_b300_vllm.sh - -set -uo pipefail -source "$(dirname "$0")/../../benchmark_lib.sh" - -MODEL="${MODEL:?MODEL env var required (e.g. zai-org/GLM-5.2-FP8)}" -SERVE_MODEL="${MODEL_PATH:-$MODEL}" -TP="${TP:-8}" -DP_ATTENTION="${DP_ATTENTION:-false}" -EP_SIZE="${EP_SIZE:-1}" -PORT="${PORT:-8888}" -GPU_MEM_UTIL="${GPU_MEM_UTIL:-0.80}" - -MTP_LIST="${MTP_LIST:-1 2 3 4 5 6 7 8}" -THINKING_MODES="${THINKING_MODES:-off on}" -CATEGORY="${CATEGORY:-coding}" -MODEL_KEY="${MODEL_KEY:-$(basename "$SERVE_MODEL" | tr '[:upper:]' '[:lower:]')}" -SPEEDBENCH_OUTPUT_LEN="${SPEEDBENCH_OUTPUT_LEN:-4096}" -CONCURRENCY="${CONCURRENCY:-1}" -TEMPERATURE="${TEMPERATURE:-1.0}" -TOP_P="${TOP_P:-0.95}" -DEFAULT_CHAT_TEMPLATE_KWARGS_ON='{"enable_thinking": true}' -DEFAULT_CHAT_TEMPLATE_KWARGS_OFF='{"enable_thinking": false}' -CHAT_TEMPLATE_KWARGS_ON="${CHAT_TEMPLATE_KWARGS_ON:-$DEFAULT_CHAT_TEMPLATE_KWARGS_ON}" -CHAT_TEMPLATE_KWARGS_OFF="${CHAT_TEMPLATE_KWARGS_OFF:-$DEFAULT_CHAT_TEMPLATE_KWARGS_OFF}" - -SPEEDBENCH_DIR="${SPEEDBENCH_DIR:-/workspace/speed_bench_data}" -RESULTS_DIR="${RESULTS_DIR:-/workspace/speedbench_results}" -OUT_YAML="${OUT_YAML:-$RESULTS_DIR/speedbench-reference-al.yaml}" - -export VLLM_ENGINE_READY_TIMEOUT_S=3600 - -mkdir -p "$RESULTS_DIR" -nvidia-smi - -# ---- Download model if not pre-staged ---- -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - echo "=== MODEL_PATH ($MODEL_PATH) is empty, downloading $MODEL ===" - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - if [[ "$SERVE_MODEL" != /* ]]; then hf download "$SERVE_MODEL"; fi -fi - -# ---- Download SPEED-Bench dataset ---- -echo "=== Downloading SPEED-Bench dataset ===" -pip install -q datasets tiktoken -curl -LsSf https://raw.githubusercontent.com/NVIDIA-NeMo/Skills/refs/heads/main/nemo_skills/dataset/speed-bench/prepare.py \ - | python3 - --config qualitative --output_dir "$SPEEDBENCH_DIR" - -if [[ ! -f "$SPEEDBENCH_DIR/qualitative.jsonl" ]]; then - echo "CRITICAL: SPEED-Bench download failed — $SPEEDBENCH_DIR/qualitative.jsonl not found" - exit 1 -fi - -# NOTE: --chat-template-kwargs is consumed natively by `vllm bench serve` here. -# GLM-5.2 only loads on the dedicated vLLM image (>=0.23), which already carries -# vllm-project/vllm#44244, so no client-side shim is needed (unlike the v0.22 -# collectors that still patch it in). - -PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") -fi -EP_ARGS=() -if [ "${EP_SIZE:-1}" -gt 1 ]; then - EP_ARGS=(--enable-expert-parallel) -fi - -fetch_metric() { - local port="$1" name="$2" - curl -s "http://localhost:${port}/metrics" \ - | grep -oP "${name}\\{[^}]*\\} \\K[0-9.]+" || echo "0" -} - -SERVER_PID="" -_descendants() { - local pid="$1" child - for child in $(pgrep -P "$pid" 2>/dev/null || true); do - echo "$child" - _descendants "$child" - done -} -cleanup_server() { - if [[ -n "$SERVER_PID" ]]; then - local descendants - descendants=$(_descendants "$SERVER_PID") - kill "$SERVER_PID" 2>/dev/null || true - wait "$SERVER_PID" 2>/dev/null || true - local pid - for pid in $descendants; do - kill -9 "$pid" 2>/dev/null || true - done - local waited=0 - while [[ $waited -lt 120 ]]; do - local used - used=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits 2>/dev/null | sort -rn | head -1) - if [[ -z "$used" || "$used" -lt 2000 ]]; then break; fi - sleep 3; waited=$((waited + 3)) - done - SERVER_PID="" - fi -} -trap 'cleanup_server' EXIT - -start_gpu_monitor - -declare -A AL_RESULT - -run_cell() { - local mode="$1" mtp="$2" - local think_args=() - if [[ "$mode" == "on" && -n "$CHAT_TEMPLATE_KWARGS_ON" ]]; then - think_args=(--chat-template-kwargs "$CHAT_TEMPLATE_KWARGS_ON") - elif [[ "$mode" == "off" && -n "$CHAT_TEMPLATE_KWARGS_OFF" ]]; then - think_args=(--chat-template-kwargs "$CHAT_TEMPLATE_KWARGS_OFF") - fi - - echo "" - echo "==========================================" - echo " Cell: thinking=$mode MTP=$mtp category=$CATEGORY" - echo "==========================================" - - local serve_args=( - --host 0.0.0.0 --port "$PORT" - "${PARALLEL_ARGS[@]}" - --pipeline-parallel-size 1 - --kv-cache-dtype fp8 - --trust-remote-code - --no-enable-prefix-caching - "${EP_ARGS[@]}" - --reasoning-parser glm45 - --tool-call-parser glm47 - --enable-auto-tool-choice - --chat-template-content-format=string - --gpu-memory-utilization "$GPU_MEM_UTIL" - --max-model-len 16384 - --speculative-config "{\"method\": \"mtp\", \"num_speculative_tokens\": $mtp}" - ) - - local server_log="$RESULTS_DIR/server_${mode}_mtp${mtp}.log" - vllm serve "$SERVE_MODEL" "${serve_args[@]}" > "$server_log" 2>&1 & - SERVER_PID=$! - - if ! wait_for_server_ready --port "$PORT" --server-log "$server_log" --server-pid "$SERVER_PID"; then - echo " -> server failed to start (thinking=$mode mtp=$mtp), recording N/A" - AL_RESULT["${mode}_${mtp}"]="N/A" - cleanup_server - return - fi - - local acc_before drf_before acc_after drf_after - acc_before=$(fetch_metric "$PORT" "vllm:spec_decode_num_accepted_tokens_total") - drf_before=$(fetch_metric "$PORT" "vllm:spec_decode_num_drafts_total") - - vllm bench serve \ - --model "$SERVE_MODEL" \ - --port "$PORT" \ - --dataset-name speed_bench \ - --dataset-path "$SPEEDBENCH_DIR" \ - --speed-bench-category "$CATEGORY" \ - --speed-bench-output-len "$SPEEDBENCH_OUTPUT_LEN" \ - --num-prompts -1 \ - --max-concurrency "$CONCURRENCY" \ - --save-result \ - --save-detailed \ - --result-dir "$RESULTS_DIR" \ - --result-filename "speedbench_${mode}_mtp${mtp}" \ - --trust-remote-code \ - --temperature "$TEMPERATURE" \ - --top-p "$TOP_P" \ - "${think_args[@]}" - - acc_after=$(fetch_metric "$PORT" "vllm:spec_decode_num_accepted_tokens_total") - drf_after=$(fetch_metric "$PORT" "vllm:spec_decode_num_drafts_total") - - local delta_acc delta_drf al - delta_acc=$(awk "BEGIN {printf \"%d\", $acc_after - $acc_before}") - delta_drf=$(awk "BEGIN {printf \"%d\", $drf_after - $drf_before}") - if [[ "$delta_drf" -gt 0 ]]; then - al=$(awk "BEGIN {printf \"%.2f\", 1 + ($delta_acc / $delta_drf)}") - else - al="N/A" - fi - echo " -> thinking=$mode MTP=$mtp AL=$al (accepted=$delta_acc drafts=$delta_drf)" - AL_RESULT["${mode}_${mtp}"]="$al" - - cleanup_server -} - -for mode in $THINKING_MODES; do - for mtp in $MTP_LIST; do - run_cell "$mode" "$mtp" - done -done - -stop_gpu_monitor - -# ---- Emit the YAML matrix ---- -emit_mode_block() { - local mode="$1" - for mtp in $MTP_LIST; do - echo " $mtp: ${AL_RESULT[${mode}_${mtp}]:-N/A}" - done -} - -{ - echo "# Acceptance Length (AL) reference values measured with SPEED-Bench." - echo "# dataset: $CATEGORY | temperature: $TEMPERATURE | top_p: $TOP_P | output_len: $SPEEDBENCH_OUTPUT_LEN" - echo "# thinking_on chat_template_kwargs: $CHAT_TEMPLATE_KWARGS_ON" - echo "# thinking_off chat_template_kwargs: $CHAT_TEMPLATE_KWARGS_OFF" - echo "# Measured on $MODEL_KEY (B300, vLLM MTP), per num_speculative_tokens." - echo "# Auto-generated by benchmarks/single_node/speedbench/glm52_fp4_b300_vllm.sh (speedbench-al.yml)." - echo "#" - echo "# key = num_speculative_tokens (MTP level); value = golden AL" - echo "${MODEL_KEY}:" - if [[ " $THINKING_MODES " == *" on "* ]]; then - echo " thinking_on:" - emit_mode_block on - fi - if [[ " $THINKING_MODES " == *" off "* ]]; then - echo " thinking_off:" - emit_mode_block off - fi -} > "$OUT_YAML" - -echo "" -echo "==========================================" -echo " SPEED-Bench AL matrix written to: $OUT_YAML" -echo "==========================================" -cat "$OUT_YAML" diff --git a/benchmarks/single_node/speedbench/glm5_fp4_b300_vllm.sh b/benchmarks/single_node/speedbench/glm5_fp4_b300_vllm.sh deleted file mode 100755 index 6265500b9..000000000 --- a/benchmarks/single_node/speedbench/glm5_fp4_b300_vllm.sh +++ /dev/null @@ -1,360 +0,0 @@ -#!/usr/bin/env bash - -# GLM-5 B300 vLLM SPEED-Bench AL matrix collector. -# -# Produces the golden acceptance-length (AL) reference matrix consumed by the -# synthetic-acceptance framework: for each thinking mode (on/off) and each MTP -# level (num_speculative_tokens), measure the REAL AL on a single SPEED-Bench -# category (default: coding) and emit a YAML matrix identical in shape to -# benchmarks/speedbench-reference-al.yaml. This measures real MTP acceptance; -# the synthetic value is injected downstream by the throughput recipe, not here. -# -# Filename *_fp4_* matches both the speedbench-al.yml path convention -# (benchmarks/single_node/speedbench/${model-prefix}_fp4_b300_vllm.sh) and the -# served checkpoint: we serve the NVFP4 build (GLM-5-NVFP4), like every model in -# this matrix. The official vLLM GLM recipe only documents FP8, but the B300 runs -# use the NVFP4 checkpoint. -# -# Adapted from speedbench/dsv4_fp4_b300_vllm.sh. Differences vs DSV4 (deepseek_v4 -# is NOT reusable for GLM): -# - reasoning-parser glm45 (was deepseek_v4) -# - tool-call-parser glm47 (was deepseek_v4) -# - --chat-template-content-format=string (GLM requirement per vLLM docs) -# - NO --tokenizer-mode deepseek_v4 (GLM uses the default/auto tokenizer) -# - --attention_config.use_fp4_indexer_cache is NOT passed (and must not be). -# Despite GLM-5 also being DSA sparse attention, that knob is wired ONLY for -# the DeepSeek dsv32 family: it is read solely by vllm/models/deepseek_v4/ -# attention.py and the MLA indexer backend (vllm/v1/attention/backends/mla/ -# indexer.py). GLM's DSA (GlmMoeDsaForCausalLM) is a separate codepath that -# never reads it, so setting it would be a no-op at best or a config error at -# worst. A GLM DSA-indexer OOM would need a GLM-specific option, not this one. -# - thinking on/off uses the enable_thinking chat_template key; thinking is ON -# by default for GLM, so the OFF cell MUST pass enable_thinking:false explicitly -# -# Checkpoint (B300 / Blackwell): NVFP4 build, basename GLM-5-NVFP4. NVIDIA's -# GLM-5-NVFP4 model card serves it with vllm/vllm-openai:latest, and the runner's -# vllm-openai:v0.21.0 (May) is newer than that 3/16 example, so it loads directly. -# For tool calling + MTP together, vLLM docs recommend a recent build. -# -# Usage (inside the GLM vLLM container, on a B300 node): -# export MODEL=/scratch/models/GLM-5-NVFP4 -# bash benchmarks/single_node/speedbench/glm5_fp4_b300_vllm.sh -# -# Tunables (env): -# MTP_LIST space-separated MTP levels (default "1 2 3 4 5 6 7 8") -# THINKING_MODES space-separated: off|on (default "off on") -# CATEGORY SPEED-Bench category (default coding) -# SPEEDBENCH_OUTPUT_LEN per-request output len (default 4096) -# OUT_YAML output matrix path (default $RESULTS_DIR/speedbench-reference-al.yaml) - -set -uo pipefail -source "$(dirname "$0")/../../benchmark_lib.sh" - -MODEL="${MODEL:?MODEL env var required (e.g. /scratch/models/GLM-5-NVFP4)}" -SERVE_MODEL="${MODEL_PATH:-$MODEL}" -TP="${TP:-8}" -DP_ATTENTION="${DP_ATTENTION:-false}" -EP_SIZE="${EP_SIZE:-1}" -PORT="${PORT:-8888}" -# NVIDIA's GLM-5-NVFP4 model card serves with 0.80; NVFP4 + DSA + MTP draft -# layers leave less headroom than DSV4, so match it to avoid startup OOM. -GPU_MEM_UTIL="${GPU_MEM_UTIL:-0.80}" - -MTP_LIST="${MTP_LIST:-1 2 3 4 5 6 7 8}" -THINKING_MODES="${THINKING_MODES:-off on}" -CATEGORY="${CATEGORY:-coding}" -MODEL_KEY="${MODEL_KEY:-$(basename "$SERVE_MODEL" | tr '[:upper:]' '[:lower:]')}" -SPEEDBENCH_OUTPUT_LEN="${SPEEDBENCH_OUTPUT_LEN:-4096}" -CONCURRENCY="${CONCURRENCY:-1}" -# Provider-recommended sampling from the GLM-5 checkpoint generation_config.json -# (temperature 1.0, top_p 0.95). vLLM's own default top_p is 1.0, so it MUST be -# passed explicitly or the measured AL is taken at the wrong sampling settings. -TEMPERATURE="${TEMPERATURE:-1.0}" -TOP_P="${TOP_P:-0.95}" -# GLM thinking toggles via the enable_thinking chat_template key (default ON). -# Use separate single-quoted defaults: an inline ${VAR:-{...}} default whose value -# contains "}" is truncated by bash brace parsing (matches upstream fix #1695). -DEFAULT_CHAT_TEMPLATE_KWARGS_ON='{"enable_thinking": true}' -DEFAULT_CHAT_TEMPLATE_KWARGS_OFF='{"enable_thinking": false}' -CHAT_TEMPLATE_KWARGS_ON="${CHAT_TEMPLATE_KWARGS_ON:-$DEFAULT_CHAT_TEMPLATE_KWARGS_ON}" -CHAT_TEMPLATE_KWARGS_OFF="${CHAT_TEMPLATE_KWARGS_OFF:-$DEFAULT_CHAT_TEMPLATE_KWARGS_OFF}" - -SPEEDBENCH_DIR="${SPEEDBENCH_DIR:-/workspace/speed_bench_data}" -# Flat results dir to match the speedbench-al.yml artifact glob -# (speedbench_results/server_*.log) and its pre-run `rm -rf speedbench_results`. -RESULTS_DIR="${RESULTS_DIR:-/workspace/speedbench_results}" -OUT_YAML="${OUT_YAML:-$RESULTS_DIR/speedbench-reference-al.yaml}" - -export VLLM_ENGINE_READY_TIMEOUT_S=3600 - -mkdir -p "$RESULTS_DIR" -nvidia-smi -if [[ "$SERVE_MODEL" != /* ]]; then hf download "$SERVE_MODEL"; fi - -# ---- Download SPEED-Bench dataset ---- -echo "=== Downloading SPEED-Bench dataset ===" -pip install -q datasets tiktoken -curl -LsSf https://raw.githubusercontent.com/NVIDIA-NeMo/Skills/refs/heads/main/nemo_skills/dataset/speed-bench/prepare.py \ - | python3 - --config qualitative --output_dir "$SPEEDBENCH_DIR" - -if [[ ! -f "$SPEEDBENCH_DIR/qualitative.jsonl" ]]; then - echo "CRITICAL: SPEED-Bench download failed — $SPEEDBENCH_DIR/qualitative.jsonl not found" - exit 1 -fi - -# ---- Temporary shim: add a real --chat-template-kwargs CLI option ---- -# Upstream gap (until vllm-project/vllm#44244 lands): speed_bench/CustomDataset -# pre-renders the chat template client-side WITHOUT chat_template_kwargs and -# posts to /v1/completions, so thinking mode cannot be enabled via --extra-body -# or --default-chat-template-kwargs. This wires a proper --chat-template-kwargs -# option through get_samples into CustomDataset.sample's apply_chat_template. -# Model agnostic (forwards whatever dict it is given). TODO: delete once #44244 -# is released in the benchmark image; idempotent (marker check), safe to leave. -apply_chat_template_kwargs_shim() { - echo "=== Patching vLLM benchmark to add --chat-template-kwargs (temporary shim) ===" - python3 - <<'PYEOF' -import vllm.benchmarks.serve as S -import vllm.benchmarks.datasets.datasets as D - -def patch(mod, edits, marker): - f = mod.__file__ - src = open(f).read() - if marker in src: - print("already patched:", f) - return - for old, new in edits: - n = src.count(old) - assert n == 1, f"anchor matched {n} times in {f}, aborting:\n{old[:80]}..." - src = src.replace(old, new, 1) - open(f, "w").write(src) - print("patched OK ->", f) - -# Edit 1: serve.py -- declare the --chat-template-kwargs argument before --extra-body -serve_old = ''' parser.add_argument( - "--extra-body",''' -serve_new = ''' parser.add_argument( - "--chat-template-kwargs", - type=json.loads, - default=None, - help="JSON dict forwarded to apply_chat_template during " - "client-side prompt rendering, e.g. to enable reasoning mode.", - ) - parser.add_argument( - "--extra-body",''' -patch(S, [(serve_old, serve_new)], marker='"--chat-template-kwargs"') - -# Edit 2: datasets.py -- forward args.chat_template_kwargs into the speed_bench .sample() call -disp_old = ''' output_len=args.speed_bench_output_len, - enable_multimodal_chat=args.enable_multimodal_chat,''' -disp_new = ''' output_len=args.speed_bench_output_len, - chat_template_kwargs=args.chat_template_kwargs, - enable_multimodal_chat=args.enable_multimodal_chat,''' - -# Edit 3: datasets.py -- forward chat_template_kwargs into CustomDataset.sample's template call -samp_old = ''' # apply template - if not skip_chat_template: - prompt = tokenizer.apply_chat_template( - [{"role": "user", "content": prompt}], - add_generation_prompt=True, - tokenize=False, - ) - - prompt_len = len(tokenizer(prompt).input_ids)''' -samp_new = ''' # apply template - if not skip_chat_template: - _ctk = kwargs.get("chat_template_kwargs") or {} - prompt = tokenizer.apply_chat_template( - [{"role": "user", "content": prompt}], - add_generation_prompt=True, - tokenize=False, - **_ctk, - ) - - prompt_len = len(tokenizer(prompt).input_ids)''' -patch(D, [(disp_old, disp_new), (samp_old, samp_new)], - marker="chat_template_kwargs=args.chat_template_kwargs") -PYEOF -} - -# Apply the shim once if any cell will pass chat_template_kwargs. -NEED_SHIM=0 -if [[ " $THINKING_MODES " == *" on "* && -n "$CHAT_TEMPLATE_KWARGS_ON" ]]; then NEED_SHIM=1; fi -if [[ " $THINKING_MODES " == *" off "* && -n "$CHAT_TEMPLATE_KWARGS_OFF" ]]; then NEED_SHIM=1; fi -if [[ "$NEED_SHIM" == "1" ]]; then - if ! apply_chat_template_kwargs_shim; then - echo "CRITICAL: --chat-template-kwargs shim failed — aborting" - exit 1 - fi -fi - -PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") -fi -EP_ARGS=() -if [ "${EP_SIZE:-1}" -gt 1 ]; then - EP_ARGS=(--enable-expert-parallel) -fi - -fetch_metric() { - local port="$1" name="$2" - curl -s "http://localhost:${port}/metrics" \ - | grep -oP "${name}\\{[^}]*\\} \\K[0-9.]+" || echo "0" -} - -SERVER_PID="" -_descendants() { - local pid="$1" child - for child in $(pgrep -P "$pid" 2>/dev/null || true); do - echo "$child" - _descendants "$child" - done -} -cleanup_server() { - if [[ -n "$SERVER_PID" ]]; then - local descendants - descendants=$(_descendants "$SERVER_PID") - kill "$SERVER_PID" 2>/dev/null || true - wait "$SERVER_PID" 2>/dev/null || true - local pid - for pid in $descendants; do - kill -9 "$pid" 2>/dev/null || true - done - local waited=0 - while [[ $waited -lt 120 ]]; do - local used - used=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits 2>/dev/null | sort -rn | head -1) - if [[ -z "$used" || "$used" -lt 2000 ]]; then break; fi - sleep 3; waited=$((waited + 3)) - done - SERVER_PID="" - fi -} -trap 'cleanup_server' EXIT - -start_gpu_monitor - -declare -A AL_RESULT - -run_cell() { - local mode="$1" mtp="$2" - local think_args=() - if [[ "$mode" == "on" && -n "$CHAT_TEMPLATE_KWARGS_ON" ]]; then - think_args=(--chat-template-kwargs "$CHAT_TEMPLATE_KWARGS_ON") - elif [[ "$mode" == "off" && -n "$CHAT_TEMPLATE_KWARGS_OFF" ]]; then - think_args=(--chat-template-kwargs "$CHAT_TEMPLATE_KWARGS_OFF") - fi - - echo "" - echo "==========================================" - echo " Cell: thinking=$mode MTP=$mtp category=$CATEGORY" - echo "==========================================" - - local serve_args=( - --host 0.0.0.0 --port "$PORT" - "${PARALLEL_ARGS[@]}" - --pipeline-parallel-size 1 - --kv-cache-dtype fp8 - --trust-remote-code - --no-enable-prefix-caching - "${EP_ARGS[@]}" - --reasoning-parser glm45 - --tool-call-parser glm47 - --enable-auto-tool-choice - --chat-template-content-format=string - --gpu-memory-utilization "$GPU_MEM_UTIL" - --max-model-len 16384 - --speculative-config "{\"method\": \"mtp\", \"num_speculative_tokens\": $mtp}" - ) - - local server_log="$RESULTS_DIR/server_${mode}_mtp${mtp}.log" - vllm serve "$SERVE_MODEL" "${serve_args[@]}" > "$server_log" 2>&1 & - SERVER_PID=$! - - if ! wait_for_server_ready --port "$PORT" --server-log "$server_log" --server-pid "$SERVER_PID"; then - echo " -> server failed to start (thinking=$mode mtp=$mtp), recording N/A" - AL_RESULT["${mode}_${mtp}"]="N/A" - cleanup_server - return - fi - - local acc_before drf_before acc_after drf_after - acc_before=$(fetch_metric "$PORT" "vllm:spec_decode_num_accepted_tokens_total") - drf_before=$(fetch_metric "$PORT" "vllm:spec_decode_num_drafts_total") - - vllm bench serve \ - --model "$SERVE_MODEL" \ - --port "$PORT" \ - --dataset-name speed_bench \ - --dataset-path "$SPEEDBENCH_DIR" \ - --speed-bench-category "$CATEGORY" \ - --speed-bench-output-len "$SPEEDBENCH_OUTPUT_LEN" \ - --num-prompts -1 \ - --max-concurrency "$CONCURRENCY" \ - --save-result \ - --save-detailed \ - --result-dir "$RESULTS_DIR" \ - --result-filename "speedbench_${mode}_mtp${mtp}" \ - --trust-remote-code \ - --temperature "$TEMPERATURE" \ - --top-p "$TOP_P" \ - "${think_args[@]}" - - acc_after=$(fetch_metric "$PORT" "vllm:spec_decode_num_accepted_tokens_total") - drf_after=$(fetch_metric "$PORT" "vllm:spec_decode_num_drafts_total") - - local delta_acc delta_drf al - delta_acc=$(awk "BEGIN {printf \"%d\", $acc_after - $acc_before}") - delta_drf=$(awk "BEGIN {printf \"%d\", $drf_after - $drf_before}") - if [[ "$delta_drf" -gt 0 ]]; then - al=$(awk "BEGIN {printf \"%.2f\", 1 + ($delta_acc / $delta_drf)}") - else - al="N/A" - fi - echo " -> thinking=$mode MTP=$mtp AL=$al (accepted=$delta_acc drafts=$delta_drf)" - AL_RESULT["${mode}_${mtp}"]="$al" - - cleanup_server -} - -for mode in $THINKING_MODES; do - for mtp in $MTP_LIST; do - run_cell "$mode" "$mtp" - done -done - -stop_gpu_monitor - -# ---- Emit the YAML matrix ---- -emit_mode_block() { - local mode="$1" - for mtp in $MTP_LIST; do - echo " $mtp: ${AL_RESULT[${mode}_${mtp}]:-N/A}" - done -} - -{ - echo "# Acceptance Length (AL) reference values measured with SPEED-Bench." - echo "# dataset: $CATEGORY | temperature: $TEMPERATURE | top_p: $TOP_P | output_len: $SPEEDBENCH_OUTPUT_LEN" - echo "# thinking_on chat_template_kwargs: $CHAT_TEMPLATE_KWARGS_ON" - echo "# thinking_off chat_template_kwargs: $CHAT_TEMPLATE_KWARGS_OFF" - echo "# Measured on $MODEL_KEY (B300, vLLM MTP), per num_speculative_tokens." - echo "# Auto-generated by benchmarks/single_node/speedbench/glm5_fp4_b300_vllm.sh (speedbench-al.yml)." - echo "#" - echo "# key = num_speculative_tokens (MTP level); value = golden AL" - echo "${MODEL_KEY}:" - if [[ " $THINKING_MODES " == *" on "* ]]; then - echo " thinking_on:" - emit_mode_block on - fi - if [[ " $THINKING_MODES " == *" off "* ]]; then - echo " thinking_off:" - emit_mode_block off - fi -} > "$OUT_YAML" - -echo "" -echo "==========================================" -echo " SPEED-Bench AL matrix written to: $OUT_YAML" -echo "==========================================" -cat "$OUT_YAML" diff --git a/benchmarks/single_node/speedbench/kimik2.5_fp4_b300_vllm.sh b/benchmarks/single_node/speedbench/kimik2.5_fp4_b300_vllm.sh deleted file mode 100755 index 890c059f9..000000000 --- a/benchmarks/single_node/speedbench/kimik2.5_fp4_b300_vllm.sh +++ /dev/null @@ -1,376 +0,0 @@ -#!/usr/bin/env bash - -# Kimi-K2.5 B300 vLLM SPEED-Bench AL matrix collector for EAGLE3 speculative -# decoding. -# -# Produces the golden acceptance-length (AL) reference matrix consumed by the -# synthetic-acceptance framework: for each thinking mode (on/off) and each -# EAGLE3 speculative-token count, measure the REAL AL on a single SPEED-Bench -# category (default: coding) and emit a YAML matrix identical in shape to -# benchmarks/speedbench-reference-al.yaml. -# -# Kimi-K2.5 uses the lightseekorg/kimi-k2.5-eagle3-mla draft head (MLA -# variant, recommended by official docs). The draft model is downloaded -# alongside the target checkpoint before the sweep begins. -# -# Differences vs the GLM-5 MTP template (glm5_fp4_b300_vllm.sh): -# - speculative-config eagle3 with external draft model (not mtp) -# - reasoning-parser kimi_k2 (was glm45) -# - tool-call-parser kimi_k2 (was glm47) -# - thinking toggle {"thinking": true/false} (was enable_thinking) -# - temperature 1.0 (thinking) / 0.6 (instant) (was fixed 1.0) -# - NO --chat-template-content-format, --tokenizer-mode, --block-size, -# or --attention_config.use_fp4_indexer_cache -# - --language-model-only (text-only benchmark, no vision) -# -# Usage (inside the Kimi vLLM container, on a B300 node): -# export MODEL=moonshotai/Kimi-K2.5-NVFP4 -# bash benchmarks/single_node/speedbench/kimik2.5_fp4_b300_vllm.sh -# -# Tunables (env): -# MTP_LIST space-separated EAGLE3 spec-token counts (default "1 2 3 4 5 6 7 8") -# THINKING_MODES space-separated: off|on (default "off on") -# CATEGORY SPEED-Bench category (default coding) -# SPEEDBENCH_OUTPUT_LEN per-request output len (default 4096) -# OUT_YAML output matrix path (default $RESULTS_DIR/speedbench-reference-al.yaml) - -set -uo pipefail -source "$(dirname "$0")/../../benchmark_lib.sh" - -MODEL="${MODEL:?MODEL env var required (e.g. moonshotai/Kimi-K2.5-NVFP4)}" -SERVE_MODEL="${MODEL_PATH:-$MODEL}" -TP="${TP:-8}" -DP_ATTENTION="${DP_ATTENTION:-false}" -EP_SIZE="${EP_SIZE:-1}" -PORT="${PORT:-8888}" -GPU_MEM_UTIL="${GPU_MEM_UTIL:-0.80}" - -DRAFT_MODEL="lightseekorg/kimi-k2.5-eagle3-mla" - -MTP_LIST="${MTP_LIST:-1 2 3 4 5 6 7 8}" -THINKING_MODES="${THINKING_MODES:-off on}" -CATEGORY="${CATEGORY:-coding}" -MODEL_KEY="${MODEL_KEY:-$(basename "$SERVE_MODEL" | tr '[:upper:]' '[:lower:]')}" -SPEEDBENCH_OUTPUT_LEN="${SPEEDBENCH_OUTPUT_LEN:-4096}" -# AL is concurrency-independent (per-token accept/reject; no spec-disable-by-batch -# is set below), so batch the SPEED-Bench pass to keep wall-time under the CI -# limit. conc=1 made Kimi-K2.5 exceed the 8h budget. 64 captures most of the -# batch-decode speedup before it saturates / KV pressure grows; override via env. -CONCURRENCY="${CONCURRENCY:-64}" -TOP_P="${TOP_P:-0.95}" -# Kimi thinking toggles via the thinking chat_template key (default ON). -DEFAULT_CHAT_TEMPLATE_KWARGS_ON='{"thinking": true}' -DEFAULT_CHAT_TEMPLATE_KWARGS_OFF='{"thinking": false}' -CHAT_TEMPLATE_KWARGS_ON="${CHAT_TEMPLATE_KWARGS_ON:-$DEFAULT_CHAT_TEMPLATE_KWARGS_ON}" -CHAT_TEMPLATE_KWARGS_OFF="${CHAT_TEMPLATE_KWARGS_OFF:-$DEFAULT_CHAT_TEMPLATE_KWARGS_OFF}" - -SPEEDBENCH_DIR="${SPEEDBENCH_DIR:-/workspace/speed_bench_data}" -RESULTS_DIR="${RESULTS_DIR:-/workspace/speedbench_results}" -OUT_YAML="${OUT_YAML:-$RESULTS_DIR/speedbench-reference-al.yaml}" - -# Blackwell NVFP4 checkpoints need FlashInfer FP4 MoE kernels; auto-enable -# when the served model name contains NVFP4 (e.g. nvidia/Kimi-K2.5-NVFP4). -if [[ "$SERVE_MODEL" == *NVFP4* || "$SERVE_MODEL" == *nvfp4* ]]; then - export VLLM_USE_FLASHINFER_MOE_FP4="${VLLM_USE_FLASHINFER_MOE_FP4:-1}" -fi -export VLLM_ENGINE_READY_TIMEOUT_S=3600 - -mkdir -p "$RESULTS_DIR" -nvidia-smi - -# ---- Download target if it is not pre-staged ---- -# A pre-staged target lands in the read-only staged mount (/scratch/models); -# only download when MODEL_PATH is an empty writable dir (non-staged run). -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - if [[ "$SERVE_MODEL" != /* ]]; then hf download "$SERVE_MODEL"; fi -fi - -# ---- Download EAGLE3 draft model to a WRITABLE dir ---- -# The draft must NOT go next to a pre-staged target: dirname(MODEL_PATH) is the -# read-only staged mount (/scratch/models), so writing the draft there fails -# with PermissionError. Use a writable workspace dir regardless of staging. -DRAFT_DIR="${DRAFT_MODEL_DIR:-/workspace/draft_models}" -mkdir -p "$DRAFT_DIR" -DRAFT_MODEL_PATH="$DRAFT_DIR/${DRAFT_MODEL##*/}" -if [[ ! -d "$DRAFT_MODEL_PATH" || -z "$(ls -A "$DRAFT_MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$DRAFT_MODEL" --local-dir "$DRAFT_MODEL_PATH" -fi - -# ---- Download SPEED-Bench dataset ---- -echo "=== Downloading SPEED-Bench dataset ===" -pip install -q datasets tiktoken -curl -LsSf https://raw.githubusercontent.com/NVIDIA-NeMo/Skills/refs/heads/main/nemo_skills/dataset/speed-bench/prepare.py \ - | python3 - --config qualitative --output_dir "$SPEEDBENCH_DIR" - -if [[ ! -f "$SPEEDBENCH_DIR/qualitative.jsonl" ]]; then - echo "CRITICAL: SPEED-Bench download failed — $SPEEDBENCH_DIR/qualitative.jsonl not found" - exit 1 -fi - -# ---- Temporary shim: add a real --chat-template-kwargs CLI option ---- -# Upstream gap (until vllm-project/vllm#44244 lands): speed_bench/CustomDataset -# pre-renders the chat template client-side WITHOUT chat_template_kwargs and -# posts to /v1/completions, so thinking mode cannot be enabled via --extra-body -# or --default-chat-template-kwargs. This wires a proper --chat-template-kwargs -# option through get_samples into CustomDataset.sample's apply_chat_template. -# Model agnostic (forwards whatever dict it is given). TODO: delete once #44244 -# is released in the benchmark image; idempotent (marker check), safe to leave. -apply_chat_template_kwargs_shim() { - echo "=== Patching vLLM benchmark to add --chat-template-kwargs (temporary shim) ===" - python3 - <<'PYEOF' -import vllm.benchmarks.serve as S -import vllm.benchmarks.datasets.datasets as D - -def patch(mod, edits, marker): - f = mod.__file__ - src = open(f).read() - if marker in src: - print("already patched:", f) - return - for old, new in edits: - n = src.count(old) - assert n == 1, f"anchor matched {n} times in {f}, aborting:\n{old[:80]}..." - src = src.replace(old, new, 1) - open(f, "w").write(src) - print("patched OK ->", f) - -# Edit 1: serve.py -- declare the --chat-template-kwargs argument before --extra-body -serve_old = ''' parser.add_argument( - "--extra-body",''' -serve_new = ''' parser.add_argument( - "--chat-template-kwargs", - type=json.loads, - default=None, - help="JSON dict forwarded to apply_chat_template during " - "client-side prompt rendering, e.g. to enable reasoning mode.", - ) - parser.add_argument( - "--extra-body",''' -patch(S, [(serve_old, serve_new)], marker='"--chat-template-kwargs"') - -# Edit 2: datasets.py -- forward args.chat_template_kwargs into the speed_bench .sample() call -disp_old = ''' output_len=args.speed_bench_output_len, - enable_multimodal_chat=args.enable_multimodal_chat,''' -disp_new = ''' output_len=args.speed_bench_output_len, - chat_template_kwargs=args.chat_template_kwargs, - enable_multimodal_chat=args.enable_multimodal_chat,''' - -# Edit 3: datasets.py -- forward chat_template_kwargs into CustomDataset.sample's template call -samp_old = ''' # apply template - if not skip_chat_template: - prompt = tokenizer.apply_chat_template( - [{"role": "user", "content": prompt}], - add_generation_prompt=True, - tokenize=False, - ) - - prompt_len = len(tokenizer(prompt).input_ids)''' -samp_new = ''' # apply template - if not skip_chat_template: - _ctk = kwargs.get("chat_template_kwargs") or {} - prompt = tokenizer.apply_chat_template( - [{"role": "user", "content": prompt}], - add_generation_prompt=True, - tokenize=False, - **_ctk, - ) - - prompt_len = len(tokenizer(prompt).input_ids)''' -patch(D, [(disp_old, disp_new), (samp_old, samp_new)], - marker="chat_template_kwargs=args.chat_template_kwargs") -PYEOF -} - -# Apply the shim once if any cell will pass chat_template_kwargs. -NEED_SHIM=0 -if [[ " $THINKING_MODES " == *" on "* && -n "$CHAT_TEMPLATE_KWARGS_ON" ]]; then NEED_SHIM=1; fi -if [[ " $THINKING_MODES " == *" off "* && -n "$CHAT_TEMPLATE_KWARGS_OFF" ]]; then NEED_SHIM=1; fi -if [[ "$NEED_SHIM" == "1" ]]; then - if ! apply_chat_template_kwargs_shim; then - echo "CRITICAL: --chat-template-kwargs shim failed — aborting" - exit 1 - fi -fi - -PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") -fi -EP_ARGS=() -if [ "${EP_SIZE:-1}" -gt 1 ]; then - EP_ARGS=(--enable-expert-parallel) -fi - -fetch_metric() { - local port="$1" name="$2" - curl -s "http://localhost:${port}/metrics" \ - | grep -oP "${name}\\{[^}]*\\} \\K[0-9.]+" || echo "0" -} - -SERVER_PID="" -_descendants() { - local pid="$1" child - for child in $(pgrep -P "$pid" 2>/dev/null || true); do - echo "$child" - _descendants "$child" - done -} -cleanup_server() { - if [[ -n "$SERVER_PID" ]]; then - local descendants - descendants=$(_descendants "$SERVER_PID") - kill "$SERVER_PID" 2>/dev/null || true - wait "$SERVER_PID" 2>/dev/null || true - local pid - for pid in $descendants; do - kill -9 "$pid" 2>/dev/null || true - done - local waited=0 - while [[ $waited -lt 120 ]]; do - local used - used=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits 2>/dev/null | sort -rn | head -1) - if [[ -z "$used" || "$used" -lt 2000 ]]; then break; fi - sleep 3; waited=$((waited + 3)) - done - SERVER_PID="" - fi -} -trap 'cleanup_server' EXIT - -start_gpu_monitor - -declare -A AL_RESULT - -run_cell() { - local mode="$1" mtp="$2" - local think_args=() - local temperature - if [[ "$mode" == "on" ]]; then - temperature=1.0 - if [[ -n "$CHAT_TEMPLATE_KWARGS_ON" ]]; then - think_args=(--chat-template-kwargs "$CHAT_TEMPLATE_KWARGS_ON") - fi - else - temperature=0.6 - if [[ -n "$CHAT_TEMPLATE_KWARGS_OFF" ]]; then - think_args=(--chat-template-kwargs "$CHAT_TEMPLATE_KWARGS_OFF") - fi - fi - - echo "" - echo "==========================================" - echo " Cell: thinking=$mode EAGLE3=$mtp category=$CATEGORY" - echo "==========================================" - - local serve_args=( - --host 0.0.0.0 --port "$PORT" - "${PARALLEL_ARGS[@]}" - --pipeline-parallel-size 1 - --kv-cache-dtype fp8 - --trust-remote-code - --language-model-only - --no-enable-prefix-caching - "${EP_ARGS[@]}" - --reasoning-parser kimi_k2 - --tool-call-parser kimi_k2 - --enable-auto-tool-choice - --gpu-memory-utilization "$GPU_MEM_UTIL" - --max-model-len 16384 - --speculative-config "{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL_PATH\", \"num_speculative_tokens\": $mtp}" - ) - - local server_log="$RESULTS_DIR/server_${mode}_mtp${mtp}.log" - vllm serve "$SERVE_MODEL" "${serve_args[@]}" > "$server_log" 2>&1 & - SERVER_PID=$! - - if ! wait_for_server_ready --port "$PORT" --server-log "$server_log" --server-pid "$SERVER_PID"; then - echo " -> server failed to start (thinking=$mode eagle3=$mtp), recording N/A" - AL_RESULT["${mode}_${mtp}"]="N/A" - cleanup_server - return - fi - - local acc_before drf_before acc_after drf_after - acc_before=$(fetch_metric "$PORT" "vllm:spec_decode_num_accepted_tokens_total") - drf_before=$(fetch_metric "$PORT" "vllm:spec_decode_num_drafts_total") - - vllm bench serve \ - --model "$SERVE_MODEL" \ - --port "$PORT" \ - --dataset-name speed_bench \ - --dataset-path "$SPEEDBENCH_DIR" \ - --speed-bench-category "$CATEGORY" \ - --speed-bench-output-len "$SPEEDBENCH_OUTPUT_LEN" \ - --num-prompts -1 \ - --max-concurrency "$CONCURRENCY" \ - --save-result \ - --save-detailed \ - --result-dir "$RESULTS_DIR" \ - --result-filename "speedbench_${mode}_mtp${mtp}" \ - --trust-remote-code \ - --temperature "$temperature" \ - --top-p "$TOP_P" \ - "${think_args[@]}" - - acc_after=$(fetch_metric "$PORT" "vllm:spec_decode_num_accepted_tokens_total") - drf_after=$(fetch_metric "$PORT" "vllm:spec_decode_num_drafts_total") - - local delta_acc delta_drf al - delta_acc=$(awk "BEGIN {printf \"%d\", $acc_after - $acc_before}") - delta_drf=$(awk "BEGIN {printf \"%d\", $drf_after - $drf_before}") - if [[ "$delta_drf" -gt 0 ]]; then - al=$(awk "BEGIN {printf \"%.2f\", 1 + ($delta_acc / $delta_drf)}") - else - al="N/A" - fi - echo " -> thinking=$mode EAGLE3=$mtp AL=$al (accepted=$delta_acc drafts=$delta_drf)" - AL_RESULT["${mode}_${mtp}"]="$al" - - cleanup_server -} - -for mode in $THINKING_MODES; do - for mtp in $MTP_LIST; do - run_cell "$mode" "$mtp" - done -done - -stop_gpu_monitor - -# ---- Emit the YAML matrix ---- -emit_mode_block() { - local mode="$1" - for mtp in $MTP_LIST; do - echo " $mtp: ${AL_RESULT[${mode}_${mtp}]:-N/A}" - done -} - -{ - echo "# Acceptance Length (AL) reference values measured with SPEED-Bench." - echo "# dataset: $CATEGORY | top_p: $TOP_P | output_len: $SPEEDBENCH_OUTPUT_LEN" - echo "# thinking_on: temperature=1.0, chat_template_kwargs: $CHAT_TEMPLATE_KWARGS_ON" - echo "# thinking_off: temperature=0.6, chat_template_kwargs: $CHAT_TEMPLATE_KWARGS_OFF" - echo "# Measured on $MODEL_KEY (B300, vLLM EAGLE3), per num_speculative_tokens." - echo "# Auto-generated by benchmarks/single_node/speedbench/kimik2.5_fp4_b300_vllm.sh (speedbench-al.yml)." - echo "#" - echo "# key = num_speculative_tokens (EAGLE3 level); value = golden AL" - echo "${MODEL_KEY}:" - if [[ " $THINKING_MODES " == *" on "* ]]; then - echo " thinking_on:" - emit_mode_block on - fi - if [[ " $THINKING_MODES " == *" off "* ]]; then - echo " thinking_off:" - emit_mode_block off - fi -} > "$OUT_YAML" - -echo "" -echo "==========================================" -echo " SPEED-Bench AL matrix written to: $OUT_YAML" -echo "==========================================" -cat "$OUT_YAML" diff --git a/benchmarks/single_node/speedbench/minimaxm3_fp4_b300_vllm.sh b/benchmarks/single_node/speedbench/minimaxm3_fp4_b300_vllm.sh deleted file mode 100755 index dac39fb53..000000000 --- a/benchmarks/single_node/speedbench/minimaxm3_fp4_b300_vllm.sh +++ /dev/null @@ -1,290 +0,0 @@ -#!/usr/bin/env bash - -# MiniMax-M3 B300 vLLM SPEED-Bench AL matrix collector for EAGLE3 speculative -# decoding. -# -# Produces the golden acceptance-length (AL) reference matrix consumed by the -# synthetic-acceptance framework: for each thinking mode (on/off) and each -# EAGLE3 level (num_speculative_tokens), measure the REAL AL on a single -# SPEED-Bench category (default: coding) and emit a YAML matrix identical in -# shape to benchmarks/speedbench-reference-al.yaml. This measures real EAGLE3 -# acceptance; the synthetic value is injected downstream by the throughput -# recipe, not here. -# -# EAGLE3 draft model: Inferact/MiniMax-M3-EAGLE3. The EAGLE3 head is MHA and -# must use FLASH_ATTN as the attention backend (FlashInfer only supports page -# size 128 through its trtllm-gen kernel requiring GQA/MQA). The target model -# keeps its default FlashInfer backend; --block-size 128 is mandatory for MSA -# sparse/index cache. The benchmark is text-only, so --language-model-only -# frees the vision encoder's VRAM. -# -# Filename *_fp4_* is ONLY a naming convention required by speedbench-al.yml -# (benchmarks/single_node/speedbench/${model-prefix}_fp4_b300_vllm.sh); it does -# NOT imply a quantized checkpoint. The staged MiniMax-M3 weights are -# unquantized BF16 (vLLM reports quantization=None, dtype=bfloat16), so no -# quantization-specific flags (e.g. --moe-backend marlin, --kv-cache-dtype fp8) -# apply here. -# -# Adapted from speedbench/glm5_fp4_b300_vllm.sh. Differences vs GLM-5 (MTP): -# - speculative method eagle3 + external draft model (was mtp, internal) -# - NO reasoning-parser / tool-call-parser (not needed for AL; matches the -# existing minimaxm3_fp8_b300_mtp.sh recipe which also omits them) -# - --block-size 128 mandatory for MSA sparse attention -# - --language-model-only (text-only benchmark, skip vision encoder) -# - --max-cudagraph-capture-size 2048 -# - NO --kv-cache-dtype fp8 (not used for M3) -# - NO --chat-template-content-format (not needed) -# - NO --tokenizer-mode (not needed) -# - NO --attention_config.use_fp4_indexer_cache (not applicable) -# - Thinking on/off uses the thinking_mode key (was enable_thinking for GLM) -# - Sampling: temperature=1.0, top_p=0.95, top_k=40 (official M3 docs) -# - EP handling: 3-way branch (DP_ATTENTION / EP / plain TP) -# -# Usage (inside the vLLM container, on a B300 node): -# export MODEL=/data/models/MiniMax-M3 -# bash benchmarks/single_node/speedbench/minimaxm3_fp4_b300_vllm.sh -# -# Tunables (env): -# MTP_LIST space-separated EAGLE3 spec-token levels (default "1 2 3 4 5 6 7 8") -# THINKING_MODES space-separated: off|on (default "off on") -# CATEGORY SPEED-Bench category (default coding) -# SPEEDBENCH_OUTPUT_LEN per-request output len (default 4096) -# OUT_YAML output matrix path (default $RESULTS_DIR/speedbench-reference-al.yaml) - -set -uo pipefail -source "$(dirname "$0")/../../benchmark_lib.sh" - -MODEL="${MODEL:?MODEL env var required (e.g. /data/models/MiniMax-M3)}" -SERVE_MODEL="${MODEL_PATH:-$MODEL}" -TP="${TP:-8}" -DP_ATTENTION="${DP_ATTENTION:-false}" -EP_SIZE="${EP_SIZE:-1}" -PORT="${PORT:-8888}" -GPU_MEM_UTIL="${GPU_MEM_UTIL:-0.90}" - -DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3" - -MTP_LIST="${MTP_LIST:-1 2 3 4 5 6 7 8}" -THINKING_MODES="${THINKING_MODES:-off on}" -CATEGORY="${CATEGORY:-coding}" -MODEL_KEY="${MODEL_KEY:-$(basename "$SERVE_MODEL" | tr '[:upper:]' '[:lower:]')}" -SPEEDBENCH_OUTPUT_LEN="${SPEEDBENCH_OUTPUT_LEN:-4096}" -CONCURRENCY="${CONCURRENCY:-1}" -# Official MiniMax-M3 sampling: temperature 1.0, top_p 0.95, top_k 40. -TEMPERATURE="${TEMPERATURE:-1.0}" -TOP_P="${TOP_P:-0.95}" -TOP_K="${TOP_K:-40}" -# M3 thinking toggles via the thinking_mode chat_template key. -DEFAULT_CHAT_TEMPLATE_KWARGS_ON='{"thinking_mode": "enabled"}' -DEFAULT_CHAT_TEMPLATE_KWARGS_OFF='{"thinking_mode": "disabled"}' -CHAT_TEMPLATE_KWARGS_ON="${CHAT_TEMPLATE_KWARGS_ON:-$DEFAULT_CHAT_TEMPLATE_KWARGS_ON}" -CHAT_TEMPLATE_KWARGS_OFF="${CHAT_TEMPLATE_KWARGS_OFF:-$DEFAULT_CHAT_TEMPLATE_KWARGS_OFF}" - -SPEEDBENCH_DIR="${SPEEDBENCH_DIR:-/workspace/speed_bench_data}" -RESULTS_DIR="${RESULTS_DIR:-/workspace/speedbench_results}" -OUT_YAML="${OUT_YAML:-$RESULTS_DIR/speedbench-reference-al.yaml}" - -export VLLM_FLOAT32_MATMUL_PRECISION="${VLLM_FLOAT32_MATMUL_PRECISION:-high}" -export VLLM_ENGINE_READY_TIMEOUT_S=3600 - -mkdir -p "$RESULTS_DIR" -nvidia-smi -if [[ "$SERVE_MODEL" != /* ]]; then hf download "$SERVE_MODEL"; fi - -# ---- Download EAGLE3 draft model to a WRITABLE dir ---- -# The draft must NOT go next to a pre-staged target: dirname(MODEL_PATH) is the -# read-only staged mount (/scratch/models), so writing the draft there fails -# with PermissionError. Use a writable workspace dir regardless of staging. -echo "=== Downloading EAGLE3 draft model ($DRAFT_MODEL) ===" -DRAFT_DIR="${DRAFT_MODEL_DIR:-/workspace/draft_models}" -mkdir -p "$DRAFT_DIR" -DRAFT_MODEL_PATH="$DRAFT_DIR/${DRAFT_MODEL##*/}" -if [[ ! -d "$DRAFT_MODEL_PATH" || -z "$(ls -A "$DRAFT_MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$DRAFT_MODEL" --local-dir "$DRAFT_MODEL_PATH" -fi - -# ---- Download SPEED-Bench dataset ---- -echo "=== Downloading SPEED-Bench dataset ===" -pip install -q datasets tiktoken -curl -LsSf https://raw.githubusercontent.com/NVIDIA-NeMo/Skills/refs/heads/main/nemo_skills/dataset/speed-bench/prepare.py \ - | python3 - --config qualitative --output_dir "$SPEEDBENCH_DIR" - -if [[ ! -f "$SPEEDBENCH_DIR/qualitative.jsonl" ]]; then - echo "CRITICAL: SPEED-Bench download failed — $SPEEDBENCH_DIR/qualitative.jsonl not found" - exit 1 -fi - -# ---- Parallel / EP args (3-way MiniMax-M3 pattern) ---- -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP" --enable-expert-parallel) -elif [ "${EP_SIZE:-1}" -gt 1 ]; then - PARALLEL_ARGS=(--tensor-parallel-size "$TP" --enable-expert-parallel) -else - # Plain TP, matching the official MiniMax-M3 recipe. Do NOT force a MoE - # backend: the staged checkpoint is unquantized BF16, for which marlin is - # rejected; let vLLM auto-select (triton / flashinfer). - PARALLEL_ARGS=(--tensor-parallel-size "$TP") -fi - -fetch_metric() { - local port="$1" name="$2" - curl -s "http://localhost:${port}/metrics" \ - | grep -oP "${name}\\{[^}]*\\} \\K[0-9.]+" || echo "0" -} - -SERVER_PID="" -_descendants() { - local pid="$1" child - for child in $(pgrep -P "$pid" 2>/dev/null || true); do - echo "$child" - _descendants "$child" - done -} -cleanup_server() { - if [[ -n "$SERVER_PID" ]]; then - local descendants - descendants=$(_descendants "$SERVER_PID") - kill "$SERVER_PID" 2>/dev/null || true - wait "$SERVER_PID" 2>/dev/null || true - local pid - for pid in $descendants; do - kill -9 "$pid" 2>/dev/null || true - done - local waited=0 - while [[ $waited -lt 120 ]]; do - local used - used=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits 2>/dev/null | sort -rn | head -1) - if [[ -z "$used" || "$used" -lt 2000 ]]; then break; fi - sleep 3; waited=$((waited + 3)) - done - SERVER_PID="" - fi -} -trap 'cleanup_server' EXIT - -start_gpu_monitor - -declare -A AL_RESULT - -run_cell() { - local mode="$1" mtp="$2" - local think_args=() - if [[ "$mode" == "on" && -n "$CHAT_TEMPLATE_KWARGS_ON" ]]; then - think_args=(--chat-template-kwargs "$CHAT_TEMPLATE_KWARGS_ON") - elif [[ "$mode" == "off" && -n "$CHAT_TEMPLATE_KWARGS_OFF" ]]; then - think_args=(--chat-template-kwargs "$CHAT_TEMPLATE_KWARGS_OFF") - fi - - echo "" - echo "==========================================" - echo " Cell: thinking=$mode EAGLE3=$mtp category=$CATEGORY" - echo "==========================================" - - local serve_args=( - --host 0.0.0.0 --port "$PORT" - "${PARALLEL_ARGS[@]}" - --pipeline-parallel-size 1 - --block-size 128 - --language-model-only - --max-cudagraph-capture-size 2048 - --trust-remote-code - --no-enable-prefix-caching - --gpu-memory-utilization "$GPU_MEM_UTIL" - --max-model-len 16384 - --max-num-batched-tokens 16384 - --stream-interval 30 - --speculative-config "{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL_PATH\", \"num_speculative_tokens\": $mtp, \"attention_backend\": \"FLASH_ATTN\"}" - ) - - local server_log="$RESULTS_DIR/server_${mode}_mtp${mtp}.log" - vllm serve "$SERVE_MODEL" "${serve_args[@]}" > "$server_log" 2>&1 & - SERVER_PID=$! - - if ! wait_for_server_ready --port "$PORT" --server-log "$server_log" --server-pid "$SERVER_PID"; then - echo " -> server failed to start (thinking=$mode eagle3=$mtp), recording N/A" - AL_RESULT["${mode}_${mtp}"]="N/A" - cleanup_server - return - fi - - local acc_before drf_before acc_after drf_after - acc_before=$(fetch_metric "$PORT" "vllm:spec_decode_num_accepted_tokens_total") - drf_before=$(fetch_metric "$PORT" "vllm:spec_decode_num_drafts_total") - - vllm bench serve \ - --model "$SERVE_MODEL" \ - --port "$PORT" \ - --dataset-name speed_bench \ - --dataset-path "$SPEEDBENCH_DIR" \ - --speed-bench-category "$CATEGORY" \ - --speed-bench-output-len "$SPEEDBENCH_OUTPUT_LEN" \ - --num-prompts -1 \ - --max-concurrency "$CONCURRENCY" \ - --save-result \ - --save-detailed \ - --result-dir "$RESULTS_DIR" \ - --result-filename "speedbench_${mode}_mtp${mtp}" \ - --trust-remote-code \ - --temperature "$TEMPERATURE" \ - --top-p "$TOP_P" \ - --top-k "$TOP_K" \ - "${think_args[@]}" - - acc_after=$(fetch_metric "$PORT" "vllm:spec_decode_num_accepted_tokens_total") - drf_after=$(fetch_metric "$PORT" "vllm:spec_decode_num_drafts_total") - - local delta_acc delta_drf al - delta_acc=$(awk "BEGIN {printf \"%d\", $acc_after - $acc_before}") - delta_drf=$(awk "BEGIN {printf \"%d\", $drf_after - $drf_before}") - if [[ "$delta_drf" -gt 0 ]]; then - al=$(awk "BEGIN {printf \"%.2f\", 1 + ($delta_acc / $delta_drf)}") - else - al="N/A" - fi - echo " -> thinking=$mode EAGLE3=$mtp AL=$al (accepted=$delta_acc drafts=$delta_drf)" - AL_RESULT["${mode}_${mtp}"]="$al" - - cleanup_server -} - -for mode in $THINKING_MODES; do - for mtp in $MTP_LIST; do - run_cell "$mode" "$mtp" - done -done - -stop_gpu_monitor - -# ---- Emit the YAML matrix ---- -emit_mode_block() { - local mode="$1" - for mtp in $MTP_LIST; do - echo " $mtp: ${AL_RESULT[${mode}_${mtp}]:-N/A}" - done -} - -{ - echo "# Acceptance Length (AL) reference values measured with SPEED-Bench." - echo "# dataset: $CATEGORY | temperature: $TEMPERATURE | top_p: $TOP_P | top_k: $TOP_K | output_len: $SPEEDBENCH_OUTPUT_LEN" - echo "# thinking_on chat_template_kwargs: $CHAT_TEMPLATE_KWARGS_ON" - echo "# thinking_off chat_template_kwargs: $CHAT_TEMPLATE_KWARGS_OFF" - echo "# Measured on $MODEL_KEY (B300, vLLM EAGLE3), per num_speculative_tokens." - echo "# Auto-generated by benchmarks/single_node/speedbench/minimaxm3_fp4_b300_vllm.sh (speedbench-al.yml)." - echo "#" - echo "# key = num_speculative_tokens (EAGLE3 level); value = golden AL" - echo "${MODEL_KEY}:" - if [[ " $THINKING_MODES " == *" on "* ]]; then - echo " thinking_on:" - emit_mode_block on - fi - if [[ " $THINKING_MODES " == *" off "* ]]; then - echo " thinking_off:" - emit_mode_block off - fi -} > "$OUT_YAML" - -echo "" -echo "==========================================" -echo " SPEED-Bench AL matrix written to: $OUT_YAML" -echo "==========================================" -cat "$OUT_YAML" diff --git a/benchmarks/single_node/speedbench/qwen3.5_fp4_b300_vllm.sh b/benchmarks/single_node/speedbench/qwen3.5_fp4_b300_vllm.sh deleted file mode 100755 index bf2bda7c8..000000000 --- a/benchmarks/single_node/speedbench/qwen3.5_fp4_b300_vllm.sh +++ /dev/null @@ -1,366 +0,0 @@ -#!/usr/bin/env bash - -# Qwen3.5-397B-A17B B300 vLLM SPEED-Bench AL matrix collector. -# -# Produces the golden acceptance-length (AL) reference matrix consumed by the -# synthetic-acceptance framework: for each thinking mode (on/off) and each MTP -# level (num_speculative_tokens), measure the REAL AL on a single SPEED-Bench -# category (default: coding) and emit a YAML matrix identical in shape to -# benchmarks/speedbench-reference-al.yaml. This measures real MTP acceptance; -# the synthetic value is injected downstream by the throughput recipe, not here. -# -# Adapted from speedbench/dsv4_fp4_b300_vllm.sh. Differences vs DSV4 (deepseek_v4 -# is NOT reusable for Qwen): -# - reasoning-parser qwen3 (was deepseek_v4) -# - tool-call-parser qwen3_coder (was deepseek_v4) -# - NO --tokenizer-mode deepseek_v4 (Qwen uses the default/auto tokenizer) -# - NO --attention_config.use_fp4_indexer_cache (DSV4 sparse-attn only) -# - --max-cudagraph-capture-size 512 (Qwen3.5 is a mamba hybrid; a large -# capture size trips the causal_conv1d -# assert, see vLLM docs / PR #34571) -# - thinking on/off uses the enable_thinking chat_template key, and OFF is -# passed explicitly (Qwen does not treat "no kwargs" as off the way deepseek does) -# -# Checkpoint (B300 / Blackwell): NVFP4 build, basename Qwen3.5-397B-A17B-NVFP4. -# NVIDIA's Qwen3.5-397B-A17B-NVFP4 model card serves it with vllm/vllm-openai:latest; -# the runner's vllm-openai:v0.21.0 (May) is newer and loads it. -# -# Usage (inside the vLLM container, on a B300 node): -# export MODEL=/scratch/models/Qwen3.5-397B-A17B-NVFP4 -# bash benchmarks/single_node/speedbench/qwen3.5_fp4_b300_vllm.sh -# -# Tunables (env): -# MTP_LIST space-separated MTP levels (default "1 2 3 4 5 6 7 8") -# THINKING_MODES space-separated: off|on (default "off on") -# CATEGORY SPEED-Bench category (default coding) -# SPEEDBENCH_OUTPUT_LEN per-request output len (default 4096) -# OUT_YAML output matrix path (default $RESULTS_DIR/speedbench-reference-al.yaml) - -set -uo pipefail -source "$(dirname "$0")/../../benchmark_lib.sh" - -MODEL="${MODEL:?MODEL env var required (e.g. /scratch/models/Qwen3.5-397B-A17B-NVFP4)}" -SERVE_MODEL="${MODEL_PATH:-$MODEL}" -TP="${TP:-8}" -DP_ATTENTION="${DP_ATTENTION:-false}" -EP_SIZE="${EP_SIZE:-1}" -PORT="${PORT:-8888}" - -MTP_LIST="${MTP_LIST:-1 2 3 4 5 6 7 8}" -THINKING_MODES="${THINKING_MODES:-off on}" -CATEGORY="${CATEGORY:-coding}" -MODEL_KEY="${MODEL_KEY:-$(basename "$SERVE_MODEL" | tr '[:upper:]' '[:lower:]')}" -SPEEDBENCH_OUTPUT_LEN="${SPEEDBENCH_OUTPUT_LEN:-4096}" -CONCURRENCY="${CONCURRENCY:-1}" -# Provider-recommended sampling — DIFFERS by mode (per the Qwen3.5 model card): -# thinking : temperature 0.6, top_p 0.95, top_k 20, presence_penalty 0.0 -# instruct : temperature 0.7, top_p 0.8, top_k 20, presence_penalty 1.5 -# (min_p 0.0 / repetition_penalty 1.0 are vLLM defaults.) These MUST be passed -# per-mode or the measured AL is taken at the wrong sampling settings. -TEMPERATURE_ON="${TEMPERATURE_ON:-0.6}"; TOP_P_ON="${TOP_P_ON:-0.95}"; TOP_K_ON="${TOP_K_ON:-20}"; PRESENCE_PENALTY_ON="${PRESENCE_PENALTY_ON:-0.0}" -TEMPERATURE_OFF="${TEMPERATURE_OFF:-0.7}"; TOP_P_OFF="${TOP_P_OFF:-0.8}"; TOP_K_OFF="${TOP_K_OFF:-20}"; PRESENCE_PENALTY_OFF="${PRESENCE_PENALTY_OFF:-1.5}" -# Optional sampling seed for run-to-run variance checks. Unset -> vLLM default -# (deterministic seed=0); set to different values to measure temperature>0 variance. -SEED="${SEED:-}" -# Optional: also save per-request completions (--save-detailed) to eyeball that -# thinking_on actually emits reasoning and thinking_off does not. Off by -# default (bloats the result JSON with all completions). Set SAVE_DETAILED=1. -SAVE_DETAILED="${SAVE_DETAILED:-}" -# Qwen thinking toggles via the enable_thinking chat_template key. -# Use separate single-quoted defaults: an inline ${VAR:-{...}} default whose value -# contains "}" is truncated by bash brace parsing (matches upstream fix #1695). -DEFAULT_CHAT_TEMPLATE_KWARGS_ON='{"enable_thinking": true}' -DEFAULT_CHAT_TEMPLATE_KWARGS_OFF='{"enable_thinking": false}' -CHAT_TEMPLATE_KWARGS_ON="${CHAT_TEMPLATE_KWARGS_ON:-$DEFAULT_CHAT_TEMPLATE_KWARGS_ON}" -CHAT_TEMPLATE_KWARGS_OFF="${CHAT_TEMPLATE_KWARGS_OFF:-$DEFAULT_CHAT_TEMPLATE_KWARGS_OFF}" - -SPEEDBENCH_DIR="${SPEEDBENCH_DIR:-/workspace/speed_bench_data}" -# Flat results dir to match the speedbench-al.yml artifact glob -# (speedbench_results/server_*.log) and its pre-run `rm -rf speedbench_results`. -RESULTS_DIR="${RESULTS_DIR:-/workspace/speedbench_results}" -OUT_YAML="${OUT_YAML:-$RESULTS_DIR/speedbench-reference-al.yaml}" - -export VLLM_ENGINE_READY_TIMEOUT_S=3600 - -mkdir -p "$RESULTS_DIR" -nvidia-smi -if [[ "$SERVE_MODEL" != /* ]]; then hf download "$SERVE_MODEL"; fi - -# ---- Download SPEED-Bench dataset ---- -echo "=== Downloading SPEED-Bench dataset ===" -pip install -q datasets tiktoken -curl -LsSf https://raw.githubusercontent.com/NVIDIA-NeMo/Skills/refs/heads/main/nemo_skills/dataset/speed-bench/prepare.py \ - | python3 - --config qualitative --output_dir "$SPEEDBENCH_DIR" - -if [[ ! -f "$SPEEDBENCH_DIR/qualitative.jsonl" ]]; then - echo "CRITICAL: SPEED-Bench download failed — $SPEEDBENCH_DIR/qualitative.jsonl not found" - exit 1 -fi - -# ---- Temporary shim: add a real --chat-template-kwargs CLI option ---- -# Upstream gap (until vllm-project/vllm#44244 lands): speed_bench/CustomDataset -# pre-renders the chat template client-side WITHOUT chat_template_kwargs and -# posts to /v1/completions, so thinking mode cannot be enabled via --extra-body -# or --default-chat-template-kwargs. This wires a proper --chat-template-kwargs -# option through get_samples into CustomDataset.sample's apply_chat_template. -# Model agnostic (forwards whatever dict it is given). TODO: delete once #44244 -# is released in the benchmark image; idempotent (marker check), safe to leave. -apply_chat_template_kwargs_shim() { - echo "=== Patching vLLM benchmark to add --chat-template-kwargs (temporary shim) ===" - python3 - <<'PYEOF' -import vllm.benchmarks.serve as S -import vllm.benchmarks.datasets.datasets as D - -def patch(mod, edits, marker): - f = mod.__file__ - src = open(f).read() - if marker in src: - print("already patched:", f) - return - for old, new in edits: - n = src.count(old) - assert n == 1, f"anchor matched {n} times in {f}, aborting:\n{old[:80]}..." - src = src.replace(old, new, 1) - open(f, "w").write(src) - print("patched OK ->", f) - -# Edit 1: serve.py -- declare the --chat-template-kwargs argument before --extra-body -serve_old = ''' parser.add_argument( - "--extra-body",''' -serve_new = ''' parser.add_argument( - "--chat-template-kwargs", - type=json.loads, - default=None, - help="JSON dict forwarded to apply_chat_template during " - "client-side prompt rendering, e.g. to enable reasoning mode.", - ) - parser.add_argument( - "--extra-body",''' -patch(S, [(serve_old, serve_new)], marker='"--chat-template-kwargs"') - -# Edit 2: datasets.py -- forward args.chat_template_kwargs into the speed_bench .sample() call -disp_old = ''' output_len=args.speed_bench_output_len, - enable_multimodal_chat=args.enable_multimodal_chat,''' -disp_new = ''' output_len=args.speed_bench_output_len, - chat_template_kwargs=args.chat_template_kwargs, - enable_multimodal_chat=args.enable_multimodal_chat,''' - -# Edit 3: datasets.py -- forward chat_template_kwargs into CustomDataset.sample's template call -samp_old = ''' # apply template - if not skip_chat_template: - prompt = tokenizer.apply_chat_template( - [{"role": "user", "content": prompt}], - add_generation_prompt=True, - tokenize=False, - ) - - prompt_len = len(tokenizer(prompt).input_ids)''' -samp_new = ''' # apply template - if not skip_chat_template: - _ctk = kwargs.get("chat_template_kwargs") or {} - prompt = tokenizer.apply_chat_template( - [{"role": "user", "content": prompt}], - add_generation_prompt=True, - tokenize=False, - **_ctk, - ) - - prompt_len = len(tokenizer(prompt).input_ids)''' -patch(D, [(disp_old, disp_new), (samp_old, samp_new)], - marker="chat_template_kwargs=args.chat_template_kwargs") -PYEOF -} - -# Apply the shim once if any cell will pass chat_template_kwargs. -NEED_SHIM=0 -if [[ " $THINKING_MODES " == *" on "* && -n "$CHAT_TEMPLATE_KWARGS_ON" ]]; then NEED_SHIM=1; fi -if [[ " $THINKING_MODES " == *" off "* && -n "$CHAT_TEMPLATE_KWARGS_OFF" ]]; then NEED_SHIM=1; fi -if [[ "$NEED_SHIM" == "1" ]]; then - if ! apply_chat_template_kwargs_shim; then - echo "CRITICAL: --chat-template-kwargs shim failed — aborting" - exit 1 - fi -fi - -PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") -fi -EP_ARGS=() -if [ "${EP_SIZE:-1}" -gt 1 ]; then - EP_ARGS=(--enable-expert-parallel) -fi - -fetch_metric() { - local port="$1" name="$2" - curl -s "http://localhost:${port}/metrics" \ - | grep -oP "${name}\\{[^}]*\\} \\K[0-9.]+" || echo "0" -} - -SERVER_PID="" -_descendants() { - local pid="$1" child - for child in $(pgrep -P "$pid" 2>/dev/null || true); do - echo "$child" - _descendants "$child" - done -} -cleanup_server() { - if [[ -n "$SERVER_PID" ]]; then - local descendants - descendants=$(_descendants "$SERVER_PID") - kill "$SERVER_PID" 2>/dev/null || true - wait "$SERVER_PID" 2>/dev/null || true - local pid - for pid in $descendants; do - kill -9 "$pid" 2>/dev/null || true - done - local waited=0 - while [[ $waited -lt 120 ]]; do - local used - used=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits 2>/dev/null | sort -rn | head -1) - if [[ -z "$used" || "$used" -lt 2000 ]]; then break; fi - sleep 3; waited=$((waited + 3)) - done - SERVER_PID="" - fi -} -trap 'cleanup_server' EXIT - -start_gpu_monitor - -declare -A AL_RESULT - -run_cell() { - local mode="$1" mtp="$2" - local think_args=() - local temp top_p top_k pp - if [[ "$mode" == "on" ]]; then - [[ -n "$CHAT_TEMPLATE_KWARGS_ON" ]] && think_args=(--chat-template-kwargs "$CHAT_TEMPLATE_KWARGS_ON") - temp="$TEMPERATURE_ON"; top_p="$TOP_P_ON"; top_k="$TOP_K_ON"; pp="$PRESENCE_PENALTY_ON" - else - [[ -n "$CHAT_TEMPLATE_KWARGS_OFF" ]] && think_args=(--chat-template-kwargs "$CHAT_TEMPLATE_KWARGS_OFF") - temp="$TEMPERATURE_OFF"; top_p="$TOP_P_OFF"; top_k="$TOP_K_OFF"; pp="$PRESENCE_PENALTY_OFF" - fi - local seed_args=() - [[ -n "$SEED" ]] && seed_args=(--seed "$SEED") - local detail_args=() - [[ -n "$SAVE_DETAILED" ]] && detail_args=(--save-detailed) - - echo "" - echo "==========================================" - echo " Cell: thinking=$mode MTP=$mtp category=$CATEGORY" - echo "==========================================" - - local serve_args=( - --host 0.0.0.0 --port "$PORT" - "${PARALLEL_ARGS[@]}" - --pipeline-parallel-size 1 - --kv-cache-dtype fp8 - --trust-remote-code - --no-enable-prefix-caching - "${EP_ARGS[@]}" - --reasoning-parser qwen3 - --tool-call-parser qwen3_coder - --enable-auto-tool-choice - --language-model-only - --max-cudagraph-capture-size 512 - --max-model-len 16384 - --speculative-config "{\"method\": \"mtp\", \"num_speculative_tokens\": $mtp}" - ) - - local server_log="$RESULTS_DIR/server_${mode}_mtp${mtp}.log" - vllm serve "$SERVE_MODEL" "${serve_args[@]}" > "$server_log" 2>&1 & - SERVER_PID=$! - - if ! wait_for_server_ready --port "$PORT" --server-log "$server_log" --server-pid "$SERVER_PID"; then - echo " -> server failed to start (thinking=$mode mtp=$mtp), recording N/A" - AL_RESULT["${mode}_${mtp}"]="N/A" - cleanup_server - return - fi - - local acc_before drf_before acc_after drf_after - acc_before=$(fetch_metric "$PORT" "vllm:spec_decode_num_accepted_tokens_total") - drf_before=$(fetch_metric "$PORT" "vllm:spec_decode_num_drafts_total") - - vllm bench serve \ - --model "$SERVE_MODEL" \ - --port "$PORT" \ - --dataset-name speed_bench \ - --dataset-path "$SPEEDBENCH_DIR" \ - --speed-bench-category "$CATEGORY" \ - --speed-bench-output-len "$SPEEDBENCH_OUTPUT_LEN" \ - --num-prompts -1 \ - --max-concurrency "$CONCURRENCY" \ - --save-result \ - --save-detailed \ - --result-dir "$RESULTS_DIR" \ - --result-filename "speedbench_${mode}_mtp${mtp}" \ - --trust-remote-code \ - --temperature "$temp" \ - --top-p "$top_p" \ - --top-k "$top_k" \ - --presence-penalty "$pp" \ - "${seed_args[@]}" \ - "${detail_args[@]}" \ - "${think_args[@]}" - - acc_after=$(fetch_metric "$PORT" "vllm:spec_decode_num_accepted_tokens_total") - drf_after=$(fetch_metric "$PORT" "vllm:spec_decode_num_drafts_total") - - local delta_acc delta_drf al - delta_acc=$(awk "BEGIN {printf \"%d\", $acc_after - $acc_before}") - delta_drf=$(awk "BEGIN {printf \"%d\", $drf_after - $drf_before}") - if [[ "$delta_drf" -gt 0 ]]; then - al=$(awk "BEGIN {printf \"%.2f\", 1 + ($delta_acc / $delta_drf)}") - else - al="N/A" - fi - echo " -> thinking=$mode MTP=$mtp AL=$al (accepted=$delta_acc drafts=$delta_drf)" - AL_RESULT["${mode}_${mtp}"]="$al" - - cleanup_server -} - -for mode in $THINKING_MODES; do - for mtp in $MTP_LIST; do - run_cell "$mode" "$mtp" - done -done - -stop_gpu_monitor - -# ---- Emit the YAML matrix ---- -emit_mode_block() { - local mode="$1" - for mtp in $MTP_LIST; do - echo " $mtp: ${AL_RESULT[${mode}_${mtp}]:-N/A}" - done -} - -{ - echo "# Acceptance Length (AL) reference values measured with SPEED-Bench." - echo "# dataset: $CATEGORY | output_len: $SPEEDBENCH_OUTPUT_LEN" - echo "# thinking_on : temp $TEMPERATURE_ON top_p $TOP_P_ON top_k $TOP_K_ON presence_penalty $PRESENCE_PENALTY_ON | chat_template_kwargs: $CHAT_TEMPLATE_KWARGS_ON" - echo "# thinking_off: temp $TEMPERATURE_OFF top_p $TOP_P_OFF top_k $TOP_K_OFF presence_penalty $PRESENCE_PENALTY_OFF | chat_template_kwargs: $CHAT_TEMPLATE_KWARGS_OFF" - echo "# Measured on $MODEL_KEY (B300, vLLM MTP), per num_speculative_tokens." - echo "# Auto-generated by benchmarks/single_node/speedbench/qwen3.5_fp4_b300_vllm.sh (speedbench-al.yml)." - echo "#" - echo "# key = num_speculative_tokens (MTP level); value = golden AL" - echo "${MODEL_KEY}:" - if [[ " $THINKING_MODES " == *" on "* ]]; then - echo " thinking_on:" - emit_mode_block on - fi - if [[ " $THINKING_MODES " == *" off "* ]]; then - echo " thinking_off:" - emit_mode_block off - fi -} > "$OUT_YAML" - -echo "" -echo "==========================================" -echo " SPEED-Bench AL matrix written to: $OUT_YAML" -echo "==========================================" -cat "$OUT_YAML" diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index a982f6699..bcfe29903 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1,2957 +1,10 @@ -dsr1-fp4-mi355x-sglang: - image: lmsysorg/sglang:v0.5.12-rocm700-mi35x - model: amd/DeepSeek-R1-0528-MXFP4-Preview - model-prefix: dsr1 - runner: mi355x - precision: fp4 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 4, conc-start: 4, conc-end: 64 } - - { tp: 8, conc-start: 4, conc-end: 64 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 4, conc-start: 4, conc-end: 64 } - - { tp: 8, conc-start: 4, conc-end: 64 } - -dsr1-fp4-mi355x-sglang-mtp: - image: lmsysorg/sglang:v0.5.12-rocm700-mi35x - model: amd/DeepSeek-R1-0528-MXFP4 - model-prefix: dsr1 - runner: mi355x - precision: fp4 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 64, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 64, spec-decoding: mtp } - -dsr1-fp4-mi355x-atom: - image: rocm/atom:rocm7.2.3_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom20260511 - model: amd/DeepSeek-R1-0528-MXFP4-Preview - model-prefix: dsr1 - runner: mi355x - precision: fp4 - framework: atom - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 4, ep: 1, conc-start: 32, conc-end: 256 } - - { tp: 8, ep: 1, conc-start: 4, conc-end: 32 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 4, ep: 1, conc-start: 4, conc-end: 256 } - - { tp: 8, ep: 1, conc-start: 4, conc-end: 4 } - -dsr1-fp4-mi355x-atom-mtp: - image: rocm/atom:rocm7.2.3_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom20260511 - model: amd/DeepSeek-R1-0528-MXFP4 - model-prefix: dsr1 - runner: mi355x - precision: fp4 - # WIP framework (no customers yet) - framework: atom - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 4, conc-start: 4, conc-end: 256, spec-decoding: mtp } - - { tp: 8, conc-start: 4, conc-end: 256, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - #- { tp: 4, conc-start: 32, conc-end: 256, spec-decoding: mtp } - - { tp: 8, conc-start: 4, conc-end: 256, spec-decoding: mtp } - -dsr1-fp8-mi300x-sglang: - image: lmsysorg/sglang:v0.5.12-rocm700-mi30x - model: deepseek-ai/DeepSeek-R1-0528 - model-prefix: dsr1 - runner: mi300x - precision: fp8 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 64 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 64 } - -dsr1-fp8-mi325x-sglang: - image: lmsysorg/sglang:v0.5.12-rocm700-mi30x - model: deepseek-ai/DeepSeek-R1-0528 - model-prefix: dsr1 - runner: mi325x - precision: fp8 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 64 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 64 } - -dsr1-fp8-mi355x-sglang: - image: lmsysorg/sglang:v0.5.12-rocm700-mi35x - model: deepseek-ai/DeepSeek-R1-0528 - model-prefix: dsr1 - runner: mi355x - precision: fp8 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 64 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 4, conc-start: 32, conc-end: 64 } - - { tp: 8, conc-start: 4, conc-end: 64 } - -dsr1-fp8-mi355x-sglang-mtp: - image: lmsysorg/sglang:v0.5.12-rocm700-mi35x - model: deepseek-ai/DeepSeek-R1-0528 - model-prefix: dsr1 - runner: mi355x - precision: fp8 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 64, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 64, spec-decoding: mtp } - -qwen3.5-bf16-mi355x-sglang: - image: lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260517 - model: Qwen/Qwen3.5-397B-A17B - model-prefix: qwen3.5 - runner: mi355x - precision: bf16 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 256 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 256 } - -qwen3.5-bf16-mi355x-sglang-mtp: - image: lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260517 - model: Qwen/Qwen3.5-397B-A17B - model-prefix: qwen3.5 - runner: mi355x - precision: bf16 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 256, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 256, spec-decoding: mtp } - -qwen3.5-bf16-mi300x-sglang: - image: lmsysorg/sglang:v0.5.12-rocm720-mi30x - model: Qwen/Qwen3.5-397B-A17B - model-prefix: qwen3.5 - runner: mi300x - precision: bf16 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 64 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 64 } - -qwen3.5-bf16-mi325x-sglang: - image: lmsysorg/sglang:v0.5.12-rocm720-mi30x - model: Qwen/Qwen3.5-397B-A17B - model-prefix: qwen3.5 - runner: mi325x - precision: bf16 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 64 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 64 } - -qwen3.5-fp8-mi325x-sglang: - image: lmsysorg/sglang:v0.5.12-rocm720-mi30x - model: Qwen/Qwen3.5-397B-A17B-FP8 - model-prefix: qwen3.5 - runner: mi325x - precision: fp8 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 64 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 64 } - -qwen3.5-fp8-mi355x-sglang: - image: lmsysorg/sglang:v0.5.14-rocm720-mi35x - model: Qwen/Qwen3.5-397B-A17B-FP8 - model-prefix: qwen3.5 - runner: mi355x - precision: fp8 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 4, ep: 1, conc-start: 4, conc-end: 256 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 4, ep: 1, conc-start: 4, conc-end: 256 } - -qwen3.5-fp8-mi355x-sglang-mtp: - image: lmsysorg/sglang:v0.5.14-rocm720-mi35x - model: Qwen/Qwen3.5-397B-A17B-FP8 - model-prefix: qwen3.5 - runner: mi355x - precision: fp8 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 4, ep: 1, conc-start: 4, conc-end: 256, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 4, ep: 1, conc-start: 4, conc-end: 256, spec-decoding: mtp } - -qwen3.5-fp8-mi355x-sglang-agentic: - image: lmsysorg/sglang-rocm:v0.5.10rc0-rocm720-mi35x-20260414 - model: Qwen/Qwen3.5-397B-A17B-FP8 - model-prefix: qwen3.5 - runner: cluster:mi355x-amds - precision: fp8 - framework: sglang - multinode: false - scenarios: - agentic-coding: - - search-space: - - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32] } - - -qwen3.5-fp8-mi355x-atom: - image: rocm/atom:rocm7.2.3_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom20260511 - model: Qwen/Qwen3.5-397B-A17B-FP8 - model-prefix: qwen3.5 - runner: mi355x - precision: fp8 - framework: atom - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 2, ep: 1, conc-start: 4, conc-end: 256 } - - { tp: 4, ep: 1, conc-start: 4, conc-end: 256 } - - { tp: 8, ep: 1, conc-start: 4, conc-end: 256 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 2, ep: 1, conc-start: 4, conc-end: 256 } - - { tp: 4, ep: 1, conc-start: 4, conc-end: 256 } - - { tp: 8, ep: 1, conc-start: 4, conc-end: 256 } - -qwen3.5-fp8-mi355x-atom-mtp: - image: rocm/atom:rocm7.2.2_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.2.post - model: Qwen/Qwen3.5-397B-A17B-FP8 - model-prefix: qwen3.5 - runner: mi355x - precision: fp8 - framework: atom - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 4, ep: 1, conc-start: 4, conc-end: 256, spec-decoding: mtp } - - { tp: 8, ep: 1, conc-start: 4, conc-end: 256, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 4, ep: 1, conc-start: 4, conc-end: 256, spec-decoding: mtp } - - { tp: 8, ep: 1, conc-start: 4, conc-end: 256, spec-decoding: mtp } - -qwen3.5-fp8-mi355x-sglang-disagg: - image: lmsysorg/sglang-rocm:v0.5.11-rocm700-mi35x-20260511 - model: Qwen/Qwen3.5-397B-A17B-FP8 - model-prefix: qwen3.5 - runner: mi355x-disagg - precision: fp8 - framework: sglang-disagg - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # Matches qwen3.5-fp8-mi355x-sglang TP8/EP1 low-concurrency sweep - - spec-decoding: "none" - conc-list: [ 8, 16, 32, 64, 128, 256, 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - - - isl: 8192 - osl: 1024 - search-space: - # 1P+1D TP8/EP1 low-concurrency sweep. - # dp-attn intentionally false (matches the 1k1k row): with - # --enable-dp-attention + --moe-a2a-backend mori, sglang auto-promotes - # moe_ep_size=tp_size=8, but is_deepep_class_backend() excludes MoRI, - # so num_shared_slots stays at the global value (1) and the - # (num_experts - num_shared_slots) % moe_ep_size assertion in - # fused_moe_triton/layer.py fires for Qwen3.5 (512 routed + 1 shared). - # Track upstream sglang for a fix; flip back to dp-attn=true once - # MoRI is added to is_deepep_class_backend() or shared-slot - # accounting is reconciled. - - spec-decoding: "none" - conc-list: [ 8, 16, 32, 64, 128, 256, 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - -qwen3.5-fp4-mi355x-sglang: - image: lmsysorg/sglang-rocm:v0.5.13-rocm720-mi35x-20260612 - model: amd/Qwen3.5-397B-A17B-MXFP4 - model-prefix: qwen3.5 - runner: mi355x - precision: fp4 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 2, conc-start: 4, conc-end: 256 } - - { tp: 4, conc-start: 4, conc-end: 16 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 2, conc-start: 4, conc-end: 256 } - - { tp: 4, conc-start: 4, conc-end: 16 } - -qwen3.5-fp4-mi355x-atom: - image: rocm/atom:rocm7.2.2_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.2.post - model: amd/Qwen3.5-397B-A17B-MXFP4 - model-prefix: qwen3.5 - runner: mi355x - precision: fp4 - framework: atom - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 2, conc-start: 4, conc-end: 256 } - - { tp: 4, conc-start: 4, conc-end: 16 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 2, conc-start: 4, conc-end: 256 } - - { tp: 4, conc-start: 4, conc-end: 16 } - -qwen3.5-fp4-mi355x-sglang-mtp: - image: lmsysorg/sglang-rocm:v0.5.13-rocm720-mi35x-20260612 - model: amd/Qwen3.5-397B-A17B-MXFP4 - model-prefix: qwen3.5 - runner: mi355x - precision: fp4 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 2, conc-start: 4, conc-end: 128, spec-decoding: mtp } - - { tp: 4, conc-start: 4, conc-end: 16, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 2, conc-start: 4, conc-end: 128, spec-decoding: mtp } - - { tp: 4, conc-start: 4, conc-end: 16, spec-decoding: mtp } - -qwen3.5-fp4-mi355x-sglang-disagg: - image: lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260523 - model: amd/Qwen3.5-397B-A17B-MXFP4 - model-prefix: qwen3.5 - runner: mi355x-disagg - precision: fp4 - framework: sglang-disagg - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # 1P1D TP8/EP1, dp-attn false; MoRI conn.py overlay via job.slurm. - - spec-decoding: "none" - conc-list: [ 8, 16, 32, 64, 128, 256, 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - - - isl: 8192 - osl: 1024 - search-space: - - spec-decoding: "none" - conc-list: [ 8, 16, 32, 64, 128, 256, 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - -qwen3.5-fp8-mi300x-sglang: - image: lmsysorg/sglang:v0.5.12-rocm720-mi30x - model: Qwen/Qwen3.5-397B-A17B-FP8 - model-prefix: qwen3.5 - runner: mi300x - precision: fp8 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 64 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 64 } - -glm5-fp8-mi355x-sglang: - image: lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260517 - model: zai-org/GLM-5-FP8 - model-prefix: glm5 - runner: mi355x - precision: fp8 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 4, conc-start: 4, conc-end: 256 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 4, conc-start: 4, conc-end: 256 } - -glm5-fp8-mi355x-sglang-mtp: - image: lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260517 - model: zai-org/GLM-5-FP8 - model-prefix: glm5 - runner: mi355x - precision: fp8 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 4, conc-start: 4, conc-end: 128, spec-decoding: mtp } - - { tp: 8, conc-start: 4, conc-end: 8, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 4, conc-start: 4, conc-end: 128, spec-decoding: mtp } - - { tp: 8, conc-start: 4, conc-end: 8, spec-decoding: mtp } - -glm5-fp8-mi355x-sglang-disagg: - image: lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260523 - model: zai-org/GLM-5-FP8 - model-prefix: glm5 - runner: mi355x-disagg - precision: fp8 - framework: sglang-disagg - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # 1P+1D TP8/EP1 CI smoke sweep (aligned with glm5-fp8-mi355x-sglang conc range) - - spec-decoding: "none" - conc-list: [ 8, 16, 32, 64, 128, 256, 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - - - isl: 8192 - osl: 1024 - search-space: - # 1P+1D TP8/EP1 CI smoke sweep; dp-attn false (NSA / MoRI path) - - spec-decoding: "none" - conc-list: [ 8, 16, 32, 64, 128, 256, 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - -glm5-fp8-mi355x-atom: - image: rocm/atom:rocm7.2.2_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.2.post - model: zai-org/GLM-5-FP8 - model-prefix: glm5 - runner: mi355x - precision: fp8 - framework: atom - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 4, conc-start: 4, conc-end: 256 } - - { tp: 8, conc-start: 4, conc-end: 256 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 4, conc-start: 4, conc-end: 256 } - - { tp: 8, conc-start: 4, conc-end: 256 } - -glm5.1-fp4-mi355x-sglang: - image: lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260529 - model: amd/GLM-5.1-MXFP4 - model-prefix: glm5.1 - runner: mi355x - precision: fp4 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 2, conc-start: 4, conc-end: 256 } - - { tp: 4, conc-start: 4, conc-end: 16 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 2, conc-start: 4, conc-end: 256 } - - { tp: 4, conc-start: 4, conc-end: 16 } - -glm5.1-fp4-mi355x-atom: - image: rocm/atom:rocm7.2.2_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.2.post - model: amd/GLM-5.1-MXFP4 - model-prefix: glm5.1 - runner: mi355x - precision: fp4 - framework: atom - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 4, conc-start: 4, conc-end: 256 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 4, conc-start: 4, conc-end: 256 } - -kimik2.5-int4-mi355x-vllm: - image: vllm/vllm-openai-rocm:v0.24.0 - model: moonshotai/Kimi-K2.5 - model-prefix: kimik2.5 - runner: mi355x - precision: int4 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 128 } - - { tp: 4, conc-start: 4, conc-end: 128 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 128 } - - { tp: 4, conc-start: 4, conc-end: 128 } - -kimik2.5-int4-mi325x-vllm: - image: vllm/vllm-openai-rocm:v0.21.0 - model: moonshotai/Kimi-K2.5 - model-prefix: kimik2.5 - runner: mi325x - precision: int4 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 64 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 64 } - -kimik2.5-int4-mi300x-vllm: - image: vllm/vllm-openai-rocm:v0.21.0 - model: moonshotai/Kimi-K2.5 - model-prefix: kimik2.5 - runner: mi300x - precision: int4 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 64 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 64 } - -kimik2.5-fp4-mi355x-vllm: - image: vllm/vllm-openai-rocm:v0.24.0 - model: amd/Kimi-K2.5-MXFP4 - model-prefix: kimik2.5 - runner: mi355x - precision: fp4 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 64 } - - { tp: 4, conc-start: 4, conc-end: 64 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 64 } - - { tp: 4, conc-start: 4, conc-end: 64 } - -kimik2.5-fp4-mi355x-vllm-agentic: - image: vllm/vllm-openai-rocm:v0.22.0 - model: amd/Kimi-K2.5-MXFP4 - model-prefix: kimik2.5 - runner: cluster:mi355x-amds - precision: fp4 - framework: vllm - multinode: false - scenarios: - agentic-coding: - - dram-utilization: 0.80 - search-space: - - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 24, 32, 40, 48] } - # DRAM offload only above the KV cliff. Lower concurrencies fit - # entirely on-GPU, so paying the offload-path overhead there would - # just slow them down without measuring anything new. - - { tp: 8, kv-offloading: dram, kv-offload-backend: native, conc-list: [32, 40, 48, 56] } - # TP=4 probe: half-node layout doubles per-GPU weight footprint - # (~62 GB on MI355X's 288 GB HBM, plenty of headroom). Restrict to - # cliff-region concurrencies on both offload modes so we can directly - # compare TP=4 vs TP=8 at the same conc points. - - { tp: 4, kv-offloading: none, conc-list: [16, 24, 32, 40] } - - { tp: 4, kv-offloading: dram, kv-offload-backend: native, conc-list: [16, 24, 32, 40] } - - -kimik2.5-fp4-mi355x-atom: - image: rocm/atom:rocm7.2.3_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom20260511 - model: amd/Kimi-K2.5-MXFP4 - model-prefix: kimik2.5 - runner: mi355x - precision: fp4 - framework: atom - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 128 } - - { tp: 4, conc-start: 4, conc-end: 128 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 128 } - - { tp: 4, conc-start: 4, conc-end: 128 } - -gptoss-fp4-mi300x-vllm: - image: vllm/vllm-openai-rocm:v0.17.0 - model: openai/gpt-oss-120b - model-prefix: gptoss - runner: mi300x - precision: fp4 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 1, conc-start: 64, conc-end: 256 } - - { tp: 2, conc-start: 4, conc-end: 64 } - - { tp: 4, conc-start: 4, conc-end: 64 } - - { tp: 8, conc-start: 1, conc-end: 16 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 1, conc-start: 4, conc-end: 64 } - - { tp: 2, conc-start: 4, conc-end: 64 } - - { tp: 4, conc-start: 4, conc-end: 64 } - - { tp: 8, conc-start: 1, conc-end: 16 } - -gptoss-fp4-mi325x-vllm: - image: vllm/vllm-openai-rocm:v0.22.0 - model: openai/gpt-oss-120b - model-prefix: gptoss - runner: mi325x - precision: fp4 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 1, conc-start: 4, conc-end: 64 } - - { tp: 2, conc-start: 4, conc-end: 64 } - - { tp: 4, conc-start: 4, conc-end: 64 } - - { tp: 8, conc-start: 4, conc-end: 64 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 1, conc-start: 4, conc-end: 64 } - - { tp: 2, conc-start: 4, conc-end: 8 } - - { tp: 4, conc-start: 4, conc-end: 8 } - - { tp: 8, conc-start: 4, conc-end: 16 } - -gptoss-fp4-mi355x-vllm: - image: vllm/vllm-openai-rocm:v0.22.0 - model: openai/gpt-oss-120b - model-prefix: gptoss - runner: mi355x - precision: fp4 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 1, conc-start: 4, conc-end: 128 } - - { tp: 4, conc-start: 4, conc-end: 8 } - - { tp: 8, conc-start: 4, conc-end: 16 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 1, conc-start: 4, conc-end: 128 } - - { tp: 4, conc-start: 4, conc-end: 4 } - - { tp: 8, conc-start: 4, conc-end: 8 } - -gptoss-fp4-mi355x-atom: - image: rocm/atom:rocm7.2.2_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.2.post - model: openai/gpt-oss-120b - model-prefix: gptoss - runner: mi355x - precision: fp4 - framework: atom - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 1, conc-start: 16, conc-end: 256 } - - { tp: 8, ep: 1, conc-start: 4, conc-end: 32 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 1, conc-start: 4, conc-end: 256 } - - { tp: 8, ep: 1, conc-start: 4, conc-end: 16 } - -dsr1-fp8-mi355x-atom: - image: rocm/atom:rocm7.2.3_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom20260511 - model: deepseek-ai/DeepSeek-R1-0528 - model-prefix: dsr1 - runner: mi355x - precision: fp8 - # WIP framework (no customers yet) - framework: atom - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 128 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 128 } - -dsr1-fp8-mi355x-atom-mtp: - image: rocm/atom:rocm7.2.4_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.3 - model: deepseek-ai/DeepSeek-R1-0528 - model-prefix: dsr1 - runner: mi355x - precision: fp8 - framework: atom - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 512, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 256, spec-decoding: mtp } - -dsr1-fp8-mi355x-sglang-disagg: - image: rocm/sgl-dev:sglang-0.5.9-rocm720-mi35x-mori-0227-2 - model: deepseek-ai/DeepSeek-R1-0528 - model-prefix: dsr1 - runner: mi355x-disagg - precision: fp8 - framework: sglang-disagg - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # non-MTP configurations - # "Top of curve" (1 prefill workers each at DEP8 and 1 decode workers at DEP16) - - spec-decoding: "none" - conc-list: [ 1024, 2048 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=0" - - # "Middle of curve" (1 prefill workers each at TP8 and 2 decode workers at DEP8) - - spec-decoding: "none" - conc-list: [ 1536, 1024, 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=0" - - # "Bottom of curve" (1 prefill worker at TEP8 and 2 decode workers at TEP8) - - spec-decoding: "none" - conc-list: [ 256, 128, 64, 32, 16, 8, 4 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=0" - - - spec-decoding: "none" - conc-list: [ 64, 32, 16, 8, 4, 2, 1 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - - - isl: 8192 - osl: 1024 - search-space: - # non-MTP configurations - # "Top of curve" (2 prefill worker at DEP8 and 1 decode worker at DEP8) - - spec-decoding: "none" - conc-list: [ 1024, 2048 ] - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=2" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - - # "Bottom of curve" (1 prefill worker at TP8 and 2 decode workers at TP8) - - spec-decoding: "none" - conc-list: [ 256, 128, 64, 32, 16, 8, 4 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=0" - - - spec-decoding: "none" - conc-list: [ 64, 32, 16, 8, 4, 2, 1 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - -dsr1-fp8-mi355x-sglang-disagg-mtp: - image: rocm/sgl-dev:sglang-0.5.9-rocm720-mi35x-mori-0227-2 - model: deepseek-ai/DeepSeek-R1-0528 - model-prefix: dsr1 - runner: mi355x-disagg - precision: fp8 - framework: sglang-disagg - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # MTP configurations - # "Top of curve" (1 prefill worker at DEP8 and 1 decode worker at DEP16) - - spec-decoding: "mtp" - conc-list: [ 1024, 2048 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=1" - - # "Middle of curve" (1 prefill worker at TP8 and 2 decode workers each at DEP8) - - spec-decoding: "mtp" - conc-list: [ 1536, 1024, 512, 256 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=1" - - # "Bottom of curve" (1 prefill worker at TEP8 and 2 decode workers at TEP8) - - spec-decoding: "mtp" - conc-list: [ 256, 128, 64, 32, 16, 8, 4 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=2" - - - spec-decoding: "mtp" - conc-list: [ 64, 32, 16, 8, 4, 2, 1 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=2" - - - isl: 8192 - osl: 1024 - search-space: - # MTP configurations - # "Top of curve" (2 prefill worker at DEP8 and 1 decode worker at DEP8) - - spec-decoding: "mtp" - conc-list: [ 1024, 2048 ] - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=2" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=1" - - # "Bottom of curve" (1 prefill worker at TP8 and 2 decode workers at TP8) - - spec-decoding: "mtp" - conc-list: [ 256, 128, 64, 32, 16, 8, 4, 2 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=2" - - - spec-decoding: "mtp" - conc-list: [ 64, 32, 16, 8, 4, 2, 1 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=2" - -kimik2.5-fp4-mi355x-vllm-disagg: - image: vllm/vllm-openai-rocm:nightly-bf610c2f56764e1b30bc6065f4ceace3d6e59036 - model: amd/Kimi-K2.5-MXFP4 - model-prefix: kimik2.5 - runner: mi355x-disagg - precision: fp4 - framework: vllm-disagg - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # 1P2D: 1 prefill node (co-located with proxy) + 2 decode nodes = 3 nodes total - - spec-decoding: "none" - conc-list: [ 8, 16, 32, 64, 128, 256, 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - "VLLM_MORIIO_CONNECTOR_READ_MODE=1" - decode: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - - isl: 8192 - osl: 1024 - search-space: - - spec-decoding: "none" - conc-list: [ 8, 16, 32, 64, 128, 256, 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - "VLLM_MORIIO_CONNECTOR_READ_MODE=1" - decode: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - -dsr1-fp4-mi355x-sglang-disagg: - image: lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260519 - model: amd/DeepSeek-R1-0528-MXFP4-v2 - model-prefix: dsr1 - runner: mi355x-disagg - precision: fp4 - framework: sglang-disagg - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # non-MTP configurations - # 1P1D TP8 - - spec-decoding: "none" - conc-list: [ 1, 2, 4, 8 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - - # 1P2D TP8 - - spec-decoding: "none" - conc-list: [ 2, 4, 8, 16, 32 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=0" - - # 1P2D TP8 - - spec-decoding: "none" - conc-list: [ 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=0" - - # 1P2D TP4 - - spec-decoding: "none" - conc-list: [ 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=0" - - # 1*DEP4+ 1*DEP8 - - spec-decoding: "none" - conc-list: [ 1024, 2048, 4096 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - - - isl: 8192 - osl: 1024 - search-space: - # non-MTP configurations - # 1P1D pure TP8 - - spec-decoding: "none" - conc-list: [ 1, 2, 4, 8 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - - # 1P2D TP8 - - spec-decoding: "none" - conc-list: [ 2, 4, 8, 16, 32 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=0" - - # 1P2D TP8 - - spec-decoding: "none" - conc-list: [ 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=0" - - # 1P2D TP4 - - spec-decoding: "none" - conc-list: [ 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=0" - - # 1*DEP8 + 1*DEP8 - - spec-decoding: "none" - conc-list: [ 128, 256, 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - - # 2*DEP8 + 1*DEP8 - - spec-decoding: "none" - conc-list: [ 1024, 2048, 4096 ] - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=2" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - -dsr1-fp4-mi355x-sglang-disagg-1k1k-mtp: - image: lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260529 - model: amd/DeepSeek-R1-0528-MXFP4-v2 - model-prefix: dsr1 - runner: mi355x-disagg - precision: fp4 - framework: sglang-disagg - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # MTP configurations - # 1P1D TP8 - - spec-decoding: "mtp" - conc-list: [ 1, 2, 4, 8 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=3" - - # 1P2D TP8 - - spec-decoding: "mtp" - conc-list: [ 2, 4, 8, 16, 32 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=3" - - # 1P2D TP8 - - spec-decoding: "mtp" - conc-list: [ 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=2" - - # 1P2D TP4 - - spec-decoding: "mtp" - conc-list: [ 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=2" - - # 1*DEP4+ 1*DEP8 - - spec-decoding: "mtp" - conc-list: [ 1024, 2048, 4096 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=1" - - -dsr1-fp4-mi355x-sglang-disagg-8k1k-mtp: - image: lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260529 - model: amd/DeepSeek-R1-0528-MXFP4-v2 - model-prefix: dsr1 - runner: mi355x-disagg - precision: fp4 - framework: sglang-disagg - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # MTP configurations - # 1P1D pure TP8 - - spec-decoding: "mtp" - conc-list: [ 1, 2, 4, 8 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=3" - - # 1P2D TP8 - - spec-decoding: "mtp" - conc-list: [ 2, 4, 8, 16, 32 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=3" - - # 1P2D TP8 - - spec-decoding: "mtp" - conc-list: [ 32, 64 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=3" - - # 1*DEP8 + 1*DEP8 - - spec-decoding: "mtp" - conc-list: [ 640, 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=3" - - # 1*DEP8 + 1*DEP8 - - spec-decoding: "mtp" - conc-list: [ 256 ] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=3" - - - # 1*DEP8 + 1*DEP8 - - spec-decoding: "mtp" - conc-list: [ 128 ] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=3" - - # 1*DEP8 + 1*DEP8 - - spec-decoding: "mtp" - conc-list: [ 64 ] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=3" - - # 2*DEP8 + 1*DEP8 - - spec-decoding: "mtp" - conc-list: [ 1024, 2048, 4096 ] - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=2" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=1" - -dsv4-fp4-mi355x-sglang: - image: lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260618 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: mi355x - precision: fp4 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, dp-attn: true, conc-start: 64, conc-end: 2048 } - - { tp: 4, dp-attn: true, conc-start: 16, conc-end: 128 } - - { tp: 4, dp-attn: false, conc-start: 1 , conc-end: 32 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, dp-attn: true, conc-start: 64, conc-end: 2048 } - - { tp: 4, dp-attn: true, conc-start: 16, conc-end: 128 } - - { tp: 4, dp-attn: false, conc-start: 1, conc-end: 32 } - - -# MTP variant of dsv4-fp4-mi355x-sglang. Mirrors the base search space and adds -# spec-decoding: mtp, which routes to dsv4_fp4_mi355x_sglang_mtp.sh (EAGLE -# speculative decoding), per sgl-project/sglang#26383 ([AMD][DSV4] DSV4 MTP -# graph + sparse triton attn optimizations, merged to main 2026-05-27). That PR -# fixes the ROCm HIP-radix MTP CUDA-graph bug (the false-EOS symptom in sgl -# #20404) and validates GSM8K 0.950 with MTP on. -# -# #26383 is on sglang `main`, NOT the amd/deepseek_v4 branch the rocm/sgl-dev:*-DSv4 -# builds are cut from (latest da28108 = f96ac98 + build fixes + an unrelated -# MLA-decode refactor, still pre-#26383 -> kv_score crash, run 26723126211). So we -# pin the mainline ROCm nightly, which carries #26383. Mainline omits deep_gemm, -# but the recipe detects that and routes the DSv4 fp8 wo_a / topk paths to their -# torch fallbacks (see dsv4_fp4_mi355x_sglang_mtp.sh). When a -DSv4 image carrying -# #26383 ships, bump to it; the recipe auto-restores the deep_gemm perf path. -dsv4-fp4-mi355x-sglang-mtp: - image: lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260601 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: mi355x - precision: fp4 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, dp-attn: true, conc-start: 64, conc-end: 2048, spec-decoding: mtp } - - { tp: 8, dp-attn: false, conc-start: 1, conc-end: 32, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, dp-attn: true, conc-start: 64, conc-end: 2048, spec-decoding: mtp } - - { tp: 8, dp-attn: false, conc-start: 1, conc-end: 32, spec-decoding: mtp } - -# DSv4 on MI355X via vLLM, using the official vllm/vllm-openai-rocm -# nightly image. DSv4 base ROCm support (vllm-project/vllm#40871) merged -# on 2026-05-05, so any nightly built after that includes the -# DeepseekV4ForCausalLM model class. -# -# IMPORTANT: pin to a digest-suffixed nightly tag rather than the -# floating `:nightly`. launch_mi355x-amds.sh caches enroot squashfs -# files keyed on the image string and short-circuits re-import if the -# file already exists, so the floating tag silently keeps a stale build -# even after Docker Hub updates `:nightly`. -# -# DeepSeek-V4-Pro is FP4+FP8 mixed (FP4 MoE expert weights, FP8 for the -# rest); InferenceX classifies this as fp4 — same as the sister sglang -# and atom DSv4 mi355x entries below. Image and serving flags follow the -# validated recipe from vllm-project/recipes#433: AITER+AITER_LINEAR, mp -# executor, triton_unfused MoE (required for the FP4 expert format), -# async scheduling, max-num-seqs=128, max-num-batched-tokens=8192, -# gpu-mem-util=0.6. TP8 sweeps conc 4-64; DEP8 has a single conc=64 -# probe to validate the ROCm DP+EP path. -dsv4-fp4-mi355x-vllm: - image: vllm/vllm-openai-rocm:nightly-09663abde0f50944a8d5ea30120666024b503faa - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: mi355x - precision: fp4 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 512 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 512 } - -# MTP variant of dsv4-fp4-mi355x-vllm. Mirrors the base recipe's search space -# and adds spec-decoding: mtp, which routes to dsv4_fp4_mi355x_vllm_mtp.sh -# (--speculative-config '{"method":"mtp","num_speculative_tokens":2}'), per -# vllm-project/vllm#43385 (ROCm DeepSeek-V4 MTP, merged 2026-05-24, included in -# v0.22.0). Full conc 4-512 range maps the complete crossover curve: MTP wins -# at low batch (PR perf data: +75% @ conc1, +38% @ conc8) and falls behind STP -# above ~conc32 (-37% @ conc32). Image reuses the base entry's v0.22.0 ROCm -# build, which already contains the MTP commit. -dsv4-fp4-mi355x-vllm-mtp: - image: vllm/vllm-openai-rocm:nightly-09663abde0f50944a8d5ea30120666024b503faa - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: mi355x - precision: fp4 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 512, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 512, spec-decoding: mtp } - -dsv4-fp4-mi355x-atom: - image: rocm/atom-dev:nightly_202606161823 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: mi355x - precision: fp4 - framework: atom - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # conc4-64, TP8 - # conc128-512, DPA - # conc1024-2048, DPA TBO - - { tp: 8, ep: 1, conc-start: 1, conc-end: 64 } - - { tp: 8, ep: 1, dp-attn: true, conc-start: 64, conc-end: 2048 } - - isl: 8192 - osl: 1024 - search-space: - # conc4-64, TP8 - # conc128, DPA - # conc256-2048, DPA TBO - - { tp: 4, ep: 1, conc-list: [8, 16, 32, 64] } - - { tp: 8, ep: 1, conc-list: [1, 2, 4, 8, 16, 32, 64] } - - { tp: 8, ep: 1, dp-attn: true, conc-start: 128, conc-end: 2048 } - -dsv4-fp4-mi355x-atom-mtp: - image: rocm/atom:rocm7.2.4_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.3 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: mi355x - precision: fp4 - framework: atom - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 1, conc-end: 1024, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 1, conc-end: 1024, spec-decoding: mtp } - -qwen3.5-bf16-mi325x-sglang-mtp: - image: lmsysorg/sglang:v0.5.12-rocm720-mi30x - model: Qwen/Qwen3.5-397B-A17B - model-prefix: qwen3.5 - runner: mi325x - precision: bf16 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 64, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 64, spec-decoding: mtp } - -dsr1-fp8-mi325x-sglang-mtp: - image: lmsysorg/sglang:v0.5.12-rocm700-mi30x - model: deepseek-ai/DeepSeek-R1-0528 - model-prefix: dsr1 - runner: mi325x - precision: fp8 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 64, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 64, spec-decoding: mtp } - -qwen3.5-fp8-mi325x-sglang-mtp: - image: lmsysorg/sglang:v0.5.12-rocm720-mi30x - model: Qwen/Qwen3.5-397B-A17B-FP8 - model-prefix: qwen3.5 - runner: mi325x - precision: fp8 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 64, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 64, spec-decoding: mtp } - -glm5-fp8-mi325x-sglang: - image: lmsysorg/sglang:v0.5.12-rocm720-mi30x - model: zai-org/GLM-5-FP8 - model-prefix: glm5 - runner: mi325x - precision: fp8 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 64 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 64 } - -glm5-fp8-mi325x-sglang-mtp: - image: lmsysorg/sglang:v0.5.12-rocm720-mi30x - model: zai-org/GLM-5-FP8 - model-prefix: glm5 - runner: mi325x - precision: fp8 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 64, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 64, spec-decoding: mtp } - -# ============================================================================ -# Net-new agentic recipes from chore/agentx-v0.3 (no overlap with main entries). -# Recipes that ALREADY existed on main were intentionally left at main's version -# to preserve main behavior; PR-branch modifications to those recipes are NOT -# brought in here. -# ============================================================================ - -qwen3.5-fp8-mi355x-sglang-agentic-hicache: - image: lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260521 - model: Qwen/Qwen3.5-397B-A17B-FP8 - model-prefix: qwen3.5 - runner: cluster:mi355x-amds - precision: fp8 - framework: sglang - multinode: false - scenarios: - agentic-coding: - - dram-utilization: 0.80 - search-space: - - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32] } - - { tp: 8, ep: 1, kv-offloading: dram, kv-offload-backend: hicache, conc-list: [16, 32, 48, 64] } - - - -# DSv4-Pro FP4 on MI355X via SGLang. Uses a rocm720 mi35x image built off the -# amd/deepseek_v4 branch in sgl-project/sglang; the SHA is encoded in the -# image tag, so bumping sglang is just an image tag bump here. Sweeps -# DP-attention on/off and EP=8. - -# CONC ranges mirror dsv4-fp4-b200-vllm-agentic for cross-hardware -# comparability. Offload sweep is none-only (SGLang has no equivalent of -# vLLM's SimpleCPUOffloadConnector path that we exercise on b200). -dsv4-fp4-mi355x-vllm-agentic: - image: vllm/vllm-openai-rocm:v0.22.0 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: cluster:mi355x-amds - precision: fp4 - framework: vllm - multinode: false - scenarios: - agentic-coding: - - search-space: - - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4] } - - { tp: 4, kv-offloading: none, conc-list: [1, 2, 4, 8, 10, 12, 16] } - - { tp: 4, ep: 4, dp-attn: true, kv-offloading: none, conc-list: [16, 24, 32, 40, 48] } - - -# DSv4-Pro FP4 on MI355X via SGLang. Uses a rocm720 mi35x image built off the -# amd/deepseek_v4 branch in sgl-project/sglang; the SHA is encoded in the -# image tag, so bumping sglang is just an image tag bump here. Sweeps -# DP-attention on/off and EP=8. - -dsr1-fp4-mi355x-sglang-disagg-mtp: - image: lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260519 - model: amd/DeepSeek-R1-0528-MXFP4-v2 - model-prefix: dsr1 - runner: mi355x-disagg - precision: fp4 - framework: sglang-disagg - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # MTP configurations - # 1P1D TP8 - - spec-decoding: "mtp" - conc-list: [ 1, 2, 4, 8 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=3" - - # 1P2D TP8 - - spec-decoding: "mtp" - conc-list: [ 2, 4, 8, 16, 32 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=3" - - # 1P2D TP8 - - spec-decoding: "mtp" - conc-list: [ 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=2" - - # 1P2D TP4 - - spec-decoding: "mtp" - conc-list: [ 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=2" - - # 1*DEP4+ 1*DEP8 - - spec-decoding: "mtp" - conc-list: [ 1024, 2048, 4096 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=1" - - - isl: 8192 - osl: 1024 - search-space: - # MTP configurations - # 1P1D pure TP8 - - spec-decoding: "mtp" - conc-list: [ 1, 2, 4, 8 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=3" - - # 1P2D TP8 - - spec-decoding: "mtp" - conc-list: [ 2, 4, 8, 16, 32 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=3" - - # 1P2D TP8 - - spec-decoding: "mtp" - conc-list: [ 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=2" - - # 1*DEP8 + 1*DEP8 - - spec-decoding: "mtp" - conc-list: [ 128, 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=1" - - # 1*DEP8 + 1*DEP8 - - spec-decoding: "mtp" - conc-list: [ 64, 256 ] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=1" - - # 2*DEP8 + 1*DEP8 - - spec-decoding: "mtp" - conc-list: [ 1024, 2048, 4096 ] - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=2" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=1" - - -# DSv4-Pro FP4 on MI355X via SGLang. Uses a rocm720 mi35x image built off the -# amd/deepseek_v4 branch in sgl-project/sglang; the SHA is encoded in the -# image tag, so bumping sglang is just an image tag bump here. Sweeps -# DP-attention on/off and EP=8. - -dsv4-fp4-mi355x-sglang-agentic: - image: rocm/sgl-dev:rocm720-mi35x-0363e6c-20260509-DSv4 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: cluster:mi355x-amds - precision: fp4 - framework: sglang - multinode: false - scenarios: - agentic-coding: - - search-space: - - { tp: 8, kv-offloading: none, conc-list: [16, 32, 64] } - - { tp: 8, dp-attn: true, kv-offloading: none, conc-list: [64, 128, 256] } - - -# MiniMax-M3 MXFP8 MI355X recipe: -# https://github.com/vllm-project/recipes/commit/2a3728ed9892debfd767a72a58ebc90b33f186e5 -# MXFP8 runs from TP=4 on gfx950; block size 128 is mandatory for MSA. -dsv4-fp4-mi355x-atom-disagg: - image: rocm/atom-dev:nightly_202606101403 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: mi355x - precision: fp4 - framework: atom-disagg - multinode: true - disagg: true - scenarios: - fixed-seq-len: - # 1P1D DPA+TP8 - - isl: 8192 - osl: 1024 - search-space: - # 2P1D DPA+TP8 - - conc-list: [ 256, 512, 768, 1024, 2048 ] - prefill: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - - "PREFILL_NODES=2" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - # 1P1D TP8 - - conc-list: [ 4, 8, 16, 32, 64, 128 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - # 1P1D TP8 - - isl: 1024 - osl: 1024 - search-space: - - conc-list: [ 4, 8, 16, 32, 64, 128, 256, 512, 1024 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - -# MiniMax-M3 MXFP8 MI355X recipe: -# https://github.com/vllm-project/recipes/commit/2a3728ed9892debfd767a72a58ebc90b33f186e5 -# MXFP8 runs from TP=4 on gfx950; block size 128 is mandatory for MSA. -minimaxm3-fp8-mi355x-vllm: - image: vllm/vllm-openai-rocm:nightly-4559c43a9526597c00cbcc4f59979496500268d1 - model: MiniMaxAI/MiniMax-M3-MXFP8 - model-prefix: minimaxm3 - runner: mi355x - precision: fp8 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, conc-start: 1, conc-end: 32 } - - { tp: 4, conc-start: 4, conc-end: 64 } - - { tp: 4, ep: 4, conc-start: 64, conc-end: 512 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, conc-start: 1, conc-end: 2 } - - { tp: 4, conc-start: 2, conc-end: 128 } - -# EAGLE3 speculative-decoding (spec-decoding: mtp) variant of -# minimaxm3-fp8-mi355x-vllm, pairing MiniMaxAI/MiniMax-M3-MXFP8 with the -# Inferact/MiniMax-M3-EAGLE3 draft head (3 speculative tokens). No -# attention_backend override is needed — the server runs on TRITON_ATTN, so -# the FlashInfer page-128/MHA limitation that forced FLASH_ATTN on Blackwell -# does not apply here. Search space mirrors the non-MTP entry trimmed at the -# extreme-concurrency end, identical to the minimaxm3-fp8-b300-vllm-mtp / -# b200-vllm-mtp precedent: spec decode pays off at low/mid concurrency while -# acceptance dilutes in big batches, and the draft weights + draft KV shave -# headroom — tp2-ep2 is dropped since its KV headroom was already thin. -minimaxm3-fp8-mi355x-vllm-mtp: - image: vllm/vllm-openai-rocm:nightly-4559c43a9526597c00cbcc4f59979496500268d1 - model: MiniMaxAI/MiniMax-M3-MXFP8 - model-prefix: minimaxm3 - runner: mi355x - precision: fp8 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 32, spec-decoding: mtp } - - { tp: 8, ep: 8, conc-start: 1, conc-end: 256, spec-decoding: mtp } - - { tp: 4, conc-start: 1, conc-end: 2, spec-decoding: mtp } - - { tp: 4, conc-start: 32, conc-end: 64, spec-decoding: mtp } - - { tp: 4, ep: 4, conc-start: 128, conc-end: 256, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 16, spec-decoding: mtp } - - { tp: 4, conc-start: 2, conc-end: 128, spec-decoding: mtp } - - { tp: 8, conc-start: 1, conc-end: 1, spec-decoding: mtp } - -# MiniMax-M3 MXFP4 MI355X vLLM disaggregated (prefill/decode) config. -minimaxm3-fp4-mi355x-vllm-disagg: - image: rocm/vllm-dev:vllm-0.23.1-rocm723-mi35x-mori-0625 - model: amd/MiniMax-M3-MXFP4 - model-prefix: minimaxm3 - runner: mi355x-disagg - precision: fp4 - framework: vllm-disagg - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # 1P TP4 + 1D TP4 (2 nodes total), conc sweep 1..512 (single job, looped) - - spec-decoding: "none" - conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128, 256, 512 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - # 2P TP4 + 1D TP4 (3 nodes total), conc 128/256/512 (single job, looped) - - spec-decoding: "none" - conc-list: [ 128, 256, 512 ] - prefill: - num-worker: 2 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=2" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" -# MiniMax-M3 MXFP4 MI355X vLLM recipe. The pinned nightly includes upstream -# MiniMax-M3 Quark MXFP4 support (vllm-project/vllm#45794). Use the text-only -# language-model path and mirror the MXFP8 MI355X search space for a direct -# precision comparison. -minimaxm3-fp4-mi355x-vllm: - image: vllm/vllm-openai-rocm:nightly-4559c43a9526597c00cbcc4f59979496500268d1 - model: amd/MiniMax-M3-MXFP4 - model-prefix: minimaxm3 - runner: mi355x - precision: fp4 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, conc-start: 1, conc-end: 64 } - - { tp: 8, ep: 8, conc-start: 1, conc-end: 512 } - - { tp: 4, conc-start: 1, conc-end: 64 } - - { tp: 4, ep: 4, conc-start: 64, conc-end: 512 } - - { tp: 2, ep: 2, conc-start: 16, conc-end: 128 } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 256, conc-end: 1024 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, conc-start: 1, conc-end: 64 } - - { tp: 8, ep: 8, conc-start: 1, conc-end: 512 } - - { tp: 4, conc-start: 1, conc-end: 128 } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 128, conc-end: 512 } - -# EAGLE3 speculative-decoding variant of minimaxm3-fp4-mi355x-vllm. Pair the -# amd/MiniMax-M3-MXFP4 target with Inferact/MiniMax-M3-EAGLE3 and three draft -# tokens. Search space mirrors the MI355X MXFP8 MTP entry, trimming the base -# FP4 sweep at extreme concurrency where speculative decoding loses value. -minimaxm3-fp4-mi355x-vllm-mtp: - image: vllm/vllm-openai-rocm:nightly-3f5a1e1733200760169ff31ebe60a271072b199e - model: amd/MiniMax-M3-MXFP4 - model-prefix: minimaxm3 - runner: mi355x - precision: fp4 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, conc-start: 1, conc-end: 64, spec-decoding: mtp } - - { tp: 8, ep: 8, conc-start: 1, conc-end: 256, spec-decoding: mtp } - - { tp: 4, conc-start: 1, conc-end: 64, spec-decoding: mtp } - - { tp: 4, ep: 4, conc-start: 64, conc-end: 256, spec-decoding: mtp } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 256, conc-end: 512, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, conc-start: 1, conc-end: 64, spec-decoding: mtp } - - { tp: 8, ep: 8, conc-start: 1, conc-end: 256, spec-decoding: mtp } - - { tp: 4, conc-start: 1, conc-end: 64, spec-decoding: mtp } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 128, conc-end: 256, spec-decoding: mtp } - -# MiniMax-M3 MXFP4 MI355X atom recipe: -# https://github.com/ROCm/ATOM/blob/5d42d49f9e4292e5b61475917e92e7ec1b1dacb7/recipes/MiniMax-M3.md -# block size 128 is mandatory for MSA. TP4 on a single gfx950 node, per the recipe. -minimaxm3-fp4-mi355x-atom: - image: rocm/atom-dev:nightly_202607011530 - model: amd/MiniMax-M3-MXFP4 - model-prefix: minimaxm3 - runner: mi355x - precision: fp4 - framework: atom - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 4, conc-start: 1, conc-end: 256 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 4, conc-start: 1, conc-end: 256 } - -minimaxm3-fp4-mi355x-atom-mtp: - image: rocm/atom-dev:nightly_202607011530 - model: amd/MiniMax-M3-MXFP4 - model-prefix: minimaxm3 - runner: mi355x - precision: fp4 - framework: atom - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 4, conc-start: 1, conc-end: 256, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 4, conc-start: 1, conc-end: 256, spec-decoding: mtp } - -minimaxm3-fp8-mi355x-atom: - image: rocm/atom-dev:nightly_202607011530 - model: MiniMaxAI/MiniMax-M3-MXFP8 - model-prefix: minimaxm3 - runner: mi355x - precision: fp8 - framework: atom - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 4, conc-start: 1, conc-end: 256 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 4, conc-start: 1, conc-end: 256 } - -minimaxm3-fp8-mi355x-atom-mtp: - image: rocm/atom-dev:nightly_202607011530 - model: MiniMaxAI/MiniMax-M3-MXFP8 - model-prefix: minimaxm3 - runner: mi355x - precision: fp8 - framework: atom - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 4, conc-start: 1, conc-end: 256, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 4, conc-start: 1, conc-end: 256, spec-decoding: mtp } - -minimaxm3-fp8-mi355x-atom-disagg: - image: rocm/atom-dev:MiniMax-M3-20260622 - model: amd/MiniMax-M3-MXFP8 - model-prefix: minimaxm3 - runner: mi355x-disagg - precision: fp8 - framework: atom-disagg - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # 1P1D TP4 - - conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128, 256, 512 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - # 1P1D TP4 - - isl: 1024 - osl: 1024 - search-space: - # 1P1D TP4 - - conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128, 256, 512 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" -minimaxm3-fp4-mi355x-atom-disagg: - image: rocm/atom-dev:MiniMax-M3-20260622 - model: amd/MiniMax-M3-MXFP4 - model-prefix: minimaxm3 +kimik2.5-fp4-mi355x-vllm-disagg: + image: vllm/vllm-openai-rocm:nightly-bf610c2f56764e1b30bc6065f4ceace3d6e59036 + model: amd/Kimi-K2.5-MXFP4 + model-prefix: kimik2.5 runner: mi355x-disagg precision: fp4 - framework: atom-disagg - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # 1P1D TP4 - - conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128, 256, 512 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - # 1P1D TP4 - - isl: 1024 - osl: 1024 - search-space: - # 1P1D TP4 - - conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128, 256, 512 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - -# MiniMax-M3 MXFP8 MI300X day-zero recipe. Reuse the dedicated ROCm image and -# MI355X serving shape, but retain the default BF16 KV cache because this -# checkpoint lacks calibrated ROCm FP8 attention scales. Use the TP8-only H100 -# search space: TP8 for latency and TP8+EP8 (TEP) at high concurrency. -minimaxm3-fp8-mi300x-vllm: - image: vllm/vllm-openai-rocm:minimax-m3 - model: MiniMaxAI/MiniMax-M3-MXFP8 - model-prefix: minimaxm3 - runner: mi300x - precision: fp8 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, conc-start: 1, conc-end: 128 } - - { tp: 8, ep: 8, conc-start: 256, conc-end: 256 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, conc-start: 1, conc-end: 64 } - - { tp: 8, ep: 8, conc-start: 128, conc-end: 256 } - -# EAGLE3 speculative-decoding (spec-decoding: mtp) variant of -# minimaxm3-fp8-mi300x-vllm, pairing MiniMaxAI/MiniMax-M3-MXFP8 with the -# Inferact/MiniMax-M3-EAGLE3 draft head (3 speculative tokens). Same TP8-only -# search space as the non-MTP MI300X entry (gfx942 192 GB is memory-tight, like -# H100), with the TP8 latency rows started at conc 1 to capture single-request -# latency — matching the H100/MI355X MTP recipes. The pinned ROCm nightly -# includes upstream SupportsEagle3 support for the AMD MiniMax-M3 model. -minimaxm3-fp8-mi300x-vllm-mtp: - image: vllm/vllm-openai-rocm:nightly-b53b1c7ffe7aebdafd0876350f30e51d1226c92a - model: MiniMaxAI/MiniMax-M3-MXFP8 - model-prefix: minimaxm3 - runner: mi300x - precision: fp8 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, conc-start: 1, conc-end: 128, spec-decoding: mtp } - - { tp: 8, ep: 8, conc-start: 256, conc-end: 256, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, conc-start: 1, conc-end: 64, spec-decoding: mtp } - - { tp: 8, ep: 8, conc-start: 128, conc-end: 256, spec-decoding: mtp } - -# MiniMax-M3 MXFP8 MI325X day-zero recipe. Reuse the dedicated ROCm image -# and serving flags validated on MI355X, with the H200 search space: TP4 and -# TP8 latency, TP4/TP8 expert parallelism, and TP8 data-parallel attention. -minimaxm3-fp8-mi325x-vllm: - image: vllm/vllm-openai-rocm:minimax-m3 - model: MiniMaxAI/MiniMax-M3-MXFP8 - model-prefix: minimaxm3 - runner: mi325x - precision: fp8 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 4, conc-start: 1, conc-end: 64 } - - { tp: 4, ep: 4, conc-start: 128, conc-end: 256 } - - { tp: 8, conc-start: 1, conc-end: 128 } - - { tp: 8, ep: 8, conc-start: 256, conc-end: 512 } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 256, conc-end: 1024 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 4, conc-start: 1, conc-end: 32 } - - { tp: 8, conc-start: 1, conc-end: 128 } - - { tp: 8, ep: 8, conc-start: 256, conc-end: 256 } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 256, conc-end: 512 } - -# EAGLE3 speculative-decoding (spec-decoding: mtp) variant of -# minimaxm3-fp8-mi325x-vllm, pairing MiniMaxAI/MiniMax-M3-MXFP8 with the -# Inferact/MiniMax-M3-EAGLE3 draft head (3 speculative tokens). Same H200-style -# search space as the non-MTP MI325X entry, trimmed at the extreme-concurrency -# end with TP-only latency rows started at conc 1 (matching the H200/MI355X MTP -# recipes). Runs with CUDA graphs (no --enforce-eager, VLLM_USE_BREAKABLE_CUDAGRAPH=0, -# BF16 KV on gfx942). The shipped ROCm image lacks SupportsEagle3 on the AMD -# MiniMax-M3 model, so the recipe applies that fix in-place at runtime -# (functionstackx/vllm#1, upstream vllm-project/vllm#45546; validated green on -# MI355X/MI300X) before serving. -minimaxm3-fp8-mi325x-vllm-mtp: - image: vllm/vllm-openai-rocm:minimax-m3 - model: MiniMaxAI/MiniMax-M3-MXFP8 - model-prefix: minimaxm3 - runner: mi325x - precision: fp8 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 4, conc-start: 1, conc-end: 64, spec-decoding: mtp } - - { tp: 4, ep: 4, conc-start: 128, conc-end: 256, spec-decoding: mtp } - - { tp: 8, conc-start: 1, conc-end: 128, spec-decoding: mtp } - - { tp: 8, ep: 8, conc-start: 256, conc-end: 256, spec-decoding: mtp } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 256, conc-end: 512, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 4, conc-start: 1, conc-end: 32, spec-decoding: mtp } - - { tp: 8, conc-start: 1, conc-end: 128, spec-decoding: mtp } - - { tp: 8, ep: 8, conc-start: 256, conc-end: 256, spec-decoding: mtp } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 256, conc-end: 256, spec-decoding: mtp } - -# MiniMax-M3 MXFP8 MI355X vLLM disaggregated (prefill/decode) smoke test on the -# day-zero ROCm image. Minimal 1 prefill (TP8) + 1 decode (TP8) at conc 1 to -# validate the MoRI-IO KV-transfer disagg pipeline end-to-end for M3. Layered on -# the MoRI-patch-removal infra (#1585). No EP (TP8 only); MoE experts are -# TP-sharded as in the single-node M3 TP8 recipe. Per-worker serve flags live in -# benchmarks/multi_node/amd_utils/models_vllm.yaml (MiniMax-M3-MXFP8). -minimaxm3-fp8-mi355x-vllm-disagg: - image: vllm/vllm-openai-rocm:nightly-556bc4e3a089378e9df2482659898192da18db15 - model: MiniMaxAI/MiniMax-M3-MXFP8 - model-prefix: minimaxm3 - runner: mi355x-disagg - precision: fp8 framework: vllm-disagg multinode: true disagg: true @@ -2960,195 +13,42 @@ minimaxm3-fp8-mi355x-vllm-disagg: - isl: 1024 osl: 1024 search-space: + # 1P2D: 1 prefill node (co-located with proxy) + 2 decode nodes = 3 nodes total - spec-decoding: "none" - conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - # Asymmetric 1P TP4 + 1D TP8 (smaller prefill, full-node decode) across - # conc 1,2,4,8,16,32,64,128,256. - - spec-decoding: "none" - conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128, 256 ] + conc-list: [ 8, 16, 32, 64, 128, 256, 512 ] prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: num-worker: 1 tp: 8 ep: 1 dp-attn: false additional-settings: - - "DECODE_NODES=1" - # Balanced half-node 1P TP4 + 1D TP4 at high conc 64,128,256,512,1024. - - spec-decoding: "none" - conc-list: [ 64, 128, 256, 512, 1024 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - "PREFILL_NODES=1" + - "VLLM_MORIIO_CONNECTOR_READ_MODE=1" decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - # 2P TP4 + 1D TP8: two half-node TP4 prefill workers (PREFILL_NODES=2) - # feeding one full-node TP8 decode, at high conc 256,512,768,1024. - - spec-decoding: "none" - conc-list: [ 256, 512, 768, 1024 ] - prefill: num-worker: 2 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=2" - decode: - num-worker: 1 tp: 8 - ep: 1 + ep: 8 dp-attn: false additional-settings: - - "DECODE_NODES=1" - # 8k1k disagg sweep across four P/D layouts (1P TP8 + 1D TP8 conc 1..1024; - # 1P TP4 + 1D TP8 conc 1..256; 1P TP4 + 1D TP4 conc 64..1024; 2P TP4 + 1D TP8 - # conc 256..1024). The multi-node eval policy (8k1k + conc >= 16) marks one - # lm-eval on the highest-max-conc layout (TP8+TP8, eval-conc=median=128) — - # validating the M3 MoRI-IO disagg pipeline's correctness end-to-end. + - "DECODE_NODES=2" + - isl: 8192 osl: 1024 search-space: - spec-decoding: "none" - conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - # Asymmetric 1P TP4 + 1D TP8 (smaller prefill, full-node decode) across - # conc 1,2,4,8,16,32,64,128,256. - - spec-decoding: "none" - conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128, 256 ] + conc-list: [ 8, 16, 32, 64, 128, 256, 512 ] prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: num-worker: 1 tp: 8 ep: 1 dp-attn: false additional-settings: - - "DECODE_NODES=1" - # Balanced half-node 1P TP4 + 1D TP4 at high conc 64,128,256,512,1024. - - spec-decoding: "none" - conc-list: [ 64, 128, 256, 512, 1024 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - "PREFILL_NODES=1" + - "VLLM_MORIIO_CONNECTOR_READ_MODE=1" decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - # 2P TP4 + 1D TP8: two half-node TP4 prefill workers (PREFILL_NODES=2) - # feeding one full-node TP8 decode, at high conc 256,512,768,1024. - - spec-decoding: "none" - conc-list: [ 256, 512, 768, 1024 ] - prefill: num-worker: 2 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=2" - decode: - num-worker: 1 tp: 8 - ep: 1 + ep: 8 dp-attn: false additional-settings: - - "DECODE_NODES=1" -minimaxm3-fp8-mi300x-vllm-agentic: - image: vllm/vllm-openai-rocm:nightly-04c2a8deac44fdb1ca3e2b5ec3e6bf16f3f6a914 - model: MiniMaxAI/MiniMax-M3-MXFP8 - model-prefix: minimaxm3 - runner: cluster:mi300x-amds - precision: fp8 - framework: vllm - multinode: false - scenarios: - agentic-coding: - # FP8 KV cache is 29,952 bytes/token/GPU for TP/TEP. Projecting the - # measured BF16 cache budget gives about 2.76M active tokens on MI300X; - # the June-21 service-time-weighted request is 269k tokens, so sample every - # integer around the expected conc-10 cliff rather than geometric jumps. - - dram-utilization: 0.80 - search-space: - - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 18, 20] } - - { tp: 8, ep: 8, kv-offloading: none, conc-list: [2, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 18, 20] } - - { tp: 8, kv-offloading: dram, kv-offload-backend: mooncake, conc-list: [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 18, 20] } - - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: mooncake, conc-list: [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 18, 20] } - -minimaxm3-fp8-mi325x-vllm-agentic: - image: vllm/vllm-openai-rocm:nightly-04c2a8deac44fdb1ca3e2b5ec3e6bf16f3f6a914 - model: MiniMaxAI/MiniMax-M3-MXFP8 - model-prefix: minimaxm3 - runner: cluster:mi325x-amds - precision: fp8 - framework: vllm - multinode: false - scenarios: - agentic-coding: - # Measured FP8 capacities: TP4 1.66M tokens, TP8 4.93M tokens, and DEP8 - # 12.69M aggregate tokens. With the June-21 service-time-weighted 269k - # active request, their expected cliffs are conc 6, 18, and 47. Use dense - # bands around each cliff; Mooncake rows overlap those bands because host - # storage improves prefix retention but does not hold active decode KV. - - dram-utilization: 0.80 - search-space: - - { tp: 4, kv-offloading: none, conc-list: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16] } - - { tp: 4, ep: 4, kv-offloading: none, conc-list: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16] } - - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 32] } - - { tp: 8, ep: 8, kv-offloading: none, conc-list: [2, 4, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 32] } - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [16, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64, 72, 80] } - - { tp: 4, ep: 4, kv-offloading: dram, kv-offload-backend: native, conc-list: [3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16] } - - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: native, conc-list: [10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 32] } - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: native, conc-list: [24, 32, 36, 40, 44, 48, 52, 56, 60, 64, 72, 80, 96] } + - "DECODE_NODES=2" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 82831764b..7a084b07e 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -1,8 +1,9 @@ -dsr1-fp4-b200-dynamo-trt: - image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post1 - model: deepseek-r1-fp4 - model-prefix: dsr1 - runner: b200-multinode + +kimik2.5-fp4-gb200-dynamo-trt: + image: nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + model: nvidia/Kimi-K2.5-NVFP4 + model-prefix: kimik2.5 + runner: gb200 precision: fp4 framework: dynamo-trt multinode: true @@ -12,11853 +13,370 @@ dsr1-fp4-b200-dynamo-trt: - isl: 1024 osl: 1024 search-space: - - spec-decoding: "mtp" - conc-list: [1214] + # Non-MTP configurations (default spec_decoding="none") + - conc-list: [ 8 ] prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/1k1k/mtp/ctx1_gen2_dep8_batch64_eplb0_mtp2.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp4/1k1k/mtp/ctx1_gen2_dep8_batch64_eplb0_mtp2.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch1_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch1_eplb0_mtp0.yaml" decode: - num-worker: 2 + num-worker: 4 tp: 8 ep: 8 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [875] + dp-attn: false + - conc-list: [ 12 ] prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/1k1k/mtp/ctx1_gen5_dep8_batch16_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp4/1k1k/mtp/ctx1_gen5_dep8_batch16_eplb0_mtp3.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch2_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch2_eplb0_mtp0.yaml" decode: - num-worker: 5 + num-worker: 4 tp: 8 ep: 8 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [6] + dp-attn: false + - conc-list: [ 24 ] prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/1k1k/mtp/ctx1_gen5_tep8_batch1_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp4/1k1k/mtp/ctx1_gen5_tep8_batch1_eplb0_mtp3.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch4_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch4_eplb0_mtp0.yaml" decode: - num-worker: 5 + num-worker: 4 tp: 8 ep: 8 dp-attn: false - - spec-decoding: "mtp" - conc-list: [10, 15, 25, 45, 90, 180] + - conc-list: [ 192 ] prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/1k1k/mtp/ctx1_gen5_tep8_batch32_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp4/1k1k/mtp/ctx1_gen5_tep8_batch32_eplb0_mtp3.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch32_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch32_eplb0_mtp0.yaml" decode: - num-worker: 5 + num-worker: 4 tp: 8 ep: 8 dp-attn: false - - spec-decoding: "mtp" - conc-list: [ 4968 ] + - conc-list: [ 30 ] prefill: - num-worker: 3 + num-worker: 1 tp: 4 ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/1k1k/mtp/ctx3_gen4_dep8_batch128_eplb0_mtp1.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp4/1k1k/mtp/ctx3_gen4_dep8_batch128_eplb0_mtp1.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen5tep4_batch4_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen5tep4_batch4_eplb0_mtp0.yaml" decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [10860] + num-worker: 5 + tp: 4 + ep: 4 + dp-attn: false + - conc-list: [ 60 ] prefill: - num-worker: 3 + num-worker: 1 tp: 4 ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/1k1k/mtp/ctx3_gen5_dep4_batch512_eplb0_mtp1.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp4/1k1k/mtp/ctx3_gen5_dep4_batch512_eplb0_mtp1.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen5tep4_batch8_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen5tep4_batch8_eplb0_mtp0.yaml" decode: num-worker: 5 tp: 4 ep: 4 - dp-attn: true - - # Non-MTP configurations - - conc-list: [4096] + dp-attn: false + - conc-list: [ 333 ] prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/1k1k/stp/ctx1_gen1_dep8_batch512_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp4/1k1k/stp/ctx1_gen1_dep8_batch512_eplb0_mtp0.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen1dep32_batch8_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen1dep32_batch8_eplb0_mtp0.yaml" decode: num-worker: 1 - tp: 8 - ep: 8 + tp: 32 + ep: 32 dp-attn: true - - conc-list: [2192] + - conc-list: [ 666 ] prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/1k1k/stp/ctx1_gen2_dep8_batch128_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp4/1k1k/stp/ctx1_gen2_dep8_batch128_eplb0_mtp0.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen1dep32_batch16_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen1dep32_batch16_eplb0_mtp0.yaml" decode: - num-worker: 2 - tp: 8 - ep: 8 + num-worker: 1 + tp: 32 + ep: 32 dp-attn: true - - conc-list: [1365] + - conc-list: [ 1127 ] prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/1k1k/stp/ctx1_gen5_dep8_batch32_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp4/1k1k/stp/ctx1_gen5_dep8_batch32_eplb0_mtp0.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen1dep32_batch32_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen1dep32_batch32_eplb0_mtp0.yaml" decode: - num-worker: 5 - tp: 8 - ep: 8 + num-worker: 1 + tp: 32 + ep: 32 dp-attn: true - - conc-list: [6] + - conc-list: [ 4301 ] prefill: - num-worker: 1 + num-worker: 2 tp: 4 ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/1k1k/stp/ctx1_gen5_tep8_batch1_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp4/1k1k/stp/ctx1_gen5_tep8_batch1_eplb0_mtp0.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL1K_OSL1K/STP/ctx2dep4_gen1dep16_batch256_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL1K_OSL1K/STP/ctx2dep4_gen1dep16_batch256_eplb0_mtp0.yaml" decode: - num-worker: 5 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [10, 15, 25, 45, 90, 180] - prefill: num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - conc-list: [ 8192 ] + prefill: + num-worker: 2 tp: 4 ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/1k1k/stp/ctx1_gen5_tep8_batch32_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp4/1k1k/stp/ctx1_gen5_tep8_batch32_eplb0_mtp0.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL1K_OSL1K/STP/ctx2dep4_gen1dep16_batch512_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL1K_OSL1K/STP/ctx2dep4_gen1dep16_batch512_eplb0_mtp0.yaml" decode: - num-worker: 5 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [450] - prefill: num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - conc-list: [ 2253 ] + prefill: + num-worker: 2 tp: 4 ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/1k1k/stp/ctx1_gen6_tep8_batch64_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp4/1k1k/stp/ctx1_gen6_tep8_batch64_eplb0_mtp0.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL1K_OSL1K/STP/ctx2dep4_gen1dep32_batch64_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL1K_OSL1K/STP/ctx2dep4_gen1dep32_batch64_eplb0_mtp0.yaml" decode: - num-worker: 6 - tp: 8 - ep: 8 - dp-attn: false - - - isl: 8192 - osl: 1024 - search-space: - - spec-decoding: "mtp" - conc-list: [90] - prefill: num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + - conc-list: [ 4301 ] + prefill: + num-worker: 2 tp: 4 ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/8k1k/mtp/ctx1_gen1_dep8_batch8_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp4/8k1k/mtp/ctx1_gen1_dep8_batch8_eplb0_mtp3.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL1K_OSL1K/STP/ctx2dep4_gen1dep32_batch128_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL1K_OSL1K/STP/ctx2dep4_gen1dep32_batch128_eplb0_mtp0.yaml" decode: num-worker: 1 - tp: 8 - ep: 8 + tp: 32 + ep: 32 dp-attn: true - - spec-decoding: "mtp" - conc-list: [66] + + - isl: 8192 + osl: 1024 + search-space: + # Non-MTP configurations (default spec_decoding="none") + - conc-list: [ 8 ] prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/8k1k/mtp/ctx1_gen3_tep8_batch16_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp4/8k1k/mtp/ctx1_gen3_tep8_batch16_eplb0_mtp3.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen4tep8_batch1_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen4tep8_batch1_eplb0_mtp0.yaml" decode: - num-worker: 3 + num-worker: 4 tp: 8 ep: 8 dp-attn: false - - spec-decoding: "mtp" - conc-list: [6] + - conc-list: [ 24 ] prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/8k1k/mtp/ctx1_gen5_tep8_batch1_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp4/8k1k/mtp/ctx1_gen5_tep8_batch1_eplb0_mtp3.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen4tep8_batch4_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen4tep8_batch4_eplb0_mtp0.yaml" decode: - num-worker: 5 + num-worker: 4 tp: 8 ep: 8 dp-attn: false - - spec-decoding: "mtp" - conc-list: [10, 15, 30, 60] + - conc-list: [ 5 ] prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/8k1k/mtp/ctx1_gen5_tep8_batch8_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp4/8k1k/mtp/ctx1_gen5_tep8_batch8_eplb0_mtp3.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch1_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch1_eplb0_mtp0.yaml" decode: num-worker: 5 - tp: 8 - ep: 8 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [548] - prefill: - num-worker: 3 tp: 4 ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/8k1k/mtp/ctx3_gen1_dep8_batch64_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp4/8k1k/mtp/ctx3_gen1_dep8_batch64_eplb0_mtp3.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [1096, 1691] + dp-attn: false + - conc-list: [ 30 ] prefill: - num-worker: 5 + num-worker: 1 tp: 4 ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/8k1k/mtp/ctx5_gen1_dep8_batch192_eplb0_mtp1.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp4/8k1k/mtp/ctx5_gen1_dep8_batch192_eplb0_mtp1.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch4_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch4_eplb0_mtp0.yaml" decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [658] - prefill: num-worker: 5 tp: 4 ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/8k1k/mtp/ctx5_gen2_dep8_batch32_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp4/8k1k/mtp/ctx5_gen2_dep8_batch32_eplb0_mtp3.yaml" - decode: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - - # Non-MTP configurations - - conc-list: [6] + dp-attn: false + - conc-list: [ 60 ] prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/8k1k/stp/ctx1_gen5_tep8_batch1_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp4/8k1k/stp/ctx1_gen5_tep8_batch1_eplb0_mtp0.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch8_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch8_eplb0_mtp0.yaml" decode: num-worker: 5 - tp: 8 - ep: 8 + tp: 4 + ep: 4 dp-attn: false - - conc-list: [10, 15, 25, 50, 100] + - conc-list: [ 105 ] prefill: num-worker: 1 tp: 4 ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/8k1k/stp/ctx1_gen5_tep8_batch8_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp4/8k1k/stp/ctx1_gen5_tep8_batch8_eplb0_mtp0.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch16_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch16_eplb0_mtp0.yaml" decode: num-worker: 5 - tp: 8 - ep: 8 + tp: 4 + ep: 4 dp-attn: false - - conc-list: [370] + - conc-list: [ 333 ] prefill: num-worker: 2 tp: 4 ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/8k1k/stp/ctx2_gen5_tep8_batch64_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp4/8k1k/stp/ctx2_gen5_tep8_batch64_eplb0_mtp0.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx2dep4_gen1dep32_batch8_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx2dep4_gen1dep32_batch8_eplb0_mtp0.yaml" decode: - num-worker: 5 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [1606] + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + - conc-list: [ 666 ] prefill: num-worker: 4 tp: 4 ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/8k1k/stp/ctx4_gen1_dep8_batch192_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp4/8k1k/stp/ctx4_gen1_dep8_batch192_eplb0_mtp0.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx4dep4_gen1dep32_batch16_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx4dep4_gen1dep32_batch16_eplb0_mtp0.yaml" decode: num-worker: 1 - tp: 8 - ep: 8 + tp: 32 + ep: 32 dp-attn: true - - conc-list: [837] + - conc-list: [ 1229 ] prefill: num-worker: 4 tp: 4 ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/8k1k/stp/ctx4_gen3_dep8_batch32_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp4/8k1k/stp/ctx4_gen3_dep8_batch32_eplb0_mtp0.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx4dep4_gen1dep16_batch64_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx4dep4_gen1dep16_batch64_eplb0_mtp0.yaml" decode: - num-worker: 3 - tp: 8 - ep: 8 + num-worker: 1 + tp: 16 + ep: 16 dp-attn: true - - conc-list: [2222] + - conc-list: [ 1229 ] prefill: - num-worker: 7 + num-worker: 6 tp: 4 ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp4/8k1k/stp/ctx7_gen2_dep8_batch128_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp4/8k1k/stp/ctx7_gen2_dep8_batch128_eplb0_mtp0.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx6dep4_gen1dep32_batch32_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx6dep4_gen1dep32_batch32_eplb0_mtp0.yaml" decode: - num-worker: 2 - tp: 8 - ep: 8 + num-worker: 1 + tp: 32 + ep: 32 dp-attn: true - - -dsr1-fp8-b200-dynamo-trt: - image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post2 - model: deepseek-ai/DeepSeek-R1-0528 - model-prefix: dsr1 - runner: b200-multinode - precision: fp8 - framework: dynamo-trt - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # MTP configurations - Low latency (TP attention) - - spec-decoding: "mtp" - conc-list: [8] + - conc-list: [ 2253 ] prefill: - num-worker: 1 - tp: 8 - ep: 8 + num-worker: 7 + tp: 4 + ep: 4 dp-attn: true additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/1k1k/mtp/ctx1_gen8_tp8_batch1_eplb0_mtp3_8.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp8/1k1k/mtp/ctx1_gen8_tp8_batch1_eplb0_mtp3_8.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx7dep4_gen1dep16_batch128_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx7dep4_gen1dep16_batch128_eplb0_mtp0.yaml" decode: - num-worker: 8 - tp: 8 - ep: 1 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [32] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/1k1k/mtp/ctx1_gen8_tp8_batch4_eplb0_mtp3_32.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp8/1k1k/mtp/ctx1_gen8_tp8_batch4_eplb0_mtp3_32.yaml" - decode: - num-worker: 8 - tp: 8 - ep: 1 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [64] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/1k1k/mtp/ctx1_gen8_tp8_batch8_eplb0_mtp3_64.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp8/1k1k/mtp/ctx1_gen8_tp8_batch8_eplb0_mtp3_64.yaml" - decode: - num-worker: 8 - tp: 8 - ep: 1 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [256] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/1k1k/mtp/ctx1_gen8_tp8_batch32_eplb0_mtp3_256.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp8/1k1k/mtp/ctx1_gen8_tp8_batch32_eplb0_mtp3_256.yaml" - decode: - num-worker: 8 - tp: 8 - ep: 1 - dp-attn: false - # MTP configurations - High throughput (DP attention) - - spec-decoding: "mtp" - conc-list: [896] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/1k1k/mtp/ctx1_gen7_dep8_batch128_eplb0_mtp3_896.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp8/1k1k/mtp/ctx1_gen7_dep8_batch128_eplb0_mtp3_896.yaml" - decode: - num-worker: 7 - tp: 8 - ep: 8 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [1024] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/1k1k/mtp/ctx1_gen4_dep8_batch256_eplb0_mtp3_1024.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp8/1k1k/mtp/ctx1_gen4_dep8_batch256_eplb0_mtp3_1024.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [1184] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/1k1k/mtp/ctx1_gen3_dep8_batch384_eplb0_mtp3_1184.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp8/1k1k/mtp/ctx1_gen3_dep8_batch384_eplb0_mtp3_1184.yaml" - decode: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [1600] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/1k1k/mtp/ctx1_gen2_dep8_batch768_eplb0_mtp2_1600.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp8/1k1k/mtp/ctx1_gen2_dep8_batch768_eplb0_mtp2_1600.yaml" - decode: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - - # Non-MTP (STP) configurations - Low latency (TP attention) - - conc-list: [4] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/1k1k/stp/ctx1_gen3_tp8_batch1024_eplb0_mtp0_4.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp8/1k1k/stp/ctx1_gen3_tp8_batch1024_eplb0_mtp0_4.yaml" - decode: - num-worker: 3 - tp: 8 - ep: 1 - dp-attn: false - - conc-list: [32] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/1k1k/stp/ctx1_gen3_tp8_batch1024_eplb0_mtp0_32.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp8/1k1k/stp/ctx1_gen3_tp8_batch1024_eplb0_mtp0_32.yaml" - decode: - num-worker: 3 - tp: 8 - ep: 1 - dp-attn: false - - conc-list: [128] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/1k1k/stp/ctx1_gen3_tp8_batch1024_eplb0_mtp0_128.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp8/1k1k/stp/ctx1_gen3_tp8_batch1024_eplb0_mtp0_128.yaml" - decode: - num-worker: 3 - tp: 8 - ep: 1 - dp-attn: false - # Non-MTP (STP) configurations - High throughput (DP attention) - - conc-list: [1920] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/1k1k/stp/ctx1_gen5_dep8_batch48_eplb0_mtp0_1920.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp8/1k1k/stp/ctx1_gen5_dep8_batch48_eplb0_mtp0_1920.yaml" - decode: - num-worker: 5 - tp: 8 - ep: 8 - dp-attn: true - - conc-list: [4096] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/1k1k/stp/ctx1_gen1_dep8_batch512_eplb0_mtp0_4096.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp8/1k1k/stp/ctx1_gen1_dep8_batch512_eplb0_mtp0_4096.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - conc-list: [5152] - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/1k1k/stp/ctx2_gen5_dep8_batch128_eplb0_mtp0_5152.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp8/1k1k/stp/ctx2_gen5_dep8_batch128_eplb0_mtp0_5152.yaml" - decode: - num-worker: 5 - tp: 8 - ep: 8 - dp-attn: true - - - isl: 8192 - osl: 1024 - search-space: - # MTP configurations - Low latency (TP attention) - - spec-decoding: "mtp" - conc-list: [8] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/mtp/ctx1_gen6_tp8_batch8_eplb0_mtp3_8.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp8/8k1k/mtp/ctx1_gen6_tp8_batch8_eplb0_mtp3_8.yaml" - decode: - num-worker: 6 - tp: 8 - ep: 1 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [8] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/mtp/ctx1_gen2_tp8_batch32_eplb0_mtp3_8.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp8/8k1k/mtp/ctx1_gen2_tp8_batch32_eplb0_mtp3_8.yaml" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [48] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/mtp/ctx1_gen6_tp8_batch8_eplb0_mtp3_48.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp8/8k1k/mtp/ctx1_gen6_tp8_batch8_eplb0_mtp3_48.yaml" - decode: - num-worker: 6 - tp: 8 - ep: 1 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [64] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/mtp/ctx1_gen4_tp8_batch16_eplb0_mtp3_64.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp8/8k1k/mtp/ctx1_gen4_tp8_batch16_eplb0_mtp3_64.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 1 - dp-attn: false - # MTP configurations - High throughput (DP attention) - - spec-decoding: "mtp" - conc-list: [224] - prefill: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/mtp/ctx2_gen3_dep8_batch8_eplb0_mtp3_224.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp8/8k1k/mtp/ctx2_gen3_dep8_batch8_eplb0_mtp3_224.yaml" - decode: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [288] - prefill: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/mtp/ctx2_gen1_dep8_batch32_eplb0_mtp3_288.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp8/8k1k/mtp/ctx2_gen1_dep8_batch32_eplb0_mtp3_288.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [1088] - prefill: - num-worker: 4 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/mtp/ctx4_gen1_dep8_batch128_eplb0_mtp2_1088.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp8/8k1k/mtp/ctx4_gen1_dep8_batch128_eplb0_mtp2_1088.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - # Non-MTP (STP) configurations - Low latency (TP attention) - - conc-list: [1] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/stp/ctx1_gen1_tp8_batch1_eplb0_mtp0_1.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp8/8k1k/stp/ctx1_gen1_tp8_batch1_eplb0_mtp0_1.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - - conc-list: [32] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/stp/ctx1_gen4_tp8_batch32_eplb0_mtp0_32.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp8/8k1k/stp/ctx1_gen4_tp8_batch32_eplb0_mtp0_32.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 1 - dp-attn: false - - conc-list: [128] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/stp/ctx1_gen4_tp8_batch32_eplb0_mtp0_128.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp8/8k1k/stp/ctx1_gen4_tp8_batch32_eplb0_mtp0_128.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 1 - dp-attn: false - - conc-list: [96] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/stp/ctx1_gen6_tp8_batch16_eplb0_mtp0_96.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp8/8k1k/stp/ctx1_gen6_tp8_batch16_eplb0_mtp0_96.yaml" - decode: - num-worker: 6 - tp: 8 - ep: 1 - dp-attn: false - # Non-MTP (STP) configurations - High throughput (DP attention) - - conc-list: [128] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/stp/ctx1_gen1_dep8_batch128_eplb0_mtp0_128.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp8/8k1k/stp/ctx1_gen1_dep8_batch128_eplb0_mtp0_128.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - conc-list: [128] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/stp/ctx1_gen2_dep8_batch64_eplb0_mtp0_128.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp8/8k1k/stp/ctx1_gen2_dep8_batch64_eplb0_mtp0_128.yaml" - decode: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - - conc-list: [256] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/stp/ctx1_gen1_dep8_batch256_eplb0_mtp0_256.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp8/8k1k/stp/ctx1_gen1_dep8_batch256_eplb0_mtp0_256.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - conc-list: [640] - prefill: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b200-fp8/8k1k/stp/ctx2_gen1_dep8_batch640_eplb0_mtp0_640.yaml - - "CONFIG_FILE=recipes/trtllm/b200-fp8/8k1k/stp/ctx2_gen1_dep8_batch640_eplb0_mtp0_640.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - -dsr1-fp4-b300-dynamo-trt: - image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post1 - model: deepseek-r1-fp4 - model-prefix: dsr1 - runner: b300 - precision: fp4 - framework: dynamo-trt - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - spec-decoding: "mtp" - conc-list: [654] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/1k1k/mtp/ctx1_gen1_dep8_batch64_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp4/1k1k/mtp/ctx1_gen1_dep8_batch64_eplb0_mtp3.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [271] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/1k1k/mtp/ctx1_gen2_dep8_batch16_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp4/1k1k/mtp/ctx1_gen2_dep8_batch16_eplb0_mtp3.yaml" - decode: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [11] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/1k1k/mtp/ctx1_gen5_tep8_batch1_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp4/1k1k/mtp/ctx1_gen5_tep8_batch1_eplb0_mtp3.yaml" - decode: - num-worker: 5 - tp: 8 - ep: 8 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [10, 20, 25, 60, 120, 200] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/1k1k/mtp/ctx1_gen5_tep8_batch32_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp4/1k1k/mtp/ctx1_gen5_tep8_batch32_eplb0_mtp3.yaml" - decode: - num-worker: 5 - tp: 8 - ep: 8 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [2342] - prefill: - num-worker: 2 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/1k1k/mtp/ctx2_gen1_dep8_batch256_eplb0_mtp1.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp4/1k1k/mtp/ctx2_gen1_dep8_batch256_eplb0_mtp1.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [8609] - prefill: - num-worker: 5 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/1k1k/mtp/ctx5_gen2_dep8_batch512_eplb0_mtp1.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp4/1k1k/mtp/ctx5_gen2_dep8_batch512_eplb0_mtp1.yaml" - decode: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [12926] - prefill: - num-worker: 5 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/1k1k/mtp/ctx5_gen2_dep8_batch768_eplb0_mtp1.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp4/1k1k/mtp/ctx5_gen2_dep8_batch768_eplb0_mtp1.yaml" - decode: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - - # Non-MTP configurations - - conc-list: [1176] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/1k1k/stp/ctx1_gen2_dep8_batch64_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp4/1k1k/stp/ctx1_gen2_dep8_batch64_eplb0_mtp0.yaml" - decode: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - - conc-list: [6] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/1k1k/stp/ctx1_gen4_tep8_batch1_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp4/1k1k/stp/ctx1_gen4_tep8_batch1_eplb0_mtp0.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [5, 10, 15, 25] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/1k1k/stp/ctx1_gen5_tep4_batch4_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp4/1k1k/stp/ctx1_gen5_tep4_batch4_eplb0_mtp0.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - conc-list: [60, 110, 195, 395] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/1k1k/stp/ctx1_gen5_tep8_batch64_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp4/1k1k/stp/ctx1_gen5_tep8_batch64_eplb0_mtp0.yaml" - decode: - num-worker: 5 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [4405] - prefill: - num-worker: 2 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/1k1k/stp/ctx2_gen1_dep8_batch512_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp4/1k1k/stp/ctx2_gen1_dep8_batch512_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - conc-list: [8192] - prefill: - num-worker: 3 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/1k1k/stp/ctx3_gen1_dep8_batch1024_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp4/1k1k/stp/ctx3_gen1_dep8_batch1024_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - conc-list: [4611] - prefill: - num-worker: 3 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/1k1k/stp/ctx3_gen2_dep8_batch256_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp4/1k1k/stp/ctx3_gen2_dep8_batch256_eplb0_mtp0.yaml" - decode: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - - - isl: 8192 - osl: 1024 - search-space: - - spec-decoding: "mtp" - conc-list: [2198] - prefill: - num-worker: 10 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/8k1k/mtp/ctx10_gen1_dep8_batch256_eplb0_mtp1.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp4/8k1k/mtp/ctx10_gen1_dep8_batch256_eplb0_mtp1.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [52] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/8k1k/mtp/ctx1_gen4_tep4_batch8_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp4/8k1k/mtp/ctx1_gen4_tep4_batch8_eplb0_mtp3.yaml" - decode: - num-worker: 4 - tp: 4 - ep: 4 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [8] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/8k1k/mtp/ctx1_gen4_tep8_batch1_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp4/8k1k/mtp/ctx1_gen4_tep8_batch1_eplb0_mtp3.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [32] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/8k1k/mtp/ctx1_gen4_tep8_batch4_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp4/8k1k/mtp/ctx1_gen4_tep8_batch4_eplb0_mtp3.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [181] - prefill: - num-worker: 3 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/8k1k/mtp/ctx3_gen1_dep8_batch16_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp4/8k1k/mtp/ctx3_gen1_dep8_batch16_eplb0_mtp3.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [1197] - prefill: - num-worker: 9 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/8k1k/mtp/ctx9_gen1_dep8_batch128_eplb0_mtp1.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp4/8k1k/mtp/ctx9_gen1_dep8_batch128_eplb0_mtp1.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - # Non-MTP configurations - - conc-list: [105] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/8k1k/stp/ctx1_gen3_tep4_batch32_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp4/8k1k/stp/ctx1_gen3_tep4_batch32_eplb0_mtp0.yaml" - decode: - num-worker: 3 - tp: 4 - ep: 4 - dp-attn: false - - conc-list: [63] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/8k1k/stp/ctx1_gen3_tep8_batch16_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp4/8k1k/stp/ctx1_gen3_tep8_batch16_eplb0_mtp0.yaml" - decode: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [4] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/8k1k/stp/ctx1_gen3_tep8_batch1_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp4/8k1k/stp/ctx1_gen3_tep8_batch1_eplb0_mtp0.yaml" - decode: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [12] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/8k1k/stp/ctx1_gen4_tep4_batch2_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp4/8k1k/stp/ctx1_gen4_tep4_batch2_eplb0_mtp0.yaml" - decode: - num-worker: 4 - tp: 4 - ep: 4 - dp-attn: false - - conc-list: [589] - prefill: - num-worker: 5 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/8k1k/stp/ctx5_gen2_dep8_batch32_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp4/8k1k/stp/ctx5_gen2_dep8_batch32_eplb0_mtp0.yaml" - decode: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - - conc-list: [1093] - prefill: - num-worker: 6 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/8k1k/stp/ctx6_gen1_dep8_batch128_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp4/8k1k/stp/ctx6_gen1_dep8_batch128_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - conc-list: [2048] - prefill: - num-worker: 8 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp4/8k1k/stp/ctx8_gen1_dep8_batch256_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp4/8k1k/stp/ctx8_gen1_dep8_batch256_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true -dsr1-fp8-b300-dynamo-trt: - image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post1 - model: deepseek-ai/DeepSeek-R1-0528 - model-prefix: dsr1 - runner: b300 - precision: fp8 - framework: dynamo-trt - multinode: true - disagg: true - scenarios: - fixed-seq-len: - # 1k1k MTP configs - - isl: 1024 - osl: 1024 - search-space: - - spec-decoding: "mtp" - conc-list: [10] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/1k1k/mtp/ctx1_gen8_tp8_batch1_eplb0_mtp3_10.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp8/1k1k/mtp/ctx1_gen8_tp8_batch1_eplb0_mtp3_10.yaml" - decode: - num-worker: 8 - tp: 8 - ep: 1 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [160] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/1k1k/mtp/ctx1_gen8_tp8_batch16_eplb0_mtp3_160.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp8/1k1k/mtp/ctx1_gen8_tp8_batch16_eplb0_mtp3_160.yaml" - decode: - num-worker: 8 - tp: 8 - ep: 1 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [3072] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/1k1k/mtp/ctx1_gen1_dp8_batch256_eplb0_mtp1_3072.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp8/1k1k/mtp/ctx1_gen1_dp8_batch256_eplb0_mtp1_3072.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [2560] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/1k1k/mtp/ctx1_gen2_dep8_batch128_eplb0_mtp1_2560.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp8/1k1k/mtp/ctx1_gen2_dep8_batch128_eplb0_mtp1_2560.yaml" - decode: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [720] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/1k1k/mtp/ctx1_gen5_dep8_batch16_eplb0_mtp2_720.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp8/1k1k/mtp/ctx1_gen5_dep8_batch16_eplb0_mtp2_720.yaml" - decode: - num-worker: 5 - tp: 8 - ep: 8 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [11264] - prefill: - num-worker: 3 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/1k1k/mtp/ctx3_gen2_dp8_batch512_eplb0_mtp1_11264.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp8/1k1k/mtp/ctx3_gen2_dp8_batch512_eplb0_mtp1_11264.yaml" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: true - # 1k1k STP configs - - isl: 1024 - osl: 1024 - search-space: - - conc-list: [2112] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/1k1k/stp/ctx1_gen1_dep8_batch256_eplb0_mtp0_2112.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp8/1k1k/stp/ctx1_gen1_dep8_batch256_eplb0_mtp0_2112.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - conc-list: [3072] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/1k1k/stp/ctx1_gen2_dp8_batch128_eplb0_mtp0_3072.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp8/1k1k/stp/ctx1_gen2_dp8_batch128_eplb0_mtp0_3072.yaml" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: true - - conc-list: [1280] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/1k1k/stp/ctx1_gen3_dp8_batch48_eplb0_mtp0_1280.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp8/1k1k/stp/ctx1_gen3_dp8_batch48_eplb0_mtp0_1280.yaml" - decode: - num-worker: 3 - tp: 8 - ep: 1 - dp-attn: true - - conc-list: [12] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/1k1k/stp/ctx1_gen8_tp8_batch64_eplb0_mtp0_12.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp8/1k1k/stp/ctx1_gen8_tp8_batch64_eplb0_mtp0_12.yaml" - decode: - num-worker: 8 - tp: 8 - ep: 1 - dp-attn: false - - conc-list: [128] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/1k1k/stp/ctx1_gen8_tp8_batch64_eplb0_mtp0_128.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp8/1k1k/stp/ctx1_gen8_tp8_batch64_eplb0_mtp0_128.yaml" - decode: - num-worker: 8 - tp: 8 - ep: 1 - dp-attn: false - - conc-list: [384] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/1k1k/stp/ctx1_gen8_tp8_batch64_eplb0_mtp0_384.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp8/1k1k/stp/ctx1_gen8_tp8_batch64_eplb0_mtp0_384.yaml" - decode: - num-worker: 8 - tp: 8 - ep: 1 - dp-attn: false - - conc-list: [16384] - prefill: - num-worker: 2 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/1k1k/stp/ctx2_gen1_dp8_batch1024_eplb0_mtp0_16384.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp8/1k1k/stp/ctx2_gen1_dp8_batch1024_eplb0_mtp0_16384.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - # 8k1k MTP configs - - isl: 8192 - osl: 1024 - search-space: - - spec-decoding: "mtp" - conc-list: [40] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/8k1k/mtp/ctx1_gen2_tp8_batch16_eplb0_mtp3_40.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp8/8k1k/mtp/ctx1_gen2_tp8_batch16_eplb0_mtp3_40.yaml" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [8] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/8k1k/mtp/ctx1_gen4_tp8_batch1_eplb0_mtp3_8.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp8/8k1k/mtp/ctx1_gen4_tp8_batch1_eplb0_mtp3_8.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 1 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [20] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/8k1k/mtp/ctx1_gen4_tp8_batch4_eplb0_mtp3_20.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp8/8k1k/mtp/ctx1_gen4_tp8_batch4_eplb0_mtp3_20.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 1 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [72] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/8k1k/mtp/ctx1_gen1_dp8_batch8_eplb0_mtp3_72.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp8/8k1k/mtp/ctx1_gen1_dp8_batch8_eplb0_mtp3_72.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [144] - prefill: - num-worker: 2 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/8k1k/mtp/ctx2_gen1_dp8_batch16_eplb0_mtp3_144.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp8/8k1k/mtp/ctx2_gen1_dp8_batch16_eplb0_mtp3_144.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [512] - prefill: - num-worker: 4 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/8k1k/mtp/ctx4_gen1_dp8_batch64_eplb0_mtp2_512.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp8/8k1k/mtp/ctx4_gen1_dp8_batch64_eplb0_mtp2_512.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - # 8k1k STP configs - - isl: 8192 - osl: 1024 - search-space: - - conc-list: [64] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/8k1k/stp/ctx1_gen4_tp8_batch16_eplb0_mtp0_64.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp8/8k1k/stp/ctx1_gen4_tp8_batch16_eplb0_mtp0_64.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 1 - dp-attn: false - - conc-list: [16] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/8k1k/stp/ctx1_gen8_tp8_batch2_eplb0_mtp0_16.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp8/8k1k/stp/ctx1_gen8_tp8_batch2_eplb0_mtp0_16.yaml" - decode: - num-worker: 8 - tp: 8 - ep: 1 - dp-attn: false - - conc-list: [256] - prefill: - num-worker: 2 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/8k1k/stp/ctx2_gen1_dp8_batch32_eplb0_mtp0_256.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp8/8k1k/stp/ctx2_gen1_dp8_batch32_eplb0_mtp0_256.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - - conc-list: [512] - prefill: - num-worker: 3 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/8k1k/stp/ctx3_gen1_dp8_batch64_eplb0_mtp0_512.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp8/8k1k/stp/ctx3_gen1_dp8_batch64_eplb0_mtp0_512.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - - conc-list: [256] - prefill: - num-worker: 3 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/8k1k/stp/ctx3_gen5_tp8_batch64_eplb0_mtp0_256.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp8/8k1k/stp/ctx3_gen5_tp8_batch64_eplb0_mtp0_256.yaml" - decode: - num-worker: 5 - tp: 8 - ep: 1 - dp-attn: false - - conc-list: [1075] - prefill: - num-worker: 5 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/8k1k/stp/ctx5_gen1_dp8_batch128_eplb0_mtp0_1075.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp8/8k1k/stp/ctx5_gen1_dp8_batch128_eplb0_mtp0_1075.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - - conc-list: [3072] - prefill: - num-worker: 7 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/b300-fp8/8k1k/stp/ctx7_gen1_dep8_batch384_eplb0_mtp0_3072.yaml - - "CONFIG_FILE=recipes/trtllm/b300-fp8/8k1k/stp/ctx7_gen1_dep8_batch384_eplb0_mtp0_3072.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true -dsr1-fp4-b200-sglang: - image: lmsysorg/sglang:v0.5.12.post1 - model: nvidia/DeepSeek-R1-0528-FP4-V2 - model-prefix: dsr1 - runner: b200 - precision: fp4 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 4, ep: 1, conc-start: 1, conc-end: 32 } - - { tp: 4, ep: 4, conc-start: 64, conc-end: 256 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 4, ep: 1, conc-start: 1, conc-end: 32 } - - { tp: 4, ep: 4, dp-attn: true, conc-start: 64, conc-end: 256 } - # agentic-coding: temporarily disabled — blocked by e2e-tests.yml artifact - # name mismatch (downloads `agentic_*` but benchmark-tmpl.yml uploads as - # `bmk_agentic_*`). Re-enable once that workflow is aligned. - # agentic-coding: - # - duration: 1800 - # search-space: - # - { tp: 4, ep: 4, offloading: none, conc-list: [1, 2, 4, 8, 12, 16, 24, 32, 48, 64, 128, 256] } - # - { tp: 8, ep: 8, offloading: none, conc-list: [1, 2, 4, 8, 12, 16, 32, 64, 128, 256, 512] } - -dsr1-fp4-b200-sglang-mtp: - image: lmsysorg/sglang:v0.5.12.post1 - model: nvidia/DeepSeek-R1-0528-FP4-V2 - model-prefix: dsr1 - runner: b200 - precision: fp4 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 4, ep: 1, conc-start: 1, conc-end: 32, spec-decoding: mtp } - - { tp: 4, ep: 4, conc-start: 64, conc-end: 256, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 4, ep: 1, conc-start: 1, conc-end: 32, spec-decoding: mtp } - - { tp: 4, ep: 4, dp-attn: true, conc-start: 64, conc-end: 256, spec-decoding: mtp } - -dsv4-fp4-b200-sglang: - image: lmsysorg/sglang:nightly-dev-cu13-20260628-da802ddc - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: b200-dsv4 - precision: fp4 - framework: sglang - multinode: false - # Two recipes from https://docs.sglang.io/cookbook/autoregressive/DeepSeek/DeepSeek-V4 - # are selected inside benchmarks/single_node/dsv4_fp4_b200.sh by DP_ATTENTION: - # low-latency (DP_ATTENTION=false): TP-only, flashinfer_mxfp4 - # DP-attention (DP_ATTENTION=true): DP-attn + DeepEP + mega_moe opts - # The DP-attention recipe covers both "balanced" (conc 64-128) and - # "max-throughput" (conc 256+) CONC ranges with identical flags; - # only --max-running-requests scales with CONC. - # ep is implicit in sglang: --moe-a2a-backend deepep forces ep_size=tp_size, - # while low-latency leaves ep_size at the default of 1. - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # low-latency (DP_ATTENTION=false) - - { tp: 8, ep: 1, conc-start: 1, conc-end: 32 } - # DP-attention (DP_ATTENTION=true) — balanced CONC range - - { tp: 8, ep: 8, dp-attn: true, conc-start: 64, conc-end: 128 } - # DP-attention (DP_ATTENTION=true) — max-throughput CONC range - - { tp: 8, ep: 8, dp-attn: true, conc-start: 256, conc-end: 1024 } - - isl: 8192 - osl: 1024 - search-space: - # low-latency (DP_ATTENTION=false) - - { tp: 8, ep: 1, conc-start: 1, conc-end: 32 } - # DP-attention (DP_ATTENTION=true) — balanced CONC range - - { tp: 8, ep: 8, dp-attn: true, conc-start: 64, conc-end: 128 } - # DP-attention (DP_ATTENTION=true) — max-throughput CONC range - - { tp: 8, ep: 8, dp-attn: true, conc-start: 256, conc-end: 1024 } - -dsv4-fp4-b200-vllm: - image: vllm/vllm-openai:nightly-3f0a91bb96f8d72e0498b95c166e817deae14d62 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: b200-dsv4 - precision: fp4 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, conc-start: 1, conc-end: 64 } - - { tp: 8, ep: 8, conc-start: 128, conc-end: 128 } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 256, conc-end: 1024 } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 4096, conc-end: 4096 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, conc-start: 1, conc-end: 32 } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 64, conc-end: 1024 } - -dsv4-fp4-b200-vllm-agentic: - image: vllm/vllm-openai:v0.23.0 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: cluster:b200-dgxc - precision: fp4 - framework: vllm - multinode: false - scenarios: - agentic-coding: - - dram-utilization: 0.80 - search-space: - # Pure TP is only competitive at very low concurrency. - - { tp: 8, kv-offloading: none, conc-list: [1, 2, 3, 4, 5] } - # Sample the useful MooncakeStore range without repeating its collapsed - # high-concurrency tail. - - { tp: 8, kv-offloading: dram, kv-offload-backend: mooncake, conc-list: [8, 10, 16] } - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [16, 32, 38, 44, 50] } - # Retain the external-cache transition and peak-throughput region. - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: mooncake, conc-list: [16, 38, 44, 56, 64, 66, 68] } - - -dsv4-fp4-b200-trt: - image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: b200-dsv4 - precision: fp4 - framework: trt - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, conc-start: 1, conc-end: 32 } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 32, conc-end: 512 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, conc-start: 1, conc-end: 32 } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 32, conc-end: 512 } - -dsv4-fp4-b200-trt-mtp: - image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: b200-dsv4 - precision: fp4 - framework: trt - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, conc-start: 1, conc-end: 32, spec-decoding: mtp } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 32, conc-end: 256, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, conc-start: 1, conc-end: 32, spec-decoding: mtp } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 32, conc-end: 256, spec-decoding: mtp } - -# MTP variant of dsv4-fp4-b200-vllm. Mirrors the base search space and adds -# --speculative-config '{"method":"mtp","num_speculative_tokens":2}'. -dsv4-fp4-b200-vllm-mtp: - image: vllm/vllm-openai:v0.21.0 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: b200-dsv4 - precision: fp4 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, conc-start: 1, conc-end: 64, spec-decoding: mtp } - - { tp: 8, ep: 8, conc-start: 128, conc-end: 128, spec-decoding: mtp } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 256, conc-end: 4096, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - # 8k/1k TP=8 caps at conc=16 (not 32) to avoid OOM observed at conc=32: - # https://github.com/SemiAnalysisAI/InferenceX/actions/runs/25134892257/job/73670854021 - - { tp: 8, conc-start: 1, conc-end: 16, spec-decoding: mtp } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 64, conc-end: 256, spec-decoding: mtp } - -# NOTE: At the time of submission, https://cookbook.sglang.io/autoregressive/DeepSeek/DeepSeek-R1 -# does not have a B300-specific recipe, so this config reuses the existing DSR1 FP4 -# B200 SGLang recipe as-is until B300-specific tuning is available. -dsr1-fp4-b300-sglang: - image: lmsysorg/sglang:v0.5.12-cu130 - model: nvidia/DeepSeek-R1-0528-FP4-V2 - model-prefix: dsr1 - runner: b300 - precision: fp4 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 4, ep: 4, conc-start: 1, conc-end: 128 } - - { tp: 8, ep: 8, conc-start: 1, conc-end: 128 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 4, ep: 4, conc-start: 1, conc-end: 128 } - - { tp: 8, ep: 8, conc-start: 1, conc-end: 16 } - -dsr1-fp4-b200-trt: - image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc14 - model: nvidia/DeepSeek-R1-0528-FP4-V2 - model-prefix: dsr1 - runner: b200 - precision: fp4 - framework: trt - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # low concurrency cases use TP only - # concurrency 64 uses TP & EP - # high concurrency cases use TP & EP & DP-ATTN - - { tp: 4, conc-start: 4, conc-end: 16 } - - { tp: 4, ep: 4, dp-attn: true, conc-start: 256, conc-end: 256 } - - { tp: 8, conc-start: 4, conc-end: 4 } - - { tp: 8, ep: 8, conc-start: 64, conc-end: 64 } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 128, conc-end: 256 } - - isl: 8192 - osl: 1024 - search-space: - # low concurrency cases use TP only - # concurrency 32 uses TP & EP - # high concurrency cases use TP & EP & DP-ATTN - - { tp: 4, conc-start: 4, conc-end: 32 } - - { tp: 4, ep: 4, conc-start: 32, conc-end: 32 } - - { tp: 4, ep: 4, dp-attn: true, conc-start: 256, conc-end: 256 } - - { tp: 8, conc-start: 4, conc-end: 4 } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 128, conc-end: 256 } - -dsr1-fp4-b200-trt-mtp: - image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc14 - model: nvidia/DeepSeek-R1-0528-FP4-V2 - model-prefix: dsr1 - runner: b200 - precision: fp4 - framework: trt - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # TP=4 configurations - - { tp: 4, conc-start: 4, conc-end: 8, spec-decoding: mtp } - - { tp: 4, ep: 4, dp-attn: true, conc-start: 256, conc-end: 256, spec-decoding: mtp } - # TP=8 configurations - - { tp: 8, conc-start: 4, conc-end: 4, spec-decoding: mtp } - - { tp: 8, conc-start: 128, conc-end: 128, spec-decoding: mtp } - - { tp: 8, ep: 8, conc-start: 32, conc-end: 128, spec-decoding: mtp } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 32, conc-end: 64, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - # TP=4 configurations - - { tp: 4, conc-start: 4, conc-end: 16, spec-decoding: mtp } - - { tp: 4, ep: 4, conc-start: 32, conc-end: 32, spec-decoding: mtp } - - { tp: 4, ep: 4, dp-attn: true, conc-start: 256, conc-end: 256, spec-decoding: mtp } - # TP=8 configurations - - { tp: 8, conc-start: 4, conc-end: 4, spec-decoding: mtp } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 64, conc-end: 256, spec-decoding: mtp } - -dsr1-fp8-b200-sglang: - image: lmsysorg/sglang:v0.5.12-cu130 - model: deepseek-ai/DeepSeek-R1-0528 - model-prefix: dsr1 - runner: b200 - precision: fp8 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 1, conc-end: 64 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 1, conc-end: 4 } - - { tp: 4, ep: 1, conc-start: 1, conc-end: 32 } - - # NOTE: At the time of submission, https://cookbook.sglang.io/autoregressive/DeepSeek/DeepSeek-R1 - # does not have a B300-specific recipe, so this config reuses the existing DSR1 FP8 - # B200 SGLang recipe as-is until B300-specific tuning is available. -dsr1-fp8-b300-sglang: - image: lmsysorg/sglang:v0.5.12-cu130 - model: deepseek-ai/DeepSeek-R1-0528 - model-prefix: dsr1 - runner: b300 - precision: fp8 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 1, conc-end: 64 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 1, conc-end: 4 } - - { tp: 4, ep: 1, conc-start: 1, conc-end: 32 } - -# DeepSeek-V4-Pro on B300 with sglang (non-MTP). -# Uses nightly image with megamoe backend for high-concurrency profiles. -dsv4-fp4-b300-sglang: - image: lmsysorg/sglang:nightly-dev-cu13-20260624-b2c8f7a2 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: b300 - precision: fp4 - framework: sglang - multinode: false - # Recipes are selected inside benchmarks/single_node/dsv4_fp4_b300_sglang.sh - # by CONC: - # CONC 1|32: TP-only, flashinfer_mxfp4 - # CONC 512: DP-attn, flashinfer_mxfp4 - # CONC 2048-8192: DP-attn, megamoe - # ep is implicit in sglang: --moe-a2a-backend megamoe forces ep_size=tp_size, - # while low-latency leaves ep_size at the default of 1. - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 1, conc-end: 1 } - - { tp: 4, ep: 1, conc-start: 32, conc-end: 32 } - - { tp: 4, ep: 1, dp-attn: true, conc-start: 512, conc-end: 512 } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 8192, conc-end: 8192 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 1, conc-end: 1 } - - { tp: 4, ep: 1, conc-start: 32, conc-end: 32 } - - { tp: 4, ep: 1, dp-attn: true, conc-start: 512, conc-end: 512 } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 2048, conc-end: 2048 } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 4096, conc-end: 4096 } - - # DeepSeek-V4-Pro on B300 with EAGLE/MTP speculative decoding. Recipe is - # selected inside benchmarks/single_node/dsv4_fp4_b300_sglang_mtp.sh by - # DP_ATTENTION: - # dp-attn: false -> TP-only + flashinfer_mxfp4 + chunked-prefill 8192 - # + EAGLE (3,1,4) + mem-fraction 0.90 - # dp-attn: true -> DP-attn + flashinfer_mxfp4 + chunked-prefill 32768 - # + EAGLE (1,1,2) + mem-fraction 0.92 + max-running 256 -dsv4-fp4-b300-sglang-mtp: - image: lmsysorg/sglang:nightly-dev-cu13-20260610-f332e526 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: b300 - precision: fp4 - framework: sglang - multinode: false - # Three CONC bands: - # A: TP=8 ep=1 -- conc 1-8 EAGLE (3,1,4) TP-only fallback - # B: TP=4 ep=1 -- conc 4-32 EAGLE (3,1,4) TP-only mid batch - # C: TP=4 ep=1 dp-attn -- conc 16-256 EAGLE (1,1,2) DP-attn flashinfer - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 1, conc-end: 8, spec-decoding: mtp } - - { tp: 4, ep: 1, conc-start: 4, conc-end: 32, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 1, conc-end: 8, spec-decoding: mtp } - - { tp: 4, ep: 1, conc-start: 4, conc-end: 32, spec-decoding: mtp } - -qwen3.5-bf16-b200-sglang: - image: lmsysorg/sglang:v0.5.14-cu130 - model: Qwen/Qwen3.5-397B-A17B - model-prefix: qwen3.5 - runner: b200 - precision: bf16 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 64 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 64 } - -qwen3.5-bf16-b200-sglang-mtp: - image: lmsysorg/sglang:v0.5.14-cu130 - model: Qwen/Qwen3.5-397B-A17B - model-prefix: qwen3.5 - runner: b200 - precision: bf16 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 64, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 64, spec-decoding: mtp } - -qwen3.5-fp8-b200-sglang: - image: lmsysorg/sglang:v0.5.14-cu130 - model: Qwen/Qwen3.5-397B-A17B-FP8 - model-prefix: qwen3.5 - runner: b200 - precision: fp8 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 4 } - - { tp: 4, ep: 1, conc-start: 4, conc-end: 256 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 4 } - - { tp: 4, ep: 1, conc-start: 4, conc-end: 256 } - -qwen3.5-fp8-b200-sglang-agentic: - image: lmsysorg/sglang:nightly-dev-20260422-de962f32 - model: Qwen/Qwen3.5-397B-A17B-FP8 - model-prefix: qwen3.5 - runner: cluster:b200-dgxc - precision: fp8 - framework: sglang - multinode: false - scenarios: - agentic-coding: - - search-space: - - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32] } - - -qwen3.5-fp4-b200-sglang: - image: lmsysorg/sglang:v0.5.14-cu130 - model: nvidia/Qwen3.5-397B-A17B-NVFP4 - model-prefix: qwen3.5 - runner: b200 - precision: fp4 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 4, ep: 1, conc-start: 4, conc-end: 4 } - - { tp: 2, ep: 1, conc-start: 4, conc-end: 128 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 4, ep: 1, conc-start: 4, conc-end: 4 } - - { tp: 2, ep: 1, conc-start: 4, conc-end: 128 } - -qwen3.5-fp4-b200-sglang-mtp: - image: lmsysorg/sglang:v0.5.14-cu130 - model: nvidia/Qwen3.5-397B-A17B-NVFP4 - model-prefix: qwen3.5 - runner: b200 - precision: fp4 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 4, ep: 1, conc-start: 4, conc-end: 4, spec-decoding: mtp } - - { tp: 2, ep: 1, conc-start: 4, conc-end: 64, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 4, ep: 1, conc-start: 4, conc-end: 4, spec-decoding: mtp } - - { tp: 2, ep: 1, conc-start: 4, conc-end: 64, spec-decoding: mtp } - -glm5-fp8-b200-sglang: - image: lmsysorg/sglang:nightly-dev-cu13-20260605-7dc73766 - model: zai-org/GLM-5-FP8 - model-prefix: glm5 - runner: b200 - precision: fp8 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 256 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 256 } - -glm5-fp8-b200-sglang-mtp: - image: lmsysorg/sglang:nightly-dev-cu13-20260605-7dc73766 - model: zai-org/GLM-5-FP8 - model-prefix: glm5 - runner: b200 - precision: fp8 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 256, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 256, spec-decoding: mtp } - - # NOTE: At the time of submission, https://cookbook.sglang.io/autoregressive/GLM/GLM-5.1 - # does not have a B300-specific recipe, so this config reuses the existing GLM5 FP8 - # B200 SGLang recipe as-is until B300-specific tuning is available. - -glm5-fp8-b300-sglang: - image: lmsysorg/sglang:v0.5.12-cu130 - model: zai-org/GLM-5-FP8 - model-prefix: glm5 - runner: b300 - precision: fp8 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 256 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 256 } - -glm5-fp8-b300-sglang-mtp: - image: lmsysorg/sglang:v0.5.12-cu130 - model: zai-org/GLM-5-FP8 - model-prefix: glm5 - runner: b300 - precision: fp8 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 256, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 256, spec-decoding: mtp } - -glm5-fp4-b200-sglang: - image: lmsysorg/sglang:nightly-dev-cu13-20260605-7dc73766 - model: nvidia/GLM-5-NVFP4 - model-prefix: glm5 - runner: b200 - precision: fp4 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 4 } - - { tp: 4, ep: 1, conc-start: 4, conc-end: 256 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 4 } - - { tp: 4, ep: 1, conc-start: 4, conc-end: 256 } - -glm5-fp4-b200-sglang-mtp: - image: lmsysorg/sglang:nightly-dev-cu13-20260605-7dc73766 - model: nvidia/GLM-5-NVFP4 - model-prefix: glm5 - runner: b200 - precision: fp4 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 4, spec-decoding: mtp } - - { tp: 4, ep: 1, conc-start: 4, conc-end: 256, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 4, spec-decoding: mtp } - - { tp: 4, ep: 1, conc-start: 4, conc-end: 256, spec-decoding: mtp } - - # NOTE: At the time of submission, https://cookbook.sglang.io/autoregressive/GLM/GLM-5 - # does not have a B300-specific recipe, so this config reuses the existing - # GLM-5 FP4 B200 SGLang recipe as-is until B300-specific tuning is available. -glm5-fp4-b300-sglang: - image: lmsysorg/sglang:v0.5.11-cu130 - model: nvidia/GLM-5-NVFP4 - model-prefix: glm5 - runner: b300 - precision: fp4 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 4 } - - { tp: 4, ep: 1, conc-start: 4, conc-end: 256 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 4 } - - { tp: 4, ep: 1, conc-start: 4, conc-end: 256 } - -glm5-fp4-b300-sglang-mtp: - image: lmsysorg/sglang:v0.5.11-cu130 - model: nvidia/GLM-5-NVFP4 - model-prefix: glm5 - runner: b300 - precision: fp4 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 4, spec-decoding: mtp } - - { tp: 4, ep: 1, conc-start: 4, conc-end: 256, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 4, spec-decoding: mtp } - - { tp: 4, ep: 1, conc-start: 4, conc-end: 256, spec-decoding: mtp } - -glm5-fp4-gb300-dynamo-trt: - image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - model: nvidia/GLM-5-NVFP4 - model-prefix: glm5 - runner: gb300-nv - precision: fp4 - framework: dynamo-trt - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # STP configurations - - conc-list: [ 4 ] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL1K_OSL1K/STP/ctx1dep2_gen4tep8_batch1_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL1K_OSL1K/STP/ctx1dep2_gen4tep8_batch1_eplb0_mtp0.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [ 5 ] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL1K_OSL1K/STP/ctx1dep2_gen5tep4_batch1_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL1K_OSL1K/STP/ctx1dep2_gen5tep4_batch1_eplb0_mtp0.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - conc-list: [ 24 ] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL1K_OSL1K/STP/ctx1dep2_gen4tep8_batch4_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL1K_OSL1K/STP/ctx1dep2_gen4tep8_batch4_eplb0_mtp0.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [ 92 ] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL1K_OSL1K/STP/ctx1dep2_gen4tep8_batch16_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL1K_OSL1K/STP/ctx1dep2_gen4tep8_batch16_eplb0_mtp0.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [ 105 ] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL1K_OSL1K/STP/ctx1dep2_gen5tep4_batch16_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL1K_OSL1K/STP/ctx1dep2_gen5tep4_batch16_eplb0_mtp0.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - conc-list: [ 336 ] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL1K_OSL1K/STP/ctx1dep2_gen4tep8_batch64_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL1K_OSL1K/STP/ctx1dep2_gen4tep8_batch64_eplb0_mtp0.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [ 666 ] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL1K_OSL1K/STP/ctx1dep2_gen1dep32_batch16_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL1K_OSL1K/STP/ctx1dep2_gen1dep32_batch16_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [ 2253 ] - prefill: - num-worker: 2 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL1K_OSL1K/STP/ctx2dep2_gen1dep16_batch128_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL1K_OSL1K/STP/ctx2dep2_gen1dep16_batch128_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - conc-list: [ 2253 ] - prefill: - num-worker: 3 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL1K_OSL1K/STP/ctx3dep2_gen1dep32_batch64_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL1K_OSL1K/STP/ctx3dep2_gen1dep32_batch64_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [ 4301 ] - prefill: - num-worker: 3 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL1K_OSL1K/STP/ctx3dep2_gen1dep16_batch256_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL1K_OSL1K/STP/ctx3dep2_gen1dep16_batch256_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - conc-list: [ 8192 ] - prefill: - num-worker: 4 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL1K_OSL1K/STP/ctx4dep2_gen1dep16_batch512_eplb256_mtp0.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL1K_OSL1K/STP/ctx4dep2_gen1dep16_batch512_eplb256_mtp0.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - isl: 8192 - osl: 1024 - search-space: - # STP configurations - - conc-list: [ 10 ] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx1dep2_gen5tep4_batch2_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx1dep2_gen5tep4_batch2_eplb0_mtp0.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - conc-list: [ 25 ] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx1dep2_gen5tep4_batch4_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx1dep2_gen5tep4_batch4_eplb0_mtp0.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - conc-list: [ 50 ] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx1dep2_gen5tep4_batch8_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx1dep2_gen5tep4_batch8_eplb0_mtp0.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - conc-list: [ 100 ] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx1dep2_gen5tep4_batch16_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx1dep2_gen5tep4_batch16_eplb0_mtp0.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - conc-list: [ 308 ] - prefill: - num-worker: 3 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx3dep2_gen1dep32_batch8_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx3dep2_gen1dep32_batch8_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [ 615 ] - prefill: - num-worker: 6 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx6dep2_gen1dep32_batch16_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx6dep2_gen1dep32_batch16_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [ 1076 ] - prefill: - num-worker: 9 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx9dep2_gen1dep16_batch64_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx9dep2_gen1dep16_batch64_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - conc-list: [ 1229 ] - prefill: - num-worker: 11 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx11dep2_gen1dep32_batch32_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx11dep2_gen1dep32_batch32_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [ 2151 ] - prefill: - num-worker: 15 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx15dep2_gen1dep16_batch128_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/GLM5/disagg/trtllm_dynamo/gb300_nvfp4/ISL8K_OSL1K/STP/ctx15dep2_gen1dep16_batch128_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - -qwen3.5-fp8-b200-sglang-mtp: - image: lmsysorg/sglang:v0.5.14-cu130 - model: Qwen/Qwen3.5-397B-A17B-FP8 - model-prefix: qwen3.5 - runner: b200 - precision: fp8 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 4, spec-decoding: mtp } - - { tp: 4, ep: 1, conc-start: 4, conc-end: 256, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 4, spec-decoding: mtp } - - { tp: 4, ep: 1, conc-start: 4, conc-end: 256, spec-decoding: mtp } - - -qwen3.5-fp8-b300-sglang-mtp: - image: lmsysorg/sglang:v0.5.12-cu130 - model: Qwen/Qwen3.5-397B-A17B-FP8 - model-prefix: qwen3.5 - runner: b300 - precision: fp8 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 4, ep: 1, conc-start: 4, conc-end: 256, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 4, ep: 1, conc-start: 4, conc-end: 256, spec-decoding: mtp } - -qwen3.5-fp8-b300-sglang: - image: lmsysorg/sglang:v0.5.12-cu130 - model: Qwen/Qwen3.5-397B-A17B-FP8 - model-prefix: qwen3.5 - runner: b300 - precision: fp8 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 4, ep: 1, conc-start: 4, conc-end: 256 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 4, ep: 1, conc-start: 4, conc-end: 256 } - -qwen3.5-fp4-b300-sglang: - image: lmsysorg/sglang:v0.5.14-cu130 - model: nvidia/Qwen3.5-397B-A17B-NVFP4 - model-prefix: qwen3.5 - runner: b300 - precision: fp4 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 4, ep: 1, conc-start: 4, conc-end: 128 } - - { tp: 2, ep: 2, conc-start: 4, conc-end: 128 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 4, ep: 1, conc-start: 4, conc-end: 128 } - - { tp: 2, ep: 2, conc-start: 4, conc-end: 128 } - -qwen3.5-fp4-b300-sglang-mtp: - image: lmsysorg/sglang:v0.5.14-cu130 - model: nvidia/Qwen3.5-397B-A17B-NVFP4 - model-prefix: qwen3.5 - runner: b300 - precision: fp4 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 4, ep: 1, conc-start: 4, conc-end: 128, spec-decoding: mtp } - - { tp: 2, ep: 2, conc-start: 4, conc-end: 128, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 4, ep: 1, conc-start: 4, conc-end: 128, spec-decoding: mtp } - - { tp: 2, ep: 2, conc-start: 4, conc-end: 128, spec-decoding: mtp } - -qwen3.5-bf16-b300-sglang: - image: lmsysorg/sglang:v0.5.12-cu130 - model: Qwen/Qwen3.5-397B-A17B - model-prefix: qwen3.5 - runner: b300 - precision: bf16 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 64 } - - { tp: 4, ep: 1, conc-start: 4, conc-end: 64 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 64 } - - { tp: 4, ep: 1, conc-start: 4, conc-end: 64 } - -qwen3.5-bf16-b300-sglang-mtp: - image: lmsysorg/sglang:v0.5.12-cu130 - model: Qwen/Qwen3.5-397B-A17B - model-prefix: qwen3.5 - runner: b300 - precision: bf16 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 64, spec-decoding: mtp } - - { tp: 4, ep: 1, conc-start: 4, conc-end: 64, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 64, spec-decoding: mtp } - - { tp: 4, ep: 1, conc-start: 4, conc-end: 64, spec-decoding: mtp } - -kimik2.5-int4-b200-vllm: - image: vllm/vllm-openai:v0.24.0 - model: moonshotai/Kimi-K2.5 - model-prefix: kimik2.5 - runner: b200 - precision: int4 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 64 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 64 } - -# NOTE: At the time of submission, https://docs.vllm.ai/projects/recipes/en/latest/moonshotai/Kimi-K2.5.html -# does not have a B300-specific recipe, so this config reuses the existing -# Kimi-K2.5 INT4 B200 vLLM recipe as-is until B300-specific tuning is available. - -kimik2.5-int4-b200-vllm-agentic: - image: vllm/vllm-openai:v0.22.0 - model: moonshotai/Kimi-K2.5 - model-prefix: kimik2.5 - runner: cluster:b200-dgxc - precision: int4 - framework: vllm - multinode: false - scenarios: - agentic-coding: - - dram-utilization: 0.80 - search-space: - - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32] } - - { tp: 8, kv-offloading: dram, kv-offload-backend: native, conc-list: [32, 64, 96, 128] } - - -kimik2.5-int4-b300-vllm: - image: vllm/vllm-openai:v0.24.0 - model: moonshotai/Kimi-K2.5 - model-prefix: kimik2.5 - runner: b300 - precision: int4 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 64 } - - { tp: 4, ep: 1, conc-start: 4, conc-end: 64 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 64 } - - { tp: 4, ep: 1, conc-start: 4, conc-end: 64 } - -kimik2.5-int4-h200-vllm: - image: vllm/vllm-openai:v0.22.0 - model: moonshotai/Kimi-K2.5 - model-prefix: kimik2.5 - runner: h200 - precision: int4 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 64 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 64 } - -kimik2.5-int4-h200-vllm-agentic: - image: vllm/vllm-openai:v0.22.0 - model: moonshotai/Kimi-K2.5 - model-prefix: kimik2.5 - runner: cluster:h200-dgxc - precision: int4 - framework: vllm - multinode: false - scenarios: - agentic-coding: - - dram-utilization: 0.80 - search-space: - - { tp: 8, kv-offloading: none, conc-list: [1, 2, 3, 4, 5, 6, 7] } - - { tp: 8, kv-offloading: dram, kv-offload-backend: native, conc-list: [6, 7, 8, 9, 10, 11, 12, 13, 14] } - - -# NOTE: At the time of submission, https://docs.vllm.ai/projects/recipes/en/latest/moonshotai/Kimi-K2.5.html -# does not have a B300-specific recipe, so this config reuses the existing -# Kimi-K2.5 FP4 B200 vLLM recipe as-is until B300-specific tuning is available. - -kimik2.5-fp4-b200-vllm: - image: vllm/vllm-openai:v0.22.0 - model: nvidia/Kimi-K2.5-NVFP4 - model-prefix: kimik2.5 - runner: b200 - precision: fp4 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 1, conc-end: 4 } - - { tp: 4, ep: 1, conc-start: 1, conc-end: 128 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 1, conc-end: 4 } - - { tp: 4, ep: 1, conc-start: 1, conc-end: 128 } - -# NOTE: At the time of submission, https://docs.vllm.ai/projects/recipes/en/latest/moonshotai/Kimi-K2.5.html -# does not have a B300-specific recipe, so this config reuses the existing -# Kimi-K2.5 FP4 B200 vLLM recipe as-is until B300-specific tuning is available. - -kimik2.5-fp4-b300-vllm: - image: vllm/vllm-openai:v0.22.0 - model: nvidia/Kimi-K2.5-NVFP4 - model-prefix: kimik2.5 - runner: b300 - precision: fp4 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 1, conc-end: 4 } - - { tp: 4, ep: 1, conc-start: 1, conc-end: 128 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 1, conc-end: 4 } - - { tp: 4, ep: 1, conc-start: 1, conc-end: 128 } - -dsr1-fp8-b200-sglang-mtp: - image: lmsysorg/sglang:v0.5.12-cu130 - model: deepseek-ai/DeepSeek-R1-0528 - model-prefix: dsr1 - runner: b200 - precision: fp8 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 1, conc-end: 512, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 1, conc-end: 512, spec-decoding: mtp } - - # NOTE: At the time of submission, https://cookbook.sglang.io/autoregressive/DeepSeek/DeepSeek-R1 - # does not have a B300-specific recipe, so this config reuses the existing DSR1 FP8 - # B200 SGLang MTP recipe as-is until B300-specific tuning is available. Image bumped - # to v0.5.10.post1-cu130 to match the standard B300 SGLang image used by other B300 configs. -dsr1-fp8-b300-sglang-mtp: - image: lmsysorg/sglang:v0.5.12-cu130 - model: deepseek-ai/DeepSeek-R1-0528 - model-prefix: dsr1 - runner: b300 - precision: fp8 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 1, conc-end: 512, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 1, conc-end: 512, spec-decoding: mtp } - -kimik2.5-fp4-b300-vllm-agentic: - # v0.20.2 (cu129) lacks the flashinfer kernels for B300's reported SM - # (sm_12x); workers hit "Only SM 10.x and 11.x are supported" in the - # trtllm_fp4_block_scale_moe path. v0.20.0-cu130 is the Blackwell-targeted - # build that has the full sm_10x/sm_11x/sm_12x kernel set and is what the - # INT4 B300 sister already uses successfully. - image: vllm/vllm-openai:v0.22.0 - model: nvidia/Kimi-K2.5-NVFP4 - model-prefix: kimik2.5 - runner: cluster:b300-nv - precision: fp4 - framework: vllm - multinode: false - scenarios: - agentic-coding: - - dram-utilization: 0.80 - search-space: - - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32, 40, 48, 56, 64] } - - { tp: 8, ep: 1, kv-offloading: dram, kv-offload-backend: native, conc-list: [1, 2, 4, 8, 16, 32, 40, 48, 56, 64] } - - -dsr1-fp8-b200-trt: - image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc14 - model: deepseek-ai/DeepSeek-R1-0528 - model-prefix: dsr1 - runner: b200 - precision: fp8 - framework: trt - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 64, conc-end: 128 } - - { tp: 4, ep: 1, conc-start: 8, conc-end: 16 } - - { tp: 8, ep: 1, conc-start: 4, conc-end: 8 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 64, conc-end: 256 } - - { tp: 4, ep: 1, conc-start: 8, conc-end: 32 } - - { tp: 8, ep: 1, conc-start: 4, conc-end: 8 } - -dsr1-fp8-b200-trt-mtp: - image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc14 - model: deepseek-ai/DeepSeek-R1-0528 - model-prefix: dsr1 - runner: b200 - precision: fp8 - framework: trt - multinode: false - scenarios: - fixed-seq-len: - # For all sequence lengths, MTP=3 (or MTP=1 when DP_ATTN=true) - - isl: 1024 - osl: 1024 - search-space: - # mostly TP8 - # If CONC == 256, then TP8, EP8, DP_ATTN=true - - { tp: 8, ep: 1, conc-start: 4, conc-end: 128, spec-decoding: mtp } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 256, conc-end: 256, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - # TP8 for all points - - { tp: 8, ep: 1, conc-start: 4, conc-end: 256, spec-decoding: mtp } - -dsr1-fp8-h200-sglang: - image: lmsysorg/sglang:v0.5.12-cu130 - model: deepseek-ai/DeepSeek-R1-0528 - model-prefix: dsr1 - runner: h200 - precision: fp8 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 64 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 64 } - -dsr1-fp8-h200-sglang-mtp: - image: lmsysorg/sglang:v0.5.12-cu130 - model: deepseek-ai/DeepSeek-R1-0528 - model-prefix: dsr1 - runner: h200 - precision: fp8 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 64, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 4, conc-end: 64, spec-decoding: mtp } - -# DeepSeek-V4-Pro H200 recipe from https://vllm.ai/blog/deepseek-v4 -# Uses the cu129 image. H200 has no FP4 path, so the FP4 indexer cache -# flag is omitted. Max-model-len is pinned at 800k per the recipe. -dsv4-fp8-h200-vllm: - image: vllm/vllm-openai:v0.21.0 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: h200 - precision: fp8 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, ep: 1, dp-attn: false, conc-start: 1, conc-end: 256 } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 1, conc-end: 256 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, ep: 1, dp-attn: false, conc-start: 1, conc-end: 256 } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 1, conc-end: 256 } - -# MTP variant of dsv4-fp8-h200-vllm. Uses the canonical v0.20.1 image -# (the non-MTP entry above is still on the deepseekv4-cu129 tag) and adds -# --speculative-config '{"method":"mtp","num_speculative_tokens":2}'. -dsv4-fp8-h200-vllm-mtp: - image: vllm/vllm-openai:v0.21.0 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: h200 - precision: fp8 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, ep: 1, dp-attn: false, conc-start: 1, conc-end: 256, spec-decoding: mtp } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 1, conc-end: 256, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, ep: 1, dp-attn: false, conc-start: 1, conc-end: 256, spec-decoding: mtp } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 1, conc-end: 256, spec-decoding: mtp } - -# DeepSeek-V4-Pro H200 single-node with SGLang (Marlin FP8, TP-only). -# Pinned to the h200-dgxc-slurm runner pool because the deepseek-v4-hopper -# image needs the /ix mount layout that only launch_h200-dgxc-slurm.sh sets up. - -dsv4-fp8-h200-vllm-agentic: - image: vllm/vllm-openai:v0.22.0 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: cluster:h200-dgxc - precision: fp8 - framework: vllm - multinode: false - scenarios: - agentic-coding: - - search-space: - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [1, 2, 4, 8, 16] } - - -# MTP variant of dsv4-fp8-h200-sglang. Mirrors the non-MTP recipe (same image, -# runner pool, search space) and adds EAGLE speculative decoding via -# --speculative-algorithm EAGLE with the (3,1,4) chain matching dsv4-fp4-b300-sglang-mtp. -dsv4-fp8-h200-sglang: - image: lmsysorg/sglang:deepseek-v4-hopper@sha256:7f19c6dc092e47a10fac2e41f47eab78970280d06648b8e50d312a82f0ae722f - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: h200-dgxc - precision: fp8 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 1, conc-end: 1 } - - { tp: 8, ep: 1, conc-start: 4, conc-end: 64 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 1, conc-end: 1 } - - { tp: 8, ep: 1, conc-start: 4, conc-end: 64 } - -# MTP variant of dsv4-fp8-h200-sglang. Mirrors the non-MTP recipe (same image, -# runner pool, search space) and adds EAGLE speculative decoding via -# --speculative-algorithm EAGLE with the (3,1,4) chain matching dsv4-fp4-b300-sglang-mtp. -dsv4-fp8-h200-sglang-mtp: - image: lmsysorg/sglang:deepseek-v4-hopper@sha256:7f19c6dc092e47a10fac2e41f47eab78970280d06648b8e50d312a82f0ae722f - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: h200-dgxc - precision: fp8 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 1, conc-end: 1, spec-decoding: mtp } - - { tp: 8, ep: 1, conc-start: 4, conc-end: 64, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 1, conc-end: 1, spec-decoding: mtp } - - { tp: 8, ep: 1, conc-start: 4, conc-end: 64, spec-decoding: mtp } - - # DeepSeek-V4-Pro B300 single-node aggregate recipe from the submitted B300 - # pareto sweep. The single-node schema has no explicit data-parallel-size - # field, so dp-attn=true is used as the existing vLLM script switch for DP4 - # layouts on 4 allocated GPUs. -dsv4-fp4-b300-vllm: - image: vllm/vllm-openai:v0.22.0 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: b300 - precision: fp4 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 4, conc-start: 1, conc-end: 128 } - - { tp: 8, conc-start: 1, conc-end: 4 } - - { tp: 4, ep: 4, dp-attn: true, conc-start: 256, conc-end: 512 } - - { tp: 4, ep: 4, dp-attn: true, conc-start: 2048, conc-end: 2048 } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 4096, conc-end: 8192 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 4, conc-start: 1, conc-end: 64 } - - { tp: 8, conc-start: 1, conc-end: 4 } - - { tp: 4, ep: 4, dp-attn: true, conc-start: 256, conc-end: 512 } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 2048, conc-end: 2048 } - -dsv4-fp4-b300-vllm-agentic: - image: vllm/vllm-openai:v0.23.0 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: cluster:b300-nv - precision: fp4 - framework: vllm - multinode: false - scenarios: - agentic-coding: - - dram-utilization: 0.80 - search-space: - # Compare native GPU-cache and MooncakeStore CPU-offload cliffs. - - { tp: 4, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 16] } - - { tp: 4, kv-offloading: dram, kv-offload-backend: mooncake, conc-list: [16, 18, 20, 24] } - # TP8 remains cache-resident through conc 52. - - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 16, 32, 40, 48, 52] } - - { tp: 8, kv-offloading: dram, kv-offload-backend: mooncake, conc-list: [52] } - - { tp: 4, ep: 4, dp-attn: true, kv-offloading: none, conc-list: [8, 16] } - - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: mooncake, conc-list: [32] } - # TP8 DEP retains representative low, peak, and transition points. - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [52, 72, 100, 128, 144] } - - -dsv4-fp4-b300-trt: - image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: b300 - precision: fp4 - framework: trt - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 4, conc-start: 1, conc-end: 64 } - - { tp: 4, ep: 4, dp-attn: true, conc-start: 64, conc-end: 256 } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 512, conc-end: 1024 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 4, conc-start: 1, conc-end: 32 } - - { tp: 4, ep: 4, dp-attn: true, conc-start: 64, conc-end: 64 } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 256, conc-end: 1024 } - -dsv4-fp4-b300-trt-mtp: - image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: b300 - precision: fp4 - framework: trt - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 4, conc-start: 1, conc-end: 64, spec-decoding: mtp } - - { tp: 4, ep: 4, dp-attn: true, conc-start: 64, conc-end: 256, spec-decoding: mtp } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 512, conc-end: 1024, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 4, conc-start: 1, conc-end: 32, spec-decoding: mtp } - - { tp: 4, ep: 4, dp-attn: true, conc-start: 64, conc-end: 64, spec-decoding: mtp } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 256, conc-end: 1024, spec-decoding: mtp } - -dsv4-fp4-b300-vllm-mtp: - image: vllm/vllm-openai:v0.21.0 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: b300 - precision: fp4 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 4, conc-start: 1, conc-end: 256, spec-decoding: mtp } - - { tp: 8, conc-start: 1, conc-end: 8, spec-decoding: mtp } - - { tp: 4, ep: 4, dp-attn: true, conc-start: 256, conc-end: 1024, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 4, conc-start: 1, conc-end: 64, spec-decoding: mtp } - - { tp: 8, conc-start: 1, conc-end: 8, spec-decoding: mtp } - - { tp: 4, ep: 4, conc-start: 64, conc-end: 256, spec-decoding: mtp } - - { tp: 4, ep: 4, dp-attn: true, conc-start: 256, conc-end: 512, spec-decoding: mtp } - -qwen3.5-fp8-h200-sglang: - image: lmsysorg/sglang:v0.5.14-cu130 - model: Qwen/Qwen3.5-397B-A17B-FP8 - model-prefix: qwen3.5 - runner: h200 - precision: fp8 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, ep: 8, conc-start: 4, conc-end: 64 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, ep: 8, conc-start: 4, conc-end: 64 } - -qwen3.5-fp8-h200-sglang-mtp: - image: lmsysorg/sglang:v0.5.14-cu130 - model: Qwen/Qwen3.5-397B-A17B-FP8 - model-prefix: qwen3.5 - runner: h200 - precision: fp8 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, ep: 8, conc-start: 4, conc-end: 128, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, ep: 8, conc-start: 4, conc-end: 128, spec-decoding: mtp } - -glm5-fp8-h200-sglang: - image: lmsysorg/sglang:v0.5.12-cu130 - model: zai-org/GLM-5-FP8 - model-prefix: glm5 - runner: h200-dgxc - precision: fp8 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 64 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 64 } - -glm5-fp8-h200-sglang-mtp: - image: lmsysorg/sglang:v0.5.12-cu130 - model: zai-org/GLM-5-FP8 - model-prefix: glm5 - runner: h200 - precision: fp8 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 64, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, conc-start: 4, conc-end: 64, spec-decoding: mtp } - -dsr1-fp8-h200-trt: - image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc14 - model: deepseek-ai/DeepSeek-R1-0528 - model-prefix: dsr1 - runner: h200 - precision: fp8 - framework: trt - multinode: false - # For all sequence lengths, EP=TP - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - # If CONC > 64, then DP_ATTN=true - search-space: - - { tp: 8, ep: 8, conc-start: 4, conc-end: 64 } - - isl: 8192 - osl: 1024 - # If CONC > 32, then DP_ATTN=true - search-space: - - { tp: 8, ep: 8, conc-start: 4, conc-end: 32 } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 64, conc-end: 64 } - -dsr1-fp8-h200-trt-mtp: - image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc14 - model: deepseek-ai/DeepSeek-R1-0528 - model-prefix: dsr1 - runner: h200 - precision: fp8 - framework: trt - multinode: false - # For all sequence lengths, EP=TP, MOE_BACKEND=CUTLASS, MTP=3 (or MTP=1 when DP_ATTN=true) - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # If CONC >= 128, then DP_ATTN=true, MTP=1 - - { tp: 8, ep: 8, conc-start: 4, conc-end: 64, spec-decoding: mtp } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 128, conc-end: 256, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - # If CONC >= 64, then DP_ATTN=true, MTP=1 - - { tp: 8, ep: 8, conc-start: 4, conc-end: 32, spec-decoding: mtp } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 64, conc-end: 256, spec-decoding: mtp } - -dsr1-fp8-h200-dynamo-trt: - image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post1 - model: deepseek-ai/DeepSeek-R1-0528 - model-prefix: dsr1 - runner: h200-multinode - precision: fp8 - framework: dynamo-trt - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # MTP configurations - - spec-decoding: "mtp" - conc-list: [1] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/1k1k/mtp/c1_ctx1_gen11_tep8_batch1_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/h200/1k1k/mtp/c1_ctx1_gen11_tep8_batch1_eplb0_mtp3.yaml" - decode: - num-worker: 11 - tp: 8 - ep: 8 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [4] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/1k1k/mtp/c4_ctx1_gen11_tep8_batch128_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/h200/1k1k/mtp/c4_ctx1_gen11_tep8_batch128_eplb0_mtp3.yaml" - decode: - num-worker: 11 - tp: 8 - ep: 8 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [8] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/1k1k/mtp/c8_ctx1_gen11_tep8_batch128_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/h200/1k1k/mtp/c8_ctx1_gen11_tep8_batch128_eplb0_mtp3.yaml" - decode: - num-worker: 11 - tp: 8 - ep: 8 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [16] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/1k1k/mtp/c16_ctx1_gen9_tep8_batch128_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/h200/1k1k/mtp/c16_ctx1_gen9_tep8_batch128_eplb0_mtp3.yaml" - decode: - num-worker: 9 - tp: 8 - ep: 8 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [32] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/1k1k/mtp/c32_ctx1_gen11_tep8_batch128_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/h200/1k1k/mtp/c32_ctx1_gen11_tep8_batch128_eplb0_mtp3.yaml" - decode: - num-worker: 11 - tp: 8 - ep: 8 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [64] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/1k1k/mtp/c64_ctx1_gen8_dep8_batch128_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/h200/1k1k/mtp/c64_ctx1_gen8_dep8_batch128_eplb0_mtp3.yaml" - decode: - num-worker: 8 - tp: 8 - ep: 8 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [128] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/1k1k/mtp/c128_ctx1_gen7_dep8_batch128_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/h200/1k1k/mtp/c128_ctx1_gen7_dep8_batch128_eplb0_mtp3.yaml" - decode: - num-worker: 7 - tp: 8 - ep: 8 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [256] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/1k1k/mtp/c256_ctx1_gen4_dep8_batch128_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/h200/1k1k/mtp/c256_ctx1_gen4_dep8_batch128_eplb0_mtp3.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [512] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/1k1k/mtp/c512_ctx1_gen2_dep8_batch256_eplb0_mtp1.yaml - - "CONFIG_FILE=recipes/trtllm/h200/1k1k/mtp/c512_ctx1_gen2_dep8_batch256_eplb0_mtp1.yaml" - decode: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - # Non-MTP configurations (STP) - - conc-list: [1] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/1k1k/stp/c1_ctx1_gen9_tep8_batch1_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h200/1k1k/stp/c1_ctx1_gen9_tep8_batch1_eplb0_mtp0.yaml" - decode: - num-worker: 9 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [4] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/1k1k/stp/c4_ctx1_gen9_tep8_batch256_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h200/1k1k/stp/c4_ctx1_gen9_tep8_batch256_eplb0_mtp0.yaml" - decode: - num-worker: 9 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [8] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/1k1k/stp/c8_ctx1_gen9_tep8_batch256_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h200/1k1k/stp/c8_ctx1_gen9_tep8_batch256_eplb0_mtp0.yaml" - decode: - num-worker: 9 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [16] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/1k1k/stp/c16_ctx1_gen9_tep8_batch256_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h200/1k1k/stp/c16_ctx1_gen9_tep8_batch256_eplb0_mtp0.yaml" - decode: - num-worker: 9 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [32] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/1k1k/stp/c32_ctx1_gen9_tep8_batch256_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h200/1k1k/stp/c32_ctx1_gen9_tep8_batch256_eplb0_mtp0.yaml" - decode: - num-worker: 9 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [64] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/1k1k/stp/c64_ctx1_gen9_tep8_batch256_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h200/1k1k/stp/c64_ctx1_gen9_tep8_batch256_eplb0_mtp0.yaml" - decode: - num-worker: 9 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [128] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/1k1k/stp/c128_ctx1_gen9_dep8_batch512_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h200/1k1k/stp/c128_ctx1_gen9_dep8_batch512_eplb0_mtp0.yaml" - decode: - num-worker: 9 - tp: 8 - ep: 8 - dp-attn: true - - conc-list: [256] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/1k1k/stp/c256_ctx1_gen6_dep8_batch512_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h200/1k1k/stp/c256_ctx1_gen6_dep8_batch512_eplb0_mtp0.yaml" - decode: - num-worker: 6 - tp: 8 - ep: 8 - dp-attn: true - - conc-list: [512] - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/1k1k/stp/c512_ctx2_gen7_dep8_batch512_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h200/1k1k/stp/c512_ctx2_gen7_dep8_batch512_eplb0_mtp0.yaml" - decode: - num-worker: 7 - tp: 8 - ep: 8 - dp-attn: true - - isl: 8192 - osl: 1024 - search-space: - # MTP configurations - - spec-decoding: "mtp" - conc-list: [1] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c1_ctx1_gen7_tep8_batch1_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/mtp/c1_ctx1_gen7_tep8_batch1_eplb0_mtp3.yaml" - decode: - num-worker: 7 - tp: 8 - ep: 8 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [4] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c4_ctx1_gen7_tep8_batch32_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/mtp/c4_ctx1_gen7_tep8_batch32_eplb0_mtp3.yaml" - decode: - num-worker: 7 - tp: 8 - ep: 8 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [8] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c8_ctx1_gen6_tep8_batch32_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/mtp/c8_ctx1_gen6_tep8_batch32_eplb0_mtp3.yaml" - decode: - num-worker: 6 - tp: 8 - ep: 8 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [16] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c16_ctx1_gen3_tep8_batch32_eplb0_mtp2.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/mtp/c16_ctx1_gen3_tep8_batch32_eplb0_mtp2.yaml" - decode: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [32] - prefill: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c32_ctx3_gen5_tep8_batch32_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/mtp/c32_ctx3_gen5_tep8_batch32_eplb0_mtp3.yaml" - decode: - num-worker: 5 - tp: 8 - ep: 8 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [64] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c64_ctx1_gen1_dep8_batch32_eplb0_mtp2.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/mtp/c64_ctx1_gen1_dep8_batch32_eplb0_mtp2.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [128] - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c128_ctx2_gen1_dep8_batch32_eplb0_mtp2.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/mtp/c128_ctx2_gen1_dep8_batch32_eplb0_mtp2.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [256] - prefill: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c256_ctx3_gen1_dep8_batch32_eplb0_mtp2.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/mtp/c256_ctx3_gen1_dep8_batch32_eplb0_mtp2.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [512] - prefill: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/mtp/c512_ctx3_gen1_dep8_batch64_eplb0_mtp1.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/mtp/c512_ctx3_gen1_dep8_batch64_eplb0_mtp1.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - # Non-MTP configurations (STP) - - conc-list: [1] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c1_ctx1_gen7_tep8_batch1_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/stp/c1_ctx1_gen7_tep8_batch1_eplb0_mtp0.yaml" - decode: - num-worker: 7 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [4] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c4_ctx1_gen7_tep8_batch32_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/stp/c4_ctx1_gen7_tep8_batch32_eplb0_mtp0.yaml" - decode: - num-worker: 7 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [8] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c8_ctx1_gen6_tep8_batch16_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/stp/c8_ctx1_gen6_tep8_batch16_eplb0_mtp0.yaml" - decode: - num-worker: 6 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [16] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c16_ctx1_gen3_tep8_batch32_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/stp/c16_ctx1_gen3_tep8_batch32_eplb0_mtp0.yaml" - decode: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [32] - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c32_ctx2_gen5_tep8_batch128_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/stp/c32_ctx2_gen5_tep8_batch128_eplb0_mtp0.yaml" - decode: - num-worker: 5 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [64] - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c64_ctx2_gen3_dep8_batch128_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/stp/c64_ctx2_gen3_dep8_batch128_eplb0_mtp0.yaml" - decode: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: true - - conc-list: [128] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c128_ctx1_gen1_dep8_batch256_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/stp/c128_ctx1_gen1_dep8_batch256_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - conc-list: [256] - prefill: - num-worker: 5 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c256_ctx5_gen3_dep8_batch256_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/stp/c256_ctx5_gen3_dep8_batch256_eplb0_mtp0.yaml" - decode: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: true - - conc-list: [512] - prefill: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h200/8k1k/stp/c512_ctx3_gen1_dep8_batch512_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h200/8k1k/stp/c512_ctx3_gen1_dep8_batch512_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - -dsr1-fp8-h100-dynamo-trt: - image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post3 - model: deepseek-ai/DeepSeek-R1-0528 - model-prefix: dsr1 - runner: h100-multinode - precision: fp8 - framework: dynamo-trt - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # MTP configurations - - spec-decoding: "mtp" - conc-list: [6] - prefill: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/1k1k/mtp/ctx1_gen3_tep16_batch1_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/h100-fp8/1k1k/mtp/ctx1_gen3_tep16_batch1_eplb0_mtp3.yaml" - decode: - num-worker: 3 - tp: 16 - ep: 16 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [9] - prefill: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/1k1k/mtp/ctx1_gen3_tep16_batch2_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/h100-fp8/1k1k/mtp/ctx1_gen3_tep16_batch2_eplb0_mtp3.yaml" - decode: - num-worker: 3 - tp: 16 - ep: 16 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [30] - prefill: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/1k1k/mtp/ctx1_gen3_tep16_batch8_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/h100-fp8/1k1k/mtp/ctx1_gen3_tep16_batch8_eplb0_mtp3.yaml" - decode: - num-worker: 3 - tp: 16 - ep: 16 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [60] - prefill: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/1k1k/mtp/ctx1_gen3_tep16_batch16_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/h100-fp8/1k1k/mtp/ctx1_gen3_tep16_batch16_eplb0_mtp3.yaml" - decode: - num-worker: 3 - tp: 16 - ep: 16 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [117] - prefill: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/1k1k/mtp/ctx1_gen3_tep16_batch32_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/h100-fp8/1k1k/mtp/ctx1_gen3_tep16_batch32_eplb0_mtp3.yaml" - decode: - num-worker: 3 - tp: 16 - ep: 16 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [231] - prefill: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/1k1k/mtp/ctx1_gen3_dep16_batch4_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/h100-fp8/1k1k/mtp/ctx1_gen3_dep16_batch4_eplb0_mtp3.yaml" - decode: - num-worker: 3 - tp: 16 - ep: 16 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [462] - prefill: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/1k1k/mtp/ctx1_gen3_tep16_batch128_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/h100-fp8/1k1k/mtp/ctx1_gen3_tep16_batch128_eplb0_mtp3.yaml" - decode: - num-worker: 3 - tp: 16 - ep: 16 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [615] - prefill: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/1k1k/mtp/ctx1_gen1_dep16_batch32_eplb0_mtp2.yaml - - "CONFIG_FILE=recipes/trtllm/h100-fp8/1k1k/mtp/ctx1_gen1_dep16_batch32_eplb0_mtp2.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [1229] - prefill: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/1k1k/mtp/ctx1_gen1_dep16_batch64_eplb0_mtp1.yaml - - "CONFIG_FILE=recipes/trtllm/h100-fp8/1k1k/mtp/ctx1_gen1_dep16_batch64_eplb0_mtp1.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - # Non-MTP configurations (STP) - - conc-list: [6] - prefill: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/1k1k/stp/ctx1_gen3_tep16_batch1_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h100-fp8/1k1k/stp/ctx1_gen3_tep16_batch1_eplb0_mtp0.yaml" - decode: - num-worker: 3 - tp: 16 - ep: 16 - dp-attn: false - - conc-list: [9] - prefill: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/1k1k/stp/ctx1_gen3_tep16_batch2_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h100-fp8/1k1k/stp/ctx1_gen3_tep16_batch2_eplb0_mtp0.yaml" - decode: - num-worker: 3 - tp: 16 - ep: 16 - dp-attn: false - - conc-list: [30] - prefill: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/1k1k/stp/ctx1_gen3_tep16_batch8_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h100-fp8/1k1k/stp/ctx1_gen3_tep16_batch8_eplb0_mtp0.yaml" - decode: - num-worker: 3 - tp: 16 - ep: 16 - dp-attn: false - - conc-list: [60] - prefill: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/1k1k/stp/ctx1_gen3_tep16_batch16_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h100-fp8/1k1k/stp/ctx1_gen3_tep16_batch16_eplb0_mtp0.yaml" - decode: - num-worker: 3 - tp: 16 - ep: 16 - dp-attn: false - - conc-list: [231] - prefill: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/1k1k/stp/ctx1_gen3_dep16_batch4_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h100-fp8/1k1k/stp/ctx1_gen3_dep16_batch4_eplb0_mtp0.yaml" - decode: - num-worker: 3 - tp: 16 - ep: 16 - dp-attn: true - - conc-list: [462] - prefill: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/1k1k/stp/ctx1_gen3_dep16_batch8_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h100-fp8/1k1k/stp/ctx1_gen3_dep16_batch8_eplb0_mtp0.yaml" - decode: - num-worker: 3 - tp: 16 - ep: 16 - dp-attn: true - - conc-list: [924] - prefill: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/1k1k/stp/ctx1_gen3_dep16_batch16_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h100-fp8/1k1k/stp/ctx1_gen3_dep16_batch16_eplb0_mtp0.yaml" - decode: - num-worker: 3 - tp: 16 - ep: 16 - dp-attn: true - - conc-list: [1845] - prefill: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/1k1k/stp/ctx1_gen3_dep16_batch32_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h100-fp8/1k1k/stp/ctx1_gen3_dep16_batch32_eplb0_mtp0.yaml" - decode: - num-worker: 3 - tp: 16 - ep: 16 - dp-attn: true - - conc-list: [4916] - prefill: - num-worker: 2 - tp: 16 - ep: 16 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/1k1k/stp/ctx2_gen1_dep16_batch256_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h100-fp8/1k1k/stp/ctx2_gen1_dep16_batch256_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - isl: 8192 - osl: 1024 - search-space: - # MTP configurations (6 points) - - spec-decoding: "mtp" - conc-list: [6] - prefill: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/mtp/ctx1_gen3_tep16_batch1_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/h100-fp8/8k1k/mtp/ctx1_gen3_tep16_batch1_eplb0_mtp3.yaml" - decode: - num-worker: 3 - tp: 16 - ep: 16 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [9] - prefill: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/mtp/ctx1_gen3_tep16_batch2_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/h100-fp8/8k1k/mtp/ctx1_gen3_tep16_batch2_eplb0_mtp3.yaml" - decode: - num-worker: 3 - tp: 16 - ep: 16 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [30] - prefill: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/mtp/ctx1_gen3_tep16_batch8_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/h100-fp8/8k1k/mtp/ctx1_gen3_tep16_batch8_eplb0_mtp3.yaml" - decode: - num-worker: 3 - tp: 16 - ep: 16 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [77] - prefill: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/mtp/ctx1_gen1_dep16_batch4_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/h100-fp8/8k1k/mtp/ctx1_gen1_dep16_batch4_eplb0_mtp3.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - # commenting out cuz it persistently causes problems - # https://github.com/InferenceMAX/InferenceMAX/actions/runs/21769314582/job/62813105509 - # - spec-decoding: "mtp" - # conc-list: [78] - # prefill: - # num-worker: 1 - # tp: 16 - # ep: 16 - # dp-attn: true - # additional-settings: - # # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/mtp/ctx1_gen2_tep16_batch32_eplb0_mtp3.yaml - # - "CONFIG_FILE=recipes/trtllm/h100-fp8/8k1k/mtp/ctx1_gen2_tep16_batch32_eplb0_mtp3.yaml" - # decode: - # num-worker: 2 - # tp: 16 - # ep: 16 - # dp-attn: false - - spec-decoding: "mtp" - conc-list: [154] - prefill: - num-worker: 2 - tp: 16 - ep: 16 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/mtp/ctx2_gen1_dep16_batch8_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/h100-fp8/8k1k/mtp/ctx2_gen1_dep16_batch8_eplb0_mtp3.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - # STP configurations (5 points) - - conc-list: [6] - prefill: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/stp/ctx1_gen3_tep16_batch1_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h100-fp8/8k1k/stp/ctx1_gen3_tep16_batch1_eplb0_mtp0.yaml" - decode: - num-worker: 3 - tp: 16 - ep: 16 - dp-attn: false - - conc-list: [9] - prefill: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/stp/ctx1_gen3_tep16_batch2_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h100-fp8/8k1k/stp/ctx1_gen3_tep16_batch2_eplb0_mtp0.yaml" - decode: - num-worker: 3 - tp: 16 - ep: 16 - dp-attn: false - - conc-list: [30] - prefill: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/stp/ctx1_gen3_tep16_batch8_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h100-fp8/8k1k/stp/ctx1_gen3_tep16_batch8_eplb0_mtp0.yaml" - decode: - num-worker: 3 - tp: 16 - ep: 16 - dp-attn: false - - conc-list: [154] - prefill: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/stp/ctx1_gen2_tep16_batch64_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h100-fp8/8k1k/stp/ctx1_gen2_tep16_batch64_eplb0_mtp0.yaml" - decode: - num-worker: 2 - tp: 16 - ep: 16 - dp-attn: false - - conc-list: [308] - prefill: - num-worker: 2 - tp: 16 - ep: 16 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/h100-fp8/8k1k/stp/ctx2_gen1_dep16_batch16_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/h100-fp8/8k1k/stp/ctx2_gen1_dep16_batch16_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - -gptoss-fp4-b200-trt: - image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc14 - model: openai/gpt-oss-120b - model-prefix: gptoss - runner: b200 - precision: fp4 - framework: trt - multinode: false - scenarios: - fixed-seq-len: - # Low ==> high TP from Left to Right of pareto - - isl: 1024 - osl: 1024 - search-space: - - { tp: 1, conc-start: 256, conc-end: 256 } - - { tp: 2, ep: 2, dp-attn: true, conc-start: 256, conc-end: 256 } - - { tp: 2, conc-start: 4, conc-end: 256 } - - { tp: 4, ep: 4, dp-attn: true, conc-start: 64, conc-end: 64 } - - { tp: 4, conc-start: 4, conc-end: 4 } - - { tp: 8, conc-start: 4, conc-end: 4 } - # Low ==> high TP from Left to Right of pareto - - isl: 8192 - osl: 1024 - search-space: - - { tp: 1, conc-start: 4, conc-end: 256} - - { tp: 2, conc-start: 4, conc-end: 256} - - { tp: 4, conc-start: 4, conc-end: 32} - - { tp: 8, conc-start: 4, conc-end: 4} - -gptoss-fp4-b200-vllm: - image: vllm/vllm-openai:v0.22.0 - model: openai/gpt-oss-120b - model-prefix: gptoss - runner: b200 - precision: fp4 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 1, conc-start: 4, conc-end: 128 } - - { tp: 2, conc-start: 4, conc-end: 128 } - - { tp: 4, conc-start: 4, conc-end: 64 } - - { tp: 8, conc-start: 4, conc-end: 8 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 1, conc-start: 4, conc-end: 128 } - - { tp: 2, conc-start: 4, conc-end: 128 } - - { tp: 4, conc-start: 4, conc-end: 64 } - - { tp: 8, conc-start: 4, conc-end: 4 } - -gptoss-fp4-h100-vllm: - image: vllm/vllm-openai:v0.21.0 - model: openai/gpt-oss-120b - model-prefix: gptoss - runner: h100 - precision: fp4 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 2, conc-start: 4, conc-end: 64 } - - { tp: 4, conc-start: 4, conc-end: 64 } - - { tp: 8, conc-start: 4, conc-end: 64 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 2, conc-start: 4, conc-end: 64 } - - { tp: 4, conc-start: 4, conc-end: 64 } - - { tp: 8, conc-start: 4, conc-end: 16 } - -# Day-zero MiniMax-M3 recipe (https://recipes.vllm.ai/MiniMaxAI/MiniMax-M3). -# M3 support has not shipped in a stable vLLM release; the dedicated -# vllm/vllm-openai:minimax-m3 image is the supported path. MXFP8 variant -# (NVIDIA-quantized, ~427 GB weights) is the lowest precision available — -# BF16 (~854 GB) does not fit 8x H100 (640 GB) at all, so H100 is TP8-only: -# weights alone take ~56 GB of each 80 GB GPU, leaving no room below TP8. -# dp-attn: true maps to the recipe's "DP8 + Expert Parallel" serve mode -# (vLLM --data-parallel-size 8 --enable-expert-parallel). -minimaxm3-fp8-h100-vllm: - image: vllm/vllm-openai:minimax-m3 - model: MiniMaxAI/MiniMax-M3-MXFP8 - model-prefix: minimaxm3 - runner: h100 - precision: fp8 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # DEP (dp-attn) omitted on H100: each DP rank replicates the ~20 GB - # BF16-dequantized attention/dense/embedding weights next to its - # ~52 GB expert shard, and KV-cache init fails at high conc (sweep - # 27441767143, conc 256/512: "No available memory for the cache - # blocks"). TEP8 covers the high-concurrency regime instead. - - { tp: 8, conc-start: 1, conc-end: 128 } - - { tp: 8, ep: 8, conc-start: 1, conc-end: 256 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, conc-start: 1, conc-end: 64 } - - { tp: 8, ep: 8, conc-start: 1, conc-end: 256 } - -dsr1-fp8-h100-dynamo-sglang: - image: lmsysorg/sglang:v0.5.8-cu130 - model: deepseek-ai/DeepSeek-R1-0528 - model-prefix: dsr1 - runner: h100-multinode - precision: fp8 - framework: dynamo-sglang - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # # STP: Max throughput TEP (1 prefill, 2 decode) - # - conc-list: [1, 2, 4, 8, 16, 32, 64, 128] - # prefill: - # num-worker: 1 - # tp: 16 - # ep: 1 - # dp-attn: false - # additional-settings: - # - "CONFIG_FILE=recipes/h100/1k1k/stp/h100-fp8-1p2d-max-tp.yaml" - # decode: - # num-worker: 2 - # tp: 16 - # ep: 1 - # dp-attn: false - # # STP: Max throughput DEP (1 prefill, 1 decode, dp-attention) - # - conc-list: [1, 2, 4, 8, 16, 32, 64] - # prefill: - # num-worker: 1 - # tp: 16 - # ep: 1 - # dp-attn: false - # additional-settings: - # - "CONFIG_FILE=recipes/h100/1k1k/stp/h100-fp8-1p1d-max-dep.yaml" - # decode: - # num-worker: 1 - # tp: 16 - # ep: 16 - # dp-attn: true - # MTP: Max throughput TEP (1 prefill, 2 decode) - - spec-decoding: "mtp" - conc-list: [1, 2, 4, 8, 16, 32, 64, 128] - prefill: - num-worker: 1 - tp: 16 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/h100/1k1k/mtp/h100-fp8-1p2d-max-tp-mtp.yaml" - decode: - num-worker: 2 - tp: 16 - ep: 1 - dp-attn: false - # MTP: Max throughput DEP (1 prefill, 1 decode, dp-attention) - - spec-decoding: "mtp" - conc-list: [1, 2, 4, 8, 16, 32, 64] - prefill: - num-worker: 1 - tp: 16 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/h100/1k1k/mtp/h100-fp8-1p1d-max-dep-mtp.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - isl: 8192 - osl: 1024 - search-space: - # # STP: Max throughput TEP (1 prefill, 1 decode) - # - conc-list: [1, 2, 4, 8, 16, 32, 64, 128] - # prefill: - # num-worker: 1 - # tp: 16 - # ep: 1 - # dp-attn: false - # additional-settings: - # - "CONFIG_FILE=recipes/h100/8k1k/stp/h100-fp8-1p1d-max-tp.yaml" - # decode: - # num-worker: 1 - # tp: 16 - # ep: 1 - # dp-attn: false - # # STP: Max throughput DEP (1 prefill, 1 decode, dp-attention) - # - conc-list: [1, 2, 4, 8, 16, 32, 64] - # prefill: - # num-worker: 1 - # tp: 16 - # ep: 1 - # dp-attn: false - # additional-settings: - # - "CONFIG_FILE=recipes/h100/8k1k/stp/h100-fp8-1p1d-max-dep.yaml" - # decode: - # num-worker: 1 - # tp: 16 - # ep: 16 - # dp-attn: true - # MTP: Max throughput TEP (1 prefill, 1 decode) - - spec-decoding: "mtp" - conc-list: [1, 2, 4, 8, 16, 32, 64, 128] - prefill: - num-worker: 1 - tp: 16 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/h100/8k1k/mtp/h100-fp8-1p1d-max-tp-mtp.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 1 - dp-attn: false - # MTP: Max throughput DEP (1 prefill, 1 decode, dp-attention) - - spec-decoding: "mtp" - conc-list: [1, 2, 4, 8, 16, 32, 64] - prefill: - num-worker: 1 - tp: 16 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/h100/8k1k/mtp/h100-fp8-1p1d-max-dep-mtp.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - -gptoss-fp4-h200-trt: - image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc14 - model: openai/gpt-oss-120b - model-prefix: gptoss - runner: h200 - precision: fp4 - framework: trt - multinode: false - # For all sequence lengths, EP=TP, DP_ATTENTION=false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 1, ep: 1, dp-attn: false, conc-start: 4, conc-end: 64 } - - { tp: 2, ep: 2, dp-attn: false, conc-start: 4, conc-end: 64 } - - { tp: 4, ep: 4, dp-attn: false, conc-start: 4, conc-end: 32 } - - { tp: 8, ep: 8, dp-attn: false, conc-start: 4, conc-end: 8 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 1, ep: 1, dp-attn: false, conc-start: 4, conc-end: 64 } - - { tp: 2, ep: 2, dp-attn: false, conc-start: 4, conc-end: 64 } - - { tp: 4, ep: 4, dp-attn: false, conc-start: 4, conc-end: 64 } - - { tp: 8, ep: 8, dp-attn: false, conc-start: 4, conc-end: 8 } - -gptoss-fp4-h200-vllm: - image: vllm/vllm-openai:v0.22.0 - model: openai/gpt-oss-120b - model-prefix: gptoss - runner: h200 - precision: fp4 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 1, conc-start: 4, conc-end: 4 } - - { tp: 2, conc-start: 4, conc-end: 64 } - - { tp: 4, conc-start: 4, conc-end: 64 } - - { tp: 8, conc-start: 4, conc-end: 64 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 1, conc-start: 4, conc-end: 64 } - - { tp: 2, conc-start: 4, conc-end: 64 } - - { tp: 4, conc-start: 4, conc-end: 64 } - - { tp: 8, conc-start: 4, conc-end: 32 } - -# Day-zero MiniMax-M3 recipe (https://recipes.vllm.ai/MiniMaxAI/MiniMax-M3). -# Dedicated vllm/vllm-openai:minimax-m3 image (no stable release has M3 yet). -# MXFP8 variant (~427 GB weights) is the lowest precision available; on -# 8x H200 (1128 GB) it leaves ample KV headroom where BF16 is a tight fit. -# TP4 (~112 GB weights/GPU) is memory-tight — swept only at low/mid conc. -# dp-attn: true maps to the recipe's "DP8 + Expert Parallel" serve mode. -minimaxm3-fp8-h200-vllm: - image: vllm/vllm-openai:minimax-m3 - model: MiniMaxAI/MiniMax-M3-MXFP8 - model-prefix: minimaxm3 - runner: h200 - precision: fp8 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 4, conc-start: 1, conc-end: 64 } - - { tp: 4, ep: 4, conc-start: 1, conc-end: 256 } - - { tp: 8, conc-start: 1, conc-end: 128 } - - { tp: 8, ep: 8, conc-start: 1, conc-end: 256 } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 256, conc-end: 1024 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 4, conc-start: 1, conc-end: 32 } - - { tp: 4, ep: 4, conc-start: 1, conc-end: 256 } - - { tp: 8, conc-start: 1, conc-end: 128 } - - { tp: 8, ep: 8, conc-start: 1, conc-end: 256 } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 256, conc-end: 512 } - -dsr1-fp4-gb200-dynamo-trt: - image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post2 - model: nvidia/DeepSeek-R1-0528-NVFP4-v2 - model-prefix: dsr1 - runner: gb200 - precision: fp4 - framework: dynamo-trt - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # MTP configurations (spec_decoding="mtp") - - spec-decoding: "mtp" - conc-list: [ 180 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/1k1k/mtp/ctx1_gen1_dep32_batch4_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp4/1k1k/mtp/ctx1_gen1_dep32_batch4_eplb0_mtp3.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [ 4, 8, 12, 24, 48 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/1k1k/mtp/ctx1_gen4_tep8_batch8_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp4/1k1k/mtp/ctx1_gen4_tep8_batch8_eplb0_mtp3.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [ 4301 ] - prefill: - num-worker: 2 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/1k1k/mtp/ctx2_gen1_dep16_batch256_eplb256_mtp1.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp4/1k1k/mtp/ctx2_gen1_dep16_batch256_eplb256_mtp1.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [ 2253 ] - prefill: - num-worker: 3 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/1k1k/mtp/ctx3_gen1_dep32_batch64_eplb288_mtp1.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp4/1k1k/mtp/ctx3_gen1_dep32_batch64_eplb288_mtp1.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [ 16130 ] - prefill: - num-worker: 3 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/1k1k/mtp/ctx3_gen5_dep4_batch768_eplb0_mtp1.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp4/1k1k/mtp/ctx3_gen5_dep4_batch768_eplb0_mtp1.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: true - - # Non-MTP configurations (default spec_decoding="none") - - conc-list: [ 4301 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/1k1k/stp/ctx1_gen1_dep8_batch512_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp4/1k1k/stp/ctx1_gen1_dep8_batch512_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - conc-list: [ 666 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/1k1k/stp/ctx1_gen1_dep32_batch16_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp4/1k1k/stp/ctx1_gen1_dep32_batch16_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [ 6144 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/1k1k/stp/ctx1_gen2_dep4_batch768_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp4/1k1k/stp/ctx1_gen2_dep4_batch768_eplb0_mtp0.yaml" - decode: - num-worker: 2 - tp: 4 - ep: 4 - dp-attn: true - - conc-list: [ 12, 24, 48, 96, 192 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/1k1k/stp/ctx1_gen4_tep8_batch32_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp4/1k1k/stp/ctx1_gen4_tep8_batch32_eplb0_mtp0.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [ 5 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/1k1k/stp/ctx1_gen4_tep8_batch1_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp4/1k1k/stp/ctx1_gen4_tep8_batch1_eplb0_mtp0.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [ 4301 ] - prefill: - num-worker: 2 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/1k1k/stp/ctx2_gen1_dep16_batch256_eplb256_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp4/1k1k/stp/ctx2_gen1_dep16_batch256_eplb256_mtp0.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - conc-list: [ 2253 ] - prefill: - num-worker: 2 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/1k1k/stp/ctx2_gen1_dep32_batch64_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp4/1k1k/stp/ctx2_gen1_dep32_batch64_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - - isl: 8192 - osl: 1024 - search-space: - # MTP configurations (spec_decoding="mtp") - - spec-decoding: "mtp" - conc-list: [ 4, 8, 12, 24, 48 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/mtp/ctx1_gen4_tep8_batch8_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp4/8k1k/mtp/ctx1_gen4_tep8_batch8_eplb0_mtp3.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [ 180 ] - prefill: - num-worker: 3 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/mtp/ctx3_gen1_dep32_batch4_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp4/8k1k/mtp/ctx3_gen1_dep32_batch4_eplb0_mtp3.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [ 1229 ] - prefill: - num-worker: 7 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/mtp/ctx7_gen1_dep16_batch64_eplb256_mtp1.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp4/8k1k/mtp/ctx7_gen1_dep16_batch64_eplb256_mtp1.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [ 666 ] - prefill: - num-worker: 8 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/mtp/ctx8_gen1_dep32_batch16_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp4/8k1k/mtp/ctx8_gen1_dep32_batch16_eplb0_mtp3.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [ 4301 ] - prefill: - num-worker: 11 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/mtp/ctx11_gen1_dep16_batch256_eplb256_mtp1.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp4/8k1k/mtp/ctx11_gen1_dep16_batch256_eplb256_mtp1.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - # Non-MTP configurations (default spec_decoding="none") - - conc-list: [ 12, 44, 76 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/stp/ctx1_gen4_tep8_batch16_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp4/8k1k/stp/ctx1_gen4_tep8_batch16_eplb0_mtp0.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [ 5 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/stp/ctx1_gen4_tep8_batch1_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp4/8k1k/stp/ctx1_gen4_tep8_batch1_eplb0_mtp0.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [ 333 ] - prefill: - num-worker: 2 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/stp/ctx2_gen1_dep32_batch8_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp4/8k1k/stp/ctx2_gen1_dep32_batch8_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [ 1229 ] - prefill: - num-worker: 7 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/stp/ctx7_gen1_dep32_batch32_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp4/8k1k/stp/ctx7_gen1_dep32_batch32_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [ 2253 ] - prefill: - num-worker: 8 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/stp/ctx8_gen1_dep16_batch128_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp4/8k1k/stp/ctx8_gen1_dep16_batch128_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - conc-list: [ 4096 ] - prefill: - num-worker: 10 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp4/8k1k/stp/ctx10_gen1_dep16_batch256_eplb256_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp4/8k1k/stp/ctx10_gen1_dep16_batch256_eplb256_mtp0.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - -dsr1-fp8-gb200-dynamo-trt: - image: nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post2 - model: deepseek-ai/DeepSeek-R1-0528 - model-prefix: dsr1 - runner: gb200 - precision: fp8 - framework: dynamo-trt - multinode: true - disagg: true - scenarios: - fixed-seq-len: - # 1k1k MTP configs - - isl: 1024 - osl: 1024 - search-space: - - spec-decoding: "mtp" - conc-list: [4301] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/1k1k/mtp/ctx1_gen1_dep8_batch512_eplb0_mtp1_4301.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/1k1k/mtp/ctx1_gen1_dep8_batch512_eplb0_mtp1_4301.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [2151] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/1k1k/mtp/ctx1_gen1_dep8_batch256_eplb0_mtp1_2151.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/1k1k/mtp/ctx1_gen1_dep8_batch256_eplb0_mtp1_2151.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [1229] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/1k1k/mtp/ctx1_gen1_dep16_batch64_eplb0_mtp1_1229.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/1k1k/mtp/ctx1_gen1_dep16_batch64_eplb0_mtp1_1229.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [615] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/1k1k/mtp/ctx1_gen1_dep32_batch16_eplb0_mtp3_615.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/1k1k/mtp/ctx1_gen1_dep32_batch16_eplb0_mtp3_615.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [36] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/1k1k/mtp/ctx1_gen3_tep8_batch8_eplb0_mtp3_36.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/1k1k/mtp/ctx1_gen3_tep8_batch8_eplb0_mtp3_36.yaml" - decode: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [18] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/1k1k/mtp/ctx1_gen3_tep8_batch4_eplb0_mtp3_18.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/1k1k/mtp/ctx1_gen3_tep8_batch4_eplb0_mtp3_18.yaml" - decode: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [9] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/1k1k/mtp/ctx1_gen3_tep8_batch2_eplb0_mtp3_9.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/1k1k/mtp/ctx1_gen3_tep8_batch2_eplb0_mtp3_9.yaml" - decode: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: false - # 1k1k STP configs - - conc-list: [6144] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/1k1k/stp/ctx1_gen1_dep8_batch768_eplb0_mtp0_6144.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/1k1k/stp/ctx1_gen1_dep8_batch768_eplb0_mtp0_6144.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - conc-list: [4301] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/1k1k/stp/ctx1_gen1_dep8_batch512_eplb0_mtp0_4301.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/1k1k/stp/ctx1_gen1_dep8_batch512_eplb0_mtp0_4301.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - conc-list: [2151] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/1k1k/stp/ctx1_gen1_dep16_batch128_eplb0_mtp0_2151.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/1k1k/stp/ctx1_gen1_dep16_batch128_eplb0_mtp0_2151.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - conc-list: [1127] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/1k1k/stp/ctx1_gen1_dep32_batch32_eplb0_mtp0_1127.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/1k1k/stp/ctx1_gen1_dep32_batch32_eplb0_mtp0_1127.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [256] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/1k1k/stp/ctx1_gen1_dep32_batch8_eplb0_mtp0_256.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/1k1k/stp/ctx1_gen1_dep32_batch8_eplb0_mtp0_256.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [27] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/1k1k/stp/ctx1_gen3_tep8_batch8_eplb0_mtp0_27.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/1k1k/stp/ctx1_gen3_tep8_batch8_eplb0_mtp0_27.yaml" - decode: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [3] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/1k1k/stp/ctx1_gen3_tep8_batch1_eplb0_mtp0_3.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/1k1k/stp/ctx1_gen3_tep8_batch1_eplb0_mtp0_3.yaml" - decode: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: false - # 8k1k MTP configs - - isl: 8192 - osl: 1024 - search-space: - - spec-decoding: "mtp" - conc-list: [666] - prefill: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/mtp/ctx3_gen1_dep8_batch64_eplb0_mtp3_666.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/mtp/ctx3_gen1_dep8_batch64_eplb0_mtp3_666.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [666] - prefill: - num-worker: 5 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/mtp/ctx5_gen1_dep16_batch32_eplb0_mtp3_666.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/mtp/ctx5_gen1_dep16_batch32_eplb0_mtp3_666.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [333] - prefill: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/mtp/ctx3_gen1_dep16_batch16_eplb0_mtp3_333.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/mtp/ctx3_gen1_dep16_batch16_eplb0_mtp3_333.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [333] - prefill: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/mtp/ctx4_gen1_dep32_batch8_eplb0_mtp3_333.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/mtp/ctx4_gen1_dep32_batch8_eplb0_mtp3_333.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [90] - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/mtp/ctx2_gen1_dep32_batch2_eplb0_mtp3_90.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/mtp/ctx2_gen1_dep32_batch2_eplb0_mtp3_90.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [15] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/mtp/ctx1_gen3_tep8_batch4_eplb0_mtp3_15.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/mtp/ctx1_gen3_tep8_batch4_eplb0_mtp3_15.yaml" - decode: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [6] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/mtp/ctx1_gen3_tep8_batch2_eplb0_mtp3_6.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/mtp/ctx1_gen3_tep8_batch2_eplb0_mtp3_6.yaml" - decode: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: false - # 8k1k STP configs - - conc-list: [1229] - prefill: - num-worker: 5 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/stp/ctx5_gen1_dep16_batch64_eplb0_mtp0_1229.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/stp/ctx5_gen1_dep16_batch64_eplb0_mtp0_1229.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - conc-list: [666] - prefill: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/stp/ctx4_gen1_dep32_batch16_eplb0_mtp0_666.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/stp/ctx4_gen1_dep32_batch16_eplb0_mtp0_666.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [615] - prefill: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/stp/ctx3_gen1_dep16_batch32_eplb0_mtp0_615.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/stp/ctx3_gen1_dep16_batch32_eplb0_mtp0_615.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - conc-list: [333] - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/stp/ctx2_gen1_dep32_batch8_eplb0_mtp0_333.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/stp/ctx2_gen1_dep32_batch8_eplb0_mtp0_333.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [63] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/stp/ctx1_gen3_tep8_batch16_eplb0_mtp0_63.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/stp/ctx1_gen3_tep8_batch16_eplb0_mtp0_63.yaml" - decode: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [18] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/stp/ctx1_gen3_tep8_batch4_eplb0_mtp0_18.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/stp/ctx1_gen3_tep8_batch4_eplb0_mtp0_18.yaml" - decode: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [6] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb200-fp8/8k1k/stp/ctx1_gen3_tep8_batch1_eplb0_mtp0_6.yaml - - "CONFIG_FILE=recipes/trtllm/gb200-fp8/8k1k/stp/ctx1_gen3_tep8_batch1_eplb0_mtp0_6.yaml" - decode: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: false - -dsr1-fp8-gb200-dynamo-sglang: - image: lmsysorg/sglang:v0.5.8.post1-cu130 - model: deepseek-ai/DeepSeek-R1-0528 - model-prefix: dsr1 - runner: gb200 - precision: fp8 - framework: dynamo-sglang - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # "Low latency" (1 prefill worker at TP4 and 1 decode worker at TP4) - - conc-list: [4, 8] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/gb200-fp8/1k1k/low-latency.yaml - - "CONFIG_FILE=recipes/gb200-fp8/1k1k/low-latency.yaml" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - - # "Mid curve" (3 prefill workers at DEP8 and 1 decode worker at DEP48) - - conc-list: [1024, 2048, 4096] - prefill: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/gb200-fp8/1k1k/mid-curve.yaml - - "CONFIG_FILE=recipes/gb200-fp8/1k1k/mid-curve.yaml" - decode: - num-worker: 1 - tp: 48 - ep: 48 - dp-attn: true - - # "Max throughput" (2 prefill workers at DEP8 and 1 decode worker at DEP32) - - conc-list: [1024, 2048, 4096, 6144] - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/gb200-fp8/1k1k/max-tpt.yaml - - "CONFIG_FILE=recipes/gb200-fp8/1k1k/max-tpt.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - # "Ultra throughput" (1 prefill workers at DEP8 and 1 decode worker at DEP8) - - conc-list: [4096] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/gb200-fp8/1k1k/ultra-tpt.yaml - - "CONFIG_FILE=recipes/gb200-fp8/1k1k/ultra-tpt.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - - isl: 8192 - osl: 1024 - search-space: - # "Low latency" (1 prefill worker at TP8 and 1 decode worker at TP8) - - conc-list: [4, 8, 16] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/gb200-fp8/8k1k/low-latency.yaml - - "CONFIG_FILE=recipes/gb200-fp8/8k1k/low-latency.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - - # "Mid curve" (5 prefill workers at DEP8 and 1 decode worker at DEP32) - - conc-list: [512, 1024, 2048, 6144] - prefill: - num-worker: 5 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/gb200-fp8/8k1k/mid-curve.yaml - - "CONFIG_FILE=recipes/gb200-fp8/8k1k/mid-curve.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - # "Max throughput" (6 prefill workers at DEP8 and 1 decode worker at DEP24) - - conc-list: [2048, 4096, 6144] - prefill: - num-worker: 6 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/gb200-fp8/8k1k/max_tpt.yaml - - "CONFIG_FILE=recipes/gb200-fp8/8k1k/max_tpt.yaml" - decode: - num-worker: 1 - tp: 24 - ep: 24 - dp-attn: true - -dsr1-fp8-gb300-dynamo-sglang: - image: lmsysorg/sglang:v0.5.8.post1-cu130 - model: deepseek-ai/DeepSeek-R1-0528 - model-prefix: dsr1 - runner: gb300 - precision: fp8 - framework: dynamo-sglang - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # "Low latency" (1 prefill worker at TP4 and 4 decode workers at TP4) - - conc-list: [4, 8, 16, 32] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/gb300-fp8/1k1k/stp/low-latency.yaml - - "CONFIG_FILE=recipes/gb300-fp8/1k1k/stp/low-latency.yaml" - decode: - num-worker: 4 - tp: 4 - ep: 1 - dp-attn: false - - # "Mid curve" (2 prefill workers at DEP8 and 1 decode worker at DEP32) - - conc-list: [1024, 2048, 4096, 6144] - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/gb300-fp8/1k1k/stp/mid.yaml - - "CONFIG_FILE=recipes/gb300-fp8/1k1k/stp/mid.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - # "Max throughput" (1 prefill worker at DEP8 and 1 decode worker at DEP8) - - conc-list: [4096, 7168, 7680] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/gb300-fp8/1k1k/stp/max.yaml - - "CONFIG_FILE=recipes/gb300-fp8/1k1k/stp/max.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - - isl: 8192 - osl: 1024 - search-space: - # "Low latency" (1 prefill worker at TP4 and 1 decode worker at TP4) - - conc-list: [4, 8] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/gb300-fp8/8k1k/stp/low-latency.yaml - - "CONFIG_FILE=recipes/gb300-fp8/8k1k/stp/low-latency.yaml" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - - # "Mid curve" (5 prefill workers at DEP8 and 1 decode worker at DEP32) - - conc-list: [128, 256, 512, 1024] - prefill: - num-worker: 5 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/gb300-fp8/8k1k/stp/mid.yaml - - "CONFIG_FILE=recipes/gb300-fp8/8k1k/stp/mid.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - # "Max throughput" (6 prefill workers at DEP8 and 1 decode worker at DEP24) - - conc-list: [2048, 4096] - prefill: - num-worker: 6 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/gb300-fp8/8k1k/stp/max.yaml - - "CONFIG_FILE=recipes/gb300-fp8/8k1k/stp/max.yaml" - decode: - num-worker: 1 - tp: 24 - ep: 24 - dp-attn: true - -dsr1-fp4-gb200-dynamo-sglang: - image: "lmsysorg/sglang:v0.5.8-cu130" - model: nvidia/DeepSeek-R1-0528-NVFP4-v2 - model-prefix: dsr1 - runner: gb200 - precision: fp4 - framework: dynamo-sglang - multinode: true - disagg: true - scenarios: - fixed-seq-len: - # 1k1k configurations - - isl: 1024 - osl: 1024 - search-space: - # Low latency (1 prefill node, 2 decode nodes) - - spec-decoding: "none" - conc-list: [ 4, 8, 32 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/gb200-fp4/1k1k/low-latency.yaml" - decode: - num-worker: 2 - tp: 4 - ep: 1 - dp-attn: false - - # Mid curve (4 prefill nodes, 8 decode nodes) - - spec-decoding: "none" - conc-list: [ 512, 2048, 4096, 8192 ] - prefill: - num-worker: 4 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/gb200-fp4/1k1k/mid-curve.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - # Max throughput (4 prefill nodes, 12 decode nodes) - - spec-decoding: "none" - conc-list: [ 2048, 4096 ] - prefill: - num-worker: 4 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/gb200-fp4/1k1k/max-tpt.yaml" - decode: - num-worker: 1 - tp: 48 - ep: 48 - dp-attn: true - - # 8k1k configurations - - isl: 8192 - osl: 1024 - search-space: - # Low latency (1 prefill node, 4 decode nodes) - - spec-decoding: "none" - conc-list: [ 4, 8 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/gb200-fp4/8k1k/low-latency.yaml" - decode: - num-worker: 4 - tp: 4 - ep: 1 - dp-attn: false - - # Mid curve (6 prefill nodes, 12 decode nodes) - - spec-decoding: "none" - conc-list: [ 512, 2048, 4096 ] - prefill: - num-worker: 6 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/gb200-fp4/8k1k/mid-curve.yaml" - decode: - num-worker: 1 - tp: 48 - ep: 48 - dp-attn: true - - # Max throughput (10 prefill nodes, 8 decode nodes) - - spec-decoding: "none" - conc-list: [ 2048 ] - prefill: - num-worker: 10 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/gb200-fp4/8k1k/max-tpt.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - -dsr1-fp4-gb300-dynamo-trt: - image: nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post2 - model: nvidia/DeepSeek-R1-0528-NVFP4-v2 - model-prefix: dsr1 - runner: gb300 - precision: fp4 - framework: dynamo-trt - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # MTP configurations - - spec-decoding: "mtp" - conc-list: [3226] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/1k1k/mtp/ctx1_gen1_dep4_batch768_eplb0_mtp1.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/1k1k/mtp/ctx1_gen1_dep4_batch768_eplb0_mtp1.yaml" - decode: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [333] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/1k1k/mtp/ctx1_gen1_dep32_batch8_eplb0_mtp.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/1k1k/mtp/ctx1_gen1_dep32_batch8_eplb0_mtp.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [5] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/1k1k/mtp/ctx1_gen4_tep8_batch1_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/1k1k/mtp/ctx1_gen4_tep8_batch1_eplb0_mtp3.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [8, 12, 24, 48] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/1k1k/mtp/ctx1_gen4_tep8_batch8_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/1k1k/mtp/ctx1_gen4_tep8_batch8_eplb0_mtp3.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [2253] - prefill: - num-worker: 3 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/1k1k/mtp/ctx3_gen1_dep16_batch128_eplb256_mtp1.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/1k1k/mtp/ctx3_gen1_dep16_batch128_eplb256_mtp1.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [1229] - prefill: - num-worker: 3 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/1k1k/mtp/ctx3_gen1_dep32_batch32_eplb288_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/1k1k/mtp/ctx3_gen1_dep32_batch32_eplb288_mtp3.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - # Non-MTP configurations (default spec_decoding="none") - - conc-list: [5] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/1k1k/stp/ctx1_gen4_tep8_batch1_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/1k1k/stp/ctx1_gen4_tep8_batch1_eplb0_mtp0.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [12, 48, 96, 192] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/1k1k/stp/ctx1_gen4_tep8_batch32_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/1k1k/stp/ctx1_gen4_tep8_batch32_eplb0_mtp0.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [8192] - prefill: - num-worker: 2 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/1k1k/stp/ctx2_gen1_dep8_batch1024_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/1k1k/stp/ctx2_gen1_dep8_batch1024_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - conc-list: [1229] - prefill: - num-worker: 2 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/1k1k/stp/ctx2_gen1_dep32_batch32_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/1k1k/stp/ctx2_gen1_dep32_batch32_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [4301] - prefill: - num-worker: 3 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/1k1k/stp/ctx3_gen1_dep16_batch256_eplb256_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/1k1k/stp/ctx3_gen1_dep16_batch256_eplb256_mtp0.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - conc-list: [2253] - prefill: - num-worker: 3 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/1k1k/stp/ctx3_gen1_dep32_batch64_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/1k1k/stp/ctx3_gen1_dep32_batch64_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - isl: 8192 - osl: 1024 - search-space: - # MTP configurations (spec_decoding="mtp") - - spec-decoding: "mtp" - conc-list: [33] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/mtp/ctx1_gen3_tep8_batch8_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/mtp/ctx1_gen3_tep8_batch8_eplb0_mtp3.yaml" - decode: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [5] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/mtp/ctx1_gen4_tep8_batch1_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/mtp/ctx1_gen4_tep8_batch1_eplb0_mtp3.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [12, 24] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/mtp/ctx1_gen4_tep8_batch4_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/mtp/ctx1_gen4_tep8_batch4_eplb0_mtp3.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [180] - prefill: - num-worker: 4 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/mtp/ctx4_gen1_dep32_batch4_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/mtp/ctx4_gen1_dep32_batch4_eplb0_mtp3.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [308] - prefill: - num-worker: 8 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/mtp/ctx8_gen1_dep32_batch8_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/mtp/ctx8_gen1_dep32_batch8_eplb0_mtp3.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [2253] - prefill: - num-worker: 10 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/mtp/ctx10_gen1_dep8_batch256_eplb0_mtp1.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/mtp/ctx10_gen1_dep8_batch256_eplb0_mtp1.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [666] - prefill: - num-worker: 10 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/mtp/ctx10_gen1_dep16_batch32_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/mtp/ctx10_gen1_dep16_batch32_eplb0_mtp3.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [1127] - prefill: - num-worker: 13 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/mtp/ctx13_gen1_dep16_batch64_eplb256_mtp3.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/mtp/ctx13_gen1_dep16_batch64_eplb256_mtp3.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - # Non-MTP configurations (default spec_decoding="none") - - conc-list: [72] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/stp/ctx1_gen3_tep8_batch16_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/stp/ctx1_gen3_tep8_batch16_eplb0_mtp0.yaml" - decode: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [5] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/stp/ctx1_gen4_tep8_batch1_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/stp/ctx1_gen4_tep8_batch1_eplb0_mtp0.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [12] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/stp/ctx1_gen4_tep8_batch2_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/stp/ctx1_gen4_tep8_batch2_eplb0_mtp0.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [5, 15, 30] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/stp/ctx1_gen5_tep4_batch4_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/stp/ctx1_gen5_tep4_batch4_eplb0_mtp0.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - conc-list: [666] - prefill: - num-worker: 7 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/stp/ctx7_gen1_dep32_batch16_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/stp/ctx7_gen1_dep32_batch16_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [1229] - prefill: - num-worker: 9 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/stp/ctx9_gen1_dep16_batch64_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/stp/ctx9_gen1_dep16_batch64_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - conc-list: [3228] - prefill: - num-worker: 11 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/stp/ctx11_gen3_dep4_batch256_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/stp/ctx11_gen3_dep4_batch256_eplb0_mtp0.yaml" - decode: - num-worker: 3 - tp: 4 - ep: 4 - dp-attn: true - - conc-list: [2253] - prefill: - num-worker: 14 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp4/8k1k/stp/ctx14_gen1_dep16_batch128_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp4/8k1k/stp/ctx14_gen1_dep16_batch128_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true -dsr1-fp4-gb300-dynamo-sglang: - image: "lmsysorg/sglang:v0.5.8.post1-cu130-runtime" - model: nvidia/DeepSeek-R1-0528-NVFP4-v2 - model-prefix: dsr1 - runner: gb300 - precision: fp4 - framework: dynamo-sglang - multinode: true - disagg: true - scenarios: - fixed-seq-len: - # 1k1k configurations - - isl: 1024 - osl: 1024 - search-space: - # Low latency (1 prefill node, 2 decode nodes) - - spec-decoding: "none" - conc-list: [ 4, 8, 32 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/gb300-fp4/1k1k/low_latency.yaml" - decode: - num-worker: 2 - tp: 4 - ep: 1 - dp-attn: false - - # Mid curve (4 prefill nodes, 8 decode nodes) - - spec-decoding: "none" - conc-list: [ 512, 2048, 4096, 8192 ] - prefill: - num-worker: 4 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/gb300-fp4/1k1k/mid_curve.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - # Max throughput (4 prefill nodes, 12 decode nodes) - - spec-decoding: "none" - conc-list: [ 512, 2048, 4096, 8192 ] - prefill: - num-worker: 4 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/gb300-fp4/1k1k/max_tpt.yaml" - decode: - num-worker: 1 - tp: 48 - ep: 48 - dp-attn: true - - # 8k1k configurations - - isl: 8192 - osl: 1024 - search-space: - # Low latency (1 prefill node, 4 decode nodes) - - spec-decoding: "none" - conc-list: [ 4, 8, 32, 64 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/gb300-fp4/8k1k/low_latency.yaml" - decode: - num-worker: 4 - tp: 4 - ep: 1 - dp-attn: false - - # Mid curve (6 prefill nodes, 12 decode nodes) - - spec-decoding: "none" - conc-list: [ 512, 2048, 4096 ] - prefill: - num-worker: 6 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/gb300-fp4/8k1k/mid_curve.yaml" - decode: - num-worker: 1 - tp: 48 - ep: 48 - dp-attn: true - - # Max throughput (10 prefill nodes, 8 decode nodes) - - spec-decoding: "none" - conc-list: [ 2048 ] - prefill: - num-worker: 10 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/gb300-fp4/8k1k/max_tpt.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - -dsr1-fp8-gb300-dynamo-trt: - image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - model: deepseek-ai/DeepSeek-R1-0528 - model-prefix: dsr1 - runner: gb300-nv - precision: fp8 - framework: dynamo-trt - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # MTP configurations (spec_decoding="mtp") - - spec-decoding: "mtp" - conc-list: [8] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/1k1k/mtp/ctx1_gen4_tep8_batch1_eplb0_mtp3_8.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/1k1k/mtp/ctx1_gen4_tep8_batch1_eplb0_mtp3_8.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [24] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/1k1k/mtp/ctx1_gen4_tep8_batch4_eplb0_mtp3_24.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/1k1k/mtp/ctx1_gen4_tep8_batch4_eplb0_mtp3_24.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [180] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/1k1k/mtp/ctx1_gen1_dep32_batch4_eplb0_mtp3_180.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/1k1k/mtp/ctx1_gen1_dep32_batch4_eplb0_mtp3_180.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [564] - prefill: - num-worker: 2 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/1k1k/mtp/ctx2_gen1_dep32_batch16_eplb0_mtp3_564.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/1k1k/mtp/ctx2_gen1_dep32_batch16_eplb0_mtp3_564.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [666] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/1k1k/mtp/ctx1_gen1_dep16_batch32_eplb0_mtp3_666.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/1k1k/mtp/ctx1_gen1_dep16_batch32_eplb0_mtp3_666.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [2253] - prefill: - num-worker: 2 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/1k1k/mtp/ctx2_gen1_dep16_batch128_eplb0_mtp1_2253.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/1k1k/mtp/ctx2_gen1_dep16_batch128_eplb0_mtp1_2253.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [8192] - prefill: - num-worker: 3 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/1k1k/mtp/ctx3_gen2_dep8_batch512_eplb0_mtp1_8192.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/1k1k/mtp/ctx3_gen2_dep8_batch512_eplb0_mtp1_8192.yaml" - decode: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - # STP configurations (no spec_decoding) - - conc-list: [4] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/1k1k/stp/ctx1_gen4_tep8_batch1_eplb0_mtp0_4.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/1k1k/stp/ctx1_gen4_tep8_batch1_eplb0_mtp0_4.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [24] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/1k1k/stp/ctx1_gen4_tep8_batch4_eplb0_mtp0_24.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/1k1k/stp/ctx1_gen4_tep8_batch4_eplb0_mtp0_24.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [84] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/1k1k/stp/ctx1_gen4_tep8_batch16_eplb0_mtp0_84.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/1k1k/stp/ctx1_gen4_tep8_batch16_eplb0_mtp0_84.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [1229] - prefill: - num-worker: 2 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/1k1k/stp/ctx2_gen1_dep32_batch32_eplb0_mtp0_1229.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/1k1k/stp/ctx2_gen1_dep32_batch32_eplb0_mtp0_1229.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [2253] - prefill: - num-worker: 2 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/1k1k/stp/ctx2_gen1_dep16_batch128_eplb0_mtp0_2253.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/1k1k/stp/ctx2_gen1_dep16_batch128_eplb0_mtp0_2253.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - conc-list: [8602] - prefill: - num-worker: 3 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/1k1k/stp/ctx3_gen2_dep8_batch512_eplb0_mtp0_8602.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/1k1k/stp/ctx3_gen2_dep8_batch512_eplb0_mtp0_8602.yaml" - decode: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - - conc-list: [12288] - prefill: - num-worker: 3 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/1k1k/stp/ctx3_gen2_dep8_batch768_eplb0_mtp0_12288.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/1k1k/stp/ctx3_gen2_dep8_batch768_eplb0_mtp0_12288.yaml" - decode: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - - isl: 8192 - osl: 1024 - search-space: - # MTP configurations (spec_decoding="mtp") - - spec-decoding: "mtp" - conc-list: [8] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/mtp/ctx1_gen4_tep8_batch1_eplb0_mtp3_8.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/8k1k/mtp/ctx1_gen4_tep8_batch1_eplb0_mtp3_8.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [24] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/mtp/ctx1_gen4_tep8_batch4_eplb0_mtp3_24.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/8k1k/mtp/ctx1_gen4_tep8_batch4_eplb0_mtp3_24.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [333] - prefill: - num-worker: 6 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/mtp/ctx6_gen1_dep32_batch8_eplb0_mtp3_333.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/8k1k/mtp/ctx6_gen1_dep32_batch8_eplb0_mtp3_333.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [666] - prefill: - num-worker: 8 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/mtp/ctx8_gen1_dep16_batch32_eplb0_mtp3_666.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/8k1k/mtp/ctx8_gen1_dep16_batch32_eplb0_mtp3_666.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [1229] - prefill: - num-worker: 10 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/mtp/ctx10_gen1_dep16_batch64_eplb0_mtp1_1229.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/8k1k/mtp/ctx10_gen1_dep16_batch64_eplb0_mtp1_1229.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [1229] - prefill: - num-worker: 7 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/mtp/ctx7_gen1_dep8_batch128_eplb0_mtp1_1229.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/8k1k/mtp/ctx7_gen1_dep8_batch128_eplb0_mtp1_1229.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - # STP configurations (no spec_decoding) - - conc-list: [4] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/stp/ctx1_gen4_tep8_batch1_eplb0_mtp0_4.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/8k1k/stp/ctx1_gen4_tep8_batch1_eplb0_mtp0_4.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [24] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/stp/ctx1_gen4_tep8_batch4_eplb0_mtp0_24.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/8k1k/stp/ctx1_gen4_tep8_batch4_eplb0_mtp0_24.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [36] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/stp/ctx1_gen4_tep8_batch8_eplb0_mtp0_36.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/8k1k/stp/ctx1_gen4_tep8_batch8_eplb0_mtp0_36.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [512] - prefill: - num-worker: 6 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/stp/ctx6_gen1_dep32_batch16_eplb0_mtp0_512.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/8k1k/stp/ctx6_gen1_dep32_batch16_eplb0_mtp0_512.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [666] - prefill: - num-worker: 4 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/stp/ctx4_gen1_dep16_batch32_eplb0_mtp0_666.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/8k1k/stp/ctx4_gen1_dep16_batch32_eplb0_mtp0_666.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - conc-list: [1229] - prefill: - num-worker: 7 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/stp/ctx7_gen1_dep16_batch64_eplb0_mtp0_1229.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/8k1k/stp/ctx7_gen1_dep16_batch64_eplb0_mtp0_1229.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - conc-list: [2151] - prefill: - num-worker: 7 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/trtllm/gb300-fp8/8k1k/stp/ctx7_gen1_dep8_batch256_eplb0_mtp0_2151.yaml - - "CONFIG_FILE=recipes/trtllm/gb300-fp8/8k1k/stp/ctx7_gen1_dep8_batch256_eplb0_mtp0_2151.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true -gptoss-fp4-gb200-dynamo-trt: - image: nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:0.7.0.post2 - model: openai/gpt-oss-120b - model-prefix: gptoss - runner: gb200 - precision: fp4 - framework: dynamo-trt - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - #Right of pareto - #P: 1xTP1 D:1xTP4 - - spec-decoding: "none" - conc-list: [ 1, 2, 4, 16, 32, 64, 128 ] - prefill: - num-worker: 1 - tp: 1 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - "PREFILL_MAX_NUM_TOKENS=20000" - - "PREFILL_MAX_BATCH_SIZE=32" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MAX_NUM_TOKENS=20000" - - "DECODE_MAX_BATCH_SIZE=256" - - "DECODE_GPU_MEM_FRACTION=0.9" - - # P: 1xTP1 D:4xTP2 - - spec-decoding: "none" - conc-list: [ 16 ] - prefill: - num-worker: 1 - tp: 1 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - "PREFILL_MAX_NUM_TOKENS=20000" - - "PREFILL_MAX_BATCH_SIZE=32" - decode: - num-worker: 4 - tp: 2 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MAX_NUM_TOKENS=20000" - - "DECODE_MAX_BATCH_SIZE=32" - - "DECODE_GPU_MEM_FRACTION=0.9" - - # P: 1xTP1 D:1xDEP2 - - spec-decoding: "none" - conc-list: [ 256, 512, 1024, 2048, 2560 ] - prefill: - num-worker: 1 - tp: 1 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - "PREFILL_MAX_NUM_TOKENS=20000" - - "PREFILL_MAX_BATCH_SIZE=32" - decode: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MAX_NUM_TOKENS=20000" - - "DECODE_MAX_BATCH_SIZE=1536" - - "DECODE_GPU_MEM_FRACTION=0.9" - - # P: 1xTP1 D:2xDEP2 - - spec-decoding: "none" - conc-list: [ 512, 1024, 2048, 2560 ] - prefill: - num-worker: 1 - tp: 1 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - "PREFILL_MAX_NUM_TOKENS=20000" - - "PREFILL_MAX_BATCH_SIZE=32" - decode: - num-worker: 2 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MAX_NUM_TOKENS=20000" - - "DECODE_MAX_BATCH_SIZE=1536" - - "DECODE_GPU_MEM_FRACTION=0.9" - - # P: 1xTP1 D:1xDEP4 - - spec-decoding: "none" - conc-list: [ 256, 1024, 1536 ] - prefill: - num-worker: 1 - tp: 1 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - "PREFILL_MAX_NUM_TOKENS=20000" - - "PREFILL_MAX_BATCH_SIZE=32" - decode: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MAX_NUM_TOKENS=20000" - - "DECODE_MAX_BATCH_SIZE=512" - - "DECODE_GPU_MEM_FRACTION=0.9" - - # P: 1xTP1 D:3xDEP4 - - spec-decoding: "none" - conc-list: [ 3072 ] - prefill: - num-worker: 1 - tp: 1 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - "PREFILL_MAX_NUM_TOKENS=20000" - - "PREFILL_MAX_BATCH_SIZE=32" - decode: - num-worker: 3 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MAX_NUM_TOKENS=20000" - - "DECODE_MAX_BATCH_SIZE=1024" - - "DECODE_GPU_MEM_FRACTION=0.9" - - - isl: 8192 - osl: 1024 - search-space: - # Right side of pareto - - spec-decoding: "none" - conc-list: [1] - prefill: - num-worker: 1 - tp: 1 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - "PREFILL_MAX_NUM_TOKENS=20000" - - "PREFILL_MAX_BATCH_SIZE=32" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MAX_NUM_TOKENS=20000" - - "DECODE_MAX_BATCH_SIZE=4" - - "DECODE_GPU_MEM_FRACTION=0.9" - - - spec-decoding: "none" - conc-list: [2, 4, 8, 16, 32, 64] - prefill: - num-worker: 1 - tp: 1 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - "PREFILL_MAX_NUM_TOKENS=20000" - - "PREFILL_MAX_BATCH_SIZE=32" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MAX_NUM_TOKENS=20000" - - "DECODE_MAX_BATCH_SIZE=128" - - "DECODE_GPU_MEM_FRACTION=0.9" - - # Middle of pareto - # P: 2xTP1 D:1xTP4 - - spec-decoding: "none" - conc-list: [128, 512] - prefill: - num-worker: 2 - tp: 1 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - "PREFILL_MAX_NUM_TOKENS=20000" - - "PREFILL_MAX_BATCH_SIZE=32" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MAX_NUM_TOKENS=20000" - - "DECODE_MAX_BATCH_SIZE=1024" - - "DECODE_GPU_MEM_FRACTION=0.9" - - # P: 2xTP1 D:1xTP2 - - spec-decoding: "none" - conc-list: [256, 384] - prefill: - num-worker: 2 - tp: 1 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - "PREFILL_MAX_NUM_TOKENS=20000" - - "PREFILL_MAX_BATCH_SIZE=32" - decode: - num-worker: 1 - tp: 2 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MAX_NUM_TOKENS=20000" - - "DECODE_MAX_BATCH_SIZE=512" - - "DECODE_GPU_MEM_FRACTION=0.9" - - # P: 2xTP1 D:1xDEP2 - - spec-decoding: "none" - conc-list: [128, 512] - prefill: - num-worker: 2 - tp: 1 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - "PREFILL_MAX_NUM_TOKENS=20000" - - "PREFILL_MAX_BATCH_SIZE=32" - decode: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MAX_NUM_TOKENS=20000" - - "DECODE_MAX_BATCH_SIZE=512" - - "DECODE_GPU_MEM_FRACTION=0.9" - -dsr1-fp8-h200-dynamo-sglang: - image: lmsysorg/sglang:v0.5.8.post1-cu130 - model: deepseek-ai/DeepSeek-R1-0528 - model-prefix: dsr1 - runner: h200-multinode - precision: fp8 - framework: dynamo-sglang - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # STP: Low latency (1 prefill, 9 decode, TEP) - - spec-decoding: "none" - conc-list: [1, 4, 8, 16, 32, 64, 128, 256] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/h200/1k1k/low-latency-1p9d.yaml" - decode: - num-worker: 9 - tp: 8 - ep: 1 - dp-attn: false - # STP: High throughput TEP (1 prefill, 6 decode) - - spec-decoding: "none" - conc-list: [512, 1024, 2048] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/h200/1k1k/bs256-1p6d-tp.yaml" - decode: - num-worker: 6 - tp: 8 - ep: 1 - dp-attn: false - # STP: High throughput DEP (1 prefill, 6 decode, dp-attention) - - spec-decoding: "none" - conc-list: [128, 256, 512, 1024, 2048] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/h200/1k1k/bs256-1p6d-dep.yaml" - decode: - num-worker: 6 - tp: 8 - ep: 8 - dp-attn: true - # MTP: Low latency (1 prefill, 9 decode, TEP) - - spec-decoding: "mtp" - conc-list: [1, 4, 8, 16, 32, 64, 128, 256] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/h200/1k1k/low-latency-1p9d-mtp.yaml" - decode: - num-worker: 9 - tp: 8 - ep: 1 - dp-attn: false - # MTP: High throughput TEP (1 prefill, 6 decode) - - spec-decoding: "mtp" - conc-list: [512, 1024, 2048] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/h200/1k1k/bs256-1p6d-tp-mtp.yaml" - decode: - num-worker: 6 - tp: 8 - ep: 1 - dp-attn: false - # MTP: High throughput DEP (1 prefill, 6 decode, dp-attention) - - spec-decoding: "mtp" - conc-list: [128, 256, 512, 1024, 2048] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/h200/1k1k/bs256-1p6d-dep-mtp.yaml" - decode: - num-worker: 6 - tp: 8 - ep: 8 - dp-attn: true - - isl: 8192 - osl: 1024 - search-space: - # STP: Low latency TEP (1 prefill, 7 decode) - - spec-decoding: "none" - conc-list: [1, 4, 8] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/h200/8k1k/bs4-1p7d.yaml" - decode: - num-worker: 7 - tp: 8 - ep: 1 - dp-attn: false - # STP: TEP (1 prefill, 6 decode) - - spec-decoding: "none" - conc-list: [4, 8, 16] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/h200/8k1k/bs8-1p6d.yaml" - decode: - num-worker: 6 - tp: 8 - ep: 1 - dp-attn: false - # STP: TEP (1 prefill, 3 decode) - - spec-decoding: "none" - conc-list: [8, 16, 32] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/h200/8k1k/bs16-1p3d.yaml" - decode: - num-worker: 3 - tp: 8 - ep: 1 - dp-attn: false - # STP: TEP (2 prefill, 3 decode) - - spec-decoding: "none" - conc-list: [32, 64, 128] - prefill: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/h200/8k1k/bs64-2p3d.yaml" - decode: - num-worker: 3 - tp: 8 - ep: 1 - dp-attn: false - # STP: High throughput DEP (1 prefill, 1 decode, dp-attention) - - spec-decoding: "none" - conc-list: [64, 128, 256] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/h200/8k1k/bs128-1p1d-dep.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - # MTP: Low latency TEP (1 prefill, 7 decode) - - spec-decoding: "mtp" - conc-list: [1, 4, 8] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/h200/8k1k/bs4-1p7d-mtp.yaml" - decode: - num-worker: 7 - tp: 8 - ep: 1 - dp-attn: false - # MTP: TEP (1 prefill, 6 decode) - - spec-decoding: "mtp" - conc-list: [2, 4, 8, 16, 32] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/h200/8k1k/bs8-1p6d-mtp.yaml" - decode: - num-worker: 6 - tp: 8 - ep: 1 - dp-attn: false - # MTP: TEP (1 prefill, 3 decode) - - spec-decoding: "mtp" - conc-list: [4, 8, 16, 32, 64] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/h200/8k1k/bs16-1p3d-mtp.yaml" - decode: - num-worker: 3 - tp: 8 - ep: 1 - dp-attn: false - # MTP: TEP (2 prefill, 3 decode) - - spec-decoding: "mtp" - conc-list: [32, 64, 128] - prefill: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/h200/8k1k/bs64-2p3d-mtp.yaml" - decode: - num-worker: 3 - tp: 8 - ep: 1 - dp-attn: false - # MTP: High throughput DEP (1 prefill, 1 decode, dp-attention) - - spec-decoding: "mtp" - conc-list: [32, 64, 128, 256, 512] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/h200/8k1k/bs128-1p1d-dep-mtp.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true -dsr1-fp4-b200-dynamo-sglang: - image: lmsysorg/sglang:v0.5.8.post1-cu130-runtime - model: deepseek-r1-fp4 - model-prefix: dsr1 - runner: b200-multinode - precision: fp4 - framework: dynamo-sglang - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # Non-MTP configurations - - conc-list: [16, 128] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/b200-fp4/1k1k.yaml:zip_override_stp_lowlat[0]" - decode: - num-worker: 5 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [32, 64, 256] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/b200-fp4/1k1k.yaml:zip_override_stp_lowlat[1]" - decode: - num-worker: 6 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [512] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/b200-fp4/1k1k.yaml:zip_override_stp_maxtpt[0]" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - conc-list: [512] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/b200-fp4/1k1k.yaml:zip_override_stp_maxtpt[1]" - decode: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - - isl: 8192 - osl: 1024 - search-space: - # Non-MTP configurations - - conc-list: [64, 128] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/b200-fp4/8k1k.yaml:zip_override_stp_lowlat[0]" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [8] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/b200-fp4/8k1k.yaml:zip_override_stp_lowlat[1]" - decode: - num-worker: 5 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [4, 128] - prefill: - num-worker: 2 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/b200-fp4/8k1k.yaml:zip_override_stp_lowlat[2]" - decode: - num-worker: 5 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [4, 8, 16, 64] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/b200-fp4/8k1k.yaml:override_stp_tp4" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - - conc-list: [1024, 2048] - prefill: - num-worker: 7 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/b200-fp4/8k1k.yaml:override_stp_maxtpt_7p2d" - decode: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true -dsr1-fp8-b200-dynamo-sglang: - image: lmsysorg/sglang:v0.5.8.post1-cu130-amd64 - model: deepseek-ai/DeepSeek-R1-0528 - model-prefix: dsr1 - runner: b200-multinode - precision: fp8 - framework: dynamo-sglang - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # Non-MTP configurations - - conc-list: [4] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/b200-fp8/1k1k.yaml:zip_override_stp_lowlat[0]" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [16, 32, 64, 128, 256] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/b200-fp8/1k1k.yaml:zip_override_stp_lowlat[1]" - decode: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [1024, 2048, 4096] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/b200-fp8/1k1k.yaml:zip_override_stp_maxtpt[0]" - decode: - num-worker: 5 - tp: 8 - ep: 8 - dp-attn: true - - conc-list: [2048, 4096] - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/b200-fp8/1k1k.yaml:zip_override_stp_maxtpt[1]" - decode: - num-worker: 5 - tp: 8 - ep: 8 - dp-attn: true - - isl: 8192 - osl: 1024 - search-space: - # STP low-latency: resolved from 8k1k.yaml zip_override_stp_lowlat - - conc-list: [128] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_stp_lowlat_0.yaml - - "CONFIG_FILE=recipes/b200-fp8/8k1k_stp_lowlat_0.yaml" - decode: - num-worker: 3 - tp: 8 - ep: 1 - dp-attn: false - - conc-list: [128] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_stp_lowlat_1.yaml - - "CONFIG_FILE=recipes/b200-fp8/8k1k_stp_lowlat_1.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 1 - dp-attn: false - - conc-list: [8, 16, 32, 64, 128] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_stp_lowlat_2.yaml - - "CONFIG_FILE=recipes/b200-fp8/8k1k_stp_lowlat_2.yaml" - decode: - num-worker: 6 - tp: 8 - ep: 1 - dp-attn: false - # STP max-throughput: resolved from 8k1k.yaml zip_override_stp_maxtpt - - conc-list: [288] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_stp_maxtpt_0.yaml - - "CONFIG_FILE=recipes/b200-fp8/8k1k_stp_maxtpt_0.yaml" - decode: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - - conc-list: [160, 288] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_stp_maxtpt_1.yaml - - "CONFIG_FILE=recipes/b200-fp8/8k1k_stp_maxtpt_1.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - conc-list: [512] - prefill: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_stp_maxtpt_2.yaml - - "CONFIG_FILE=recipes/b200-fp8/8k1k_stp_maxtpt_2.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - conc-list: [1024] - prefill: - num-worker: 3 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_stp_maxtpt_3.yaml - - "CONFIG_FILE=recipes/b200-fp8/8k1k_stp_maxtpt_3.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - -dsr1-fp8-b200-dynamo-sglang-mtp: - image: lmsysorg/sglang:v0.5.8.post1-cu130-amd64 - model: deepseek-ai/DeepSeek-R1-0528 - model-prefix: dsr1 - runner: b200-multinode - precision: fp8 - framework: dynamo-sglang - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # MTP low-latency: 1P1D - - spec-decoding: "mtp" - conc-list: [4, 64] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/b200-fp8/1k1k.yaml:zip_override_mtp_lowlat[0]" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: false - # MTP low-latency: 1P3D - - spec-decoding: "mtp" - conc-list: [4, 8, 16, 32, 128] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/b200-fp8/1k1k.yaml:zip_override_mtp_lowlat[1]" - decode: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: false - # MTP max-tpt: 1P5D - - spec-decoding: "mtp" - conc-list: [512, 4096] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/b200-fp8/1k1k.yaml:zip_override_mtp_maxtpt[1]" - decode: - num-worker: 5 - tp: 8 - ep: 8 - dp-attn: true - # MTP max-tpt: 2P5D - - spec-decoding: "mtp" - conc-list: [1024, 2048, 4096] - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/b200-fp8/1k1k.yaml:zip_override_mtp_maxtpt[2]" - decode: - num-worker: 5 - tp: 8 - ep: 8 - dp-attn: true - # MTP max-tpt: 1P2D - - spec-decoding: "mtp" - conc-list: [512, 1024, 2048] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/b200-fp8/1k1k.yaml:override_mtp_maxtpt_1p2d" - decode: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - - isl: 8192 - osl: 1024 - search-space: - # MTP low-latency: resolved from 8k1k.yaml zip_override_mtp_lowlat - - spec-decoding: "mtp" - conc-list: [128] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_mtp_lowlat_0.yaml - - "CONFIG_FILE=recipes/b200-fp8/8k1k_mtp_lowlat_0.yaml" - decode: - num-worker: 3 - tp: 8 - ep: 1 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [128] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_mtp_lowlat_1.yaml - - "CONFIG_FILE=recipes/b200-fp8/8k1k_mtp_lowlat_1.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 1 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [8, 16, 32, 64, 128] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_mtp_lowlat_2.yaml - - "CONFIG_FILE=recipes/b200-fp8/8k1k_mtp_lowlat_2.yaml" - decode: - num-worker: 6 - tp: 8 - ep: 1 - dp-attn: false - # MTP max-throughput: resolved from 8k1k.yaml zip_override_mtp_maxtpt - - spec-decoding: "mtp" - conc-list: [288] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_mtp_maxtpt_0.yaml - - "CONFIG_FILE=recipes/b200-fp8/8k1k_mtp_maxtpt_0.yaml" - decode: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [160, 288] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_mtp_maxtpt_1.yaml - - "CONFIG_FILE=recipes/b200-fp8/8k1k_mtp_maxtpt_1.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [512] - prefill: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_mtp_maxtpt_2.yaml - - "CONFIG_FILE=recipes/b200-fp8/8k1k_mtp_maxtpt_2.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [1024] - prefill: - num-worker: 3 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/b200-fp8/8k1k_mtp_maxtpt_3.yaml - - "CONFIG_FILE=recipes/b200-fp8/8k1k_mtp_maxtpt_3.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - -dsr1-fp4-b200-dynamo-sglang-mtp: - image: "lmsysorg/sglang:v0.5.12.post1" - model: deepseek-r1-fp4 - model-prefix: dsr1 - runner: b200-multinode - precision: fp4 - framework: dynamo-sglang - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - spec-decoding: "mtp" - conc-list: [16, 512] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/b200-fp4/1k1k.yaml:zip_override_mtp_lowlat[0]" - decode: - num-worker: 5 - tp: 8 - ep: 8 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [32, 64, 256, 512] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/b200-fp4/1k1k.yaml:zip_override_mtp_lowlat[1]" - decode: - num-worker: 6 - tp: 8 - ep: 8 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [512, 1024] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/b200-fp4/1k1k.yaml:zip_override_mtp_maxtpt[0]" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [512] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/b200-fp4/1k1k.yaml:zip_override_mtp_maxtpt[1]" - decode: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - - isl: 8192 - osl: 1024 - search-space: - # 1p5d low-latency (decode-heavy). - - spec-decoding: "mtp" - conc-list: [4, 8, 16, 32, 64] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp_lowlat_0.yaml" - decode: - num-worker: 5 - tp: 8 - ep: 1 - dp-attn: false - # 1p3d low-latency. - - spec-decoding: "mtp" - conc-list: [32, 64, 128, 256, 512] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp_lowlat_1.yaml" - decode: - num-worker: 3 - tp: 8 - ep: 1 - dp-attn: false - # 1p1d low-latency. - - spec-decoding: "mtp" - conc-list: [32, 64, 128, 256, 512] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp_lowlat_2.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - # MTP2 high-throughput (DEP4 prefill / DEP8 decode), one Pareto point each. - # 2p1d throughput. - - spec-decoding: "mtp" - conc-list: [768] - prefill: - num-worker: 2 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp2_throughput_2p1d.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - # 3p1d throughput. - - spec-decoding: "mtp" - conc-list: [1024] - prefill: - num-worker: 3 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp2_throughput_3p1d.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - # 5p1d throughput. - - spec-decoding: "mtp" - conc-list: [2048] - prefill: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp2_throughput_5p1d.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - -kimik2.5-fp4-gb200-dynamo-trt: - image: nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - model: nvidia/Kimi-K2.5-NVFP4 - model-prefix: kimik2.5 - runner: gb200 - precision: fp4 - framework: dynamo-trt - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # Non-MTP configurations (default spec_decoding="none") - - conc-list: [ 8 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch1_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch1_eplb0_mtp0.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [ 12 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch2_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch2_eplb0_mtp0.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [ 24 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch4_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch4_eplb0_mtp0.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [ 192 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch32_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch32_eplb0_mtp0.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [ 30 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen5tep4_batch4_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen5tep4_batch4_eplb0_mtp0.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - conc-list: [ 60 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen5tep4_batch8_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen5tep4_batch8_eplb0_mtp0.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - conc-list: [ 333 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen1dep32_batch8_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen1dep32_batch8_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [ 666 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen1dep32_batch16_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen1dep32_batch16_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [ 1127 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen1dep32_batch32_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen1dep32_batch32_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [ 4301 ] - prefill: - num-worker: 2 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL1K_OSL1K/STP/ctx2dep4_gen1dep16_batch256_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL1K_OSL1K/STP/ctx2dep4_gen1dep16_batch256_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - conc-list: [ 8192 ] - prefill: - num-worker: 2 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL1K_OSL1K/STP/ctx2dep4_gen1dep16_batch512_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL1K_OSL1K/STP/ctx2dep4_gen1dep16_batch512_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - conc-list: [ 2253 ] - prefill: - num-worker: 2 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL1K_OSL1K/STP/ctx2dep4_gen1dep32_batch64_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL1K_OSL1K/STP/ctx2dep4_gen1dep32_batch64_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [ 4301 ] - prefill: - num-worker: 2 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL1K_OSL1K/STP/ctx2dep4_gen1dep32_batch128_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL1K_OSL1K/STP/ctx2dep4_gen1dep32_batch128_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - - isl: 8192 - osl: 1024 - search-space: - # Non-MTP configurations (default spec_decoding="none") - - conc-list: [ 8 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen4tep8_batch1_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen4tep8_batch1_eplb0_mtp0.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [ 24 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen4tep8_batch4_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen4tep8_batch4_eplb0_mtp0.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [ 5 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch1_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch1_eplb0_mtp0.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - conc-list: [ 30 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch4_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch4_eplb0_mtp0.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - conc-list: [ 60 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch8_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch8_eplb0_mtp0.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - conc-list: [ 105 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch16_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch16_eplb0_mtp0.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - conc-list: [ 333 ] - prefill: - num-worker: 2 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx2dep4_gen1dep32_batch8_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx2dep4_gen1dep32_batch8_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [ 666 ] - prefill: - num-worker: 4 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx4dep4_gen1dep32_batch16_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx4dep4_gen1dep32_batch16_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [ 1229 ] - prefill: - num-worker: 4 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx4dep4_gen1dep16_batch64_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx4dep4_gen1dep16_batch64_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - conc-list: [ 1229 ] - prefill: - num-worker: 6 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx6dep4_gen1dep32_batch32_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx6dep4_gen1dep32_batch32_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [ 2253 ] - prefill: - num-worker: 7 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx7dep4_gen1dep16_batch128_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx7dep4_gen1dep16_batch128_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - conc-list: [ 4301 ] - prefill: - num-worker: 9 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx9dep4_gen1dep16_batch256_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx9dep4_gen1dep16_batch256_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - -kimik2.5-fp4-gb300-dynamo-trt: - image: nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 - model: nvidia/Kimi-K2.5-NVFP4 - model-prefix: kimik2.5 - runner: gb300 - precision: fp4 - framework: dynamo-trt - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # Non-MTP configurations (default spec_decoding="none") - - conc-list: [ 8 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch1_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch1_eplb0_mtp0.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [ 12 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch2_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch2_eplb0_mtp0.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [ 24 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch4_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch4_eplb0_mtp0.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [ 30 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen5tep4_batch4_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen5tep4_batch4_eplb0_mtp0.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - conc-list: [ 60 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen5tep4_batch8_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen5tep4_batch8_eplb0_mtp0.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - conc-list: [ 333 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen1dep32_batch8_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen1dep32_batch8_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [ 666 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen1dep32_batch16_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen1dep32_batch16_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [ 1229 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen1dep32_batch32_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen1dep32_batch32_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [ 4301 ] - prefill: - num-worker: 2 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL1K_OSL1K/STP/ctx2dep4_gen1dep16_batch256_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL1K_OSL1K/STP/ctx2dep4_gen1dep16_batch256_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - conc-list: [ 8192 ] - prefill: - num-worker: 2 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL1K_OSL1K/STP/ctx2dep4_gen1dep16_batch512_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL1K_OSL1K/STP/ctx2dep4_gen1dep16_batch512_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - conc-list: [ 2253 ] - prefill: - num-worker: 2 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL1K_OSL1K/STP/ctx2dep4_gen1dep32_batch64_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL1K_OSL1K/STP/ctx2dep4_gen1dep32_batch64_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [ 4301 ] - prefill: - num-worker: 2 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL1K_OSL1K/STP/ctx2dep4_gen1dep32_batch128_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL1K_OSL1K/STP/ctx2dep4_gen1dep32_batch128_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - - isl: 8192 - osl: 1024 - search-space: - # Non-MTP configurations (default spec_decoding="none") - - conc-list: [ 4 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen4tep8_batch1_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen4tep8_batch1_eplb0_mtp0.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [ 12 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen4tep8_batch2_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen4tep8_batch2_eplb0_mtp0.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [ 24 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen4tep8_batch4_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen4tep8_batch4_eplb0_mtp0.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [ 30 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch4_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch4_eplb0_mtp0.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - conc-list: [ 60 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch8_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch8_eplb0_mtp0.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - conc-list: [ 115 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch16_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch16_eplb0_mtp0.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - conc-list: [ 333 ] - prefill: - num-worker: 2 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx2dep4_gen1dep32_batch8_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx2dep4_gen1dep32_batch8_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [ 615 ] - prefill: - num-worker: 3 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx3dep4_gen1dep32_batch16_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx3dep4_gen1dep32_batch16_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [ 1229 ] - prefill: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx5dep4_gen1dep32_batch32_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx5dep4_gen1dep32_batch32_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [ 2253 ] - prefill: - num-worker: 6 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx6dep4_gen1dep16_batch128_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx6dep4_gen1dep16_batch128_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - conc-list: [ 2151 ] - prefill: - num-worker: 8 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx8dep4_gen1dep32_batch64_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx8dep4_gen1dep32_batch64_eplb0_mtp0.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - -kimik2.5-fp4-gb200-dynamo-vllm: - image: vllm/vllm-openai:v0.21.0 - model: nvidia/Kimi-K2.5-NVFP4 - model-prefix: kimik2.5 - runner: gb200 - precision: fp4 - framework: dynamo-vllm - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - conc-list: [4096, 12288] - prefill: - num-worker: 1 - tp: 1 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k2.5-fp4/1k1k/disagg-gb200-1p1d-dep4-dep8.yaml" - decode: - num-worker: 1 - tp: 1 - ep: 8 - dp-attn: true - - conc-list: [4, 8, 32, 128] - prefill: - num-worker: 1 - tp: 1 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k2.5-fp4/1k1k/disagg-gb200-1p4d-dep4-tp8.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 1 - dp-attn: false - - conc-list: [4096, 6144] - prefill: - num-worker: 1 - tp: 1 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k2.5-fp4/1k1k/disagg-gb200-1p1d-dep4-dep16.yaml" - decode: - num-worker: 1 - tp: 1 - ep: 16 - dp-attn: true - - isl: 8192 - osl: 1024 - search-space: - - conc-list: [128] - prefill: - num-worker: 1 - tp: 1 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb200-1p4d-dep4-tep4.yaml" - decode: - num-worker: 4 - tp: 4 - ep: 4 - dp-attn: false - - conc-list: [4, 8, 16, 32, 256] - prefill: - num-worker: 1 - tp: 1 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb200-1p4d-dep4-tp8.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 1 - dp-attn: false - - conc-list: [1024] - prefill: - num-worker: 3 - tp: 1 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb200-3p1d-dep4-dep16.yaml" - decode: - num-worker: 1 - tp: 1 - ep: 16 - dp-attn: true - - conc-list: [3072] - prefill: - num-worker: 6 - tp: 1 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb200-6p1d-dep4-dep16.yaml" - decode: - num-worker: 1 - tp: 1 - ep: 16 - dp-attn: true - - conc-list: [6144] - prefill: - num-worker: 8 - tp: 1 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb200-8p1d-dep4-dep16.yaml" - decode: - num-worker: 1 - tp: 1 - ep: 16 - dp-attn: true - -dsv4-fp4-b200-dynamo-vllm: - image: vllm/vllm-openai:v0.23.0 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: b200-multinode - precision: fp4 - framework: dynamo-vllm - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # B200 adaptation of the DSV4 GB200 vLLM disagg recipes. Each worker - # maps to one full 8-GPU B200 node. - - conc-list: [1] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-latency-c1.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - - conc-list: [32, 128] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-latency-c32-c128.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - - conc-list: [64] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-latency-c64.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - - conc-list: [256] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-middle-c256.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - conc-list: [512] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-b200-low-middle-c512.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - -dsv4-fp4-gb200-dynamo-vllm: - image: vllm/vllm-openai:v0.20.0-ubuntu2404 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: gb200 - precision: fp4 - framework: dynamo-vllm - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # Validated 8k/1k points mirrored from NVIDIA/srt-slurm - # aflowers/vllm-gb200-v0.20.0 history. conc-list values match each - # recipe's benchmark.concurrencies. - - # Low latency: 1 prefill (DEP=8) + 1 decode (TP=8). 5 nodes total with - # a dedicated NATS/etcd infra node. - - conc-list: [1] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb200-low-latency.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - - # Low-middle curve: 1 prefill (DEP=8) + 4 decode (TP=8). 11 nodes total - # with a dedicated NATS/etcd infra node. - - conc-list: [256, 512] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb200-low-middle-curve.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 1 - dp-attn: false - - # MegaMOE mid curve: 1 prefill (DEP=8) + 1 decode (DEP=8). 5 nodes - # total with a dedicated NATS/etcd infra node. - - conc-list: [256, 512, 1024] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb200-mid-curve-megamoe.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - # MegaMOE high throughput: 2 prefill (DEP=8 each) + 1 decode (DEP=8). - # 7 nodes total with a dedicated NATS/etcd infra node. - - conc-list: [4096] - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb200-high-tpt-megamoe.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - # MegaMOE max throughput: 3 prefill (DEP=8 each) + 1 decode (DEP=8). - # 9 nodes total with a dedicated NATS/etcd infra node. - - conc-list: [4096] - prefill: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb200-max-tpt-megamoe.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - -# MTP2 variant of dsv4-fp4-gb200-dynamo-vllm. Uses the vLLM 0.20.1 image -# and hand-picked 8k/1k Pareto points mirrored from NVIDIA/srt-slurm. -dsv4-fp4-gb200-dynamo-vllm-mtp2: - image: vllm/vllm-openai:v0.20.1-ubuntu2404 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: gb200 - precision: fp4 - framework: dynamo-vllm - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # Aggregate low latency: TP=8, max-num-seqs=4. - - conc-list: [1] - spec-decoding: mtp - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/agg-gb200-low-latency-mtp2.yaml" - decode: - num-worker: 0 - tp: 8 - ep: 1 - dp-attn: false - - # Low-latency bridge: 1 prefill (DEP=8) + 4 decode (TP=8), no offload. - - conc-list: [16, 32, 64] - spec-decoding: mtp - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb200-low-latency-mtp2.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 1 - dp-attn: false - - # MegaMOE mid curve: 1 prefill (DEP=8) + 1 decode (DEP=8). - # 5 nodes total with a dedicated NATS/etcd infra node. - - conc-list: [128, 256, 512, 1024] - spec-decoding: mtp - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb200-mid-curve-megamoe-mtp2.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - # MegaMOE high throughput: 2 prefill (DEP=8 each) + 1 decode (DEP=8). - # 7 nodes total with a dedicated NATS/etcd infra node. - - conc-list: [1024] - spec-decoding: mtp - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb200-high-tpt-megamoe-mtp2.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - -dsv4-fp4-gb200-dynamo-sglang: - image: lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: gb200 - precision: fp4 - framework: dynamo-sglang - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # Low latency: 1p1d-tp8-tp8. 4 nodes. - - conc-list: [1] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p1d-tp8-tp8-4-c1.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - # 1p4d-dep8-tp8. 10 nodes. - - conc-list: [64] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p4d-dep8-tp8-10-c64.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 1 - dp-attn: false - # WideEP TP=16 decode: 1p2d-dep8-dep16. 10 nodes. - - conc-list: [256] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p2d-dep8-dep16-10-c256.yaml" - decode: - num-worker: 2 - tp: 16 - ep: 16 - dp-attn: true - # WideEP TP=16 decode: 1p1d-dep8-dep16. 6 nodes. - - conc-list: [512] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-1p1d-dep8-dep16-6-c512.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - # WideEP TP=16 decode: 2p1d-dep8-dep16. 8 nodes. - - conc-list: [1536] - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-2p1d-dep8-dep16-8-c1536.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - # WideEP TP=16 decode: 4p1d-dep8-dep16. 12 nodes. - - conc-list: [4096] - prefill: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-4p1d-dep8-dep16-12-c4096.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - # WideEP TP=16 decode: 5p1d-dep8-dep16. 14 nodes. - - conc-list: [8192] - prefill: - num-worker: 5 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-5p1d-dep8-dep16-14-c8192.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - # WideEP TP=12 decode: 6p1d-dep8-dep12. 15 nodes. - - conc-list: [8192] - prefill: - num-worker: 6 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-6p1d-dep8-dep12-15-c8192.yaml" - decode: - num-worker: 1 - tp: 12 - ep: 12 - dp-attn: true - -qwen3.5-fp8-gb200-dynamo-sglang: - image: lmsysorg/sglang:nightly-dev-cu13-20260608-303757cc - model: Qwen/Qwen3.5-397B-A17B-FP8 - model-prefix: qwen3.5 - runner: gb200 - precision: fp8 - framework: dynamo-sglang - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # 1P1D STP: TP4 prefill + TP4 decode (pure tensor parallel). 2 nodes (1+1). - - spec-decoding: "none" - conc-list: [1, 2, 4, 8, 16, 32, 64] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb200-fp8/1k1k/1p1d-tp4-tp4.yaml" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - # 1P1D wide-EP: prefill DEP4 + decode DEP16. 5 nodes (1+4). - - spec-decoding: "none" - conc-list: [512, 1024, 2048] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb200-fp8/1k1k/1p1d-dep4-dep16.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - # 2P1D wide-EP: 2 prefill DEP4 + decode DEP16. 6 nodes (2+4). - - spec-decoding: "none" - conc-list: [4096] - prefill: - num-worker: 2 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb200-fp8/1k1k/2p1d-dep4-dep16.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - isl: 8192 - osl: 1024 - search-space: - # 1P1D STP: TP4 prefill + TP4 decode (pure tensor parallel). 2 nodes (1+1). - - spec-decoding: "none" - conc-list: [1, 2, 4, 8, 16, 32, 64, 128] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb200-fp8/8k1k/1p1d-tp4-tp4.yaml" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - # 4P1D wide-EP: 4 prefill DEP4 + decode DEP16. 8 nodes (4+4). - - spec-decoding: "none" - conc-list: [1024] - prefill: - num-worker: 4 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb200-fp8/8k1k/4p1d-dep4-dep16.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - # 8P1D wide-EP: 8 prefill DEP4 + decode DEP16. 12 nodes (8+4). - - spec-decoding: "none" - conc-list: [2048, 4096] - prefill: - num-worker: 8 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb200-fp8/8k1k/8p1d-dep4-dep16.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - -# MTP variant of dsv4-fp4-gb200-dynamo-sglang. -dsv4-fp4-gb200-dynamo-sglang-mtp: - image: lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: gb200 - precision: fp4 - framework: dynamo-sglang - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # Low-latency baseline: 1p1d-tp8-tp8. 4 nodes. - - spec-decoding: "mtp" - conc-list: [1] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-low-latency-1p1d-tp8-tp8-mtp.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - # Low-latency 1p6d-dep8-tp8: 1P (DEP=8) + 6 TP=8 decode workers. 14 nodes. - # Recipe runs concurrencies=32x64x128; matrix tracks the max. - - spec-decoding: "mtp" - conc-list: [128] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-low-latency-1p6d-dep8-tp8-mtp.yaml" - decode: - num-worker: 6 - tp: 8 - ep: 1 - dp-attn: false - # Mid curve 1p1d-dep8-dep16. 6 nodes. - - spec-decoding: "mtp" - conc-list: [1024] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-1p1d-dep8-dep16-mtp.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - # Mid curve 2p1d-dep8-dep16. 8 nodes. - - spec-decoding: "mtp" - conc-list: [2048] - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-2p1d-dep8-dep16-mtp.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - # Mid curve 3p1d-dep8-dep16. 10 nodes. - - spec-decoding: "mtp" - conc-list: [3072] - prefill: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-3p1d-dep8-dep16-mtp.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - # Mid curve 4p1d-dep8-dep16. 12 nodes. - - spec-decoding: "mtp" - conc-list: [6144] - prefill: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-4p1d-dep8-dep16-mtp.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - # Mid curve 5p1d-dep8-dep16. 14 nodes. - - spec-decoding: "mtp" - conc-list: [8192] - prefill: - num-worker: 5 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-5p1d-dep8-dep16-mtp.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - # Mid curve 6p1d-dep8-dep16. 16 nodes. - - spec-decoding: "mtp" - conc-list: [16384] - prefill: - num-worker: 6 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb200-mid-curve-6p1d-dep8-dep16-mtp.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true -dsv4-fp4-b300-dynamo-vllm: - image: vllm/vllm-openai:v0.23.0 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: b300 - precision: fp4 - framework: dynamo-vllm - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # B300 adaptation of the DSV4 B200/GB200 vLLM disagg recipes. Each - # prefill/decode worker maps to one full 8-GPU B300 node. - - conc-list: [1, 32, 64, 128] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-b300-low-latency.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - - conc-list: [256, 1024] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-b300-mid-curve-megamoe.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - conc-list: [4096] - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-b300-high-tpt-megamoe.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - -dsv4-fp4-gb300-dynamo-vllm: - image: vllm/vllm-openai:v0.20.0-ubuntu2404 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: gb300-nv - precision: fp4 - framework: dynamo-vllm - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - - conc-list: [192] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb300-1p6d-dep4-tp4.yaml" - decode: - num-worker: 6 - tp: 4 - ep: 1 - dp-attn: false - - conc-list: [18] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb300-1p17d-tep4-tp4.yaml" - decode: - num-worker: 17 - tp: 4 - ep: 1 - dp-attn: false - - conc-list: [4096] - prefill: - num-worker: 4 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb300-4p1d-dep4-dep8-24-c4096.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - conc-list: [4096] - prefill: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb300-5p1d-dep4-dep8-28-c4096.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - conc-list: [4096] - prefill: - num-worker: 6 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb300-6p1d-dep4-dep8-32-c4096.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - conc-list: [3072] - prefill: - num-worker: 7 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/disagg-gb300-7p2d-dep4-dep16.yaml" - decode: - num-worker: 2 - tp: 16 - ep: 16 - dp-attn: true - -dsv4-fp4-gb300-dynamo-trt: - image: nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-deepseek-v4-dev.1 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: gb300-nv - precision: fp4 - framework: dynamo-trt - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - conc-list: [4] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch1_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch1_eplb0_mtp0.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [5] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/STP/ctx1dep4_gen5tep4_batch1_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/STP/ctx1dep4_gen5tep4_batch1_eplb0_mtp0.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - conc-list: [15] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/STP/ctx1dep4_gen5tep4_batch2_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/STP/ctx1dep4_gen5tep4_batch2_eplb0_mtp0.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - conc-list: [25] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/STP/ctx1dep4_gen5tep4_batch4_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/STP/ctx1dep4_gen5tep4_batch4_eplb0_mtp0.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - conc-list: [55] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/STP/ctx1dep4_gen5tep4_batch8_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/STP/ctx1dep4_gen5tep4_batch8_eplb0_mtp0.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - conc-list: [167] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/STP/ctx1dep4_gen1dep32_batch4_eplb384_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/STP/ctx1dep4_gen1dep32_batch4_eplb384_mtp0.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [333] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/STP/ctx1dep4_gen1dep32_batch8_eplb384_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/STP/ctx1dep4_gen1dep32_batch8_eplb384_mtp0.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [666] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/STP/ctx1dep4_gen1dep32_batch16_eplb384_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/STP/ctx1dep4_gen1dep32_batch16_eplb384_mtp0.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [1229] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/STP/ctx1dep4_gen1dep32_batch32_eplb384_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/STP/ctx1dep4_gen1dep32_batch32_eplb384_mtp0.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [2253] - prefill: - num-worker: 2 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/STP/ctx2dep4_gen1dep32_batch64_eplb384_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/STP/ctx2dep4_gen1dep32_batch64_eplb384_mtp0.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [4301] - prefill: - num-worker: 3 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/STP/ctx3dep4_gen1dep32_batch128_eplb384_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/STP/ctx3dep4_gen1dep32_batch128_eplb384_mtp0.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [8192] - prefill: - num-worker: 3 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/STP/ctx3dep4_gen1dep16_batch512_eplb384_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/STP/ctx3dep4_gen1dep16_batch512_eplb384_mtp0.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - conc-list: [8192] - prefill: - num-worker: 4 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/STP/ctx4dep4_gen1dep32_batch256_eplb384_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/STP/ctx4dep4_gen1dep32_batch256_eplb384_mtp0.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [16384] - prefill: - num-worker: 4 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/STP/ctx4dep4_gen1dep16_batch1024_eplb384_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/STP/ctx4dep4_gen1dep16_batch1024_eplb384_mtp0.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - isl: 8192 - osl: 1024 - search-space: - - conc-list: [4] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx1dep4_gen4tep8_batch1_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx1dep4_gen4tep8_batch1_eplb0_mtp0.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [5] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch1_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch1_eplb0_mtp0.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - conc-list: [15] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch2_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch2_eplb0_mtp0.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - conc-list: [25] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch4_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch4_eplb0_mtp0.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - conc-list: [55] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch8_eplb0_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch8_eplb0_mtp0.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - conc-list: [154] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx1dep4_gen1dep32_batch4_eplb384_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx1dep4_gen1dep32_batch4_eplb384_mtp0.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [308] - prefill: - num-worker: 2 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx2dep4_gen1dep32_batch8_eplb384_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx2dep4_gen1dep32_batch8_eplb384_mtp0.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [615] - prefill: - num-worker: 4 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx4dep4_gen1dep32_batch16_eplb384_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx4dep4_gen1dep32_batch16_eplb384_mtp0.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [1127] - prefill: - num-worker: 6 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx6dep4_gen1dep32_batch32_eplb384_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx6dep4_gen1dep32_batch32_eplb384_mtp0.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [1229] - prefill: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx5dep4_gen1dep16_batch64_eplb384_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx5dep4_gen1dep16_batch64_eplb384_mtp0.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - conc-list: [2253] - prefill: - num-worker: 6 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx6dep4_gen1dep8_batch256_eplb384_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx6dep4_gen1dep8_batch256_eplb384_mtp0.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - conc-list: [2253] - prefill: - num-worker: 9 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx9dep4_gen1dep16_batch128_eplb384_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx9dep4_gen1dep16_batch128_eplb384_mtp0.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - conc-list: [4301] - prefill: - num-worker: 10 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx10dep4_gen1dep8_batch512_eplb384_mtp0.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/STP/ctx10dep4_gen1dep8_batch512_eplb384_mtp0.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - -dsv4-fp4-gb300-dynamo-trt-mtp: - image: nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-deepseek-v4-dev.1 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: gb300-nv - precision: fp4 - framework: dynamo-trt - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - spec-decoding: "mtp" - conc-list: [8] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch1_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen4tep8_batch1_eplb0_mtp3.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [10] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [15] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen5tep4_batch2_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen5tep4_batch2_eplb0_mtp3.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [25] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen5tep4_batch4_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen5tep4_batch4_eplb0_mtp3.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [90] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch2_eplb384_mtp3.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch2_eplb384_mtp3.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [167] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch4_eplb384_mtp3.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch4_eplb384_mtp3.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [333] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch8_eplb384_mtp3.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch8_eplb384_mtp3.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [615] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch16_eplb384_mtp3.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/MTP/ctx1dep4_gen1dep32_batch16_eplb384_mtp3.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [1229] - prefill: - num-worker: 2 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep32_batch32_eplb384_mtp3.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep32_batch32_eplb384_mtp3.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [2253] - prefill: - num-worker: 3 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/MTP/ctx3dep4_gen1dep32_batch64_eplb384_mtp3.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/MTP/ctx3dep4_gen1dep32_batch64_eplb384_mtp3.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [4301] - prefill: - num-worker: 3 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/MTP/ctx3dep4_gen1dep16_batch256_eplb384_mtp1.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/MTP/ctx3dep4_gen1dep16_batch256_eplb384_mtp1.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [4301] - prefill: - num-worker: 4 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/MTP/ctx4dep4_gen1dep32_batch128_eplb384_mtp1.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/MTP/ctx4dep4_gen1dep32_batch128_eplb384_mtp1.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [8192] - prefill: - num-worker: 2 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep8_batch1024_eplb384_mtp1.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/MTP/ctx2dep4_gen1dep8_batch1024_eplb384_mtp1.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [8192] - prefill: - num-worker: 3 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/MTP/ctx3dep4_gen1dep16_batch512_eplb384_mtp1.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL1K_OSL1K/MTP/ctx3dep4_gen1dep16_batch512_eplb384_mtp1.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - isl: 8192 - osl: 1024 - search-space: - - spec-decoding: "mtp" - conc-list: [8] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen4tep8_batch1_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen4tep8_batch1_eplb0_mtp3.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [10] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch1_eplb0_mtp3.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [15] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch2_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch2_eplb0_mtp3.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [30] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch4_eplb0_mtp3.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx1dep4_gen5tep4_batch4_eplb0_mtp3.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: false - - spec-decoding: "mtp" - conc-list: [84] - prefill: - num-worker: 2 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx2dep4_gen1dep32_batch2_eplb384_mtp3.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx2dep4_gen1dep32_batch2_eplb384_mtp3.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [180] - prefill: - num-worker: 3 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx3dep4_gen1dep32_batch4_eplb384_mtp3.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx3dep4_gen1dep32_batch4_eplb384_mtp3.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [333] - prefill: - num-worker: 4 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx4dep4_gen1dep32_batch8_eplb384_mtp3.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx4dep4_gen1dep32_batch8_eplb384_mtp3.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [615] - prefill: - num-worker: 8 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx8dep4_gen1dep32_batch16_eplb384_mtp3.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx8dep4_gen1dep32_batch16_eplb384_mtp3.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [666] - prefill: - num-worker: 6 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx6dep4_gen1dep16_batch32_eplb384_mtp3.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx6dep4_gen1dep16_batch32_eplb384_mtp3.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [1229] - prefill: - num-worker: 7 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx7dep4_gen1dep8_batch128_eplb384_mtp3.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx7dep4_gen1dep8_batch128_eplb384_mtp3.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [1229] - prefill: - num-worker: 10 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx10dep4_gen1dep16_batch64_eplb384_mtp3.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx10dep4_gen1dep16_batch64_eplb384_mtp3.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [2253] - prefill: - num-worker: 9 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx9dep4_gen1dep8_batch256_eplb384_mtp1.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx9dep4_gen1dep8_batch256_eplb384_mtp1.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - spec-decoding: "mtp" - conc-list: [4301] - prefill: - num-worker: 12 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx12dep4_gen1dep8_batch512_eplb384_mtp1.yaml - - "CONFIG_FILE=recipes/DeepSeek-V4-Pro/disagg/trtllm_dynamo/gb300_mxfp4/ISL8K_OSL1K/MTP/ctx12dep4_gen1dep8_batch512_eplb384_mtp1.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - -dsv4-fp4-gb300-dynamo-sglang: - image: lmsysorg/sglang:nightly-dev-cu13-20260520-425dffbd - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: gb300 - precision: fp4 - framework: dynamo-sglang - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # WideEP TP=16 decode: 1p1d-dep4-dep16. 5 nodes (4P + 16D = 20 GPUs). - - conc-list: [1024] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb300-1p1d-dep4-dep16-5-c1024.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - # Low concurrency: 1p1d-tp4-tp4. 2 nodes. - - conc-list: [1] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb300-1p1d-tp4-tp4-2-c1.yaml" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - # --- Weiliang wide-EP sweep (srt-slurm PR#173), 18 nodes total --- - # EP=12: 15P+3D, conc=12000. - - conc-list: [12000] - prefill: - num-worker: 15 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb300-15p1d-dep4-dep12-18-c12000.yaml" - decode: - num-worker: 1 - tp: 12 - ep: 12 - dp-attn: true - # EP=16: 14P+4D, conc=8192. - - conc-list: [8192] - prefill: - num-worker: 14 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb300-14p1d-dep4-dep16-18-c8192.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - # EP=24: 12P+6D, conc=3000. - - conc-list: [3000] - prefill: - num-worker: 12 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb300-12p1d-dep4-dep24-18-c3000.yaml" - decode: - num-worker: 1 - tp: 24 - ep: 24 - dp-attn: true - # EP=32: 10P+8D, conc=2500. - - conc-list: [2500] - prefill: - num-worker: 10 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb300-10p1d-dep4-dep32-18-c2500.yaml" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - # EP=40: 8P+10D, conc=2048. - - conc-list: [2048] - prefill: - num-worker: 8 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-gb300-8p1d-dep4-dep40-18-c2048.yaml" - decode: - num-worker: 1 - tp: 40 - ep: 40 - dp-attn: true - -glm5-fp8-b200-dynamo-sglang: - image: lmsysorg/sglang:v0.5.11-cu130 - model: zai-org/GLM-5-FP8 - model-prefix: glm5 - runner: b200-dgxc - precision: fp8 - framework: dynamo-sglang - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - conc-list: [2576] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/b200-fp8/glm5.yaml:zip_override_1k1k_hightpt[0]" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - - conc-list: [1248] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/b200-fp8/glm5.yaml:zip_override_1k1k_hightpt[1]" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: true - - conc-list: [800] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/b200-fp8/glm5.yaml:zip_override_1k1k_hightpt[2]" - decode: - num-worker: 3 - tp: 8 - ep: 1 - dp-attn: true - - conc-list: [576] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/b200-fp8/glm5.yaml:zip_override_1k1k_hightpt[3]" - decode: - num-worker: 4 - tp: 8 - ep: 1 - dp-attn: true - - conc-list: [512, 256, 128, 64, 32] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/b200-fp8/glm5.yaml:zip_override_1k1k_lowlat[0]" - decode: - num-worker: 8 - tp: 8 - ep: 1 - dp-attn: false - - conc-list: [16] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/b200-fp8/glm5.yaml:zip_override_1k1k_lowlat[1]" - decode: - num-worker: 8 - tp: 8 - ep: 1 - dp-attn: false - - isl: 8192 - osl: 1024 - search-space: - - conc-list: [560] - prefill: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/b200-fp8/glm5.yaml:zip_override_8k1k_hightpt[0]" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - - conc-list: [240] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/b200-fp8/glm5.yaml:zip_override_8k1k_hightpt[1]" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - - conc-list: [224] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/b200-fp8/glm5.yaml:zip_override_8k1k_hightpt[2]" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: true - - conc-list: [256] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/b200-fp8/glm5.yaml:zip_override_8k1k_lowlat[0]" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - - conc-list: [256] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/b200-fp8/glm5.yaml:zip_override_8k1k_lowlat[1]" - decode: - num-worker: 3 - tp: 8 - ep: 1 - dp-attn: false - - conc-list: [200] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/b200-fp8/glm5.yaml:zip_override_8k1k_lowlat[2]" - decode: - num-worker: 4 - tp: 8 - ep: 1 - dp-attn: false - - conc-list: [128] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/b200-fp8/glm5.yaml:zip_override_8k1k_lowlat[3]" - decode: - num-worker: 5 - tp: 8 - ep: 1 - dp-attn: false - - conc-list: [64] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/b200-fp8/glm5.yaml:zip_override_8k1k_lowlat[4]" - decode: - num-worker: 7 - tp: 8 - ep: 1 - dp-attn: false - - conc-list: [12] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/b200-fp8/glm5.yaml:zip_override_8k1k_lowlat[5]" - decode: - num-worker: 8 - tp: 8 - ep: 1 - dp-attn: false - -# MTP variant of dsv4-fp4-gb300-dynamo-sglang. -dsv4-fp4-gb300-dynamo-sglang-mtp: - image: lmsysorg/sglang:nightly-dev-20260527-14f81a67 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: gb300 - precision: fp4 - framework: dynamo-sglang - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # Low-latency baseline: 1p1d-tp4-tp4. 2 nodes. - - spec-decoding: "mtp" - conc-list: [1] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p1d-tp4-tp4-mtp.yaml" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - # Low-latency 1p6d-dep4-tp4: 1P (DEP=4) + 6 TP=4 decode workers. 7 nodes. - # Recipe runs concurrencies=8x32x64; matrix tracks the max. - - spec-decoding: "mtp" - conc-list: [64] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-low-latency-1p6d-dep4-tp4-mtp.yaml" - decode: - num-worker: 6 - tp: 4 - ep: 1 - dp-attn: false - # Mid curve 1p1d-dep4-dep8. 3 nodes. - - spec-decoding: "mtp" - conc-list: [256] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep8-mtp.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - # Mid curve 1p1d-dep4-dep16. 5 nodes. - - spec-decoding: "mtp" - conc-list: [256] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-1p1d-dep4-dep16-mtp.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - # Mid curve 2p1d-dep4-dep8. 4 nodes. - - spec-decoding: "mtp" - conc-list: [512] - prefill: - num-worker: 2 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-2p1d-dep4-dep8-mtp.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - # Mid curve 4p1d-dep4-dep8. 6 nodes. - - spec-decoding: "mtp" - conc-list: [1024] - prefill: - num-worker: 4 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-mid-curve-4p1d-dep4-dep8-mtp.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - # High concurrency 6p1d-dep4-dep8. 8 nodes. - - spec-decoding: "mtp" - conc-list: [4096] - prefill: - num-worker: 6 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-high-conc-6p1d-dep4-dep8-mtp.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - # High concurrency 8p1d-dep4-dep8. 10 nodes. - - spec-decoding: "mtp" - conc-list: [8192] - prefill: - num-worker: 8 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/deepseek-v4/8k1k/disagg-high-conc-8p1d-dep4-dep8-mtp.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - -kimik2.5-int4-h100-vllm: - image: vllm/vllm-openai:v0.22.0 - model: moonshotai/Kimi-K2.5 - model-prefix: kimik2.5 - runner: cluster:h100-dgxc - precision: int4 - framework: vllm - multinode: false - scenarios: - # H100 has 80 GB HBM per GPU (smallest in this set); the KV cliff arrives - # early. Sweep saturates conc=20 to keep total HBM headroom. - agentic-coding: - - dram-utilization: 0.80 - search-space: - - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 8, 12, 16, 20] } - - { tp: 8, kv-offloading: dram, kv-offload-backend: native, conc-list: [1, 2, 4, 8, 12, 16, 20] } - - -qwen3.5-fp8-h100-sglang: - image: lmsysorg/sglang:v0.5.14-cu130 - model: Qwen/Qwen3.5-397B-A17B-FP8 - model-prefix: qwen3.5 - runner: h100 - precision: fp8 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 1, conc-end: 8 } - - { tp: 8, ep: 8, conc-start: 16, conc-end: 256 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, ep: 1, conc-start: 1, conc-end: 8 } - - { tp: 8, ep: 8, conc-start: 16, conc-end: 256 } - -qwen3.5-fp8-h100-sglang-mtp: - image: lmsysorg/sglang:v0.5.14-cu130 - model: Qwen/Qwen3.5-397B-A17B-FP8 - model-prefix: qwen3.5 - runner: h100 - precision: fp8 - framework: sglang - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, ep: 8, conc-start: 4, conc-end: 32, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, ep: 8, conc-start: 4, conc-end: 32, spec-decoding: mtp } - -qwen3.5-fp4-gb300-dynamo-sglang: - image: lmsysorg/sglang:nightly-dev-cu13-20260624-b2c8f7a2 - model: nvidia/Qwen3.5-397B-A17B-NVFP4 - model-prefix: qwen3.5 - runner: gb300 - precision: fp4 - framework: dynamo-sglang - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # 1P1D TP4: 1 prefill worker at TP4 and 1 decode worker at TP4 - # Pure tensor parallel (STP), 8k1k baseline-low-latency sweep. - # Total: 8 GB300 GPUs. - - spec-decoding: "none" - conc-list: [1, 4, 8, 16, 32, 64, 256] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - # 5P1D wide-EP: 5 prefill workers @ DEP4 + 1 decode worker @ DEP16. - # NIXL transfer. Total: 36 GB300 GPUs (5*4 + 4*4). - - spec-decoding: "none" - conc-list: [2048] - prefill: - num-worker: 5 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_0.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - # 6P1D wide-EP: 6 prefill workers @ DEP4 + 1 decode worker @ DEP16. - # Mooncake transfer. Total: 40 GB300 GPUs (6*4 + 4*4). - - spec-decoding: "none" - conc-list: [5120] - prefill: - num-worker: 6 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_1.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - # 7P1D wide-EP: 7 prefill workers @ DEP4 + 1 decode worker @ DEP16. - # Mooncake transfer. Total: 44 GB300 GPUs (7*4 + 4*4). - - spec-decoding: "none" - conc-list: [5120] - prefill: - num-worker: 7 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_maxtpt_2.yaml" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - -glm5-fp4-gb300-dynamo-sglang: - image: lmsysorg/sglang:v0.5.11-cu130 - model: nvidia/GLM-5-NVFP4 - model-prefix: glm5 - runner: gb300-nv - precision: fp4 - framework: dynamo-sglang - multinode: true - disagg: true - scenarios: - fixed-seq-len: - # ---------- 8k1k high-throughput (wide-EP TP=32 decode) ---------- - - isl: 8192 - osl: 1024 - search-space: - # 5p1d wide-EP. 13 nodes (5P @ TP=4 + 1D @ TP=32 on 8 nodes). - - conc-list: [2048] - prefill: - num-worker: 5 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/gb300-fp4/glm5.yaml:zip_override_8k1k_hightpt[0]" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - # 7p1d wide-EP. 15 nodes. - - conc-list: [3072] - prefill: - num-worker: 7 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/gb300-fp4/glm5.yaml:zip_override_8k1k_hightpt[1]" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - # 10p1d wide-EP. 18 nodes. - - conc-list: [4096] - prefill: - num-worker: 10 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/gb300-fp4/glm5.yaml:zip_override_8k1k_hightpt[2]" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - # ---------- 8k1k low-latency (per-node TP=4 decode workers) ---------- - - isl: 8192 - osl: 1024 - search-space: - # 1p3d. 4 nodes (1P + 3 D workers @ 1 node each). - - conc-list: [128] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_0.yaml" - decode: - num-worker: 3 - tp: 4 - ep: 1 - dp-attn: false - # 1p5d. 6 nodes. - - conc-list: [64] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_1.yaml" - decode: - num-worker: 5 - tp: 4 - ep: 1 - dp-attn: false - # 1p9d. 10 nodes. - - conc-list: [32] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_2.yaml" - decode: - num-worker: 9 - tp: 4 - ep: 1 - dp-attn: false - # 1p15d. 16 nodes. - - conc-list: [16] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_3.yaml" - decode: - num-worker: 15 - tp: 4 - ep: 1 - dp-attn: false - # 1p17d. 18 nodes. - - conc-list: [12] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/sglang/glm5/gb300-fp4/8k1k/disagg/stp/8k1k_stp_lowlat_4.yaml" - decode: - num-worker: 17 - tp: 4 - ep: 1 - dp-attn: false - # ---------- 1k1k high-throughput (wide-EP TP=32 decode) ---------- - - isl: 1024 - osl: 1024 - search-space: - # 3p1d wide-EP. 11 nodes. conc 16500. - - conc-list: [16500] - prefill: - num-worker: 3 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/gb300-fp4/glm5.yaml:zip_override_1k1k_hightpt[0]" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - # 2p1d wide-EP. 10 nodes. conc 8300. - - conc-list: [8300] - prefill: - num-worker: 2 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/gb300-fp4/glm5.yaml:zip_override_1k1k_hightpt[1]" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - # 1p1d wide-EP. 9 nodes. conc sweep 2500x1024x512x256. - - conc-list: [2500, 1024, 512, 256] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/gb300-fp4/glm5.yaml:zip_override_1k1k_hightpt[2]" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - # ---------- 1k1k low-latency (per-node TP=4 decode workers) ---------- - - isl: 1024 - osl: 1024 - search-space: - # 1p17d low-latency, bs=32 sweep. 18 nodes. - - conc-list: [512, 256, 128, 64] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/gb300-fp4/glm5.yaml:zip_override_1k1k_lowlat[0]" - decode: - num-worker: 17 - tp: 4 - ep: 1 - dp-attn: false - # 1p17d low-latency, bs=1 (single-stream). 18 nodes. - - conc-list: [32] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/gb300-fp4/glm5.yaml:zip_override_1k1k_lowlat[1]" - decode: - num-worker: 17 - tp: 4 - ep: 1 - dp-attn: false - -glm5-fp8-gb300-dynamo-sglang: - image: lmsysorg/sglang:v0.5.11-cu130 - model: zai-org/GLM-5-FP8 - model-prefix: glm5 - runner: gb300-nv - precision: fp8 - framework: dynamo-sglang - multinode: true - disagg: true - scenarios: - fixed-seq-len: - # ---------- 8k1k high-throughput (wide-EP decode) ---------- - - isl: 8192 - osl: 1024 - search-space: - - conc-list: [2800] - prefill: - num-worker: 14 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/gb300-fp8/glm5.yaml:zip_override_8k1k_hightpt[0]" - decode: - num-worker: 1 - tp: 16 - ep: 16 - dp-attn: true - - conc-list: [1700] - prefill: - num-worker: 12 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/gb300-fp8/glm5.yaml:zip_override_8k1k_hightpt[1]" - decode: - num-worker: 1 - tp: 24 - ep: 24 - dp-attn: true - - conc-list: [1300] - prefill: - num-worker: 10 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/gb300-fp8/glm5.yaml:zip_override_8k1k_hightpt[2]" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [900] - prefill: - num-worker: 8 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/gb300-fp8/glm5.yaml:zip_override_8k1k_hightpt[3]" - decode: - num-worker: 1 - tp: 40 - ep: 40 - dp-attn: true - # ---------- 8k1k low-latency (per-node TP=4 decode workers) ---------- - - isl: 8192 - osl: 1024 - search-space: - - conc-list: [150] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/gb300-fp8/glm5.yaml:zip_override_8k1k_lowlat[0]" - decode: - num-worker: 9 - tp: 4 - ep: 1 - dp-attn: false - - conc-list: [128, 64, 32] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/gb300-fp8/glm5.yaml:zip_override_8k1k_lowlat[1]" - decode: - num-worker: 17 - tp: 4 - ep: 1 - dp-attn: false - - conc-list: [24] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/gb300-fp8/glm5.yaml:zip_override_8k1k_lowlat[2]" - decode: - num-worker: 17 - tp: 4 - ep: 1 - dp-attn: false - # ---------- 1k1k high-throughput (wide-EP decode) ---------- - - isl: 1024 - osl: 1024 - search-space: - - conc-list: [8192] - prefill: - num-worker: 12 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/gb300-fp8/glm5.yaml:zip_override_1k1k_hightpt[0]" - decode: - num-worker: 1 - tp: 24 - ep: 24 - dp-attn: true - - conc-list: [7500] - prefill: - num-worker: 10 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/gb300-fp8/glm5.yaml:zip_override_1k1k_hightpt[1]" - decode: - num-worker: 1 - tp: 32 - ep: 32 - dp-attn: true - - conc-list: [7300] - prefill: - num-worker: 8 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/gb300-fp8/glm5.yaml:zip_override_1k1k_hightpt[2]" - decode: - num-worker: 1 - tp: 40 - ep: 40 - dp-attn: true - - conc-list: [6500] - prefill: - num-worker: 6 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/gb300-fp8/glm5.yaml:zip_override_1k1k_hightpt[3]" - decode: - num-worker: 1 - tp: 48 - ep: 48 - dp-attn: true - - conc-list: [5700] - prefill: - num-worker: 4 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/gb300-fp8/glm5.yaml:zip_override_1k1k_hightpt[4]" - decode: - num-worker: 1 - tp: 56 - ep: 56 - dp-attn: true - # ---------- 1k1k low-latency (per-node TP=4 decode workers) ---------- - - isl: 1024 - osl: 1024 - search-space: - - conc-list: [512, 256, 128, 64] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/gb300-fp8/glm5.yaml:zip_override_1k1k_lowlat[0]" - decode: - num-worker: 17 - tp: 4 - ep: 1 - dp-attn: false - - conc-list: [32] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/gb300-fp8/glm5.yaml:zip_override_1k1k_lowlat[1]" - decode: - num-worker: 17 - tp: 4 - ep: 1 - dp-attn: false - -# ============================================================================ -# Net-new agentic recipes from chore/agentx-v0.3 (no overlap with main entries). -# Recipes that ALREADY existed on main were intentionally left at main's version -# to preserve main behavior; PR-branch modifications to those recipes are NOT -# brought in here. -# ============================================================================ - -qwen3.5-fp8-b300-sglang-agentic-hicache: - image: lmsysorg/sglang:nightly-dev-cu13-20260520-425dffbd - model: Qwen/Qwen3.5-397B-A17B-FP8 - model-prefix: qwen3.5 - runner: cluster:b300-nv - precision: fp8 - framework: sglang - multinode: false - scenarios: - agentic-coding: - - dram-utilization: 0.80 - search-space: - - { tp: 4, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32] } - - { tp: 4, ep: 1, kv-offloading: dram, kv-offload-backend: hicache, conc-list: [16, 32, 48, 64] } - - -kimik2.5-fp4-b200-vllm-agentic-lmcache: - image: vllm/vllm-openai:v0.22.0 - model: nvidia/Kimi-K2.5-NVFP4 - model-prefix: kimik2.5 - runner: cluster:b200-dgxc - precision: fp4 - framework: vllm - multinode: false - scenarios: - agentic-coding: - - dram-utilization: 0.80 - search-space: - - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 24] } - - { tp: 8, ep: 1, kv-offloading: dram, kv-offload-backend: lmcache, conc-list: [16, 24, 32, 36] } - - { tp: 4, ep: 1, kv-offloading: none, conc-list: [8, 12, 14, 16, 18, 20] } - - { tp: 4, ep: 1, kv-offloading: dram, kv-offload-backend: lmcache, conc-list: [12, 14, 16, 18, 20, 22, 24, 32] } - - -# CONC range conservative for H100's 80 GB HBM3 under the long-ISL with- -# subagents corpus. hicache arm capped at conc 16 since high-conc + hicache -# tends to flake on first runs and conc 16 covers the cliff. The bench script -# sets WEKA_LOADER_OVERRIDE to the 256k-capped corpus variant. -dsv4-fp4-gb300-dynamo-vllm-agentic: - image: vllm/vllm-openai:v0.21.0-ubuntu2404 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: cluster:gb300-nv - precision: fp4 - framework: dynamo-vllm - multinode: true - disagg: true - scenarios: - agentic-coding: - - search-space: - # Low-latency: same 1p6d shape as the mid tier but at much lower conc - # (32 vs 192). 32/6 ≈ 5 seqs per decode worker — well below saturation, - # so each request gets ~6× the per-request decode compute it would get - # at conc=192. Reuses the 1p6d recipe; no separate recipe file needed. - - spec-decoding: none - conc-list: [32] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p6d-dep4-tp4-agentic.yaml" - decode: - num-worker: 6 - tp: 4 - ep: 1 - dp-attn: false - # Mid: 1 prefill (DEP=4) + 6 decode (TP=4). 7 nodes / 28 GPUs. - # Mirrors fixed-seq-len conc=192 entry. - - spec-decoding: none - conc-list: [192] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p6d-dep4-tp4-agentic.yaml" - decode: - num-worker: 6 - tp: 4 - ep: 1 - dp-attn: false - # High-throughput: 4 prefill (DEP=4 each) + 1 decode (DEP=8). 6 nodes / - # 24 GPUs. Smallest 4096-class shape in fixed-seq-len; deep_gemm_mega_moe - # on both sides. Mirrors fixed-seq-len conc=4096 entry (4p1d variant). - - spec-decoding: none - conc-list: [4096] - prefill: - num-worker: 4 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-4p1d-dep4-dep8-24-c4096-agentic.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - - -qwen3.5-fp8-h100-sglang-agentic: - image: lmsysorg/sglang:v0.5.12-cu130 - model: Qwen/Qwen3.5-397B-A17B-FP8 - model-prefix: qwen3.5 - runner: cluster:h100-dgxc - precision: fp8 - framework: sglang - multinode: false - scenarios: - agentic-coding: - - dram-utilization: 0.80 - search-space: - - { tp: 8, ep: 8, kv-offloading: none, conc-list: [1, 2, 4, 8, 12, 14, 16] } - - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: hicache, conc-list: [12, 14, 16, 20, 24, 28, 32, 42] } - - -# MiniMax-M3 NVFP4 disagg sweep on the same B300 topology matrix as the MXFP8 -# baseline above. The image includes vLLM PR #46380, so no runtime patch is -# needed. -minimaxm3-fp8-b300-dynamo-vllm: - image: vllm/vllm-openai:minimax-m3-0618-x86_64-cu130 - model: MiniMaxAI/MiniMax-M3-MXFP8 - model-prefix: minimaxm3 - runner: b300 - precision: fp8 - framework: dynamo-vllm - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - conc-list: [4, 16, 64, 128, 4096] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp8/1k1k/1p1d-dep2-tep8-1k1k.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [1, 4, 8, 16] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp8/1k1k/1p1d-dep2-tp4-marlin-1k1k.yaml" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - - conc-list: [2048] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp8/1k1k/1p2d-dep2-dep4-1k1k.yaml" - decode: - num-worker: 2 - tp: 4 - ep: 4 - dp-attn: true - - conc-list: [512, 4096] - prefill: - num-worker: 2 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp8/1k1k/2p1d-dep2-dep8-1k1k.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - conc-list: [32] - prefill: - num-worker: 2 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp8/1k1k/2p1d-dep2-tep8-1k1k.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [16] - prefill: - num-worker: 2 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp8/1k1k/2p2d-dep2-tep8-1k1k.yaml" - decode: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [4] - prefill: - num-worker: 3 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp8/1k1k/3p2d-dep2-tep8-1k1k.yaml" - decode: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: false - - isl: 8192 - osl: 1024 - search-space: - - conc-list: [256, 512] - prefill: - num-worker: 2 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp8/8k1k/2p2d-dep2-dep8-8k1k.yaml" - decode: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - - conc-list: [16] - prefill: - num-worker: 2 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp8/8k1k/2p2d-dep2-tep8-8k1k.yaml" - decode: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [4096] - prefill: - num-worker: 4 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp8/8k1k/4p2d-dep2-dep8-8k1k.yaml" - decode: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - - conc-list: [1, 4, 8, 16] - prefill: num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp8/8k1k/1p1d-dep2-tp4-marlin-8k1k.yaml" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - - conc-list: [4096] - prefill: - num-worker: 4 - tp: 2 - ep: 2 + tp: 16 + ep: 16 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp8/8k1k/4p2d-dep2-tep4-8k1k.yaml" - decode: - num-worker: 2 - tp: 4 - ep: 4 - dp-attn: false - - conc-list: [16, 32, 64, 128] + - conc-list: [ 4301 ] prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp8/8k1k/1p4d-dep2-tep4-8k1k.yaml" - decode: - num-worker: 4 + num-worker: 9 tp: 4 ep: 4 - dp-attn: false - - conc-list: [16] - prefill: - num-worker: 1 - tp: 2 - ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp8/8k1k/1p2d-dep2-tep4-8k1k.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx9dep4_gen1dep16_batch256_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb200Nvfp4/ISL8K_OSL1K/STP/ctx9dep4_gen1dep16_batch256_eplb0_mtp0.yaml" decode: - num-worker: 2 - tp: 4 - ep: 4 - dp-attn: false - - conc-list: [4] - prefill: num-worker: 1 - tp: 2 - ep: 2 + tp: 16 + ep: 16 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp8/8k1k/1p4d-dep2-tep8-8k1k.yaml" - decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false -# MiniMax-M3 NVFP4 disagg sweep on the same B300 topology matrix as the MXFP8 -# baseline above. The image includes vLLM PR #46380, so no runtime patch is -# needed. -minimaxm3-fp4-b300-dynamo-vllm: - image: vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41 - model: nvidia/MiniMax-M3-NVFP4 - model-prefix: minimaxm3 - runner: b300 +kimik2.5-fp4-gb300-dynamo-trt: + image: nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 + model: nvidia/Kimi-K2.5-NVFP4 + model-prefix: kimik2.5 + runner: gb300 precision: fp4 - framework: dynamo-vllm + framework: dynamo-trt multinode: true disagg: true scenarios: @@ -11866,1037 +384,457 @@ minimaxm3-fp4-b300-dynamo-vllm: - isl: 1024 osl: 1024 search-space: - - conc-list: [4, 16, 64, 128, 4096] + # Non-MTP configurations (default spec_decoding="none") + - conc-list: [ 8 ] prefill: num-worker: 1 - tp: 2 - ep: 2 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp4/1k1k/1p1d-dep2-tep8-1k1k.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch1_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch1_eplb0_mtp0.yaml" decode: - num-worker: 1 + num-worker: 4 tp: 8 ep: 8 dp-attn: false - - conc-list: [1, 4, 8, 16] - prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp4/1k1k/1p1d-dep2-tp4-1k1k.yaml" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - - conc-list: [2048] + - conc-list: [ 12 ] prefill: num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp4/1k1k/1p2d-dep2-dep4-1k1k.yaml" - decode: - num-worker: 2 tp: 4 ep: 4 dp-attn: true - - conc-list: [512, 4096] - prefill: - num-worker: 2 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp4/1k1k/2p1d-dep2-dep8-1k1k.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - conc-list: [32] - prefill: - num-worker: 2 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp4/1k1k/2p1d-dep2-tep8-1k1k.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [16] - prefill: - num-worker: 2 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp4/1k1k/2p2d-dep2-tep8-1k1k.yaml" - decode: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [4] - prefill: - num-worker: 3 - tp: 2 - ep: 2 - dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp4/1k1k/3p2d-dep2-tep8-1k1k.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch2_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch2_eplb0_mtp0.yaml" decode: - num-worker: 2 + num-worker: 4 tp: 8 ep: 8 dp-attn: false - - isl: 8192 - osl: 1024 - search-space: - - conc-list: [256, 512] - prefill: - num-worker: 2 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp4/8k1k/2p2d-dep2-dep8-8k1k.yaml" - decode: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - - conc-list: [16] + - conc-list: [ 24 ] prefill: - num-worker: 2 - tp: 2 - ep: 2 + num-worker: 1 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp4/8k1k/2p2d-dep2-tep8-8k1k.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch4_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen4tep8_batch4_eplb0_mtp0.yaml" decode: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: false - - conc-list: [4096] - prefill: num-worker: 4 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp4/8k1k/4p2d-dep2-dep8-8k1k.yaml" - decode: - num-worker: 2 tp: 8 ep: 8 - dp-attn: true - - conc-list: [1, 4, 8, 16] + dp-attn: false + - conc-list: [ 30 ] prefill: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp4/8k1k/1p1d-dep2-tp4-8k1k.yaml" - decode: num-worker: 1 tp: 4 - ep: 1 - dp-attn: false - - conc-list: [4096] - prefill: - num-worker: 4 - tp: 2 - ep: 2 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp4/8k1k/4p2d-dep2-tep4-8k1k.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen5tep4_batch4_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen5tep4_batch4_eplb0_mtp0.yaml" decode: - num-worker: 2 + num-worker: 5 tp: 4 ep: 4 dp-attn: false - - conc-list: [16, 32, 64, 128] + - conc-list: [ 60 ] prefill: num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp4/8k1k/1p4d-dep2-tep4-8k1k.yaml" - decode: - num-worker: 4 tp: 4 ep: 4 - dp-attn: false - - conc-list: [16] - prefill: - num-worker: 1 - tp: 2 - ep: 2 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp4/8k1k/1p2d-dep2-tep4-8k1k.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen5tep4_batch8_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen5tep4_batch8_eplb0_mtp0.yaml" decode: - num-worker: 2 + num-worker: 5 tp: 4 ep: 4 dp-attn: false - - conc-list: [4] + - conc-list: [ 333 ] prefill: num-worker: 1 - tp: 2 - ep: 2 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3/b300-fp4/8k1k/1p4d-dep2-tep8-8k1k.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen1dep32_batch8_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen1dep32_batch8_eplb0_mtp0.yaml" decode: - num-worker: 4 - tp: 8 - ep: 8 - dp-attn: false - -# MiniMax-M3 GB300 disagg sweep — refreshed recipe set (no Marlin variants). -# All prefill DEP2 (TP1 DP2 EP, 2 GPU/worker). Decode: DEP4, TEP8, DEP8, TEP4. -# 4 GPU/node (GB300 NVL72). kv-cache-dtype=fp8. srun_options mem=0 required. -minimaxm3-fp8-gb300-dynamo-vllm: - image: vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223 - model: MiniMaxAI/MiniMax-M3-MXFP8 - model-prefix: minimaxm3 - runner: gb300-nv - precision: fp8 - framework: dynamo-vllm - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # 1p1d DEP2+DEP4, 2n: conc 8192 - - conc-list: [8192] + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + - conc-list: [ 666 ] prefill: num-worker: 1 - tp: 2 - ep: 2 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/1k1k/1p1d-dep2-dep4-1k1k.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen1dep32_batch16_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen1dep32_batch16_eplb0_mtp0.yaml" decode: num-worker: 1 - tp: 4 - ep: 4 + tp: 32 + ep: 32 dp-attn: true - - # 1p2d DEP2+TEP8, 5n: conc 4,16,64,128,256 - - conc-list: [4, 16, 64, 128, 256] + - conc-list: [ 1229 ] prefill: num-worker: 1 - tp: 2 - ep: 2 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/1k1k/1p2d-dep2-tep8-1k1k.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen1dep32_batch32_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL1K_OSL1K/STP/ctx1dep4_gen1dep32_batch32_eplb0_mtp0.yaml" decode: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: false - - # 2p2d DEP2+TEP8, 5n: conc 32 - - conc-list: [32] + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + - conc-list: [ 4301 ] prefill: num-worker: 2 - tp: 2 - ep: 2 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/1k1k/2p2d-dep2-tep8-1k1k.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL1K_OSL1K/STP/ctx2dep4_gen1dep16_batch256_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL1K_OSL1K/STP/ctx2dep4_gen1dep16_batch256_eplb0_mtp0.yaml" decode: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: false - - # 2p3d DEP2+DEP4, 4n: conc 8192 - - conc-list: [8192] + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - conc-list: [ 8192 ] prefill: num-worker: 2 - tp: 2 - ep: 2 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/1k1k/2p3d-dep2-dep4-1k1k.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL1K_OSL1K/STP/ctx2dep4_gen1dep16_batch512_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL1K_OSL1K/STP/ctx2dep4_gen1dep16_batch512_eplb0_mtp0.yaml" decode: - num-worker: 3 + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + - conc-list: [ 2253 ] + prefill: + num-worker: 2 tp: 4 ep: 4 dp-attn: true - - # 2p4d DEP2+DEP4, 5n: conc 8192 - - conc-list: [8192] + additional-settings: + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL1K_OSL1K/STP/ctx2dep4_gen1dep32_batch64_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL1K_OSL1K/STP/ctx2dep4_gen1dep32_batch64_eplb0_mtp0.yaml" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + - conc-list: [ 4301 ] prefill: num-worker: 2 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/1k1k/2p4d-dep2-dep4-1k1k.yaml" - decode: - num-worker: 4 tp: 4 ep: 4 dp-attn: true - - # 4p2d DEP2+DEP8, 6n: conc 1024,4096 - - conc-list: [1024, 4096] - prefill: - num-worker: 4 - tp: 2 - ep: 2 - dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/1k1k/4p2d-dep2-dep8-1k1k.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL1K_OSL1K/STP/ctx2dep4_gen1dep32_batch128_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL1K_OSL1K/STP/ctx2dep4_gen1dep32_batch128_eplb0_mtp0.yaml" decode: - num-worker: 2 - tp: 8 - ep: 8 + num-worker: 1 + tp: 32 + ep: 32 dp-attn: true - isl: 8192 osl: 1024 search-space: - # 1p1d DEP2+DEP8, 3n: conc 256 - - conc-list: [256] + # Non-MTP configurations (default spec_decoding="none") + - conc-list: [ 4 ] prefill: num-worker: 1 - tp: 2 - ep: 2 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/8k1k/1p1d-dep2-dep8-8k1k.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen4tep8_batch1_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen4tep8_batch1_eplb0_mtp0.yaml" decode: - num-worker: 1 + num-worker: 4 tp: 8 ep: 8 - dp-attn: true - - # 1p1d DEP2+TEP8, 3n: conc 128 - - conc-list: [128] + dp-attn: false + - conc-list: [ 12 ] prefill: num-worker: 1 - tp: 2 - ep: 2 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/8k1k/1p1d-dep2-tep8-8k1k.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen4tep8_batch2_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen4tep8_batch2_eplb0_mtp0.yaml" decode: - num-worker: 1 + num-worker: 4 tp: 8 ep: 8 dp-attn: false - - # 1p2d DEP2+TEP8, 5n: conc 32,64,128 - - conc-list: [32, 64, 128] + - conc-list: [ 24 ] prefill: num-worker: 1 - tp: 2 - ep: 2 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/8k1k/1p2d-dep2-tep8-8k1k.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen4tep8_batch4_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen4tep8_batch4_eplb0_mtp0.yaml" decode: - num-worker: 2 + num-worker: 4 tp: 8 ep: 8 dp-attn: false - - # 2p1d DEP2+DEP8, 3n: conc 512 - - conc-list: [512] + - conc-list: [ 30 ] prefill: - num-worker: 2 - tp: 2 - ep: 2 + num-worker: 1 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/8k1k/2p1d-dep2-dep8-8k1k.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch4_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch4_eplb0_mtp0.yaml" decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - # 2p2d DEP2+TEP8, 5n: conc 16 - - conc-list: [16] + num-worker: 5 + tp: 4 + ep: 4 + dp-attn: false + - conc-list: [ 60 ] prefill: - num-worker: 2 - tp: 2 - ep: 2 + num-worker: 1 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/8k1k/2p2d-dep2-tep8-8k1k.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch8_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch8_eplb0_mtp0.yaml" decode: - num-worker: 2 - tp: 8 - ep: 8 + num-worker: 5 + tp: 4 + ep: 4 dp-attn: false - - # 2p4d DEP2+TEP4, 5n: conc 4 - - conc-list: [4] + - conc-list: [ 115 ] prefill: - num-worker: 2 - tp: 2 - ep: 2 + num-worker: 1 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/8k1k/2p4d-dep2-tep4-8k1k.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch16_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx1dep4_gen5tep4_batch16_eplb0_mtp0.yaml" decode: - num-worker: 4 + num-worker: 5 tp: 4 ep: 4 dp-attn: false - - # 3p1d DEP2+DEP8, 4n: conc 1024 - - conc-list: [1024] + - conc-list: [ 333 ] prefill: - num-worker: 3 - tp: 2 - ep: 2 + num-worker: 2 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/8k1k/3p1d-dep2-dep8-8k1k.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx2dep4_gen1dep32_batch8_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx2dep4_gen1dep32_batch8_eplb0_mtp0.yaml" decode: num-worker: 1 - tp: 8 - ep: 8 + tp: 32 + ep: 32 dp-attn: true - - # 3p2d DEP2+DEP8, 6n: conc 512 - - conc-list: [512] + - conc-list: [ 615 ] prefill: num-worker: 3 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/8k1k/3p2d-dep2-dep8-8k1k.yaml" - decode: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - - # 6p1d DEP2+DEP8, 5n: conc 2048 - - conc-list: [2048] - prefill: - num-worker: 6 - tp: 2 - ep: 2 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/8k1k/6p1d-dep2-dep8-8k1k.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx3dep4_gen1dep32_batch16_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx3dep4_gen1dep32_batch16_eplb0_mtp0.yaml" decode: num-worker: 1 - tp: 8 - ep: 8 + tp: 32 + ep: 32 dp-attn: true - -qwen3.5-fp4-b200-trt: - image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc18 - model: nvidia/Qwen3.5-397B-A17B-NVFP4 - model-prefix: qwen3.5 - runner: b200 - precision: fp4 - framework: trt - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 4, ep: 1, conc-list: [8, 16] } - - { tp: 4, ep: 4, conc-list: [64, 128] } - - { tp: 8, ep: 8, conc-list: [4, 64] } - - { tp: 4, ep: 4, dp-attn: true, conc-list: [1024] } - - { tp: 8, ep: 8, dp-attn: true, conc-list: [512, 1024] } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 2, ep: 1, conc-list: [4, 16] } - - { tp: 4, ep: 1, conc-list: [4] } - - { tp: 2, ep: 2, conc-list: [8, 32] } - - { tp: 8, ep: 8, conc-list: [4] } - - { tp: 4, ep: 4, dp-attn: true, conc-list: [1024] } - - { tp: 8, ep: 8, dp-attn: true, conc-list: [256, 512, 1024] } - -# MiniMax-M3 GB200 disagg sweep — adapted from NV B300 PR #1863. -# All prefill DEP4 (TP1 DP4 EP, 4 GPU/worker). Decode: Marlin, TEP8, DEP8, TEP4. -# 4 GPU/node (GB200 NVL72). FLASHINFER attention with FP8 KV cache. -minimaxm3-fp8-gb200-dynamo-vllm: - image: vllm/vllm-openai:minimax-m3-perf-arm64-13.0.1-7a67223 - model: MiniMaxAI/MiniMax-M3-MXFP8 - model-prefix: minimaxm3 - runner: gb200 - precision: fp8 - framework: dynamo-vllm - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # 1p1d DEP4+DEP8, 3n: conc 1024,4096 - - conc-list: [1024, 4096] + - conc-list: [ 1229 ] prefill: - num-worker: 1 + num-worker: 5 tp: 4 ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb200-fp8/1k1k/1p1d-dep4-dep8-1k1k.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx5dep4_gen1dep32_batch32_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx5dep4_gen1dep32_batch32_eplb0_mtp0.yaml" decode: num-worker: 1 - tp: 8 - ep: 8 + tp: 32 + ep: 32 dp-attn: true - - # 1p1d DEP4+TEP8, 3n: conc 128,256 - - conc-list: [128, 256] + - conc-list: [ 2253 ] prefill: - num-worker: 1 + num-worker: 6 tp: 4 ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb200-fp8/1k1k/1p1d-dep4-tep8-1k1k.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx6dep4_gen1dep16_batch128_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx6dep4_gen1dep16_batch128_eplb0_mtp0.yaml" decode: num-worker: 1 - tp: 8 - ep: 8 - dp-attn: false - - # 1p2d DEP4+DEP8, 5n: conc 1024,4096 - - conc-list: [1024, 4096] + tp: 16 + ep: 16 + dp-attn: true + - conc-list: [ 2151 ] prefill: - num-worker: 1 + num-worker: 8 tp: 4 ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb200-fp8/1k1k/1p2d-dep4-dep8-1k1k.yaml" + # https://github.com/NVIDIA/srt-slurm/blob/sa-submission-q2-2026/recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx8dep4_gen1dep32_batch64_eplb0_mtp0.yaml + - "CONFIG_FILE=recipes/kimi2.5/trtllm_dynamo/disagg/gb300Nvfp4/ISL8K_OSL1K/STP/ctx8dep4_gen1dep32_batch64_eplb0_mtp0.yaml" decode: - num-worker: 2 - tp: 8 - ep: 8 + num-worker: 1 + tp: 32 + ep: 32 dp-attn: true - # 1p2d DEP4+TEP8, 5n: conc 4,16,64 - - conc-list: [4, 16, 64] +kimik2.5-fp4-gb200-dynamo-vllm: + image: vllm/vllm-openai:v0.21.0 + model: nvidia/Kimi-K2.5-NVFP4 + model-prefix: kimik2.5 + runner: gb200 + precision: fp4 + framework: dynamo-vllm + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 1024 + osl: 1024 + search-space: + - conc-list: [4096, 12288] prefill: num-worker: 1 - tp: 4 + tp: 1 ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb200-fp8/1k1k/1p2d-dep4-tep8-1k1k.yaml" + - "CONFIG_FILE=recipes/vllm/kimi-k2.5-fp4/1k1k/disagg-gb200-1p1d-dep4-dep8.yaml" decode: - num-worker: 2 - tp: 8 + num-worker: 1 + tp: 1 ep: 8 - dp-attn: false - - # 1p4d DEP4+TP4 Marlin, 5n: conc 32 - - conc-list: [32] + dp-attn: true + - conc-list: [4, 8, 32, 128] prefill: num-worker: 1 - tp: 4 + tp: 1 ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb200-fp8/1k1k/1p4d-dep4-tp4-marlin-1k1k.yaml" + - "CONFIG_FILE=recipes/vllm/kimi-k2.5-fp4/1k1k/disagg-gb200-1p4d-dep4-tp8.yaml" decode: num-worker: 4 - tp: 4 + tp: 8 ep: 1 dp-attn: false - - - isl: 8192 - osl: 1024 - search-space: - # 1p2d DEP4+DEP8, 5n: conc 512 - - conc-list: [512] + - conc-list: [4096, 6144] prefill: num-worker: 1 - tp: 4 + tp: 1 ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb200-fp8/8k1k/1p2d-dep4-dep8-8k1k.yaml" + - "CONFIG_FILE=recipes/vllm/kimi-k2.5-fp4/1k1k/disagg-gb200-1p1d-dep4-dep16.yaml" decode: - num-worker: 2 - tp: 8 - ep: 8 + num-worker: 1 + tp: 1 + ep: 16 dp-attn: true - - # 1p2d DEP4+TEP4, 3n: conc 4 - - conc-list: [4] + - isl: 8192 + osl: 1024 + search-space: + - conc-list: [128] prefill: num-worker: 1 - tp: 4 + tp: 1 ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb200-fp8/8k1k/1p2d-dep4-tep4-8k1k.yaml" + - "CONFIG_FILE=recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb200-1p4d-dep4-tep4.yaml" decode: - num-worker: 2 + num-worker: 4 tp: 4 ep: 4 dp-attn: false - - # 1p2d DEP4+TEP8, 5n: conc 4,16,32,64,128 - - conc-list: [4, 16, 32, 64, 128] + - conc-list: [4, 8, 16, 32, 256] prefill: num-worker: 1 - tp: 4 + tp: 1 ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb200-fp8/8k1k/1p2d-dep4-tep8-8k1k.yaml" + - "CONFIG_FILE=recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb200-1p4d-dep4-tp8.yaml" decode: - num-worker: 2 + num-worker: 4 tp: 8 - ep: 8 + ep: 1 dp-attn: false - - # 2p2d DEP4+DEP8, 6n: conc 512,1024 - - conc-list: [512, 1024] + - conc-list: [1024] prefill: - num-worker: 2 - tp: 4 + num-worker: 3 + tp: 1 ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb200-fp8/8k1k/2p2d-dep4-dep8-8k1k.yaml" + - "CONFIG_FILE=recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb200-3p1d-dep4-dep16.yaml" decode: - num-worker: 2 - tp: 8 - ep: 8 + num-worker: 1 + tp: 1 + ep: 16 dp-attn: true - - # 3p2d DEP4+DEP8, 7n: conc 4096 - - conc-list: [4096] + - conc-list: [3072] prefill: - num-worker: 3 - tp: 4 + num-worker: 6 + tp: 1 ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb200-fp8/8k1k/3p2d-dep4-dep8-8k1k.yaml" + - "CONFIG_FILE=recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb200-6p1d-dep4-dep16.yaml" decode: - num-worker: 2 - tp: 8 - ep: 8 + num-worker: 1 + tp: 1 + ep: 16 dp-attn: true - - # 5p2d DEP4+DEP8, 9n: conc 4096 - - conc-list: [4096] + - conc-list: [6144] prefill: - num-worker: 5 - tp: 4 + num-worker: 8 + tp: 1 ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/minimax-m3-gb200-fp8/8k1k/5p2d-dep4-dep8-8k1k.yaml" + - "CONFIG_FILE=recipes/vllm/kimi-k2.5-fp4/8k1k/disagg-gb200-8p1d-dep4-dep16.yaml" decode: - num-worker: 2 - tp: 8 - ep: 8 + num-worker: 1 + tp: 1 + ep: 16 dp-attn: true -qwen3.5-fp4-b200-trt-mtp: - image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc18 - model: nvidia/Qwen3.5-397B-A17B-NVFP4 - model-prefix: qwen3.5 - runner: b200 - precision: fp4 - framework: trt - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 2, ep: 1, spec-decoding: "mtp", conc-list: [8] } - - { tp: 2, ep: 2, spec-decoding: "mtp", conc-list: [4] } - - { tp: 8, ep: 8, spec-decoding: "mtp", conc-list: [4] } - - { tp: 8, ep: 8, dp-attn: true, spec-decoding: "mtp", conc-list: [64, 128, 256, 512, 1024] } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 2, ep: 1, spec-decoding: "mtp", conc-list: [4] } - - { tp: 2, ep: 2, spec-decoding: "mtp", conc-list: [8, 16] } - - { tp: 4, ep: 4, spec-decoding: "mtp", conc-list: [4] } - - { tp: 8, ep: 8, spec-decoding: "mtp", conc-list: [4] } - - { tp: 8, ep: 8, dp-attn: true, spec-decoding: "mtp", conc-list: [128, 256, 1024] } - -# MiniMax-M3 day-zero (https://recipes.vllm.ai/MiniMaxAI/MiniMax-M3). -# 427B total / 26B active MoE with MSA sparse attention; MXFP8 checkpoint -# (MiniMaxAI/MiniMax-M3-MXFP8, ~444 GB) quantized by NVIDIA — native MX tensor -# cores on Blackwell. M3 support has not shipped in a stable vLLM release; -# the dedicated vllm/vllm-openai:minimax-m3-0618-x86_64-cu130 image is built -# from the m3_release branch (vllm-project/vllm#45381). --block-size 128 is mandatory (MSA -# sparse/index cache alignment). Weights are NOT SRE-staged; b300 falls back -# to writable /data/models (see launch_b300-nv.sh MODEL_PATH split). -minimaxm3-fp8-b300-vllm: - image: vllm/vllm-openai:minimax-m3-0618-x86_64-cu130 - model: MiniMaxAI/MiniMax-M3-MXFP8 - model-prefix: minimaxm3 - runner: b300 - precision: fp8 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, conc-start: 1, conc-end: 64 } - - { tp: 8, ep: 8, conc-start: 1, conc-end: 512 } - - { tp: 4, conc-start: 1, conc-end: 64 } - - { tp: 4, ep: 4, conc-start: 64, conc-end: 512 } - - { tp: 4, ep: 4, dp-attn: true, conc-start: 128, conc-end: 512 } - # tp2 fits MXFP8 weights (~222 GB/GPU of 288) but KV headroom is thin; - # 1k1k only, drop if it OOMs at the high end. - - { tp: 2, ep: 2, conc-start: 16, conc-end: 128 } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 256, conc-end: 1024 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, conc-start: 1, conc-end: 64 } - - { tp: 8, ep: 8, conc-start: 1, conc-end: 512 } - - { tp: 4, conc-start: 1, conc-end: 128 } - - { tp: 4, ep: 4, conc-start: 64, conc-end: 256 } - - { tp: 4, ep: 4, dp-attn: true, conc-start: 64, conc-end: 128 } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 128, conc-end: 512 } - -# MiniMax-M3 NVFP4 (nvidia/MiniMax-M3-NVFP4) B300 single-node vLLM — FP4 variant -# of minimaxm3-fp8-b300-vllm. MiniMax-M3 modelopt NVFP4 support (vllm-project/vllm -# PR #46380) is baked into the perf container image, so no runtime patch is -# needed. --block-size 128 is mandatory (MSA sparse/index cache); -# weights are pre-staged read-only at /scratch/models/MiniMax-M3-NVFP4 (added to -# the STAGED_MODELS allow-list in launch_b300-nv.sh). -minimaxm3-fp4-b300-vllm: - image: vllm/vllm-openai:nightly-93d8f834dd8acf33eb0e2a75b2711b628cb6e226 - model: nvidia/MiniMax-M3-NVFP4 - model-prefix: minimaxm3 - runner: b300 - precision: fp4 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, conc-start: 1, conc-end: 2 } - - { tp: 4, conc-start: 1, conc-end: 2 } - - { tp: 2, conc-start: 4, conc-end: 256 } - - { tp: 4, conc-start: 64, conc-end: 64 } - - { tp: 4, ep: 4, conc-start: 64, conc-end: 512 } - - { tp: 2, ep: 2, dp-attn: true, conc-start: 512, conc-end: 512 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, conc-start: 1, conc-end: 2 } - - { tp: 4, conc-start: 1, conc-end: 2 } - - { tp: 2, conc-start: 4, conc-end: 256 } - - { tp: 2, ep: 2, dp-attn: true, conc-start: 512, conc-end: 512 } - -# EAGLE3 speculative-decoding (spec-decoding: mtp) variant of MiniMax-M3 NVFP4 -# (nvidia/MiniMax-M3-NVFP4) B300 single-node vLLM, pairing the target with the -# Inferact/MiniMax-M3-EAGLE3 draft head (3 speculative tokens). MiniMax-M3 -# modelopt NVFP4 support (vllm-project/vllm PR #46380) is baked into the perf -# container image, so no runtime patch is needed; prompts are routed through the -# chat template. Target weights are pre-staged read-only at -# /scratch/models/MiniMax-M3-NVFP4 (added to the STAGED_MODELS allow-list in -# launch_b300-nv.sh); the EAGLE3 draft is downloaded to the writable models dir. -minimaxm3-fp4-b300-vllm-mtp: - image: vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41 - model: nvidia/MiniMax-M3-NVFP4 - model-prefix: minimaxm3 - runner: b300 - precision: fp4 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, conc-start: 1, conc-end: 64, spec-decoding: mtp } - - { tp: 8, ep: 8, conc-start: 1, conc-end: 256, spec-decoding: mtp } - - { tp: 4, conc-start: 1, conc-end: 64, spec-decoding: mtp } - - { tp: 4, ep: 4, conc-start: 64, conc-end: 256, spec-decoding: mtp } - - { tp: 4, ep: 4, dp-attn: true, conc-start: 128, conc-end: 512, spec-decoding: mtp } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 256, conc-end: 512, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, conc-start: 1, conc-end: 64, spec-decoding: mtp } - - { tp: 8, ep: 8, conc-start: 1, conc-end: 256, spec-decoding: mtp } - - { tp: 4, conc-start: 1, conc-end: 64, spec-decoding: mtp } - - { tp: 4, ep: 4, conc-start: 64, conc-end: 256, spec-decoding: mtp } - - { tp: 4, ep: 4, dp-attn: true, conc-start: 64, conc-end: 128, spec-decoding: mtp } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 128, conc-end: 256, spec-decoding: mtp } - -# MiniMax-M3 day-zero (https://recipes.vllm.ai/MiniMaxAI/MiniMax-M3). -# 427B total / 26B active MoE with MSA sparse attention; MXFP8 checkpoint -# (MiniMaxAI/MiniMax-M3-MXFP8, ~444 GB) quantized by NVIDIA — native MX tensor -# cores on Blackwell. M3 support has not shipped in a stable vLLM release; -# the dedicated vllm/vllm-openai:minimax-m3-0618-x86_64-cu130 image is built -# from the m3_release branch (vllm-project/vllm#45381). --block-size 128 is mandatory (MSA -# sparse/index cache alignment). Weights are NOT SRE-staged: b200-dgxc reads -# /lustre/fsw/gharunners/models/MiniMax-M3-MXFP8 (pre-downloaded, see -# launch_b200-dgxc.sh). -minimaxm3-fp8-b200-vllm: - image: vllm/vllm-openai:minimax-m3-0618-x86_64-cu130 - model: MiniMaxAI/MiniMax-M3-MXFP8 - model-prefix: minimaxm3 - runner: b200-dgxc - precision: fp8 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, conc-start: 1, conc-end: 64 } - - { tp: 8, ep: 8, conc-start: 1, conc-end: 512 } - - { tp: 4, conc-start: 1, conc-end: 64 } - - { tp: 4, ep: 4, conc-start: 64, conc-end: 512 } - - { tp: 4, ep: 4, dp-attn: true, conc-start: 128, conc-end: 512 } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 256, conc-end: 1024 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, conc-start: 1, conc-end: 64 } - - { tp: 8, ep: 8, conc-start: 1, conc-end: 256 } - - { tp: 4, conc-start: 1, conc-end: 64 } - - { tp: 4, ep: 4, conc-start: 64, conc-end: 256 } - - { tp: 4, ep: 4, dp-attn: true, conc-start: 64, conc-end: 128 } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 128, conc-end: 256 } - -# MiniMax-M3 NVFP4 (nvidia/MiniMax-M3-NVFP4) B200 single-node vLLM — FP4 variant -# of minimaxm3-fp8-b200-vllm, running on the b200-dgxc cluster. MiniMax-M3 -# modelopt NVFP4 support (vllm-project/vllm PR #46380) is baked into the perf -# container image, so no runtime patch is needed. --block-size 128 is mandatory -# (MSA sparse/index cache); weights are pre-staged at /scratch/fsw/models/MiniMax-M3-NVFP4 -# (launch_b200-dgxc.sh resolves MODEL_PATH for minimaxm3-fp4). -minimaxm3-fp4-b200-vllm: - image: vllm/vllm-openai:nightly-93d8f834dd8acf33eb0e2a75b2711b628cb6e226 - model: nvidia/MiniMax-M3-NVFP4 - model-prefix: minimaxm3 - runner: b200-dgxc - precision: fp4 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, conc-start: 1, conc-end: 8 } - - { tp: 2, conc-start: 4, conc-end: 8 } - - { tp: 2, conc-start: 256, conc-end: 2048 } - - { tp: 4, conc-start: 32, conc-end: 256 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, conc-start: 1, conc-end: 4 } - - { tp: 4, conc-start: 1, conc-end: 4 } - - { tp: 2, conc-start: 1, conc-end: 128 } - - { tp: 4, conc-start: 256, conc-end: 1024 } - - { tp: 4, ep: 4, dp-attn: true, conc-start: 256, conc-end: 1024 } - -# EAGLE3 speculative-decoding (spec-decoding: mtp) variant of -# minimaxm3-fp8-b200-vllm, pairing MiniMaxAI/MiniMax-M3-MXFP8 with the -# Inferact/MiniMax-M3-EAGLE3 draft head (3 speculative tokens, drafter pinned -# to TRITON_ATTN). Search space mirrors the non-MTP entry trimmed at the -# extreme-concurrency end, identical to the minimaxm3-fp8-b300-vllm-mtp -# precedent: spec decode pays off at low/mid concurrency while acceptance -# dilutes in big batches, and the draft weights + draft KV shave headroom. -minimaxm3-fp8-b200-vllm-mtp: - image: vllm/vllm-openai:minimax-m3-0618-x86_64-cu130 - model: MiniMaxAI/MiniMax-M3-MXFP8 - model-prefix: minimaxm3 - runner: b200-dgxc - precision: fp8 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, conc-start: 1, conc-end: 64, spec-decoding: mtp } - - { tp: 8, ep: 8, conc-start: 1, conc-end: 256, spec-decoding: mtp } - - { tp: 4, conc-start: 1, conc-end: 64, spec-decoding: mtp } - - { tp: 4, ep: 4, conc-start: 64, conc-end: 256, spec-decoding: mtp } - - { tp: 4, ep: 4, dp-attn: true, conc-start: 128, conc-end: 512, spec-decoding: mtp } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 256, conc-end: 512, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, conc-start: 1, conc-end: 64, spec-decoding: mtp } - - { tp: 8, ep: 8, conc-start: 1, conc-end: 256, spec-decoding: mtp } - - { tp: 4, conc-start: 1, conc-end: 64, spec-decoding: mtp } - - { tp: 4, ep: 4, conc-start: 64, conc-end: 256, spec-decoding: mtp } - - { tp: 4, ep: 4, dp-attn: true, conc-start: 64, conc-end: 128, spec-decoding: mtp } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 128, conc-end: 256, spec-decoding: mtp } - -# EAGLE3 speculative-decoding (spec-decoding: mtp) variant of MiniMax-M3 NVFP4 -# (nvidia/MiniMax-M3-NVFP4) B200 single-node vLLM, pairing the target with the -# Inferact/MiniMax-M3-EAGLE3 draft head (3 speculative tokens). Runs on the -# b200-dgxc cluster. MiniMax-M3 modelopt NVFP4 support (vllm-project/vllm -# PR #46380) is baked into the perf container image, so no runtime patch is -# needed; prompts are routed through the chat template. Target weights are -# pre-staged at /scratch/fsw/models/MiniMax-M3-NVFP4 (launch_b200-dgxc.sh -# resolves MODEL_PATH for minimaxm3-fp4); the EAGLE3 draft is fetched next to -# the target weights. -minimaxm3-fp4-b200-vllm-mtp: - image: vllm/vllm-openai:vllm-minimax-m3-perf-x86_64-13.0.1-8b00f41 - model: nvidia/MiniMax-M3-NVFP4 - model-prefix: minimaxm3 - runner: b200-dgxc - precision: fp4 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, conc-start: 1, conc-end: 64, spec-decoding: mtp } - - { tp: 8, ep: 8, conc-start: 1, conc-end: 256, spec-decoding: mtp } - - { tp: 4, conc-start: 1, conc-end: 64, spec-decoding: mtp } - - { tp: 4, ep: 4, conc-start: 64, conc-end: 256, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, conc-start: 1, conc-end: 64, spec-decoding: mtp } - - { tp: 8, ep: 8, conc-start: 1, conc-end: 256, spec-decoding: mtp } - - { tp: 4, conc-start: 1, conc-end: 64, spec-decoding: mtp } - - { tp: 4, ep: 4, conc-start: 64, conc-end: 256, spec-decoding: mtp } - -# EAGLE3 speculative-decoding (spec-decoding: mtp) variant of -# minimaxm3-fp8-b300-vllm, pairing MiniMaxAI/MiniMax-M3-MXFP8 with the -# Inferact/MiniMax-M3-EAGLE3 draft head (3 speculative tokens). -# Search space mirrors the non-MTP entry trimmed at -# the extreme-concurrency end, per the dsv4-fp4-b300-vllm-mtp precedent: -# spec decode pays off at low/mid concurrency while acceptance dilutes in -# big batches, and the draft weights + draft KV shave headroom — tp2-ep2 is -# dropped entirely since its KV headroom was already thin without a draft. -minimaxm3-fp8-b300-vllm-mtp: - image: vllm/vllm-openai:minimax-m3-0618-x86_64-cu130 - model: MiniMaxAI/MiniMax-M3-MXFP8 - model-prefix: minimaxm3 - runner: b300 - precision: fp8 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, conc-start: 1, conc-end: 64, spec-decoding: mtp } - - { tp: 8, ep: 8, conc-start: 1, conc-end: 256, spec-decoding: mtp } - - { tp: 4, conc-start: 1, conc-end: 64, spec-decoding: mtp } - - { tp: 4, ep: 4, conc-start: 64, conc-end: 256, spec-decoding: mtp } - - { tp: 4, ep: 4, dp-attn: true, conc-start: 128, conc-end: 512, spec-decoding: mtp } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 256, conc-end: 512, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, conc-start: 1, conc-end: 64, spec-decoding: mtp } - - { tp: 8, ep: 8, conc-start: 1, conc-end: 256, spec-decoding: mtp } - - { tp: 4, conc-start: 1, conc-end: 64, spec-decoding: mtp } - - { tp: 4, ep: 4, conc-start: 64, conc-end: 256, spec-decoding: mtp } - - { tp: 4, ep: 4, dp-attn: true, conc-start: 64, conc-end: 128, spec-decoding: mtp } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 128, conc-end: 256, spec-decoding: mtp } - -# EAGLE3 speculative-decoding (spec-decoding: mtp) variant of -# minimaxm3-fp8-h200-vllm (PR #1731), pairing MiniMaxAI/MiniMax-M3-MXFP8 with -# the Inferact/MiniMax-M3-EAGLE3 draft head (3 speculative tokens, drafter -# pinned to FLASH_ATTN since the head is MHA and FlashInfer needs GQA/MQA at -# page size 128). Search space mirrors the non-MTP entry trimmed at the -# extreme-concurrency end, per the dsv4-fp4-b300-vllm-mtp / minimaxm3 b300-mtp -# precedent: spec decode pays off at low/mid concurrency while acceptance -# dilutes in big batches, and the draft weights + draft KV shave headroom. -minimaxm3-fp8-h200-vllm-mtp: - image: vllm/vllm-openai:minimax-m3 - model: MiniMaxAI/MiniMax-M3-MXFP8 - model-prefix: minimaxm3 - runner: h200 - precision: fp8 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 4, conc-start: 1, conc-end: 64, spec-decoding: mtp } - - { tp: 4, ep: 4, conc-start: 128, conc-end: 256, spec-decoding: mtp } - - { tp: 8, conc-start: 1, conc-end: 128, spec-decoding: mtp } - - { tp: 8, ep: 8, conc-start: 256, conc-end: 256, spec-decoding: mtp } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 256, conc-end: 512, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 4, conc-start: 1, conc-end: 32, spec-decoding: mtp } - - { tp: 8, conc-start: 1, conc-end: 128, spec-decoding: mtp } - - { tp: 8, ep: 8, conc-start: 256, conc-end: 256, spec-decoding: mtp } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 256, conc-end: 256, spec-decoding: mtp } - -# EAGLE3 speculative-decoding (spec-decoding: mtp) variant of -# minimaxm3-fp8-h100-vllm (PR #1731). Same TP8-only layout (H100 80 GB has no -# room below TP8 once ~56 GB of weights land per GPU) plus the -# Inferact/MiniMax-M3-EAGLE3 draft head (3 speculative tokens, FLASH_ATTN -# drafter). DEP stays omitted as on the non-MTP entry — KV-cache init already -# failed at high conc, and the draft head + draft KV only tighten it further. -minimaxm3-fp8-h100-vllm-mtp: - image: vllm/vllm-openai:minimax-m3 - model: MiniMaxAI/MiniMax-M3-MXFP8 - model-prefix: minimaxm3 - runner: h100 - precision: fp8 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 8, conc-start: 1, conc-end: 128, spec-decoding: mtp } - - { tp: 8, ep: 8, conc-start: 256, conc-end: 256, spec-decoding: mtp } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 8, conc-start: 1, conc-end: 64, spec-decoding: mtp } - - { tp: 8, ep: 8, conc-start: 128, conc-end: 256, spec-decoding: mtp } - kimik2.5-fp4-gb300-dynamo-vllm: image: vllm/vllm-openai:v0.21.0 model: nvidia/Kimi-K2.5-NVFP4 @@ -13057,206 +995,26 @@ kimik2.5-fp4-gb300-dynamo-vllm: tp: 1 ep: 24 dp-attn: true -minimaxm3-fp8-h100-vllm-agentic: - image: vllm/vllm-openai:nightly-04c2a8deac44fdb1ca3e2b5ec3e6bf16f3f6a914 - model: MiniMaxAI/MiniMax-M3-MXFP8 - model-prefix: minimaxm3 - runner: cluster:h100-dgxc - precision: fp8 - framework: vllm - multinode: false - scenarios: - agentic-coding: - # FP8 KV cache is 29,952 bytes/token/GPU for TP/TEP: 60 layers of - # sharded FP8 K/V plus 57 layers of BF16 indexer cache. The June-21 trace - # has a 269k-token service-time-weighted active request, placing the H100 - # 1.64M-token HBM cliff near conc 6. Sample every integer through the - # cliff; Mooncake extends prefix retention, not active-request HBM. - - dram-utilization: 0.80 - search-space: - - { tp: 8, kv-offloading: none, conc-list: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16] } - - { tp: 8, ep: 8, kv-offloading: none, conc-list: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16] } - - { tp: 8, kv-offloading: dram, kv-offload-backend: mooncake, conc-list: [3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16] } - - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: mooncake, conc-list: [3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16] } - -minimaxm3-fp8-h200-vllm-agentic: - image: vllm/vllm-openai:nightly-04c2a8deac44fdb1ca3e2b5ec3e6bf16f3f6a914 - model: MiniMaxAI/MiniMax-M3-MXFP8 - model-prefix: minimaxm3 - runner: cluster:h200-dgxc +# Single-node DeepSeek-R1 FP8 sglang smoke on the CoreWeave B300 cluster +# (runner cluster:b300-cw). Cloned from dsr1-fp8-b300-sglang; the only +# difference is the runner label pinning it to the CoreWeave fleet. +dsr1-fp8-b300-cw-sglang: + image: lmsysorg/sglang:v0.5.12-cu130 + model: deepseek-ai/DeepSeek-R1-0528 + model-prefix: dsr1 + runner: cluster:b300-cw precision: fp8 - framework: vllm - multinode: false - scenarios: - agentic-coding: - # The same 29,952-byte FP8 TP/TEP cache layout gives H200 about 2.56M - # active tokens. Against the June-21 service-time-weighted 269k-token - # request, the expected HBM cliff is near conc 10; sample densely from - # 5-14 and retain post-cliff points through 20. - - dram-utilization: 0.80 - search-space: - - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 18, 20] } - - { tp: 8, ep: 8, kv-offloading: none, conc-list: [2, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 18, 20] } - - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: mooncake, conc-list: [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 18, 20] } - - - -dsv4-fp4-b200-sglang-agentic-hicache: - image: lmsysorg/sglang:v0.5.13-cu130 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: cluster:b200-dgxc - precision: fp4 framework: sglang multinode: false scenarios: - agentic-coding: - - dram-utilization: 0.80 + fixed-seq-len: + - isl: 1024 + osl: 1024 search-space: - - { tp: 8, kv-offloading: none, conc-list: [1, 2, 3, 4, 5] } - - { tp: 8, kv-offloading: dram, kv-offload-backend: hicache, conc-list: [8, 10, 16, 32, 40, 44] } - # DEP without HiCache is already degraded at conc 52 (2,098 tok/s/GPU), - # so resolve its pre-52 cliff instead of repeating the collapsed tail. - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [16, 24, 32, 38, 44, 48, 50, 52] } - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: hicache, conc-list: [16, 32, 38, 44, 50, 56, 64, 66, 68] } - - - -# GB200 DeepSeek-V4 disaggregated AgentX frontier. The 3P/2D TEP8/TP8 curve -# covers the middle/high-interactivity range omitted by the one-decode DEP -# throughput curves below. Each engine start carries at most four concurrencies. -dsv4-fp4-b300-sglang-agentic-hicache: - image: lmsysorg/sglang:v0.5.13-cu130 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: cluster:b300-nv - precision: fp4 - framework: sglang - multinode: false - scenarios: - agentic-coding: - - dram-utilization: 0.80 + - { tp: 8, ep: 1, conc-start: 1, conc-end: 64 } + - isl: 8192 + osl: 1024 search-space: - - { tp: 4, kv-offloading: none, conc-list: [1, 4, 8, 16, 20, 24, 32] } - - { tp: 8, kv-offloading: none, conc-list: [1, 4, 8, 16, 32, 40, 48, 52, 56, 60, 64, 72] } - - { tp: 4, ep: 4, dp-attn: true, kv-offloading: none, conc-list: [8, 16, 24, 32, 40, 64] } - - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: hicache, conc-list: [32, 40, 48, 56, 64, 72, 80, 88, 96, 128] } - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [52, 72, 100, 128, 144, 196, 512] } - - -# DEP8 prefill uses an 8K batch because 16K OOMs in the FP4 MoE intermediate; -# decode uses FULL_DECODE_ONLY after the controlled graph test restored decode -# throughput. Dynamo KV routing and AIPerf conversation-aware routing remain -# enabled by framework=dynamo-vllm. -dsv4-fp4-gb200-dynamo-vllm-agentic-3p2d-tep8-tp8: - image: vllm/vllm-openai:v0.23.0 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: cluster:gb200-nv - precision: fp4 - framework: dynamo-vllm - multinode: true - disagg: true - scenarios: - agentic-coding: - - search-space: - # Ultra-high-interactivity probes below the historical c16 endpoint. - - spec-decoding: none - conc-list: [4, 8] - prefill: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p2d-tep8-tp8-agentic.yaml" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - - spec-decoding: none - conc-list: [16, 24, 32, 40] - prefill: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p2d-tep8-tp8-agentic.yaml" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - - spec-decoding: none - conc-list: [48, 56, 64, 80] - prefill: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p2d-tep8-tp8-agentic.yaml" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - -dsv4-fp4-gb200-dynamo-vllm-agentic-2p1d-dep8-dep8: - image: vllm/vllm-openai:v0.23.0 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: cluster:gb200-nv - precision: fp4 - framework: dynamo-vllm - multinode: true - disagg: true - scenarios: - agentic-coding: - - search-space: - - spec-decoding: none - conc-list: [32, 48, 64, 80] - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-agentic.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - spec-decoding: none - conc-list: [96, 128, 160] - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-agentic.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - # Exploratory tail beyond the measured c160 normalized-throughput peak. - - spec-decoding: none - conc-list: [192, 224, 256] - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-agentic.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true + - { tp: 8, ep: 1, conc-start: 1, conc-end: 4 } + - { tp: 4, ep: 1, conc-start: 1, conc-end: 32 } diff --git a/configs/runners.yaml b/configs/runners.yaml index 93da791b8..4517d3e62 100644 --- a/configs/runners.yaml +++ b/configs/runners.yaml @@ -306,6 +306,9 @@ hardware: cluster:b300-nv: available-cpu-dram-mib: 2_964_436 gpus-per-node: 8 + cluster:b300-cw: + available-cpu-dram-mib: 3_774_464 + gpus-per-node: 8 cluster:gb200-nv: available-cpu-dram-mib: 860_160 gpus-per-node: 4 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c9ebf8249..556342bc5 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4530,3 +4530,9 @@ description: - "Update vLLM ROCm image from vllm/vllm-openai-rocm:nightly-b8336c3c7c298e0878f22a7bf70f4e295b2f4e01 to vllm/vllm-openai-rocm:v0.24.0" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2077 + +- config-keys: + - dsr1-fp8-b300-cw-sglang + description: + - "Add DeepSeek-R1-0528 FP8 sglang single-node benchmark on the CoreWeave B300 cluster (runner cluster:b300-cw), on lmsysorg/sglang:v0.5.12-cu130; validates the single-node pipeline on the CoreWeave B300 fleet" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2086 diff --git a/runners/launch_b300-cw.sh b/runners/launch_b300-cw.sh new file mode 100755 index 000000000..d2f91ca46 --- /dev/null +++ b/runners/launch_b300-cw.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash + +# Single-node launcher for the CoreWeave B300 cluster (runner b300-cw). +# Cloned from launch_b200-cw.sh (the proven CoreWeave template): allocate one +# node, import the container to that node's local /tmp under flock, then srun +# the benchmark in the same allocation. Importing to node-local /tmp (rather +# than shared root-squash NFS) avoids the enroot aufs-whiteout failures NFS +# triggers. + +export HF_HUB_CACHE_MOUNT="/tmp/gharunner/hf-hub-cache" +export PORT=8888 + +MODEL_CODE="${EXP_NAME%%_*}" +FRAMEWORK_SUFFIX=$([[ "$FRAMEWORK" == "trt" ]] && printf '_trt' || printf '') +SPEC_SUFFIX=$([[ "$SPEC_DECODING" == "mtp" ]] && printf '_mtp' || printf '') +# Prefer a framework-tagged script (e.g. dsv4_fp4_b300_vllm.sh) so models +# with multiple inference engines can coexist; fall back to the historical +# name without an engine suffix (`_trt` for trt, bare for everyone else). +BENCH_BASE="benchmarks/single_node/${SCENARIO_SUBDIR}${MODEL_CODE}_${PRECISION}_b300" +BENCH_SCRIPT="${BENCH_BASE}_${FRAMEWORK}${SPEC_SUFFIX}.sh" +if [[ ! -f "$BENCH_SCRIPT" ]]; then + BENCH_SCRIPT="${BENCH_BASE}${FRAMEWORK_SUFFIX}${SPEC_SUFFIX}.sh" +fi + +PARTITION="b300" +SQUASH_FILE="/tmp/gharunner/squash/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" +LOCK_FILE="${SQUASH_FILE}.lock" + +CONTAINER_MOUNT_DIR=/workspace + +set -x + +JOB_ID=$(salloc --partition=$PARTITION --gres=gpu:b300:$TP --time=180 --no-shell --job-name="$RUNNER_NAME" 2>&1 | tee /dev/stderr | grep -oP 'Granted job allocation \K[0-9]+') + +if [ -z "$JOB_ID" ]; then + echo "ERROR: salloc failed to allocate a job" + exit 1 +fi + +# Use Docker image directly for openai/gpt-oss-120b with trt, otherwise use squash file +if [[ "$MODEL" == "openai/gpt-oss-120b" && "$FRAMEWORK" == "trt" ]]; then + CONTAINER_IMAGE=$IMAGE +else + # Use flock to serialize concurrent imports to the same squash file. + # mkdir on the worker first: /tmp/gharunner is node-local and may not + # exist on a freshly allocated CoreWeave node. + srun --jobid=$JOB_ID --job-name="$RUNNER_NAME" bash -c " + mkdir -p /tmp/gharunner/squash \"$HF_HUB_CACHE_MOUNT\" + exec 9>\"$LOCK_FILE\" + flock -w 600 9 || { echo 'Failed to acquire lock for $SQUASH_FILE'; exit 1; } + if unsquashfs -l \"$SQUASH_FILE\" > /dev/null 2>&1; then + echo 'Squash file already exists and is valid, skipping import' + else + rm -f \"$SQUASH_FILE\" + enroot import -o \"$SQUASH_FILE\" docker://$IMAGE + fi + " + # Squash file lives on the allocated worker node's /tmp, which is not + # visible from the host, so realpath on the host would return empty. + # Pass the path as-is; srun resolves it inside the job. + CONTAINER_IMAGE=$SQUASH_FILE +fi + +srun --jobid=$JOB_ID \ +--container-image=$CONTAINER_IMAGE \ +--container-mounts=$GITHUB_WORKSPACE:$CONTAINER_MOUNT_DIR,$HF_HUB_CACHE_MOUNT:$HF_HUB_CACHE \ +--container-mount-home \ +--container-workdir=$CONTAINER_MOUNT_DIR \ +--no-container-entrypoint --export=ALL \ +bash "$BENCH_SCRIPT" + +scancel $JOB_ID