From 36754c1d644928ebfdd062bb9f12d29558122ad2 Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Tue, 7 Jul 2026 13:57:39 -0700 Subject: [PATCH 1/6] Add glm5-fp4-gb200-dynamo-sglang-mtp config --- .../sglang/glm5/gb200-fp4/glm5-mtp.yaml | 235 +++++++++++++++ configs/nvidia-master.yaml | 284 ++++++++++++++++++ perf-changelog.yaml | 7 + runners/launch_gb200-nv.sh | 13 + 4 files changed, 539 insertions(+) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml new file mode 100644 index 0000000000..55e777edea --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml @@ -0,0 +1,235 @@ +base: + name: gb200-fp4-glm5-mtp + model: + path: glm-5-fp4 + container: v0.5.13.post1 + precision: fp4 + identity: + model: + repo: nvidia/GLM-5-NVFP4 + revision: dc54ff55a7e9e71b85db953d8bc22eca894b44c6 + frameworks: + dynamo: 1.2.1 + sglang: 0.5.13.post1 + resources: + gpu_type: gb200 + gpus_per_node: 4 + frontend: + type: dynamo + dynamo: + version: 1.2.1 + backend: + type: sglang + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' + SGLANG_MOE_NVFP4_DISPATCH: '1' + sglang_config: + prefill: + served-model-name: GLM-5-FP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + max-running-requests: 136 + cuda-graph-max-bs: 136 + mem-fraction-static: 0.812 + context-length: 9280 + chunked-prefill-size: 65536 + max-prefill-tokens: 16384 + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + fp4-gemm-backend: flashinfer_cutlass + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + decode: + served-model-name: GLM-5-FP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + speculative-algorithm: EAGLE + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + load-balance-method: round_robin + max-running-requests: 22 + cuda-graph-max-bs: 22 + chunked-prefill-size: 64 + context-length: 9280 + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + fp4-gemm-backend: flashinfer_cutlass + mem-fraction-static: 0.918 + skip-tokenizer-init: true + stream-interval: 30 + disable-radix-cache: true + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + health_check: + max_attempts: 360 + interval_seconds: 10 + benchmark: + type: sa-bench + req_rate: inf +# ################# 8k1k ################# +zip_override_mtp_8k1k_hightpt: + name: [8k1k-1p1d_dep16, 8k1k-2p1d_dep16, 8k1k-4p1d_dep16, 8k1k-8p1d_dep16] + backend: + decode_environment: + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + sglang_config: + decode: + cuda-graph-max-bs: [91, 196, 451, 1354] + data-parallel-size: 16 + deepep-config: /configs/deepep_config.json + deepep-mode: low_latency + enable-dp-attention: true + enable-dp-lm-head: true + enable-flashinfer-allreduce-fusion: false + ep-dispatch-algorithm: static + ep-num-redundant-experts: 0 + expert-parallel-size: 16 + load-balance-method: null + max-running-requests: [91, 196, 451, 1354] + mem-fraction-static: 0.858 + moe-a2a-backend: deepep + moe-dense-tp-size: 1 + moe-runner-backend: flashinfer_cutedsl + speculative-moe-a2a-backend: deepep + speculative-moe-runner-backend: deep_gemm + tensor-parallel-size: 16 + benchmark: + concurrencies: [[109], [398], [853], [2132]] + isl: 8192 + osl: 1024 + resources: + decode_nodes: 4 + decode_workers: 1 + prefill_nodes: [1, 2, 4, 8] + prefill_workers: [1, 2, 4, 8] +zip_override_mtp_8k1k_lowlat: + name: [8k1k-1p4d_tp4, 8k1k-1p4d_tp4-conc16, 8k1k-1p8d_tp4, 8k1k-1p8d_tp4-conc4, 8k1k-1p16d_tp4, 8k1k-1p16d_tp4-conc1] + backend: + sglang_config: + decode: + cuda-graph-max-bs: [22, 15, 9, 4, 4, 1] + max-running-requests: [22, 15, 9, 4, 4, 1] + benchmark: + concurrencies: [[104], [82], [87], [47], [74], [26]] + isl: 8192 + osl: 1024 + resources: + decode_nodes: [4, 4, 8, 8, 16, 16] + decode_workers: [4, 4, 8, 8, 16, 16] + prefill_nodes: 1 + prefill_workers: 1 +# ################# 1k1k ################# +zip_override_mtp_1k1k_hightpt: + name: [1k1k-1p1d_dep16, 1k1k-1p1d_dep16-conc1024, 1k1k-1p1d_dep16-conc256, 1k1k-1p1d_dep32] + backend: + decode_environment: + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + sglang_config: + decode: + context-length: 2112 + cuda-graph-max-bs: [1627, 958, 239, 1122] + data-parallel-size: [16, 16, 16, 32] + deepep-config: /configs/deepep_config.json + deepep-mode: low_latency + enable-dp-attention: true + enable-dp-lm-head: true + enable-flashinfer-allreduce-fusion: false + ep-dispatch-algorithm: static + ep-num-redundant-experts: 0 + expert-parallel-size: [16, 16, 16, 32] + max-running-requests: [1627, 958, 239, 1122] + mem-fraction-static: [0.858, 0.858, 0.858, 0.845] + moe-a2a-backend: deepep + moe-dense-tp-size: 1 + moe-runner-backend: flashinfer_cutedsl + speculative-moe-a2a-backend: deepep + speculative-moe-runner-backend: deep_gemm + tensor-parallel-size: [16, 16, 16, 32] + prefill: + context-length: 2112 + cuda-graph-max-bs: 512 + disable-cuda-graph: true + max-running-requests: 512 + mem-fraction-static: 0.83 + benchmark: + concurrencies: [[1742], [1150], [302], [1263]] + isl: 1024 + osl: 1024 + resources: + decode_nodes: [4, 4, 4, 8] + decode_workers: 1 + prefill_nodes: 1 + prefill_workers: 1 +zip_override_mtp_1k1k_lowlat: + name: [1k1k-1p16d_tp4-conc1, 1k1k-1p16d_tp4-conc16, 1k1k-1p16d_tp4-conc4] + backend: + sglang_config: + decode: + context-length: 2112 + cuda-graph-max-bs: [1, 16, 4] + max-running-requests: [1, 16, 4] + prefill: + context-length: 2112 + cuda-graph-max-bs: 512 + disable-cuda-graph: true + max-running-requests: 512 + mem-fraction-static: 0.83 + benchmark: + concurrencies: [[26], [343], [94]] + isl: 1024 + osl: 1024 + resources: + decode_nodes: 16 + decode_workers: 16 + prefill_nodes: 1 + prefill_workers: 1 diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 8d84bd0dd7..9d0947cd5f 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -9092,6 +9092,290 @@ dsv4-fp4-gb200-dynamo-sglang-mtp: tp: 16 ep: 16 dp-attn: true + +glm5-fp4-gb200-dynamo-sglang-mtp: + image: lmsysorg/sglang:v0.5.13.post1-cu130 + model: nvidia/GLM-5-NVFP4 + model-prefix: glm5 + runner: gb200 + precision: fp4 + framework: dynamo-sglang + multinode: true + disagg: true + scenarios: + fixed-seq-len: + # ---------- 8k1k high-throughput (wide-EP TP=16 decode, MTP) ---------- + - isl: 8192 + osl: 1024 + search-space: + # 1p1d dep16. 5 nodes (1P + 4D). + - spec-decoding: "mtp" + conc-list: [109] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_hightpt[0]" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # 2p1d dep16. 6 nodes (2P + 4D). + - spec-decoding: "mtp" + conc-list: [398] + prefill: + num-worker: 2 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_hightpt[1]" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # 4p1d dep16. 8 nodes (4P + 4D). + - spec-decoding: "mtp" + conc-list: [853] + prefill: + num-worker: 4 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_hightpt[2]" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # 8p1d dep16. 12 nodes (8P + 4D). + - spec-decoding: "mtp" + conc-list: [2132] + prefill: + num-worker: 8 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_hightpt[3]" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # ---------- 8k1k low-latency (per-node TP=4 decode workers, MTP) ---------- + - isl: 8192 + osl: 1024 + search-space: + # 1p4d tp4. 5 nodes. + - spec-decoding: "mtp" + conc-list: [104] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[0]" + decode: + num-worker: 4 + tp: 4 + ep: 1 + dp-attn: false + # 1p4d tp4, conc16. 5 nodes. + - spec-decoding: "mtp" + conc-list: [82] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[1]" + decode: + num-worker: 4 + tp: 4 + ep: 1 + dp-attn: false + # 1p8d tp4. 9 nodes. + - spec-decoding: "mtp" + conc-list: [87] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[2]" + decode: + num-worker: 8 + tp: 4 + ep: 1 + dp-attn: false + # 1p8d tp4, conc4. 9 nodes. + - spec-decoding: "mtp" + conc-list: [47] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[3]" + decode: + num-worker: 8 + tp: 4 + ep: 1 + dp-attn: false + # 1p16d tp4. 17 nodes. + - spec-decoding: "mtp" + conc-list: [74] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[4]" + decode: + num-worker: 16 + tp: 4 + ep: 1 + dp-attn: false + # 1p16d tp4, conc1. 17 nodes. + - spec-decoding: "mtp" + conc-list: [26] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[5]" + decode: + num-worker: 16 + tp: 4 + ep: 1 + dp-attn: false + # ---------- 1k1k high-throughput (wide-EP decode, MTP) ---------- + - isl: 1024 + osl: 1024 + search-space: + # 1p1d dep16. 5 nodes. + - spec-decoding: "mtp" + conc-list: [1742] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_1k1k_hightpt[0]" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # 1p1d dep16, conc1024. 5 nodes. + - spec-decoding: "mtp" + conc-list: [1150] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_1k1k_hightpt[1]" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # 1p1d dep16, conc256. 5 nodes. + - spec-decoding: "mtp" + conc-list: [302] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_1k1k_hightpt[2]" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # 1p1d dep32. 9 nodes. + - spec-decoding: "mtp" + conc-list: [1263] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_1k1k_hightpt[3]" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + # ---------- 1k1k low-latency (per-node TP=4 decode workers, MTP) ---------- + - isl: 1024 + osl: 1024 + search-space: + # 1p16d tp4, conc1. 17 nodes. + - spec-decoding: "mtp" + conc-list: [26] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_1k1k_lowlat[0]" + decode: + num-worker: 16 + tp: 4 + ep: 1 + dp-attn: false + # 1p16d tp4, conc16. 17 nodes. + - spec-decoding: "mtp" + conc-list: [343] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_1k1k_lowlat[1]" + decode: + num-worker: 16 + tp: 4 + ep: 1 + dp-attn: false + # 1p16d tp4, conc4. 17 nodes. + - spec-decoding: "mtp" + conc-list: [94] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_1k1k_lowlat[2]" + decode: + num-worker: 16 + tp: 4 + ep: 1 + dp-attn: false + dsv4-fp4-b300-dynamo-vllm: image: vllm/vllm-openai:v0.23.0 model: deepseek-ai/DeepSeek-V4-Pro diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6e1249adfc..6b16c163ce 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4581,3 +4581,10 @@ - "Pin the EAGLE3 drafter to TRITON_ATTN via the speculative-config attention_backend override; without it the drafter falls back to a slow default backend." - "Speeds up the draft forward measured on MI355X TP4 MXFP8, 8k/1k throughput gains growing with concurrency vs no override." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2107 + +- config-keys: + - glm5-fp4-gb200-dynamo-sglang-mtp + description: + - "Add GLM-5 NVFP4 GB200 disaggregated dynamo-sglang MTP config covering 8k1k and 1k1k, high-throughput and low-latency topologies" + - "Image: lmsysorg/sglang:v0.5.13.post1-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index ba5fffa83e..fddb6f146f 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -24,6 +24,11 @@ if [[ $FRAMEWORK == "dynamo-sglang" ]]; then elif [[ $MODEL_PREFIX == "qwen3.5" && $PRECISION == "fp8" ]]; then export MODEL_PATH="/mnt/lustre01/models/Qwen3.5-397B-A17B-FP8" export SRT_SLURM_MODEL_PREFIX="qwen3.5-fp8" + elif [[ $MODEL_PREFIX == "glm5" && $PRECISION == "fp4" ]]; then + # SRT_SLURM_MODEL_PREFIX matches the model.path alias in our + # GLM-5 sglang recipes (glm-5-fp4). + export MODEL_PATH="/mnt/lustre01/models/GLM-5-NVFP4" + export SRT_SLURM_MODEL_PREFIX="glm-5-fp4" else export MODEL_PATH=$MODEL fi @@ -307,6 +312,13 @@ elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "qwen3.5" ]]; then cd "$SRT_REPO_DIR" mkdir -p recipes/sglang/qwen3.5 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5" recipes/sglang/qwen3.5 +elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "glm5" ]]; then + # Stay on NVIDIA/srt-slurm:main (default) and overlay our version-controlled + # GLM-5 gb200 sglang recipe onto it. + git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" + mkdir -p recipes/sglang/glm5/gb200-fp4 + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4" recipes/sglang/glm5/gb200-fp4 elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp8" ]]; then git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" || exit 1 cd "$SRT_REPO_DIR" || exit 1 @@ -409,6 +421,7 @@ model_paths: containers: dynamo-trtllm: ${SQUASH_FILE} dynamo-sglang: ${SQUASH_FILE} + v0.5.13.post1: ${SQUASH_FILE} "${IMAGE}": ${SQUASH_FILE} nginx-sqsh: ${NGINX_SQUASH_FILE} # srtctl defaults this to true, which adds #SBATCH --segment=. From 6fac6b17c6833c5cb9fec35485c75b0a55f917b3 Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Tue, 7 Jul 2026 13:57:58 -0700 Subject: [PATCH 2/6] Update perf-changelog pr-link for #2115 --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6b16c163ce..88cb45426b 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4587,4 +4587,4 @@ description: - "Add GLM-5 NVFP4 GB200 disaggregated dynamo-sglang MTP config covering 8k1k and 1k1k, high-throughput and low-latency topologies" - "Image: lmsysorg/sglang:v0.5.13.post1-cu130" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2115 From 39b6b4b5e5e35df2d29bb11b10db4f41f9f2ff79 Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Tue, 7 Jul 2026 17:09:17 -0700 Subject: [PATCH 3/6] Pass full CONFIG_FILE selector to srtctl in launch_gb200-nv.sh --- runners/launch_gb200-nv.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index fddb6f146f..3fb809ccb5 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -494,7 +494,10 @@ fi SRTCTL_APPLY_ARGS=( "${PREFLIGHT_ARGS[@]}" - -f "$CONFIG_PATH" + # Pass the full CONFIG_FILE (not the stripped CONFIG_PATH): srtctl needs the + # ":zip_override_...[i]" selector to pick the recipe block. For plain-file + # recipes CONFIG_FILE == CONFIG_PATH, so this is a no-op for them. + -f "$CONFIG_FILE" --tags "gb200,${MODEL_PREFIX},${PRECISION},${ISL}x${OSL},infmax-$(date +%Y%m%d)" ) if [[ "$FRAMEWORK" == "dynamo-sglang" ]]; then From 19de179bd25d2b73e27e166043fb1afb9b618907 Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Tue, 7 Jul 2026 18:27:11 -0700 Subject: [PATCH 4/6] Point gb200 MTP config at GLM-5.1 NVFP4 weights --- configs/nvidia-master.yaml | 2 +- perf-changelog.yaml | 2 +- runners/launch_gb200-nv.sh | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 9d0947cd5f..a33ae5aaf1 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -9095,7 +9095,7 @@ dsv4-fp4-gb200-dynamo-sglang-mtp: glm5-fp4-gb200-dynamo-sglang-mtp: image: lmsysorg/sglang:v0.5.13.post1-cu130 - model: nvidia/GLM-5-NVFP4 + model: nvidia/GLM-5.1-NVFP4 model-prefix: glm5 runner: gb200 precision: fp4 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 88cb45426b..b0689f11a6 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4585,6 +4585,6 @@ - config-keys: - glm5-fp4-gb200-dynamo-sglang-mtp description: - - "Add GLM-5 NVFP4 GB200 disaggregated dynamo-sglang MTP config covering 8k1k and 1k1k, high-throughput and low-latency topologies" + - "Add GLM-5.1 NVFP4 GB200 disaggregated dynamo-sglang MTP config covering 8k1k and 1k1k, high-throughput and low-latency topologies" - "Image: lmsysorg/sglang:v0.5.13.post1-cu130" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2115 diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index 3fb809ccb5..73740b693e 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -26,8 +26,8 @@ if [[ $FRAMEWORK == "dynamo-sglang" ]]; then export SRT_SLURM_MODEL_PREFIX="qwen3.5-fp8" elif [[ $MODEL_PREFIX == "glm5" && $PRECISION == "fp4" ]]; then # SRT_SLURM_MODEL_PREFIX matches the model.path alias in our - # GLM-5 sglang recipes (glm-5-fp4). - export MODEL_PATH="/mnt/lustre01/models/GLM-5-NVFP4" + # GLM-5.1 sglang recipe (glm-5-fp4). + export MODEL_PATH="/mnt/lustre01/models/GLM-5.1-NVFP4" export SRT_SLURM_MODEL_PREFIX="glm-5-fp4" else export MODEL_PATH=$MODEL From 037d31b374795ecf4ff21e39b6e17448d28baa30 Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Tue, 7 Jul 2026 18:37:58 -0700 Subject: [PATCH 5/6] Use glm5.1 model-prefix for gb200 MTP config (match PR #1780) --- configs/nvidia-master.yaml | 2 +- runners/launch_gb200-nv.sh | 15 +++++++-------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index a33ae5aaf1..43bc6b444a 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -9096,7 +9096,7 @@ dsv4-fp4-gb200-dynamo-sglang-mtp: glm5-fp4-gb200-dynamo-sglang-mtp: image: lmsysorg/sglang:v0.5.13.post1-cu130 model: nvidia/GLM-5.1-NVFP4 - model-prefix: glm5 + model-prefix: glm5.1 runner: gb200 precision: fp4 framework: dynamo-sglang diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index 73740b693e..a1bd3144a8 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -24,9 +24,9 @@ if [[ $FRAMEWORK == "dynamo-sglang" ]]; then elif [[ $MODEL_PREFIX == "qwen3.5" && $PRECISION == "fp8" ]]; then export MODEL_PATH="/mnt/lustre01/models/Qwen3.5-397B-A17B-FP8" export SRT_SLURM_MODEL_PREFIX="qwen3.5-fp8" - elif [[ $MODEL_PREFIX == "glm5" && $PRECISION == "fp4" ]]; then - # SRT_SLURM_MODEL_PREFIX matches the model.path alias in our - # GLM-5.1 sglang recipe (glm-5-fp4). + elif [[ $MODEL_PREFIX == "glm5.1" && $PRECISION == "fp4" ]]; then + # SRT_SLURM_MODEL_PREFIX matches the model.path alias ("glm-5-fp4") + # in our GLM-5.1 sglang recipes. export MODEL_PATH="/mnt/lustre01/models/GLM-5.1-NVFP4" export SRT_SLURM_MODEL_PREFIX="glm-5-fp4" else @@ -312,13 +312,12 @@ elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "qwen3.5" ]]; then cd "$SRT_REPO_DIR" mkdir -p recipes/sglang/qwen3.5 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5" recipes/sglang/qwen3.5 -elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "glm5" ]]; then - # Stay on NVIDIA/srt-slurm:main (default) and overlay our version-controlled - # GLM-5 gb200 sglang recipe onto it. +elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "glm5.1" ]]; then git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" - mkdir -p recipes/sglang/glm5/gb200-fp4 - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4" recipes/sglang/glm5/gb200-fp4 + git checkout sa-submission-q2-2026 + mkdir -p recipes/sglang/glm5 + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5" recipes/sglang/glm5 elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp8" ]]; then git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" || exit 1 cd "$SRT_REPO_DIR" || exit 1 From 6da26cd66af35c570adda467ab1b01514e548f3e Mon Sep 17 00:00:00 2001 From: Rohit Pujar Nagraj Date: Tue, 7 Jul 2026 18:48:40 -0700 Subject: [PATCH 6/6] Align gb200 recipe model.container with image string; drop launcher alias --- .../srt-slurm-recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml | 2 +- runners/launch_gb200-nv.sh | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml index 55e777edea..b4a8864faf 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml @@ -2,7 +2,7 @@ base: name: gb200-fp4-glm5-mtp model: path: glm-5-fp4 - container: v0.5.13.post1 + container: "lmsysorg/sglang:v0.5.13.post1-cu130" precision: fp4 identity: model: diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index a1bd3144a8..1a77a066cf 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -420,7 +420,6 @@ model_paths: containers: dynamo-trtllm: ${SQUASH_FILE} dynamo-sglang: ${SQUASH_FILE} - v0.5.13.post1: ${SQUASH_FILE} "${IMAGE}": ${SQUASH_FILE} nginx-sqsh: ${NGINX_SQUASH_FILE} # srtctl defaults this to true, which adds #SBATCH --segment=.