diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml new file mode 100644 index 000000000..b4a8864fa --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml @@ -0,0 +1,235 @@ +base: + name: gb200-fp4-glm5-mtp + model: + path: glm-5-fp4 + container: "lmsysorg/sglang:v0.5.13.post1-cu130" + precision: fp4 + identity: + model: + repo: nvidia/GLM-5-NVFP4 + revision: dc54ff55a7e9e71b85db953d8bc22eca894b44c6 + frameworks: + dynamo: 1.2.1 + sglang: 0.5.13.post1 + resources: + gpu_type: gb200 + gpus_per_node: 4 + frontend: + type: dynamo + dynamo: + version: 1.2.1 + backend: + type: sglang + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + MC_TE_METRIC: 'true' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' + SGLANG_MOE_NVFP4_DISPATCH: '1' + sglang_config: + prefill: + served-model-name: GLM-5-FP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + max-running-requests: 136 + cuda-graph-max-bs: 136 + mem-fraction-static: 0.812 + context-length: 9280 + chunked-prefill-size: 65536 + max-prefill-tokens: 16384 + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + load-balance-method: total_tokens + nsa-decode-backend: trtllm + nsa-prefill-backend: trtllm + moe-runner-backend: flashinfer_trtllm + fp4-gemm-backend: flashinfer_cutlass + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + decode: + served-model-name: GLM-5-FP4 + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + speculative-algorithm: EAGLE + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + load-balance-method: round_robin + max-running-requests: 22 + cuda-graph-max-bs: 22 + chunked-prefill-size: 64 + context-length: 9280 + nsa-prefill-backend: trtllm + nsa-decode-backend: trtllm + moe-runner-backend: flashinfer_trtllm + fp4-gemm-backend: flashinfer_cutlass + mem-fraction-static: 0.918 + skip-tokenizer-init: true + stream-interval: 30 + disable-radix-cache: true + enable-flashinfer-allreduce-fusion: true + weight-loader-prefetch-checkpoints: true + model-loader-extra-config: '{"enable_multithread_load": true}' + health_check: + max_attempts: 360 + interval_seconds: 10 + benchmark: + type: sa-bench + req_rate: inf +# ################# 8k1k ################# +zip_override_mtp_8k1k_hightpt: + name: [8k1k-1p1d_dep16, 8k1k-2p1d_dep16, 8k1k-4p1d_dep16, 8k1k-8p1d_dep16] + backend: + decode_environment: + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + sglang_config: + decode: + cuda-graph-max-bs: [91, 196, 451, 1354] + data-parallel-size: 16 + deepep-config: /configs/deepep_config.json + deepep-mode: low_latency + enable-dp-attention: true + enable-dp-lm-head: true + enable-flashinfer-allreduce-fusion: false + ep-dispatch-algorithm: static + ep-num-redundant-experts: 0 + expert-parallel-size: 16 + load-balance-method: null + max-running-requests: [91, 196, 451, 1354] + mem-fraction-static: 0.858 + moe-a2a-backend: deepep + moe-dense-tp-size: 1 + moe-runner-backend: flashinfer_cutedsl + speculative-moe-a2a-backend: deepep + speculative-moe-runner-backend: deep_gemm + tensor-parallel-size: 16 + benchmark: + concurrencies: [[109], [398], [853], [2132]] + isl: 8192 + osl: 1024 + resources: + decode_nodes: 4 + decode_workers: 1 + prefill_nodes: [1, 2, 4, 8] + prefill_workers: [1, 2, 4, 8] +zip_override_mtp_8k1k_lowlat: + name: [8k1k-1p4d_tp4, 8k1k-1p4d_tp4-conc16, 8k1k-1p8d_tp4, 8k1k-1p8d_tp4-conc4, 8k1k-1p16d_tp4, 8k1k-1p16d_tp4-conc1] + backend: + sglang_config: + decode: + cuda-graph-max-bs: [22, 15, 9, 4, 4, 1] + max-running-requests: [22, 15, 9, 4, 4, 1] + benchmark: + concurrencies: [[104], [82], [87], [47], [74], [26]] + isl: 8192 + osl: 1024 + resources: + decode_nodes: [4, 4, 8, 8, 16, 16] + decode_workers: [4, 4, 8, 8, 16, 16] + prefill_nodes: 1 + prefill_workers: 1 +# ################# 1k1k ################# +zip_override_mtp_1k1k_hightpt: + name: [1k1k-1p1d_dep16, 1k1k-1p1d_dep16-conc1024, 1k1k-1p1d_dep16-conc256, 1k1k-1p1d_dep32] + backend: + decode_environment: + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + sglang_config: + decode: + context-length: 2112 + cuda-graph-max-bs: [1627, 958, 239, 1122] + data-parallel-size: [16, 16, 16, 32] + deepep-config: /configs/deepep_config.json + deepep-mode: low_latency + enable-dp-attention: true + enable-dp-lm-head: true + enable-flashinfer-allreduce-fusion: false + ep-dispatch-algorithm: static + ep-num-redundant-experts: 0 + expert-parallel-size: [16, 16, 16, 32] + max-running-requests: [1627, 958, 239, 1122] + mem-fraction-static: [0.858, 0.858, 0.858, 0.845] + moe-a2a-backend: deepep + moe-dense-tp-size: 1 + moe-runner-backend: flashinfer_cutedsl + speculative-moe-a2a-backend: deepep + speculative-moe-runner-backend: deep_gemm + tensor-parallel-size: [16, 16, 16, 32] + prefill: + context-length: 2112 + cuda-graph-max-bs: 512 + disable-cuda-graph: true + max-running-requests: 512 + mem-fraction-static: 0.83 + benchmark: + concurrencies: [[1742], [1150], [302], [1263]] + isl: 1024 + osl: 1024 + resources: + decode_nodes: [4, 4, 4, 8] + decode_workers: 1 + prefill_nodes: 1 + prefill_workers: 1 +zip_override_mtp_1k1k_lowlat: + name: [1k1k-1p16d_tp4-conc1, 1k1k-1p16d_tp4-conc16, 1k1k-1p16d_tp4-conc4] + backend: + sglang_config: + decode: + context-length: 2112 + cuda-graph-max-bs: [1, 16, 4] + max-running-requests: [1, 16, 4] + prefill: + context-length: 2112 + cuda-graph-max-bs: 512 + disable-cuda-graph: true + max-running-requests: 512 + mem-fraction-static: 0.83 + benchmark: + concurrencies: [[26], [343], [94]] + isl: 1024 + osl: 1024 + resources: + decode_nodes: 16 + decode_workers: 16 + prefill_nodes: 1 + prefill_workers: 1 diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 8376b5b93..c3c6e7be4 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -9137,6 +9137,290 @@ dsv4-fp4-gb200-dynamo-sglang-mtp: tp: 16 ep: 16 dp-attn: true + +glm5-fp4-gb200-dynamo-sglang-mtp: + image: lmsysorg/sglang:v0.5.13.post1-cu130 + model: nvidia/GLM-5.1-NVFP4 + model-prefix: glm5.1 + runner: gb200 + precision: fp4 + framework: dynamo-sglang + multinode: true + disagg: true + scenarios: + fixed-seq-len: + # ---------- 8k1k high-throughput (wide-EP TP=16 decode, MTP) ---------- + - isl: 8192 + osl: 1024 + search-space: + # 1p1d dep16. 5 nodes (1P + 4D). + - spec-decoding: "mtp" + conc-list: [109] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_hightpt[0]" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # 2p1d dep16. 6 nodes (2P + 4D). + - spec-decoding: "mtp" + conc-list: [398] + prefill: + num-worker: 2 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_hightpt[1]" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # 4p1d dep16. 8 nodes (4P + 4D). + - spec-decoding: "mtp" + conc-list: [853] + prefill: + num-worker: 4 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_hightpt[2]" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # 8p1d dep16. 12 nodes (8P + 4D). + - spec-decoding: "mtp" + conc-list: [2132] + prefill: + num-worker: 8 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_hightpt[3]" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # ---------- 8k1k low-latency (per-node TP=4 decode workers, MTP) ---------- + - isl: 8192 + osl: 1024 + search-space: + # 1p4d tp4. 5 nodes. + - spec-decoding: "mtp" + conc-list: [104] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[0]" + decode: + num-worker: 4 + tp: 4 + ep: 1 + dp-attn: false + # 1p4d tp4, conc16. 5 nodes. + - spec-decoding: "mtp" + conc-list: [82] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[1]" + decode: + num-worker: 4 + tp: 4 + ep: 1 + dp-attn: false + # 1p8d tp4. 9 nodes. + - spec-decoding: "mtp" + conc-list: [87] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[2]" + decode: + num-worker: 8 + tp: 4 + ep: 1 + dp-attn: false + # 1p8d tp4, conc4. 9 nodes. + - spec-decoding: "mtp" + conc-list: [47] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[3]" + decode: + num-worker: 8 + tp: 4 + ep: 1 + dp-attn: false + # 1p16d tp4. 17 nodes. + - spec-decoding: "mtp" + conc-list: [74] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[4]" + decode: + num-worker: 16 + tp: 4 + ep: 1 + dp-attn: false + # 1p16d tp4, conc1. 17 nodes. + - spec-decoding: "mtp" + conc-list: [26] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[5]" + decode: + num-worker: 16 + tp: 4 + ep: 1 + dp-attn: false + # ---------- 1k1k high-throughput (wide-EP decode, MTP) ---------- + - isl: 1024 + osl: 1024 + search-space: + # 1p1d dep16. 5 nodes. + - spec-decoding: "mtp" + conc-list: [1742] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_1k1k_hightpt[0]" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # 1p1d dep16, conc1024. 5 nodes. + - spec-decoding: "mtp" + conc-list: [1150] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_1k1k_hightpt[1]" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # 1p1d dep16, conc256. 5 nodes. + - spec-decoding: "mtp" + conc-list: [302] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_1k1k_hightpt[2]" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # 1p1d dep32. 9 nodes. + - spec-decoding: "mtp" + conc-list: [1263] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_1k1k_hightpt[3]" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + # ---------- 1k1k low-latency (per-node TP=4 decode workers, MTP) ---------- + - isl: 1024 + osl: 1024 + search-space: + # 1p16d tp4, conc1. 17 nodes. + - spec-decoding: "mtp" + conc-list: [26] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_1k1k_lowlat[0]" + decode: + num-worker: 16 + tp: 4 + ep: 1 + dp-attn: false + # 1p16d tp4, conc16. 17 nodes. + - spec-decoding: "mtp" + conc-list: [343] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_1k1k_lowlat[1]" + decode: + num-worker: 16 + tp: 4 + ep: 1 + dp-attn: false + # 1p16d tp4, conc4. 17 nodes. + - spec-decoding: "mtp" + conc-list: [94] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/glm5/gb200-fp4/glm5-mtp.yaml:zip_override_mtp_1k1k_lowlat[2]" + decode: + num-worker: 16 + tp: 4 + ep: 1 + dp-attn: false + dsv4-fp4-b300-dynamo-vllm: image: vllm/vllm-openai:v0.23.0 model: deepseek-ai/DeepSeek-V4-Pro diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 910e730e8..70f45732e 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4620,3 +4620,10 @@ - "Add 8k/1k MTP disagg high-throughput points: 1p1d and 4p1d (DEP4 prefill / DEP8 decode) at conc 512." - "Add an 8k/1k MTP disagg 4p1d (DEP4 prefill / DEP8 decode) high-throughput point at conc 2048 with a dedicated engine tune (max-running-requests 2048, decode scheduler-recv-interval 1, prefill/decode stream-interval 100/34, SGLANG_HACK_SEQ_BOOTSTRAP_ROOM)." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2113 + +- config-keys: + - glm5-fp4-gb200-dynamo-sglang-mtp + description: + - "Add GLM-5.1 NVFP4 GB200 disaggregated dynamo-sglang MTP config covering 8k1k and 1k1k, high-throughput and low-latency topologies" + - "Image: lmsysorg/sglang:v0.5.13.post1-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2115 diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index 17b377819..b60f4f353 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -29,6 +29,11 @@ if [[ $FRAMEWORK == "dynamo-sglang" ]]; then elif [[ $MODEL_PREFIX == "qwen3.5" && $PRECISION == "fp8" ]]; then export MODEL_PATH="/mnt/lustre01/models/Qwen3.5-397B-A17B-FP8" export SRT_SLURM_MODEL_PREFIX="qwen3.5-fp8" + elif [[ $MODEL_PREFIX == "glm5.1" && $PRECISION == "fp4" ]]; then + # SRT_SLURM_MODEL_PREFIX matches the model.path alias ("glm-5-fp4") + # in our GLM-5.1 sglang recipes. + export MODEL_PATH="/mnt/lustre01/models/GLM-5.1-NVFP4" + export SRT_SLURM_MODEL_PREFIX="glm-5-fp4" elif [[ $MODEL_PREFIX == "glm5.1" && $PRECISION == "fp8" ]]; then # SRT_SLURM_MODEL_PREFIX matches the model.path alias ("glm-5.1-fp8") # in our GLM-5.1 sglang recipes. @@ -502,7 +507,10 @@ fi SRTCTL_APPLY_ARGS=( "${PREFLIGHT_ARGS[@]}" - -f "$CONFIG_PATH" + # Pass the full CONFIG_FILE (not the stripped CONFIG_PATH): srtctl needs the + # ":zip_override_...[i]" selector to pick the recipe block. For plain-file + # recipes CONFIG_FILE == CONFIG_PATH, so this is a no-op for them. + -f "$CONFIG_FILE" --tags "gb200,${MODEL_PREFIX},${PRECISION},${ISL}x${OSL},infmax-$(date +%Y%m%d)" ) if [[ "$FRAMEWORK" == "dynamo-sglang" ]]; then