diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index c3c6e7be4..68a125b5e 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -11690,6 +11690,274 @@ glm5-fp8-gb200-dynamo-sglang: ep: 1 dp-attn: false +glm5-fp4-gb300-dynamo-sglang-mtp: + image: lmsysorg/sglang:v0.5.13.post1-cu130 + model: nvidia/GLM-5.1-NVFP4 + model-prefix: glm5.1 + runner: gb300-nv + precision: fp4 + framework: dynamo-sglang + multinode: true + disagg: true + scenarios: + fixed-seq-len: + # ---------- 8k1k high-throughput (wide-EP TP=16 decode, MTP) ---------- + - isl: 8192 + osl: 1024 + search-space: + # 4p1d dep16. 6 nodes (2P + 4D). + - spec-decoding: "mtp" + conc-list: [364] + prefill: + num-worker: 4 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/gb300-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_hightpt[0]" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # 8p1d dep16. 8 nodes (4P + 4D). + - spec-decoding: "mtp" + conc-list: [512] + prefill: + num-worker: 8 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/gb300-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_hightpt[1]" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # ---------- 8k1k low-latency (per-node TP=4 decode workers, MTP) ---------- + - isl: 8192 + osl: 1024 + search-space: + # 1p2d tp4. 3 nodes. + - spec-decoding: "mtp" + conc-list: [66] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/gb300-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[0]" + decode: + num-worker: 2 + tp: 4 + ep: 1 + dp-attn: false + # 1p2d tp4, conc16. 3 nodes. + - spec-decoding: "mtp" + conc-list: [41] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/gb300-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[1]" + decode: + num-worker: 2 + tp: 4 + ep: 1 + dp-attn: false + # 1p4d tp4. 5 nodes. + - spec-decoding: "mtp" + conc-list: [55] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/gb300-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[2]" + decode: + num-worker: 4 + tp: 4 + ep: 1 + dp-attn: false + # 1p4d tp4, conc4. 5 nodes. + - spec-decoding: "mtp" + conc-list: [23] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/gb300-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[3]" + decode: + num-worker: 4 + tp: 4 + ep: 1 + dp-attn: false + # 1p8d tp4. 9 nodes. + - spec-decoding: "mtp" + conc-list: [44] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/gb300-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[4]" + decode: + num-worker: 8 + tp: 4 + ep: 1 + dp-attn: false + # 1p8d tp4, conc1. 9 nodes. + - spec-decoding: "mtp" + conc-list: [12] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/gb300-fp4/glm5-mtp.yaml:zip_override_mtp_8k1k_lowlat[5]" + decode: + num-worker: 8 + tp: 4 + ep: 1 + dp-attn: false + # ---------- 1k1k high-throughput (wide-EP decode, MTP) ---------- + - isl: 1024 + osl: 1024 + search-space: + # 1p1d dep16. 5 nodes. + - spec-decoding: "mtp" + conc-list: [773] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/gb300-fp4/glm5-mtp.yaml:zip_override_mtp_1k1k_hightpt[0]" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # 1p1d dep16, conc512. 5 nodes. + - spec-decoding: "mtp" + conc-list: [578] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/gb300-fp4/glm5-mtp.yaml:zip_override_mtp_1k1k_hightpt[1]" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # 1p1d dep32. 9 nodes. + - spec-decoding: "mtp" + conc-list: [677] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/gb300-fp4/glm5-mtp.yaml:zip_override_mtp_1k1k_hightpt[2]" + decode: + num-worker: 1 + tp: 32 + ep: 32 + dp-attn: true + # 2p1d dep16. 5 nodes. + - spec-decoding: "mtp" + conc-list: [2048] + prefill: + num-worker: 2 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/gb300-fp4/glm5-mtp.yaml:zip_override_mtp_1k1k_hightpt[3]" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # 2p1d dep16, conc1024. 5 nodes. + - spec-decoding: "mtp" + conc-list: [1362] + prefill: + num-worker: 2 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/gb300-fp4/glm5-mtp.yaml:zip_override_mtp_1k1k_hightpt[4]" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # ---------- 1k1k low-latency (per-node TP=4 decode workers, MTP) ---------- + - isl: 1024 + osl: 1024 + search-space: + # 1p8d tp4, conc1. 9 nodes. + - spec-decoding: "mtp" + conc-list: [13] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/gb300-fp4/glm5-mtp.yaml:zip_override_mtp_1k1k_lowlat[0]" + decode: + num-worker: 8 + tp: 4 + ep: 1 + dp-attn: false + # 1p8d tp4, conc16. 9 nodes. + - spec-decoding: "mtp" + conc-list: [162] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/gb300-fp4/glm5-mtp.yaml:zip_override_mtp_1k1k_lowlat[1]" + decode: + num-worker: 8 + tp: 4 + ep: 1 + dp-attn: false + # 1p8d tp4, conc4. 9 nodes. + - spec-decoding: "mtp" + conc-list: [44] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/gb300-fp4/glm5-mtp.yaml:zip_override_mtp_1k1k_lowlat[2]" + decode: + num-worker: 8 + tp: 4 + ep: 1 + dp-attn: false + glm5-fp8-gb300-dynamo-sglang: image: lmsysorg/sglang:v0.5.11-cu130 model: zai-org/GLM-5-FP8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 70f45732e..fc7965030 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4627,3 +4627,10 @@ - "Add GLM-5.1 NVFP4 GB200 disaggregated dynamo-sglang MTP config covering 8k1k and 1k1k, high-throughput and low-latency topologies" - "Image: lmsysorg/sglang:v0.5.13.post1-cu130" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2115 + +- config-keys: + - glm5-fp4-gb300-dynamo-sglang-mtp + description: + - "Add GLM-5.1 NVFP4 GB300 disaggregated dynamo-sglang MTP config covering 8k1k and 1k1k, high-throughput and low-latency topologies" + - "Image: lmsysorg/sglang:v0.5.13.post1-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2114 diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index aaf48134f..3180d9d21 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -46,6 +46,11 @@ elif [[ $MODEL_PREFIX == "glm5" && $PRECISION == "fp4" && $FRAMEWORK == "dynamo- export SERVED_MODEL_NAME="glm-5-nvfp4" export MODEL_PATH=/scratch/models/GLM-5-NVFP4 export SRT_SLURM_MODEL_PREFIX="nvidia/GLM-5-NVFP4" +elif [[ $MODEL_PREFIX == "glm5.1" && $PRECISION == "fp4" ]]; then + # SRT_SLURM_MODEL_PREFIX matches the model.path alias ("glm-5-fp4") + # in our GLM-5.1 sglang recipes. + export MODEL_PATH=/scratch/models/GLM-5.1-NVFP4 + export SRT_SLURM_MODEL_PREFIX="glm-5-fp4" elif [[ $MODEL_PREFIX == "glm5" && $PRECISION == "fp4" ]]; then export MODEL_PATH=/scratch/models/GLM-5-NVFP4 export SRT_SLURM_MODEL_PREFIX="glm-5-fp4" @@ -164,6 +169,12 @@ elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "glm5" ]]; then mkdir -p recipes/sglang/glm5/gb300-fp4 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb300-fp4" recipes/sglang/glm5/gb300-fp4 fi +elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "glm5.1" ]]; then + # GLM-5.1 MTP recipe (recipes/gb300-fp4/glm5-mtp.yaml) lives on + # NVIDIA/srt-slurm:main — check it out; no in-repo overlay needed. + git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" + git checkout main elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "qwen3.5" ]]; then # Overlay our version-controlled Qwen3.5 recipes onto the submission branch. git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" @@ -255,6 +266,7 @@ containers: dynamo-trtllm: ${SQUASH_FILE} dynamo-sglang: ${SQUASH_FILE} v0.5.11: ${SQUASH_FILE} + v0.5.13.post1: ${SQUASH_FILE} "${IMAGE}": ${SQUASH_FILE} nginx-sqsh: ${NGINX_SQUASH_FILE} use_segment_sbatch_directive: false @@ -277,20 +289,28 @@ if [[ -z "$CONFIG_FILE" ]]; then exit 1 fi -# Override the job name in the config file with the runner name -sed -i "s/^name:.*/name: \"${RUNNER_NAME}\"/" "$CONFIG_FILE" - -# --no-preflight is only safe on the agentic path, where the recipe -# resolves model.path to /scratch (compute-node-only NVMe) and the -# srtctl process running on the GHA runner pod can't see it. Fixed- -# seq-len recipes still resolve model.path to an NFS-visible location -# where the precheck is a useful sanity guard, so keep enforcement on -# for them. +# Override the job name in the config file with the runner name. +# CONFIG_FILE may carry a ":zip_override_...[i]" selector suffix that only +# `srtctl apply -f` parses; strip it to the real path for the sed. srtctl +# below still receives the full CONFIG_FILE (with selector). +CONFIG_PATH="${CONFIG_FILE%%:*}" +sed -i "s/^name:.*/name: \"${RUNNER_NAME}\"/" "$CONFIG_PATH" + +# --no-preflight skips srtctl's pre-submit model-path stat, which runs on +# the GHA runner host (im-gb300-login-02, an x86 login node). It's required +# whenever model.path resolves to the node-local /scratch NVMe that the login +# node can't see: +# - the agentic path (DSv4-Pro checkpoint), and +# - glm5.1, whose GLM-5.1-NVFP4 weights are prestaged on the compute-node +# /scratch/models. +# The engine still fails loudly at runtime if the path is genuinely missing on +# the compute node. Other fixed-seq-len recipes resolve model.path to a +# login-visible location, so keep the precheck enforced for them. SRTCTL_APPLY_ARGS=( -f "$CONFIG_FILE" --tags "gb300,${MODEL_PREFIX},${PRECISION},${ISL}x${OSL},infmax-$(date +%Y%m%d)" ) -if [[ "$IS_AGENTIC" == "1" ]]; then +if [[ "$IS_AGENTIC" == "1" || "$MODEL_PREFIX" == "glm5.1" ]]; then SRTCTL_APPLY_ARGS+=(--no-preflight) fi if [[ -n "$SRTCTL_SETUP_SCRIPT" ]]; then