diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/1k1k/disagg/mtp/1k1k_mtp_lowlat_0.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/1k1k/disagg/mtp/1k1k_mtp_lowlat_0.yaml new file mode 100644 index 0000000000..11f56972ee --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/1k1k/disagg/mtp/1k1k_mtp_lowlat_0.yaml @@ -0,0 +1,123 @@ +# Derived from the srt-slurm b200-fp4 1k1k recipe (recipes/b200-fp4/1k1k.yaml +# base + zip_override_mtp_lowlat[0]): 1p5d low-latency (dep4 prefill / tep8 decode, 5 decode nodes). +# One flat YAML per concrete topology, matching the 8k1k local recipe layout +# (sglang//-//disagg//...). + +name: b200-fp4-mtp-low-latency-dep4-1p-tep8-5d +model: + path: dsr1 + container: dynamo-sglang + precision: fp4 + +dynamo: + hash: "5b4bc1dd70965017a737c71b19db5a0aeaa88727" + install: true + +resources: + gpu_type: b200 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_nodes: 5 + decode_workers: 5 + gpus_per_node: 8 +backend: + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + DYN_REQUEST_PLANE: nats + SGLANG_ENABLE_SPEC_V2: '1' + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + DYN_REQUEST_PLANE: nats + SGLANG_ENABLE_SPEC_V2: '1' + sglang_config: + prefill: + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: modelopt_fp4 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + mem-fraction-static: 0.85 + max-prefill-tokens: 32768 + chunked-prefill-size: 32768 + context-length: 2200 + max-running-requests: 512 + disable-cuda-graph: true + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + moe-runner-backend: flashinfer_trtllm + moe-dense-tp-size: 1 + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + fp4-gemm-backend: flashinfer_trtllm + decode: + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: modelopt_fp4 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + mem-fraction-static: 0.85 + max-prefill-tokens: 32768 + chunked-prefill-size: 32768 + context-length: 2200 + max-running-requests: 512 + cuda-graph-max-bs: 512 + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 8 + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + moe-runner-backend: flashinfer_trtllm + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + fp4-gemm-backend: flashinfer_trtllm + speculative-algorithm: EAGLE + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 +health_check: + max_attempts: 360 + interval_seconds: 10 +benchmark: + type: sa-bench + isl: 1024 + osl: 1024 + req_rate: inf + concurrencies: 16x512 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/1k1k/disagg/mtp/1k1k_mtp_lowlat_1.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/1k1k/disagg/mtp/1k1k_mtp_lowlat_1.yaml new file mode 100644 index 0000000000..acbec6f11a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/1k1k/disagg/mtp/1k1k_mtp_lowlat_1.yaml @@ -0,0 +1,123 @@ +# Derived from the srt-slurm b200-fp4 1k1k recipe (recipes/b200-fp4/1k1k.yaml +# base + zip_override_mtp_lowlat[1]): 1p6d low-latency (dep4 prefill / tep8 decode, 6 decode nodes). +# One flat YAML per concrete topology, matching the 8k1k local recipe layout +# (sglang//-//disagg//...). + +name: b200-fp4-mtp-low-latency-dep4-1p-tep8-6d +model: + path: dsr1 + container: dynamo-sglang + precision: fp4 + +dynamo: + hash: "5b4bc1dd70965017a737c71b19db5a0aeaa88727" + install: true + +resources: + gpu_type: b200 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_nodes: 6 + decode_workers: 6 + gpus_per_node: 8 +backend: + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + DYN_REQUEST_PLANE: nats + SGLANG_ENABLE_SPEC_V2: '1' + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + DYN_REQUEST_PLANE: nats + SGLANG_ENABLE_SPEC_V2: '1' + sglang_config: + prefill: + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: modelopt_fp4 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + mem-fraction-static: 0.85 + max-prefill-tokens: 32768 + chunked-prefill-size: 32768 + context-length: 2200 + max-running-requests: 512 + disable-cuda-graph: true + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + moe-runner-backend: flashinfer_trtllm + moe-dense-tp-size: 1 + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + fp4-gemm-backend: flashinfer_trtllm + decode: + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: modelopt_fp4 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + mem-fraction-static: 0.85 + max-prefill-tokens: 32768 + chunked-prefill-size: 32768 + context-length: 2200 + max-running-requests: 512 + cuda-graph-max-bs: 512 + tensor-parallel-size: 8 + data-parallel-size: 1 + expert-parallel-size: 8 + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + moe-runner-backend: flashinfer_trtllm + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + fp4-gemm-backend: flashinfer_trtllm + speculative-algorithm: EAGLE + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 +health_check: + max_attempts: 360 + interval_seconds: 10 +benchmark: + type: sa-bench + isl: 1024 + osl: 1024 + req_rate: inf + concurrencies: 32x64x256x512 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/1k1k/disagg/mtp/1k1k_mtp_maxtpt_0.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/1k1k/disagg/mtp/1k1k_mtp_maxtpt_0.yaml new file mode 100644 index 0000000000..5f67dfb50f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/1k1k/disagg/mtp/1k1k_mtp_maxtpt_0.yaml @@ -0,0 +1,128 @@ +# Derived from the srt-slurm b200-fp4 1k1k recipe (recipes/b200-fp4/1k1k.yaml +# base + zip_override_mtp_maxtpt[0]): 1p1d max-throughput (dep4 prefill / dep8 decode, mem-fraction 0.75). +# One flat YAML per concrete topology, matching the 8k1k local recipe layout +# (sglang//-//disagg//...). + +name: b200-fp4-mtp-max-tpt-dep4-1p-dep8-1d +model: + path: dsr1 + container: dynamo-sglang + precision: fp4 + +dynamo: + hash: "5b4bc1dd70965017a737c71b19db5a0aeaa88727" + install: true + +resources: + gpu_type: b200 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_nodes: 1 + decode_workers: 1 + gpus_per_node: 8 +backend: + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + DYN_REQUEST_PLANE: nats + SGLANG_ENABLE_SPEC_V2: '1' + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + DYN_REQUEST_PLANE: nats + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_ENABLE_SPEC_V2: '1' + sglang_config: + prefill: + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: modelopt_fp4 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + mem-fraction-static: 0.85 + max-prefill-tokens: 32768 + chunked-prefill-size: 32768 + context-length: 2200 + max-running-requests: 1024 + disable-cuda-graph: true + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + moe-runner-backend: flashinfer_trtllm + moe-dense-tp-size: 1 + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + fp4-gemm-backend: flashinfer_trtllm + decode: + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: modelopt_fp4 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + mem-fraction-static: 0.75 + max-prefill-tokens: 32768 + chunked-prefill-size: 32768 + context-length: 2200 + max-running-requests: 1024 + cuda-graph-max-bs: 1024 + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + moe-runner-backend: flashinfer_trtllm + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + fp4-gemm-backend: flashinfer_trtllm + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + speculative-algorithm: EAGLE + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 +health_check: + max_attempts: 360 + interval_seconds: 10 +benchmark: + type: sa-bench + isl: 1024 + osl: 1024 + req_rate: inf + concurrencies: 512x1024 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/1k1k/disagg/mtp/1k1k_mtp_maxtpt_1.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/1k1k/disagg/mtp/1k1k_mtp_maxtpt_1.yaml new file mode 100644 index 0000000000..36ceac6122 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/1k1k/disagg/mtp/1k1k_mtp_maxtpt_1.yaml @@ -0,0 +1,128 @@ +# Derived from the srt-slurm b200-fp4 1k1k recipe (recipes/b200-fp4/1k1k.yaml +# base + zip_override_mtp_maxtpt[1]): 1p2d max-throughput (dep4 prefill / dep8 decode, mem-fraction 0.85). +# One flat YAML per concrete topology, matching the 8k1k local recipe layout +# (sglang//-//disagg//...). + +name: b200-fp4-mtp-max-tpt-dep4-1p-dep8-2d +model: + path: dsr1 + container: dynamo-sglang + precision: fp4 + +dynamo: + hash: "5b4bc1dd70965017a737c71b19db5a0aeaa88727" + install: true + +resources: + gpu_type: b200 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_nodes: 2 + decode_workers: 2 + gpus_per_node: 8 +backend: + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + DYN_REQUEST_PLANE: nats + SGLANG_ENABLE_SPEC_V2: '1' + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + DYN_REQUEST_PLANE: nats + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_ENABLE_SPEC_V2: '1' + sglang_config: + prefill: + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: modelopt_fp4 + disaggregation-mode: prefill + disaggregation-transfer-backend: nixl + mem-fraction-static: 0.85 + max-prefill-tokens: 32768 + chunked-prefill-size: 32768 + context-length: 2200 + max-running-requests: 512 + disable-cuda-graph: true + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + moe-runner-backend: flashinfer_trtllm + moe-dense-tp-size: 1 + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + fp4-gemm-backend: flashinfer_trtllm + decode: + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: modelopt_fp4 + disaggregation-mode: decode + disaggregation-transfer-backend: nixl + mem-fraction-static: 0.85 + max-prefill-tokens: 32768 + chunked-prefill-size: 32768 + context-length: 2200 + max-running-requests: 512 + cuda-graph-max-bs: 512 + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + moe-runner-backend: flashinfer_trtllm + stream-interval: 30 + watchdog-timeout: 1000000 + enable-flashinfer-allreduce-fusion: true + disable-radix-cache: true + fp4-gemm-backend: flashinfer_trtllm + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + speculative-algorithm: EAGLE + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 +health_check: + max_attempts: 360 + interval_seconds: 10 +benchmark: + type: sa-bench + isl: 1024 + osl: 1024 + req_rate: inf + concurrencies: '512' + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp2_throughput_1p1d.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp2_throughput_1p1d.yaml new file mode 100644 index 0000000000..13a1a7d9d0 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp2_throughput_1p1d.yaml @@ -0,0 +1,143 @@ +name: "b200-fp4-mtp2-throughput-1p-dep4-1d-dep8" + +# Derived from the srt-slurm b200-fp4 8k1k recipe (recipes/b200-fp4/8k1k.yaml +# base + override_mtp2_throughput_1p1d, DEP4 prefill / DEP8 decode, MTP2). +# One flat YAML per concrete topology, matching the InferenceX glm5 disagg +# layout (sglang//-//disagg//...). + +dynamo: + hash: "5b4bc1dd70965017a737c71b19db5a0aeaa88727" + install: true + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 3 + nginx_container: nginx-sqsh + +model: + path: "dsr1" + container: "lmsysorg/sglang:v0.5.12.post1" + precision: "fp4" + +resources: + gpu_type: "b200" + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_node: 8 + +backend: + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_ENABLE_JIT_DEEPGEMM: "false" + SGLANG_ENABLE_SPEC_V2: "1" + UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" + + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_ENABLE_JIT_DEEPGEMM: "false" + SGLANG_ENABLE_SPEC_V2: "1" + UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" + + sglang_config: + prefill: + disaggregation-mode: "prefill" + served-model-name: "deepseek-ai/DeepSeek-R1" + trust-remote-code: true + disable-radix-cache: true + kv-cache-dtype: "fp8_e4m3" + attention-backend: "trtllm_mla" + quantization: "modelopt_fp4" + moe-runner-backend: "flashinfer_trtllm" + stream-interval: 50 + watchdog-timeout: 1000000 + context-length: 9600 + mem-fraction-static: 0.95 + max-total-tokens: 32768 + chunked-prefill-size: 24576 + cuda-graph-max-bs: 256 + max-running-requests: 2048 + scheduler-recv-interval: 1 + load-balance-method: "round_robin" + disaggregation-bootstrap-port: null + data-parallel-size: 4 + tensor-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + fp4-gemm-backend: "flashinfer_trtllm" + disaggregation-transfer-backend: nixl + speculative-algorithm: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + + decode: + disaggregation-mode: "decode" + served-model-name: "deepseek-ai/DeepSeek-R1" + prefill-round-robin-balance: true + trust-remote-code: true + disable-radix-cache: true + kv-cache-dtype: "fp8_e4m3" + attention-backend: "trtllm_mla" + quantization: "modelopt_fp4" + moe-runner-backend: "flashinfer_trtllm" + disaggregation-bootstrap-port: 30001 + stream-interval: 50 + watchdog-timeout: 1000000 + context-length: 9600 + mem-fraction-static: 0.85 + chunked-prefill-size: 8192 + cuda-graph-max-bs: 1024 + max-running-requests: 2048 + scheduler-recv-interval: 1 + data-parallel-size: 8 + tensor-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + fp4-gemm-backend: "flashinfer_trtllm" + disaggregation-transfer-backend: nixl + speculative-algorithm: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + req_rate: 300 + concurrencies: "512" + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp2_throughput_4p1d.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp2_throughput_4p1d.yaml new file mode 100644 index 0000000000..a3923b02db --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp2_throughput_4p1d.yaml @@ -0,0 +1,143 @@ +name: "b200-fp4-mtp2-throughput-4p-dep4-1d-dep8" + +# Derived from the srt-slurm b200-fp4 8k1k recipe (recipes/b200-fp4/8k1k.yaml +# base + override_mtp2_throughput_4p1d, DEP4 prefill / DEP8 decode, MTP2). +# One flat YAML per concrete topology, matching the InferenceX glm5 disagg +# layout (sglang//-//disagg//...). + +dynamo: + hash: "5b4bc1dd70965017a737c71b19db5a0aeaa88727" + install: true + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 3 + nginx_container: nginx-sqsh + +model: + path: "dsr1" + container: "lmsysorg/sglang:v0.5.12.post1" + precision: "fp4" + +resources: + gpu_type: "b200" + prefill_nodes: 2 + decode_nodes: 1 + prefill_workers: 4 + decode_workers: 1 + gpus_per_prefill: 4 + gpus_per_node: 8 + +backend: + prefill_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_ENABLE_JIT_DEEPGEMM: "false" + SGLANG_ENABLE_SPEC_V2: "1" + UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" + + decode_environment: + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + PYTHONUNBUFFERED: "1" + DYN_SKIP_SGLANG_LOG_FORMATTING: "1" + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: "0" + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: "1" + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: "100000" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: "1000" + MC_FORCE_MNNVL: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + SGLANG_ENABLE_JIT_DEEPGEMM: "false" + SGLANG_ENABLE_SPEC_V2: "1" + UCX_TLS: "rc,cuda_ipc,cuda_copy,tcp,self" + + sglang_config: + prefill: + disaggregation-mode: "prefill" + served-model-name: "deepseek-ai/DeepSeek-R1" + trust-remote-code: true + disable-radix-cache: true + kv-cache-dtype: "fp8_e4m3" + attention-backend: "trtllm_mla" + quantization: "modelopt_fp4" + moe-runner-backend: "flashinfer_trtllm" + stream-interval: 50 + watchdog-timeout: 1000000 + context-length: 9600 + mem-fraction-static: 0.95 + max-total-tokens: 32768 + chunked-prefill-size: 24576 + cuda-graph-max-bs: 256 + max-running-requests: 2048 + scheduler-recv-interval: 1 + load-balance-method: "round_robin" + disaggregation-bootstrap-port: null + data-parallel-size: 4 + tensor-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + fp4-gemm-backend: "flashinfer_trtllm" + disaggregation-transfer-backend: nixl + speculative-algorithm: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + + decode: + disaggregation-mode: "decode" + served-model-name: "deepseek-ai/DeepSeek-R1" + prefill-round-robin-balance: true + trust-remote-code: true + disable-radix-cache: true + kv-cache-dtype: "fp8_e4m3" + attention-backend: "trtllm_mla" + quantization: "modelopt_fp4" + moe-runner-backend: "flashinfer_trtllm" + disaggregation-bootstrap-port: 30001 + stream-interval: 50 + watchdog-timeout: 1000000 + context-length: 9600 + mem-fraction-static: 0.85 + chunked-prefill-size: 8192 + cuda-graph-max-bs: 1024 + max-running-requests: 2048 + scheduler-recv-interval: 1 + data-parallel-size: 8 + tensor-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + fp4-gemm-backend: "flashinfer_trtllm" + disaggregation-transfer-backend: nixl + speculative-algorithm: "EAGLE" + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 + +health_check: + max_attempts: 360 + interval_seconds: 10 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + req_rate: 300 + concurrencies: "512" + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp_4p1d_c2048.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp_4p1d_c2048.yaml new file mode 100644 index 0000000000..fc2b0f3366 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp_4p1d_c2048.yaml @@ -0,0 +1,119 @@ +# DeepSeek-R1-0528 FP4 8k/1k B200 dynamo-sglang disagg — MTP, 4P:1D (c2048) +# Topology: 4 prefill workers (TP4/EP4 dp-attn) + 1 decode worker (TP8/EP8 dp-attn) = 24 GPU. +# Dedicated high-concurrency tune: +# SGLANG_HACK_SEQ_BOOTSTRAP_ROOM performance knob (not output-affecting; standard in GB200 recipes). +# max-running-requests raised to 2048 to avoid admission throttle at high concurrency. +# scheduler-recv-interval: 1 on decode. +# symm-mem + allreduce fusion retained. +# prefill stream-interval: 100; decode stream-interval: 34 (tuned for the c2048 batch cadence). +name: b200-fp4-dsr1_8k1k_mtp_4p1d_c2048 +model: + path: "dsr1" + container: "lmsysorg/sglang:v0.5.12.post1" + precision: fp4 +resources: + gpu_type: b200 + gpus_per_node: 8 + prefill_nodes: 2 + prefill_workers: 4 + gpus_per_prefill: 4 + decode_nodes: 1 + decode_workers: 1 +frontend: + type: dynamo +dynamo: + hash: "5b4bc1dd70965017a737c71b19db5a0aeaa88727" + install: true +backend: + prefill_environment: &env + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + PYTHONUNBUFFERED: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + DYN_REQUEST_PLANE: nats + SGLANG_ENABLE_JIT_DEEPGEMM: 'false' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_PER_TOKEN_GROUP_QUANT_8BIT_V2: '1' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + MC_FORCE_MNNVL: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' + decode_environment: *env + sglang_config: + prefill: + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: modelopt_fp4 + disaggregation-mode: prefill + load-balance-method: round_robin + disaggregation-transfer-backend: nixl + mem-fraction-static: 0.85 + max-prefill-tokens: 32768 + chunked-prefill-size: 32768 + context-length: 9600 + max-running-requests: 2048 + disable-cuda-graph: true + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + moe-runner-backend: flashinfer_trtllm + moe-dense-tp-size: 1 + enable-flashinfer-allreduce-fusion: true + enable-symm-mem: true + disable-radix-cache: true + stream-interval: 100 + watchdog-timeout: 1000000 + decode: + served-model-name: deepseek-ai/DeepSeek-R1 + trust-remote-code: true + quantization: modelopt_fp4 + disaggregation-mode: decode + prefill-round-robin-balance: true + disaggregation-transfer-backend: nixl + mem-fraction-static: 0.75 + max-prefill-tokens: 32768 + chunked-prefill-size: 32768 + context-length: 9600 + max-running-requests: 2048 + cuda-graph-max-bs: 1024 + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + attention-backend: trtllm_mla + kv-cache-dtype: fp8_e4m3 + moe-runner-backend: flashinfer_trtllm + moe-dense-tp-size: 1 + enable-flashinfer-allreduce-fusion: true + enable-symm-mem: true + disable-radix-cache: true + stream-interval: 34 + scheduler-recv-interval: 1 + watchdog-timeout: 1000000 + speculative-algorithm: EAGLE + speculative-num-steps: 2 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 3 +health_check: + max_attempts: 360 + interval_seconds: 10 +benchmark: + type: sa-bench + req_rate: inf + isl: 8192 + osl: 1024 + concurrencies: '2048' + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp_lowlat_0.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp_lowlat_0.yaml index 160854ebc4..62bbddf84d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp_lowlat_0.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp_lowlat_0.yaml @@ -136,5 +136,5 @@ benchmark: isl: 8192 osl: 1024 req_rate: 300 - concurrencies: "4x8x16x32x64" + concurrencies: "4x8x16x32" use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp_lowlat_1.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp_lowlat_1.yaml index fd1d4a4f53..521eddf931 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp_lowlat_1.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp_lowlat_1.yaml @@ -136,5 +136,5 @@ benchmark: isl: 8192 osl: 1024 req_rate: 300 - concurrencies: "32x64x128x256x512" + concurrencies: "32x64" use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp_lowlat_2.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp_lowlat_2.yaml index bcceaf872d..9dfc6331a8 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp_lowlat_2.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp_lowlat_2.yaml @@ -136,5 +136,5 @@ benchmark: isl: 8192 osl: 1024 req_rate: 300 - concurrencies: "32x64x128x256x512" + concurrencies: "32" use_chat_template: true diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index b6f68c6483..bc5dfb7939 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7480,7 +7480,7 @@ dsr1-fp4-b200-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/b200-fp4/1k1k.yaml:zip_override_mtp_lowlat[0]" + - "CONFIG_FILE=recipes/sglang/dsr1/b200-fp4/1k1k/disagg/mtp/1k1k_mtp_lowlat_0.yaml" decode: num-worker: 5 tp: 8 @@ -7494,7 +7494,7 @@ dsr1-fp4-b200-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/b200-fp4/1k1k.yaml:zip_override_mtp_lowlat[1]" + - "CONFIG_FILE=recipes/sglang/dsr1/b200-fp4/1k1k/disagg/mtp/1k1k_mtp_lowlat_1.yaml" decode: num-worker: 6 tp: 8 @@ -7508,7 +7508,7 @@ dsr1-fp4-b200-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/b200-fp4/1k1k.yaml:zip_override_mtp_maxtpt[0]" + - "CONFIG_FILE=recipes/sglang/dsr1/b200-fp4/1k1k/disagg/mtp/1k1k_mtp_maxtpt_0.yaml" decode: num-worker: 1 tp: 8 @@ -7522,7 +7522,7 @@ dsr1-fp4-b200-dynamo-sglang-mtp: ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/b200-fp4/1k1k.yaml:zip_override_mtp_maxtpt[1]" + - "CONFIG_FILE=recipes/sglang/dsr1/b200-fp4/1k1k/disagg/mtp/1k1k_mtp_maxtpt_1.yaml" decode: num-worker: 2 tp: 8 @@ -7533,7 +7533,7 @@ dsr1-fp4-b200-dynamo-sglang-mtp: search-space: # 1p5d low-latency (decode-heavy). - spec-decoding: "mtp" - conc-list: [4, 8, 16, 32, 64] + conc-list: [4, 8, 16, 32] prefill: num-worker: 1 tp: 4 @@ -7548,7 +7548,7 @@ dsr1-fp4-b200-dynamo-sglang-mtp: dp-attn: false # 1p3d low-latency. - spec-decoding: "mtp" - conc-list: [32, 64, 128, 256, 512] + conc-list: [32, 64] prefill: num-worker: 1 tp: 4 @@ -7563,7 +7563,7 @@ dsr1-fp4-b200-dynamo-sglang-mtp: dp-attn: false # 1p1d low-latency. - spec-decoding: "mtp" - conc-list: [32, 64, 128, 256, 512] + conc-list: [32] prefill: num-worker: 1 tp: 4 @@ -7577,6 +7577,21 @@ dsr1-fp4-b200-dynamo-sglang-mtp: ep: 1 dp-attn: false # MTP2 high-throughput (DEP4 prefill / DEP8 decode), one Pareto point each. + # 1p1d throughput. + - spec-decoding: "mtp" + conc-list: [512] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp2_throughput_1p1d.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true # 2p1d throughput. - spec-decoding: "mtp" conc-list: [768] @@ -7607,6 +7622,21 @@ dsr1-fp4-b200-dynamo-sglang-mtp: tp: 8 ep: 8 dp-attn: true + # 4p1d throughput. + - spec-decoding: "mtp" + conc-list: [512] + prefill: + num-worker: 4 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp2_throughput_4p1d.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true # 5p1d throughput. - spec-decoding: "mtp" conc-list: [2048] @@ -7622,6 +7652,21 @@ dsr1-fp4-b200-dynamo-sglang-mtp: tp: 8 ep: 8 dp-attn: true + # 4p1d high-throughput (dedicated c2048 tune). + - spec-decoding: "mtp" + conc-list: [2048] + prefill: + num-worker: 4 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/dsr1/b200-fp4/8k1k/disagg/mtp/8k1k_mtp_4p1d_c2048.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true kimik2.5-fp4-gb200-dynamo-trt: image: nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 336edc9a61..bb7440ca4d 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4589,3 +4589,11 @@ - "Image: lmsysorg/sglang:v0.5.12" - "14 topologies across 1k/1k and 8k/1k: prefill TP8 STP + decode wide-EP (DEP16/DEP32 high-throughput) and per-node TP8 low-latency, recipes under benchmarks/multi_node/srt-slurm-recipes/sglang/glm5/gb200-fp8/" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1895 + +- config-keys: + - dsr1-fp4-b200-dynamo-sglang-mtp + description: + - "Prune the 8k/1k MTP disagg low-latency points: 1p5d-tp8 to conc 4-32, 1p3d-tp8 to conc 32-64, and 1p1d-tp8 to conc 32." + - "Add 8k/1k MTP disagg high-throughput points: 1p1d and 4p1d (DEP4 prefill / DEP8 decode) at conc 512." + - "Add an 8k/1k MTP disagg 4p1d (DEP4 prefill / DEP8 decode) high-throughput point at conc 2048 with a dedicated engine tune (max-running-requests 2048, decode scheduler-recv-interval 1, prefill/decode stream-interval 100/34, SGLANG_HACK_SEQ_BOOTSTRAP_ROOM)." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2113 diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index e08441ebcc..8c7aa76bb2 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -120,7 +120,12 @@ if [[ "$IS_MULTINODE" == "true" ]]; then elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "dsr1" && $PRECISION == "fp4" ]]; then git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" || exit 1 - git checkout main + # Pin srt-slurm: newer commits stopped honoring the hash-pinned dynamo + # build and fall back to a dynamo release that is incompatible with this + # sglang image (worker fails at import). This is the last commit before + # that change. Do not float on main -- the srtctl + dynamo-install + # toolchain is unpinned there. + git checkout a98738de9b2233459b5456e9ed71af09ce893f92 mkdir -p recipes/sglang/dsr1/b200-fp4 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4" recipes/sglang/dsr1/b200-fp4 else