diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep16-conc8192.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep16-conc8192.yaml new file mode 100644 index 0000000000..021e430ebe --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep16-conc8192.yaml @@ -0,0 +1,200 @@ +# 1k1k 1P1D / DEP16 / total 5 nodes +# conc sweep: 2048, 4096, 8192 (c/4, c/2, c where c = X * 8192) +# stack: lmsysorg/sglang nightly-dev-cu13-20260707-b4155233 (megamoe) + dynamo 81d0555e + +name: "disagg-1p1d-dep16-conc8192" + +slurm: + time_limit: "03:00:00" + +sbatch_directives: + cpus-per-task: '144' + mem: '0' + +# Backend workers build dynamo from source; that install runs +# `apt-get install libclang-dev`, which needs root. Run the worker container +# as remapped root so dpkg works (pyxis maps container-root back to the +# invoking user on the host, so files on mounted dirs stay caller-owned). +# Without this, workers run non-root and the install dies with exit 100 +# (`dpkg: requested operation requires superuser privilege`), hidden by the +# install's `> /dev/null 2>&1`. +srun_options: + container-remap-root: "" + # Step-level --mem=0: give each worker srun step all node RAM so the + # DSV4-Pro weight load does not hit the step cgroup limit and stall/OOM + # (job-level `sbatch --mem=0` does not lift the per-step memory cgroup). + mem: "0" + +dynamo: + hash: "81d0555ee23519cea80a42b4fe824e30368b7300" + install: true + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + env: + DYN_ROUTER_LOAD_BLOCK_SIZE: "1" + args: + router-mode: "kv" + router-kv-overlap-score-weight: 0 + router-queue-threshold: 64 + router-temperature: 0.5 + no-kv-events: true + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260707-b4155233" + precision: fp4 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_nodes: 4 + decode_workers: 1 + gpus_per_decode: 16 + +backend: + type: sglang + + prefill_environment: + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + PYTHONUNBUFFERED: '1' + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '0' + SGLANG_ENABLE_THINKING: '1' + SGLANG_REASONING_EFFORT: max + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_FORCE_MNNVL: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + DYN_LOG: 'info,dynamo_runtime::pipeline::network::ingress::push_handler=warn' + + decode_environment: + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + PYTHONUNBUFFERED: '1' + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '0' + SGLANG_ENABLE_THINKING: '1' + SGLANG_REASONING_EFFORT: max + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '1152' + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" + MC_FORCE_MNNVL: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + DYN_LOG: 'info,dynamo_runtime::pipeline::network::ingress::push_handler=warn' + + sglang_config: + prefill: + served-model-name: deepseek-ai/DeepSeek-V4-Pro + trust-remote-code: true + weight-loader-prefetch-checkpoints: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + moe-dense-tp-size: 1 + enable-dp-lm-head: true + moe-a2a-backend: megamoe + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + mem-fraction-static: 0.9 + max-running-requests: 512 + cuda-graph-max-bs: 512 + chunked-prefill-size: 32768 + + decode: + served-model-name: deepseek-ai/DeepSeek-V4-Pro + trust-remote-code: true + weight-loader-prefetch-checkpoints: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + moe-a2a-backend: megamoe + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + mem-fraction-static: 0.94 + swa-full-tokens-ratio: 0.12 + context-length: 16384 + max-running-requests: 18432 + cuda-graph-max-bs: 1152 + +benchmark: + type: custom + command: | + set -e + ulimit -n 1048576 + REPO=/configs/upstream-sa-bench/InferenceX + [ -d "$REPO" ] || git clone https://github.com/SemiAnalysisAI/InferenceX.git "$REPO" + cd "$REPO/utils/bench_serving" + + echo "=== block conc=2048 prompts=6144 warmups=2048 ===" + python3 benchmark_serving.py \ + --backend vllm --model deepseek-ai/DeepSeek-V4-Pro --tokenizer /model \ + --host 127.0.0.1 --port 8000 --endpoint /v1/completions \ + --dataset-name random \ + --random-input-len 1024 --random-output-len 1024 --random-range-ratio 0.8 \ + --random-num-workers 96 \ + --num-prompts 6144 --max-concurrency 2048 --request-rate inf \ + --num-warmups 2048 \ + --ignore-eos --trust-remote-code \ + --percentile-metrics ttft,tpot,itl,e2el \ + --save-result --result-dir /logs --result-filename results_conc2048.json + + echo "=== block conc=4096 prompts=12288 warmups=4096 ===" + python3 benchmark_serving.py \ + --backend vllm --model deepseek-ai/DeepSeek-V4-Pro --tokenizer /model \ + --host 127.0.0.1 --port 8000 --endpoint /v1/completions \ + --dataset-name random \ + --random-input-len 1024 --random-output-len 1024 --random-range-ratio 0.8 \ + --random-num-workers 96 \ + --num-prompts 12288 --max-concurrency 4096 --request-rate inf \ + --num-warmups 4096 \ + --ignore-eos --trust-remote-code \ + --percentile-metrics ttft,tpot,itl,e2el \ + --save-result --result-dir /logs --result-filename results_conc4096.json + + echo "=== block conc=8192 prompts=24576 warmups=8192 ===" + python3 benchmark_serving.py \ + --backend vllm --model deepseek-ai/DeepSeek-V4-Pro --tokenizer /model \ + --host 127.0.0.1 --port 8000 --endpoint /v1/completions \ + --dataset-name random \ + --random-input-len 1024 --random-output-len 1024 --random-range-ratio 0.8 \ + --random-num-workers 96 \ + --num-prompts 24576 --max-concurrency 8192 --request-rate inf \ + --num-warmups 8192 \ + --ignore-eos --trust-remote-code \ + --percentile-metrics ttft,tpot,itl,e2el \ + --save-result --result-dir /logs --result-filename results_conc8192.json diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep8-conc8192.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep8-conc8192.yaml new file mode 100644 index 0000000000..2232e6f817 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep8-conc8192.yaml @@ -0,0 +1,201 @@ +# 1k1k 1P1D / DEP8 / total 3 nodes +# conc sweep: 2048, 4096, 8192 (c/4, c/2, c where c = X * 8192) +# stack: lmsysorg/sglang nightly-dev-cu13-20260707-b4155233 (megamoe) + dynamo 81d0555e + +name: "disagg-1p1d-dep8-conc8192" + +slurm: + time_limit: "03:00:00" + +sbatch_directives: + cpus-per-task: '144' + mem: '0' + +# Backend workers build dynamo from source; that install runs +# `apt-get install libclang-dev`, which needs root. Run the worker container +# as remapped root so dpkg works (pyxis maps container-root back to the +# invoking user on the host, so files on mounted dirs stay caller-owned). +# Without this, workers run non-root and the install dies with exit 100 +# (`dpkg: requested operation requires superuser privilege`), hidden by the +# install's `> /dev/null 2>&1`. +srun_options: + container-remap-root: "" + # Step-level --mem=0: give each worker srun step all node RAM so the + # DSV4-Pro weight load does not hit the step cgroup limit and stall/OOM + # (job-level `sbatch --mem=0` does not lift the per-step memory cgroup). + mem: "0" + +dynamo: + hash: "81d0555ee23519cea80a42b4fe824e30368b7300" + install: true + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + env: + DYN_ROUTER_LOAD_BLOCK_SIZE: "1" + args: + router-mode: "kv" + router-kv-overlap-score-weight: 0 + router-queue-threshold: 64 + router-temperature: 0.5 + no-kv-events: true + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260707-b4155233" + precision: fp4 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_nodes: 2 + decode_workers: 1 + gpus_per_decode: 8 + +backend: + type: sglang + + prefill_environment: + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + PYTHONUNBUFFERED: '1' + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '0' + SGLANG_ENABLE_THINKING: '1' + SGLANG_REASONING_EFFORT: max + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_FORCE_MNNVL: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + DYN_LOG: 'info,dynamo_runtime::pipeline::network::ingress::push_handler=warn' + + decode_environment: + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + PYTHONUNBUFFERED: '1' + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '0' + SGLANG_ENABLE_THINKING: '1' + SGLANG_REASONING_EFFORT: max + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '1152' + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" + MC_FORCE_MNNVL: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + DYN_LOG: 'info,dynamo_runtime::pipeline::network::ingress::push_handler=warn' + + sglang_config: + prefill: + served-model-name: deepseek-ai/DeepSeek-V4-Pro + trust-remote-code: true + weight-loader-prefetch-checkpoints: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + moe-dense-tp-size: 1 + enable-dp-lm-head: true + moe-a2a-backend: megamoe + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + mem-fraction-static: 0.9 + max-running-requests: 512 + cuda-graph-max-bs: 512 + chunked-prefill-size: 32768 + + decode: + served-model-name: deepseek-ai/DeepSeek-V4-Pro + trust-remote-code: true + weight-loader-prefetch-checkpoints: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + tensor-parallel-size: 8 + data-parallel-size: 8 + expert-parallel-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + moe-a2a-backend: megamoe + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + mem-fraction-static: 0.94 + swa-full-tokens-ratio: 0.12 + context-length: 16384 + max-running-requests: 9216 + cuda-graph-max-bs: 1152 + +benchmark: + type: custom + command: | + set -e + ulimit -n 1048576 + REPO=/configs/upstream-sa-bench/InferenceX + [ -d "$REPO" ] || git clone https://github.com/SemiAnalysisAI/InferenceX.git "$REPO" + cd "$REPO/utils/bench_serving" + + echo "=== block conc=2048 prompts=6144 warmups=2048 ===" + python3 benchmark_serving.py \ + --backend vllm --model deepseek-ai/DeepSeek-V4-Pro --tokenizer /model \ + --host 127.0.0.1 --port 8000 --endpoint /v1/completions \ + --dataset-name random \ + --random-input-len 1024 --random-output-len 1024 --random-range-ratio 0.8 \ + --random-num-workers 96 \ + --num-prompts 6144 --max-concurrency 2048 --request-rate inf \ + --num-warmups 2048 \ + --ignore-eos --trust-remote-code \ + --percentile-metrics ttft,tpot,itl,e2el \ + --save-result --result-dir /logs --result-filename results_conc2048.json + + echo "=== block conc=4096 prompts=12288 warmups=4096 ===" + python3 benchmark_serving.py \ + --backend vllm --model deepseek-ai/DeepSeek-V4-Pro --tokenizer /model \ + --host 127.0.0.1 --port 8000 --endpoint /v1/completions \ + --dataset-name random \ + --random-input-len 1024 --random-output-len 1024 --random-range-ratio 0.8 \ + --random-num-workers 96 \ + --num-prompts 12288 --max-concurrency 4096 --request-rate inf \ + --num-warmups 4096 \ + --ignore-eos --trust-remote-code \ + --percentile-metrics ttft,tpot,itl,e2el \ + --save-result --result-dir /logs --result-filename results_conc4096.json + + echo "=== block conc=8192 prompts=24576 warmups=8192 ===" + python3 benchmark_serving.py \ + --backend vllm --model deepseek-ai/DeepSeek-V4-Pro --tokenizer /model \ + --host 127.0.0.1 --port 8000 --endpoint /v1/completions \ + --dataset-name random \ + --random-input-len 1024 --random-output-len 1024 --random-range-ratio 0.8 \ + --random-num-workers 96 \ + --num-prompts 24576 --max-concurrency 8192 --request-rate inf \ + --num-warmups 8192 \ + --ignore-eos --trust-remote-code \ + --percentile-metrics ttft,tpot,itl,e2el \ + --save-result --result-dir /logs --result-filename results_conc8192.json + diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p4d-dep4-tp4.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p4d-dep4-tp4.yaml new file mode 100644 index 0000000000..3ec6102c25 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p4d-dep4-tp4.yaml @@ -0,0 +1,127 @@ +name: "dsv4-pro-gb300-disagg-1k1k-1p4d-dep4-tp4" + +# DeepSeek-V4-Pro 1k/1k disagg on GB300 — 1P + 4D, DEP4 prefill / TP4 decode. +# +# Single-config recipe derived from disagg-low-latency-dep4.yaml zip override +# variant [2] (decode_nodes=4). Concurrencies pruned to the Pareto-frontier +# points from the original sweep. + +# GB300 host-RAM: give the job all node memory and each srun step --mem=0 +# so the DSV4-Pro weight load is not constrained by the per-step memory +# cgroup (job-level mem=0 alone does not lift it). Matches minimax GB300. +sbatch_directives: + mem: "0" + cpus-per-task: "144" +srun_options: + mem: "0" + +frontend: + type: sglang + enable_multiple_frontends: false + args: + policy: "cache_aware" + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260707-b4155233" + precision: "mxfp4" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + decode_nodes: 4 + decode_workers: 4 + +backend: + type: sglang + + prefill_environment: + PYTHONUNBUFFERED: "1" + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + SGLANG_ENABLE_THINKING: "1" + SGLANG_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + decode_environment: + PYTHONUNBUFFERED: "1" + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + SGLANG_ENABLE_THINKING: "1" + SGLANG_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + sglang_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + weight-loader-prefetch-checkpoints: true + disable-radix-cache: true + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + + enable-dp-attention: true + enable-dp-lm-head: true + + moe-a2a-backend: "megamoe" + + mem-fraction-static: 0.90 + max-running-requests: 512 + cuda-graph-max-bs: 512 + chunked-prefill-size: 4096 + + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + weight-loader-prefetch-checkpoints: true + disable-radix-cache: true + + disaggregation-mode: "decode" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + mem-fraction-static: 0.9 + max-running-requests: 128 + cuda-graph-max-bs: 128 + swa-full-tokens-ratio: 0.1 + context-length: 4096 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "8x64" + req_rate: "inf" + use_chat_template: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p6d-dep4-tp4.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p6d-dep4-tp4.yaml new file mode 100644 index 0000000000..6a8767baae --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-1p6d-dep4-tp4.yaml @@ -0,0 +1,126 @@ +name: "dsv4-pro-gb300-disagg-1k1k-1p6d-dep4-tp4" +# DeepSeek-V4-Pro 1k/1k disagg on GB300 — 1P + 6D, DEP4 prefill / TP4 decode. +# +# Single-config recipe derived from disagg-low-latency-dep4.yaml zip override +# variant [3] (decode_nodes=6). Concurrencies pruned to the Pareto-frontier +# points from the original sweep + +# GB300 host-RAM: give the job all node memory and each srun step --mem=0 +# so the DSV4-Pro weight load is not constrained by the per-step memory +# cgroup (job-level mem=0 alone does not lift it). Matches minimax GB300. +sbatch_directives: + mem: "0" + cpus-per-task: "144" +srun_options: + mem: "0" + +frontend: + type: sglang + enable_multiple_frontends: false + args: + policy: "cache_aware" + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260707-b4155233" + precision: "mxfp4" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + decode_nodes: 6 + decode_workers: 6 + +backend: + type: sglang + + prefill_environment: + PYTHONUNBUFFERED: "1" + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + SGLANG_ENABLE_THINKING: "1" + SGLANG_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "9216" + + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + decode_environment: + PYTHONUNBUFFERED: "1" + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "0" + SGLANG_ENABLE_THINKING: "1" + SGLANG_REASONING_EFFORT: "max" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_CUMEM_ENABLE: "1" + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: "True" + MC_FORCE_MNNVL: "1" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "100000" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "100000" + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: "1" + + sglang_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + weight-loader-prefetch-checkpoints: true + disable-radix-cache: true + + disaggregation-mode: "prefill" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + + enable-dp-attention: true + enable-dp-lm-head: true + + moe-a2a-backend: "megamoe" + + mem-fraction-static: 0.90 + max-running-requests: 512 + cuda-graph-max-bs: 512 + chunked-prefill-size: 4096 + + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + model-path: "/model/" + trust-remote-code: true + weight-loader-prefetch-checkpoints: true + disable-radix-cache: true + + disaggregation-mode: "decode" + disaggregation-transfer-backend: mooncake + + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + + moe-runner-backend: "flashinfer_mxfp4" + disable-flashinfer-autotune: true + + mem-fraction-static: 0.9 + max-running-requests: 128 + cuda-graph-max-bs: 128 + swa-full-tokens-ratio: 0.1 + context-length: 4096 + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + random_range_ratio: 0.8 + concurrencies: "32x64" + req_rate: "inf" + use_chat_template: false diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-2p1d-dep12-conc18432.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-2p1d-dep12-conc18432.yaml new file mode 100644 index 0000000000..ef78f369b3 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-2p1d-dep12-conc18432.yaml @@ -0,0 +1,144 @@ +# 1k1k 2P1D / DEP12 / total 5 nodes +# rate-match point between 2:4 and 3:4; c8bd stack, single conc=18432, mrr=13824 +# stack: lmsysorg/sglang nightly-dev-cu13-20260707-b4155233 (megamoe) + dynamo 81d0555e + +name: "disagg-2p1d-dep12-conc18432" +slurm: + time_limit: 03:00:00 +sbatch_directives: + cpus-per-task: '144' + mem: '0' +# Backend workers build dynamo from source; that install runs +# `apt-get install libclang-dev`, which needs root. Run the worker container +# as remapped root so dpkg works (pyxis maps container-root back to the +# invoking user on the host, so files on mounted dirs stay caller-owned). +# Without this, workers run non-root and the install dies with exit 100 +# (`dpkg: requested operation requires superuser privilege`), hidden by the +# install's `> /dev/null 2>&1`. +srun_options: + container-remap-root: "" + # Step-level --mem=0: give each worker srun step all node RAM so the + # DSV4-Pro weight load does not hit the step cgroup limit and stall/OOM + # (job-level `sbatch --mem=0` does not lift the per-step memory cgroup). + mem: "0" + +dynamo: + hash: 81d0555ee23519cea80a42b4fe824e30368b7300 + install: true +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + env: + DYN_ROUTER_LOAD_BLOCK_SIZE: '1' + args: + router-mode: kv + router-kv-overlap-score-weight: 0 + router-queue-threshold: 64 + router-temperature: 0.5 + no-kv-events: true +model: + path: "deepseek-v4-pro" + container: lmsysorg/sglang:nightly-dev-cu13-20260707-b4155233 + precision: fp4 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 2 + prefill_workers: 2 + gpus_per_prefill: 4 + decode_nodes: 3 + decode_workers: 1 + gpus_per_decode: 12 +backend: + type: sglang + prefill_environment: + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + PYTHONUNBUFFERED: '1' + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '0' + SGLANG_ENABLE_THINKING: '1' + SGLANG_REASONING_EFFORT: max + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '8192' + SGLANG_OPT_USE_ONLINE_COMPRESS: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_FORCE_MNNVL: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn + decode_environment: + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + PYTHONUNBUFFERED: '1' + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '0' + SGLANG_ENABLE_THINKING: '1' + SGLANG_REASONING_EFFORT: max + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '1152' + SGLANG_OPT_USE_ONLINE_COMPRESS: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + MC_FORCE_MNNVL: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn + sglang_config: + prefill: + served-model-name: deepseek-ai/DeepSeek-V4-Pro + trust-remote-code: true + weight-loader-prefetch-checkpoints: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + moe-dense-tp-size: 1 + enable-dp-lm-head: true + moe-a2a-backend: megamoe + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + mem-fraction-static: 0.9 + max-running-requests: 512 + cuda-graph-max-bs: 512 + chunked-prefill-size: 32768 + decode: + served-model-name: deepseek-ai/DeepSeek-V4-Pro + trust-remote-code: true + weight-loader-prefetch-checkpoints: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + tensor-parallel-size: 12 + data-parallel-size: 12 + expert-parallel-size: 12 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + moe-a2a-backend: megamoe + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + mem-fraction-static: 0.94 + swa-full-tokens-ratio: 0.12 + context-length: 16384 + max-running-requests: 13824 + cuda-graph-max-bs: 1152 +benchmark: + type: custom + command: "set -e\nulimit -n 1048576\nREPO=/configs/upstream-sa-bench/InferenceX\n[ -d \"$REPO\" ] || git clone https://github.com/SemiAnalysisAI/InferenceX.git \"$REPO\"\ncd \"$REPO/utils/bench_serving\"\n\necho \"=== 2P1D DEP12 rate-match point: conc=18432 prompts=55296 warmups=18432 mrr=13824 ===\"\npython3 benchmark_serving.py \\\n --backend vllm --model deepseek-ai/DeepSeek-V4-Pro --tokenizer /model \\\n --host 127.0.0.1 --port 8000 --endpoint /v1/completions \\\n --dataset-name random \\\n --random-input-len 1024 --random-output-len 1024 --random-range-ratio 0.8 \\\n --random-num-workers 96 \\\n --num-prompts 55296 --max-concurrency 18432 --request-rate inf \\\n --num-warmups 18432 \\\n --ignore-eos --trust-remote-code \\\n --percentile-metrics ttft,tpot,itl,e2el \\\n --save-result --result-dir /logs --result-filename results_conc18432_mrr13824.json\n" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-2p1d-dep16-conc16384.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-2p1d-dep16-conc16384.yaml new file mode 100644 index 0000000000..2826b7d9a7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-2p1d-dep16-conc16384.yaml @@ -0,0 +1,200 @@ +# 1k1k 2P1D / DEP16 / total 6 nodes +# conc sweep: 4096, 8192, 16384 (c/4, c/2, c where c = X * 8192) +# stack: lmsysorg/sglang nightly-dev-cu13-20260707-b4155233 (megamoe) + dynamo 81d0555e + +name: "disagg-2p1d-dep16-conc16384" + +slurm: + time_limit: "03:00:00" + +sbatch_directives: + cpus-per-task: '144' + mem: '0' + +# Backend workers build dynamo from source; that install runs +# `apt-get install libclang-dev`, which needs root. Run the worker container +# as remapped root so dpkg works (pyxis maps container-root back to the +# invoking user on the host, so files on mounted dirs stay caller-owned). +# Without this, workers run non-root and the install dies with exit 100 +# (`dpkg: requested operation requires superuser privilege`), hidden by the +# install's `> /dev/null 2>&1`. +srun_options: + container-remap-root: "" + # Step-level --mem=0: give each worker srun step all node RAM so the + # DSV4-Pro weight load does not hit the step cgroup limit and stall/OOM + # (job-level `sbatch --mem=0` does not lift the per-step memory cgroup). + mem: "0" + +dynamo: + hash: "81d0555ee23519cea80a42b4fe824e30368b7300" + install: true + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + env: + DYN_ROUTER_LOAD_BLOCK_SIZE: "1" + args: + router-mode: "kv" + router-kv-overlap-score-weight: 0 + router-queue-threshold: 64 + router-temperature: 0.5 + no-kv-events: true + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260707-b4155233" + precision: fp4 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 2 + prefill_workers: 2 + gpus_per_prefill: 4 + decode_nodes: 4 + decode_workers: 1 + gpus_per_decode: 16 + +backend: + type: sglang + + prefill_environment: + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + PYTHONUNBUFFERED: '1' + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '0' + SGLANG_ENABLE_THINKING: '1' + SGLANG_REASONING_EFFORT: max + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_FORCE_MNNVL: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + DYN_LOG: 'info,dynamo_runtime::pipeline::network::ingress::push_handler=warn' + + decode_environment: + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + PYTHONUNBUFFERED: '1' + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '0' + SGLANG_ENABLE_THINKING: '1' + SGLANG_REASONING_EFFORT: max + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '1152' + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" + MC_FORCE_MNNVL: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + DYN_LOG: 'info,dynamo_runtime::pipeline::network::ingress::push_handler=warn' + + sglang_config: + prefill: + served-model-name: deepseek-ai/DeepSeek-V4-Pro + trust-remote-code: true + weight-loader-prefetch-checkpoints: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + moe-dense-tp-size: 1 + enable-dp-lm-head: true + moe-a2a-backend: megamoe + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + mem-fraction-static: 0.9 + max-running-requests: 512 + cuda-graph-max-bs: 512 + chunked-prefill-size: 32768 + + decode: + served-model-name: deepseek-ai/DeepSeek-V4-Pro + trust-remote-code: true + weight-loader-prefetch-checkpoints: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + tensor-parallel-size: 16 + data-parallel-size: 16 + expert-parallel-size: 16 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + moe-a2a-backend: megamoe + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + mem-fraction-static: 0.94 + swa-full-tokens-ratio: 0.12 + context-length: 16384 + max-running-requests: 18432 + cuda-graph-max-bs: 1152 + +benchmark: + type: custom + command: | + set -e + ulimit -n 1048576 + REPO=/configs/upstream-sa-bench/InferenceX + [ -d "$REPO" ] || git clone https://github.com/SemiAnalysisAI/InferenceX.git "$REPO" + cd "$REPO/utils/bench_serving" + + echo "=== block conc=4096 prompts=12288 warmups=4096 ===" + python3 benchmark_serving.py \ + --backend vllm --model deepseek-ai/DeepSeek-V4-Pro --tokenizer /model \ + --host 127.0.0.1 --port 8000 --endpoint /v1/completions \ + --dataset-name random \ + --random-input-len 1024 --random-output-len 1024 --random-range-ratio 0.8 \ + --random-num-workers 96 \ + --num-prompts 12288 --max-concurrency 4096 --request-rate inf \ + --num-warmups 4096 \ + --ignore-eos --trust-remote-code \ + --percentile-metrics ttft,tpot,itl,e2el \ + --save-result --result-dir /logs --result-filename results_conc4096.json + + echo "=== block conc=8192 prompts=24576 warmups=8192 ===" + python3 benchmark_serving.py \ + --backend vllm --model deepseek-ai/DeepSeek-V4-Pro --tokenizer /model \ + --host 127.0.0.1 --port 8000 --endpoint /v1/completions \ + --dataset-name random \ + --random-input-len 1024 --random-output-len 1024 --random-range-ratio 0.8 \ + --random-num-workers 96 \ + --num-prompts 24576 --max-concurrency 8192 --request-rate inf \ + --num-warmups 8192 \ + --ignore-eos --trust-remote-code \ + --percentile-metrics ttft,tpot,itl,e2el \ + --save-result --result-dir /logs --result-filename results_conc8192.json + + echo "=== block conc=16384 prompts=49152 warmups=16384 ===" + python3 benchmark_serving.py \ + --backend vllm --model deepseek-ai/DeepSeek-V4-Pro --tokenizer /model \ + --host 127.0.0.1 --port 8000 --endpoint /v1/completions \ + --dataset-name random \ + --random-input-len 1024 --random-output-len 1024 --random-range-ratio 0.8 \ + --random-num-workers 96 \ + --num-prompts 49152 --max-concurrency 16384 --request-rate inf \ + --num-warmups 16384 \ + --ignore-eos --trust-remote-code \ + --percentile-metrics ttft,tpot,itl,e2el \ + --save-result --result-dir /logs --result-filename results_conc16384.json diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-2p1d-dep48-conc16384.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-2p1d-dep48-conc16384.yaml new file mode 100644 index 0000000000..f5e56890af --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/disagg-2p1d-dep48-conc16384.yaml @@ -0,0 +1,202 @@ +# 1k1k 2P1D / DEP48 / total 14 nodes +# conc sweep: 4096, 8192, 16384 (c/4, c/2, c where c = X * 8192) +# stack: lmsysorg/sglang nightly-dev-cu13-20260707-b4155233 (megamoe) + dynamo 81d0555e + +name: "disagg-2p1d-dep48-conc16384" + +slurm: + time_limit: "03:00:00" + +sbatch_directives: + cpus-per-task: '144' + mem: '0' + +# Backend workers build dynamo from source; that install runs +# `apt-get install libclang-dev`, which needs root. Run the worker container +# as remapped root so dpkg works (pyxis maps container-root back to the +# invoking user on the host, so files on mounted dirs stay caller-owned). +# Without this, workers run non-root and the install dies with exit 100 +# (`dpkg: requested operation requires superuser privilege`), hidden by the +# install's `> /dev/null 2>&1`. +srun_options: + container-remap-root: "" + # Step-level --mem=0: give each worker srun step all node RAM so the + # DSV4-Pro weight load does not hit the step cgroup limit and stall/OOM + # (job-level `sbatch --mem=0` does not lift the per-step memory cgroup). + mem: "0" + +dynamo: + hash: "81d0555ee23519cea80a42b4fe824e30368b7300" + install: true + +frontend: + type: dynamo + enable_multiple_frontends: true + num_additional_frontends: 8 + env: + DYN_ROUTER_LOAD_BLOCK_SIZE: "1" + args: + router-mode: "kv" + router-kv-overlap-score-weight: 0 + router-queue-threshold: 64 + router-temperature: 0.5 + no-kv-events: true + +model: + path: "deepseek-v4-pro" + container: "lmsysorg/sglang:nightly-dev-cu13-20260707-b4155233" + precision: fp4 + +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 2 + prefill_workers: 2 + gpus_per_prefill: 4 + decode_nodes: 12 + decode_workers: 1 + gpus_per_decode: 48 + +backend: + type: sglang + + prefill_environment: + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + PYTHONUNBUFFERED: '1' + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '0' + SGLANG_ENABLE_THINKING: '1' + SGLANG_REASONING_EFFORT: max + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: "8192" + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + MC_FORCE_MNNVL: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + DYN_LOG: 'info,dynamo_runtime::pipeline::network::ingress::push_handler=warn' + + decode_environment: + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + PYTHONUNBUFFERED: '1' + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '0' + SGLANG_ENABLE_THINKING: '1' + SGLANG_REASONING_EFFORT: max + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '1152' + SGLANG_OPT_USE_ONLINE_COMPRESS: "1" + NCCL_MNNVL_ENABLE: '1' + NCCL_CUMEM_ENABLE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: "8" + MC_FORCE_MNNVL: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + DYN_LOG: 'info,dynamo_runtime::pipeline::network::ingress::push_handler=warn' + + sglang_config: + prefill: + served-model-name: deepseek-ai/DeepSeek-V4-Pro + trust-remote-code: true + weight-loader-prefetch-checkpoints: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + moe-dense-tp-size: 1 + enable-dp-lm-head: true + moe-a2a-backend: megamoe + disaggregation-mode: prefill + disaggregation-transfer-backend: mooncake + mem-fraction-static: 0.9 + max-running-requests: 512 + cuda-graph-max-bs: 512 + chunked-prefill-size: 32768 + + decode: + served-model-name: deepseek-ai/DeepSeek-V4-Pro + trust-remote-code: true + weight-loader-prefetch-checkpoints: true + watchdog-timeout: 86400 + skip-tokenizer-init: true + stream-interval: 60 + tensor-parallel-size: 48 + data-parallel-size: 48 + expert-parallel-size: 48 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + moe-a2a-backend: megamoe + disaggregation-mode: decode + disaggregation-transfer-backend: mooncake + mem-fraction-static: 0.94 + swa-full-tokens-ratio: 0.12 + context-length: 16384 + max-running-requests: 55296 + cuda-graph-max-bs: 1152 + +benchmark: + type: custom + command: | + set -e + ulimit -n 1048576 + REPO=/configs/upstream-sa-bench/InferenceX + [ -d "$REPO" ] || git clone https://github.com/SemiAnalysisAI/InferenceX.git "$REPO" + cd "$REPO/utils/bench_serving" + + echo "=== block conc=4096 prompts=12288 warmups=4096 ===" + python3 benchmark_serving.py \ + --backend vllm --model deepseek-ai/DeepSeek-V4-Pro --tokenizer /model \ + --host 127.0.0.1 --port 8000 --endpoint /v1/completions \ + --dataset-name random \ + --random-input-len 1024 --random-output-len 1024 --random-range-ratio 0.8 \ + --random-num-workers 96 \ + --num-prompts 12288 --max-concurrency 4096 --request-rate inf \ + --num-warmups 4096 \ + --ignore-eos --trust-remote-code \ + --percentile-metrics ttft,tpot,itl,e2el \ + --save-result --result-dir /logs --result-filename results_conc4096.json + + echo "=== block conc=8192 prompts=24576 warmups=8192 ===" + python3 benchmark_serving.py \ + --backend vllm --model deepseek-ai/DeepSeek-V4-Pro --tokenizer /model \ + --host 127.0.0.1 --port 8000 --endpoint /v1/completions \ + --dataset-name random \ + --random-input-len 1024 --random-output-len 1024 --random-range-ratio 0.8 \ + --random-num-workers 96 \ + --num-prompts 24576 --max-concurrency 8192 --request-rate inf \ + --num-warmups 8192 \ + --ignore-eos --trust-remote-code \ + --percentile-metrics ttft,tpot,itl,e2el \ + --save-result --result-dir /logs --result-filename results_conc8192.json + + echo "=== block conc=16384 prompts=49152 warmups=16384 ===" + python3 benchmark_serving.py \ + --backend vllm --model deepseek-ai/DeepSeek-V4-Pro --tokenizer /model \ + --host 127.0.0.1 --port 8000 --endpoint /v1/completions \ + --dataset-name random \ + --random-input-len 1024 --random-output-len 1024 --random-range-ratio 0.8 \ + --random-num-workers 96 \ + --num-prompts 49152 --max-concurrency 16384 --request-rate inf \ + --num-warmups 16384 \ + --ignore-eos --trust-remote-code \ + --percentile-metrics ttft,tpot,itl,e2el \ + --save-result --result-dir /logs --result-filename results_conc16384.json + + diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 8d84bd0dd7..fd2feb0ade 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -10173,6 +10173,108 @@ dsv4-fp4-gb300-dynamo-sglang: tp: 40 ep: 40 dp-attn: true + # 1k1k STP disagg configs. + - isl: 1024 + osl: 1024 + search-space: + # 1p1d-dep8. 3 nodes. + - conc-list: [8192] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep8-conc8192.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + # 1p1d-dep16. 5 nodes. + - conc-list: [8192] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/1k1k/disagg-1p1d-dep16-conc8192.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # 1p4d-dep4-tp4. 5 nodes. sa-bench conc 8x64. + - conc-list: [64] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/1k1k/disagg-1p4d-dep4-tp4.yaml" + decode: + num-worker: 4 + tp: 4 + ep: 1 + dp-attn: false + # 1p6d-dep4-tp4. 7 nodes. sa-bench conc 32x64. + - conc-list: [64] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/1k1k/disagg-1p6d-dep4-tp4.yaml" + decode: + num-worker: 6 + tp: 4 + ep: 1 + dp-attn: false + # 2p1d-dep12. 5 nodes. + - conc-list: [18432] + prefill: + num-worker: 2 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/1k1k/disagg-2p1d-dep12-conc18432.yaml" + decode: + num-worker: 1 + tp: 12 + ep: 12 + dp-attn: true + # 2p1d-dep16. 6 nodes. + - conc-list: [16384] + prefill: + num-worker: 2 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/1k1k/disagg-2p1d-dep16-conc16384.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # 2p1d-dep48. 14 nodes. + - conc-list: [16384] + prefill: + num-worker: 2 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/sglang/deepseek-v4/1k1k/disagg-2p1d-dep48-conc16384.yaml" + decode: + num-worker: 1 + tp: 48 + ep: 48 + dp-attn: true glm5-fp8-b200-dynamo-sglang: image: lmsysorg/sglang:v0.5.11-cu130 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6e1249adfc..4d30e92b22 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4581,3 +4581,12 @@ - "Pin the EAGLE3 drafter to TRITON_ATTN via the speculative-config attention_backend override; without it the drafter falls back to a slow default backend." - "Speeds up the draft forward measured on MI355X TP4 MXFP8, 8k/1k throughput gains growing with concurrency vs no override." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2107 + +- config-keys: + - dsv4-fp4-gb300-dynamo-sglang + description: + - "Add DeepSeek-V4-Pro NVFP4 GB300 1k1k STP disagg scenarios to the existing dsv4-fp4-gb300-dynamo-sglang key (7 fixed-seq-len entries alongside the existing 8k1k scenarios)." + - "Topologies: 1P1D DEP8 and DEP16 (custom conc sweep), 1P4D and 1P6D TP4 (sa-bench conc sweep), 2P1D DEP12/DEP16/DEP48 (custom conc sweep)." + - "Recipes live under benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/1k1k/, container aligned with the existing 8k1k image (lmsysorg/sglang:nightly-dev-cu13-20260520-425dffbd)." + - "launch_gb300-nv.sh gains a dynamo-sglang + dsv4 overlay branch (srt-slurm sa-submission-q2-2026 base, in-tree recipes overlaid on top)." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2099 diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index aaf48134fa..4731a47adc 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -156,6 +156,12 @@ elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "dsv4" ]]; then git checkout aflowers/gb200-dsv4-recipes mkdir -p recipes/vllm/deepseek-v4 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4" recipes/vllm/deepseek-v4 +elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "dsv4" ]]; then + git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" + git checkout sa-submission-q2-2026 + mkdir -p recipes/sglang/deepseek-v4 + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4" recipes/sglang/deepseek-v4 elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "glm5" ]]; then git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR"