From 753dbc8aa0508d55f22cdc5f67e28f4d82ea3027 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 6 Jul 2026 14:55:49 -0400 Subject: [PATCH] Deprecate gpt-oss-120b benchmark configs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Following the MiniMax M2.5/M2.7 deprecation pattern (#1874): all 10 gptoss config entries removed from the active master configs and archived under configs/deprecated/{nvidia,amd}-gptoss-master.yaml; the 9 single-node scripts and the gb200 dynamo-trt multi-node script moved into deprecated/ subdirectories. Unlike #1874, the moved scripts' benchmark_lib.sh source paths are corrected for the extra directory level (the #1860 bug class). 中文：按 MiniMax M2.5/M2.7 弃用模式（#1874）弃用 gpt-oss-120b 基准 测试配置 - 10 个配置条目从活动 master 配置中移除并归档至 configs/deprecated/，全部脚本移入 deprecated/ 子目录；与 #1874 不同， 本次已同步修正移动脚本中 benchmark_lib.sh 的相对引用路径。 Co-Authored-By: Claude Fable 5 --- .../gptoss_fp4_gb200_dynamo-trt.sh | 2 +- .../{ => deprecated}/gptoss_fp4_b200.sh | 2 +- .../{ => deprecated}/gptoss_fp4_b200_trt.sh | 2 +- .../{ => deprecated}/gptoss_fp4_h100.sh | 2 +- .../{ => deprecated}/gptoss_fp4_h200.sh | 2 +- .../{ => deprecated}/gptoss_fp4_h200_trt.sh | 2 +- .../{ => deprecated}/gptoss_fp4_mi300x.sh | 2 +- .../{ => deprecated}/gptoss_fp4_mi325x.sh | 2 +- .../{ => deprecated}/gptoss_fp4_mi355x.sh | 2 +- .../gptoss_fp4_mi355x_atom.sh | 2 +- configs/amd-master.yaml | 104 ----- configs/deprecated/amd-gptoss-master.yaml | 96 ++++ configs/deprecated/nvidia-gptoss-master.yaml | 417 +++++++++++++++++ configs/nvidia-master.yaml | 436 ------------------ 14 files changed, 523 insertions(+), 550 deletions(-) rename benchmarks/multi_node/{ => deprecated}/gptoss_fp4_gb200_dynamo-trt.sh (97%) rename benchmarks/single_node/fixed_seq_len/{ => deprecated}/gptoss_fp4_b200.sh (97%) rename benchmarks/single_node/fixed_seq_len/{ => deprecated}/gptoss_fp4_b200_trt.sh (98%) rename benchmarks/single_node/fixed_seq_len/{ => deprecated}/gptoss_fp4_h100.sh (96%) rename benchmarks/single_node/fixed_seq_len/{ => deprecated}/gptoss_fp4_h200.sh (97%) rename benchmarks/single_node/fixed_seq_len/{ => deprecated}/gptoss_fp4_h200_trt.sh (97%) rename benchmarks/single_node/fixed_seq_len/{ => deprecated}/gptoss_fp4_mi300x.sh (97%) rename benchmarks/single_node/fixed_seq_len/{ => deprecated}/gptoss_fp4_mi325x.sh (97%) rename benchmarks/single_node/fixed_seq_len/{ => deprecated}/gptoss_fp4_mi355x.sh (97%) rename benchmarks/single_node/fixed_seq_len/{ => deprecated}/gptoss_fp4_mi355x_atom.sh (97%) create mode 100644 configs/deprecated/amd-gptoss-master.yaml create mode 100644 configs/deprecated/nvidia-gptoss-master.yaml diff --git a/benchmarks/multi_node/gptoss_fp4_gb200_dynamo-trt.sh b/benchmarks/multi_node/deprecated/gptoss_fp4_gb200_dynamo-trt.sh similarity index 97% rename from benchmarks/multi_node/gptoss_fp4_gb200_dynamo-trt.sh rename to benchmarks/multi_node/deprecated/gptoss_fp4_gb200_dynamo-trt.sh index a08e6f4b01..efa587beda 100644 --- a/benchmarks/multi_node/gptoss_fp4_gb200_dynamo-trt.sh +++ b/benchmarks/multi_node/deprecated/gptoss_fp4_gb200_dynamo-trt.sh @@ -2,7 +2,7 @@ set -x -source "$(dirname "$0")/../benchmark_lib.sh" +source "$(dirname "$0")/../../benchmark_lib.sh" check_env_vars \ CONC_LIST \ diff --git a/benchmarks/single_node/fixed_seq_len/gptoss_fp4_b200.sh b/benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_b200.sh similarity index 97% rename from benchmarks/single_node/fixed_seq_len/gptoss_fp4_b200.sh rename to benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_b200.sh index 743974df39..4a41811bfd 100644 --- a/benchmarks/single_node/fixed_seq_len/gptoss_fp4_b200.sh +++ b/benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_b200.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -source "$(dirname "$0")/../../benchmark_lib.sh" +source "$(dirname "$0")/../../../benchmark_lib.sh" check_env_vars \ MODEL \ diff --git a/benchmarks/single_node/fixed_seq_len/gptoss_fp4_b200_trt.sh b/benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_b200_trt.sh similarity index 98% rename from benchmarks/single_node/fixed_seq_len/gptoss_fp4_b200_trt.sh rename to benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_b200_trt.sh index ced9162f9d..2871774cbe 100644 --- a/benchmarks/single_node/fixed_seq_len/gptoss_fp4_b200_trt.sh +++ b/benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_b200_trt.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # Source benchmark utilities early -source "$(dirname "$0")/../../benchmark_lib.sh" +source "$(dirname "$0")/../../../benchmark_lib.sh" check_env_vars \ MODEL \ diff --git a/benchmarks/single_node/fixed_seq_len/gptoss_fp4_h100.sh b/benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_h100.sh similarity index 96% rename from benchmarks/single_node/fixed_seq_len/gptoss_fp4_h100.sh rename to benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_h100.sh index dfd842a885..8b373b4468 100644 --- a/benchmarks/single_node/fixed_seq_len/gptoss_fp4_h100.sh +++ b/benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_h100.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -source "$(dirname "$0")/../../benchmark_lib.sh" +source "$(dirname "$0")/../../../benchmark_lib.sh" check_env_vars \ MODEL \ diff --git a/benchmarks/single_node/fixed_seq_len/gptoss_fp4_h200.sh b/benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_h200.sh similarity index 97% rename from benchmarks/single_node/fixed_seq_len/gptoss_fp4_h200.sh rename to benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_h200.sh index b65c86782f..ffe225f0b6 100644 --- a/benchmarks/single_node/fixed_seq_len/gptoss_fp4_h200.sh +++ b/benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_h200.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -source "$(dirname "$0")/../../benchmark_lib.sh" +source "$(dirname "$0")/../../../benchmark_lib.sh" check_env_vars \ MODEL \ diff --git a/benchmarks/single_node/fixed_seq_len/gptoss_fp4_h200_trt.sh b/benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_h200_trt.sh similarity index 97% rename from benchmarks/single_node/fixed_seq_len/gptoss_fp4_h200_trt.sh rename to benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_h200_trt.sh index 02dd05bc99..84eaa67370 100644 --- a/benchmarks/single_node/fixed_seq_len/gptoss_fp4_h200_trt.sh +++ b/benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_h200_trt.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -source "$(dirname "$0")/../../benchmark_lib.sh" +source "$(dirname "$0")/../../../benchmark_lib.sh" check_env_vars \ MODEL \ diff --git a/benchmarks/single_node/fixed_seq_len/gptoss_fp4_mi300x.sh b/benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_mi300x.sh similarity index 97% rename from benchmarks/single_node/fixed_seq_len/gptoss_fp4_mi300x.sh rename to benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_mi300x.sh index c18a5a3ee2..833718b5ad 100644 --- a/benchmarks/single_node/fixed_seq_len/gptoss_fp4_mi300x.sh +++ b/benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_mi300x.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -source "$(dirname "$0")/../../benchmark_lib.sh" +source "$(dirname "$0")/../../../benchmark_lib.sh" check_env_vars \ MODEL \ diff --git a/benchmarks/single_node/fixed_seq_len/gptoss_fp4_mi325x.sh b/benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_mi325x.sh similarity index 97% rename from benchmarks/single_node/fixed_seq_len/gptoss_fp4_mi325x.sh rename to benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_mi325x.sh index c18a5a3ee2..833718b5ad 100644 --- a/benchmarks/single_node/fixed_seq_len/gptoss_fp4_mi325x.sh +++ b/benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_mi325x.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -source "$(dirname "$0")/../../benchmark_lib.sh" +source "$(dirname "$0")/../../../benchmark_lib.sh" check_env_vars \ MODEL \ diff --git a/benchmarks/single_node/fixed_seq_len/gptoss_fp4_mi355x.sh b/benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_mi355x.sh similarity index 97% rename from benchmarks/single_node/fixed_seq_len/gptoss_fp4_mi355x.sh rename to benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_mi355x.sh index 14dedb1411..b4a4d03732 100644 --- a/benchmarks/single_node/fixed_seq_len/gptoss_fp4_mi355x.sh +++ b/benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_mi355x.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -source "$(dirname "$0")/../../benchmark_lib.sh" +source "$(dirname "$0")/../../../benchmark_lib.sh" check_env_vars \ MODEL \ diff --git a/benchmarks/single_node/fixed_seq_len/gptoss_fp4_mi355x_atom.sh b/benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_mi355x_atom.sh similarity index 97% rename from benchmarks/single_node/fixed_seq_len/gptoss_fp4_mi355x_atom.sh rename to benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_mi355x_atom.sh index d3a8a66a13..e60c389d3b 100644 --- a/benchmarks/single_node/fixed_seq_len/gptoss_fp4_mi355x_atom.sh +++ b/benchmarks/single_node/fixed_seq_len/deprecated/gptoss_fp4_mi355x_atom.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -source "$(dirname "$0")/../../benchmark_lib.sh" +source "$(dirname "$0")/../../../benchmark_lib.sh" check_env_vars \ MODEL \ diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index dd85cd81db..9851e8d6d0 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -304,7 +304,6 @@ qwen3.5-fp8-mi355x-sglang-agentic: - search-space: - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32] } - qwen3.5-fp8-mi355x-atom: image: rocm/atom:rocm7.2.3_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom20260511 model: Qwen/Qwen3.5-397B-A17B-FP8 @@ -809,7 +808,6 @@ kimik2.5-fp4-mi355x-vllm-agentic: - { tp: 4, kv-offloading: none, conc-list: [16, 24, 32, 40] } - { tp: 4, kv-offloading: dram, kv-offload-backend: native, conc-list: [16, 24, 32, 40] } - kimik2.5-fp4-mi355x-atom: image: rocm/atom:rocm7.2.3_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom20260511 model: amd/Kimi-K2.5-MXFP4 @@ -831,100 +829,6 @@ kimik2.5-fp4-mi355x-atom: - { tp: 8, conc-start: 4, conc-end: 128 } - { tp: 4, conc-start: 4, conc-end: 128 } -gptoss-fp4-mi300x-vllm: - image: vllm/vllm-openai-rocm:v0.17.0 - model: openai/gpt-oss-120b - model-prefix: gptoss - runner: mi300x - precision: fp4 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 1, conc-start: 64, conc-end: 256 } - - { tp: 2, conc-start: 4, conc-end: 64 } - - { tp: 4, conc-start: 4, conc-end: 64 } - - { tp: 8, conc-start: 1, conc-end: 16 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 1, conc-start: 4, conc-end: 64 } - - { tp: 2, conc-start: 4, conc-end: 64 } - - { tp: 4, conc-start: 4, conc-end: 64 } - - { tp: 8, conc-start: 1, conc-end: 16 } - -gptoss-fp4-mi325x-vllm: - image: vllm/vllm-openai-rocm:v0.22.0 - model: openai/gpt-oss-120b - model-prefix: gptoss - runner: mi325x - precision: fp4 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 1, conc-start: 4, conc-end: 64 } - - { tp: 2, conc-start: 4, conc-end: 64 } - - { tp: 4, conc-start: 4, conc-end: 64 } - - { tp: 8, conc-start: 4, conc-end: 64 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 1, conc-start: 4, conc-end: 64 } - - { tp: 2, conc-start: 4, conc-end: 8 } - - { tp: 4, conc-start: 4, conc-end: 8 } - - { tp: 8, conc-start: 4, conc-end: 16 } - -gptoss-fp4-mi355x-vllm: - image: vllm/vllm-openai-rocm:v0.22.0 - model: openai/gpt-oss-120b - model-prefix: gptoss - runner: mi355x - precision: fp4 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 1, conc-start: 4, conc-end: 128 } - - { tp: 4, conc-start: 4, conc-end: 8 } - - { tp: 8, conc-start: 4, conc-end: 16 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 1, conc-start: 4, conc-end: 128 } - - { tp: 4, conc-start: 4, conc-end: 4 } - - { tp: 8, conc-start: 4, conc-end: 8 } - -gptoss-fp4-mi355x-atom: - image: rocm/atom:rocm7.2.2_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.2.post - model: openai/gpt-oss-120b - model-prefix: gptoss - runner: mi355x - precision: fp4 - framework: atom - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 1, conc-start: 16, conc-end: 256 } - - { tp: 8, ep: 1, conc-start: 4, conc-end: 32 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 1, conc-start: 4, conc-end: 256 } - - { tp: 8, ep: 1, conc-start: 4, conc-end: 16 } - dsr1-fp8-mi355x-atom: image: rocm/atom:rocm7.2.3_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom20260511 model: deepseek-ai/DeepSeek-R1-0528 @@ -1664,7 +1568,6 @@ dsr1-fp4-mi355x-sglang-disagg-1k1k-mtp: - "DECODE_NODES=1" - "DECODE_MTP_SIZE=1" - dsr1-fp4-mi355x-sglang-disagg-8k1k-mtp: image: lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260529 model: amd/DeepSeek-R1-0528-MXFP4-v2 @@ -1775,7 +1678,6 @@ dsr1-fp4-mi355x-sglang-disagg-8k1k-mtp: - "DECODE_NODES=1" - "DECODE_MTP_SIZE=3" - # 1*DEP8 + 1*DEP8 - spec-decoding: "mtp" conc-list: [ 128 ] @@ -1856,7 +1758,6 @@ dsv4-fp4-mi355x-sglang: - { tp: 4, dp-attn: true, conc-start: 16, conc-end: 128 } - { tp: 4, dp-attn: false, conc-start: 1, conc-end: 32 } - # MTP variant of dsv4-fp4-mi355x-sglang. Mirrors the base search space and adds # spec-decoding: mtp, which routes to dsv4_fp4_mi355x_sglang_mtp.sh (EAGLE # speculative decoding), per sgl-project/sglang#26383 ([AMD][DSV4] DSV4 MTP @@ -2121,8 +2022,6 @@ qwen3.5-fp8-mi355x-sglang-agentic-hicache: - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32] } - { tp: 8, ep: 1, kv-offloading: dram, kv-offload-backend: hicache, conc-list: [16, 32, 48, 64] } - - # DSv4-Pro FP4 on MI355X via SGLang. Uses a rocm720 mi35x image built off the # amd/deepseek_v4 branch in sgl-project/sglang; the SHA is encoded in the # image tag, so bumping sglang is just an image tag bump here. Sweeps @@ -2146,7 +2045,6 @@ dsv4-fp4-mi355x-vllm-agentic: - { tp: 4, kv-offloading: none, conc-list: [1, 2, 4, 8, 10, 12, 16] } - { tp: 4, ep: 4, dp-attn: true, kv-offloading: none, conc-list: [16, 24, 32, 40, 48] } - # DSv4-Pro FP4 on MI355X via SGLang. Uses a rocm720 mi35x image built off the # amd/deepseek_v4 branch in sgl-project/sglang; the SHA is encoded in the # image tag, so bumping sglang is just an image tag bump here. Sweeps @@ -2380,7 +2278,6 @@ dsr1-fp4-mi355x-sglang-disagg-mtp: - "DECODE_NODES=1" - "DECODE_MTP_SIZE=1" - # DSv4-Pro FP4 on MI355X via SGLang. Uses a rocm720 mi35x image built off the # amd/deepseek_v4 branch in sgl-project/sglang; the SHA is encoded in the # image tag, so bumping sglang is just an image tag bump here. Sweeps @@ -2400,7 +2297,6 @@ dsv4-fp4-mi355x-sglang-agentic: - { tp: 8, kv-offloading: none, conc-list: [16, 32, 64] } - { tp: 8, dp-attn: true, kv-offloading: none, conc-list: [64, 128, 256] } - # MiniMax-M3 MXFP8 MI355X recipe: # https://github.com/vllm-project/recipes/commit/2a3728ed9892debfd767a72a58ebc90b33f186e5 # MXFP8 runs from TP=4 on gfx950; block size 128 is mandatory for MSA. diff --git a/configs/deprecated/amd-gptoss-master.yaml b/configs/deprecated/amd-gptoss-master.yaml new file mode 100644 index 0000000000..874b4d5614 --- /dev/null +++ b/configs/deprecated/amd-gptoss-master.yaml @@ -0,0 +1,96 @@ +# Deprecated gpt-oss-120b entries archived from amd-master.yaml. +# Removed from the active master config so sweep generation no longer selects them. + +gptoss-fp4-mi300x-vllm: + image: vllm/vllm-openai-rocm:v0.17.0 + model: openai/gpt-oss-120b + model-prefix: gptoss + runner: mi300x + precision: fp4 + framework: vllm + multinode: false + scenarios: + fixed-seq-len: + - isl: 1024 + osl: 1024 + search-space: + - { tp: 1, conc-start: 64, conc-end: 256 } + - { tp: 2, conc-start: 4, conc-end: 64 } + - { tp: 4, conc-start: 4, conc-end: 64 } + - { tp: 8, conc-start: 1, conc-end: 16 } + - isl: 8192 + osl: 1024 + search-space: + - { tp: 1, conc-start: 4, conc-end: 64 } + - { tp: 2, conc-start: 4, conc-end: 64 } + - { tp: 4, conc-start: 4, conc-end: 64 } + - { tp: 8, conc-start: 1, conc-end: 16 } + +gptoss-fp4-mi325x-vllm: + image: vllm/vllm-openai-rocm:v0.22.0 + model: openai/gpt-oss-120b + model-prefix: gptoss + runner: mi325x + precision: fp4 + framework: vllm + multinode: false + scenarios: + fixed-seq-len: + - isl: 1024 + osl: 1024 + search-space: + - { tp: 1, conc-start: 4, conc-end: 64 } + - { tp: 2, conc-start: 4, conc-end: 64 } + - { tp: 4, conc-start: 4, conc-end: 64 } + - { tp: 8, conc-start: 4, conc-end: 64 } + - isl: 8192 + osl: 1024 + search-space: + - { tp: 1, conc-start: 4, conc-end: 64 } + - { tp: 2, conc-start: 4, conc-end: 8 } + - { tp: 4, conc-start: 4, conc-end: 8 } + - { tp: 8, conc-start: 4, conc-end: 16 } + +gptoss-fp4-mi355x-vllm: + image: vllm/vllm-openai-rocm:v0.22.0 + model: openai/gpt-oss-120b + model-prefix: gptoss + runner: mi355x + precision: fp4 + framework: vllm + multinode: false + scenarios: + fixed-seq-len: + - isl: 1024 + osl: 1024 + search-space: + - { tp: 1, conc-start: 4, conc-end: 128 } + - { tp: 4, conc-start: 4, conc-end: 8 } + - { tp: 8, conc-start: 4, conc-end: 16 } + - isl: 8192 + osl: 1024 + search-space: + - { tp: 1, conc-start: 4, conc-end: 128 } + - { tp: 4, conc-start: 4, conc-end: 4 } + - { tp: 8, conc-start: 4, conc-end: 8 } + +gptoss-fp4-mi355x-atom: + image: rocm/atom:rocm7.2.2_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.2.post + model: openai/gpt-oss-120b + model-prefix: gptoss + runner: mi355x + precision: fp4 + framework: atom + multinode: false + scenarios: + fixed-seq-len: + - isl: 1024 + osl: 1024 + search-space: + - { tp: 1, conc-start: 16, conc-end: 256 } + - { tp: 8, ep: 1, conc-start: 4, conc-end: 32 } + - isl: 8192 + osl: 1024 + search-space: + - { tp: 1, conc-start: 4, conc-end: 256 } + - { tp: 8, ep: 1, conc-start: 4, conc-end: 16 } diff --git a/configs/deprecated/nvidia-gptoss-master.yaml b/configs/deprecated/nvidia-gptoss-master.yaml new file mode 100644 index 0000000000..7a59147b6e --- /dev/null +++ b/configs/deprecated/nvidia-gptoss-master.yaml @@ -0,0 +1,417 @@ +# Deprecated gpt-oss-120b entries archived from nvidia-master.yaml. +# Removed from the active master config so sweep generation no longer selects them. + +gptoss-fp4-b200-trt: + image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc14 + model: openai/gpt-oss-120b + model-prefix: gptoss + runner: b200 + precision: fp4 + framework: trt + multinode: false + scenarios: + fixed-seq-len: + # Low ==> high TP from Left to Right of pareto + - isl: 1024 + osl: 1024 + search-space: + - { tp: 1, conc-start: 256, conc-end: 256 } + - { tp: 2, ep: 2, dp-attn: true, conc-start: 256, conc-end: 256 } + - { tp: 2, conc-start: 4, conc-end: 256 } + - { tp: 4, ep: 4, dp-attn: true, conc-start: 64, conc-end: 64 } + - { tp: 4, conc-start: 4, conc-end: 4 } + - { tp: 8, conc-start: 4, conc-end: 4 } + # Low ==> high TP from Left to Right of pareto + - isl: 8192 + osl: 1024 + search-space: + - { tp: 1, conc-start: 4, conc-end: 256} + - { tp: 2, conc-start: 4, conc-end: 256} + - { tp: 4, conc-start: 4, conc-end: 32} + - { tp: 8, conc-start: 4, conc-end: 4} + +gptoss-fp4-b200-vllm: + image: vllm/vllm-openai:v0.22.0 + model: openai/gpt-oss-120b + model-prefix: gptoss + runner: b200 + precision: fp4 + framework: vllm + multinode: false + scenarios: + fixed-seq-len: + - isl: 1024 + osl: 1024 + search-space: + - { tp: 1, conc-start: 4, conc-end: 128 } + - { tp: 2, conc-start: 4, conc-end: 128 } + - { tp: 4, conc-start: 4, conc-end: 64 } + - { tp: 8, conc-start: 4, conc-end: 8 } + - isl: 8192 + osl: 1024 + search-space: + - { tp: 1, conc-start: 4, conc-end: 128 } + - { tp: 2, conc-start: 4, conc-end: 128 } + - { tp: 4, conc-start: 4, conc-end: 64 } + - { tp: 8, conc-start: 4, conc-end: 4 } + +gptoss-fp4-h100-vllm: + image: vllm/vllm-openai:v0.21.0 + model: openai/gpt-oss-120b + model-prefix: gptoss + runner: h100 + precision: fp4 + framework: vllm + multinode: false + scenarios: + fixed-seq-len: + - isl: 1024 + osl: 1024 + search-space: + - { tp: 2, conc-start: 4, conc-end: 64 } + - { tp: 4, conc-start: 4, conc-end: 64 } + - { tp: 8, conc-start: 4, conc-end: 64 } + - isl: 8192 + osl: 1024 + search-space: + - { tp: 2, conc-start: 4, conc-end: 64 } + - { tp: 4, conc-start: 4, conc-end: 64 } + - { tp: 8, conc-start: 4, conc-end: 16 } + +# Day-zero MiniMax-M3 recipe (https://recipes.vllm.ai/MiniMaxAI/MiniMax-M3). +# M3 support has not shipped in a stable vLLM release; the dedicated +# vllm/vllm-openai:minimax-m3 image is the supported path. MXFP8 variant +# (NVIDIA-quantized, ~427 GB weights) is the lowest precision available — +# BF16 (~854 GB) does not fit 8x H100 (640 GB) at all, so H100 is TP8-only: +# weights alone take ~56 GB of each 80 GB GPU, leaving no room below TP8. +# dp-attn: true maps to the recipe's "DP8 + Expert Parallel" serve mode +# (vLLM --data-parallel-size 8 --enable-expert-parallel). + +gptoss-fp4-h200-trt: + image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc14 + model: openai/gpt-oss-120b + model-prefix: gptoss + runner: h200 + precision: fp4 + framework: trt + multinode: false + # For all sequence lengths, EP=TP, DP_ATTENTION=false + scenarios: + fixed-seq-len: + - isl: 1024 + osl: 1024 + search-space: + - { tp: 1, ep: 1, dp-attn: false, conc-start: 4, conc-end: 64 } + - { tp: 2, ep: 2, dp-attn: false, conc-start: 4, conc-end: 64 } + - { tp: 4, ep: 4, dp-attn: false, conc-start: 4, conc-end: 32 } + - { tp: 8, ep: 8, dp-attn: false, conc-start: 4, conc-end: 8 } + - isl: 8192 + osl: 1024 + search-space: + - { tp: 1, ep: 1, dp-attn: false, conc-start: 4, conc-end: 64 } + - { tp: 2, ep: 2, dp-attn: false, conc-start: 4, conc-end: 64 } + - { tp: 4, ep: 4, dp-attn: false, conc-start: 4, conc-end: 64 } + - { tp: 8, ep: 8, dp-attn: false, conc-start: 4, conc-end: 8 } + +gptoss-fp4-h200-vllm: + image: vllm/vllm-openai:v0.22.0 + model: openai/gpt-oss-120b + model-prefix: gptoss + runner: h200 + precision: fp4 + framework: vllm + multinode: false + scenarios: + fixed-seq-len: + - isl: 1024 + osl: 1024 + search-space: + - { tp: 1, conc-start: 4, conc-end: 4 } + - { tp: 2, conc-start: 4, conc-end: 64 } + - { tp: 4, conc-start: 4, conc-end: 64 } + - { tp: 8, conc-start: 4, conc-end: 64 } + - isl: 8192 + osl: 1024 + search-space: + - { tp: 1, conc-start: 4, conc-end: 64 } + - { tp: 2, conc-start: 4, conc-end: 64 } + - { tp: 4, conc-start: 4, conc-end: 64 } + - { tp: 8, conc-start: 4, conc-end: 32 } + +# Day-zero MiniMax-M3 recipe (https://recipes.vllm.ai/MiniMaxAI/MiniMax-M3). +# Dedicated vllm/vllm-openai:minimax-m3 image (no stable release has M3 yet). +# MXFP8 variant (~427 GB weights) is the lowest precision available; on +# 8x H200 (1128 GB) it leaves ample KV headroom where BF16 is a tight fit. +# TP4 (~112 GB weights/GPU) is memory-tight — swept only at low/mid conc. +# dp-attn: true maps to the recipe's "DP8 + Expert Parallel" serve mode. + +gptoss-fp4-gb200-dynamo-trt: + image: nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:0.7.0.post2 + model: openai/gpt-oss-120b + model-prefix: gptoss + runner: gb200 + precision: fp4 + framework: dynamo-trt + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 1024 + osl: 1024 + search-space: + #Right of pareto + #P: 1xTP1 D:1xTP4 + - spec-decoding: "none" + conc-list: [ 1, 2, 4, 16, 32, 64, 128 ] + prefill: + num-worker: 1 + tp: 1 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_NODES=1" + - "PREFILL_MAX_NUM_TOKENS=20000" + - "PREFILL_MAX_BATCH_SIZE=32" + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "DECODE_NODES=1" + - "DECODE_MAX_NUM_TOKENS=20000" + - "DECODE_MAX_BATCH_SIZE=256" + - "DECODE_GPU_MEM_FRACTION=0.9" + + # P: 1xTP1 D:4xTP2 + - spec-decoding: "none" + conc-list: [ 16 ] + prefill: + num-worker: 1 + tp: 1 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_NODES=1" + - "PREFILL_MAX_NUM_TOKENS=20000" + - "PREFILL_MAX_BATCH_SIZE=32" + decode: + num-worker: 4 + tp: 2 + ep: 1 + dp-attn: false + additional-settings: + - "DECODE_NODES=2" + - "DECODE_MAX_NUM_TOKENS=20000" + - "DECODE_MAX_BATCH_SIZE=32" + - "DECODE_GPU_MEM_FRACTION=0.9" + + # P: 1xTP1 D:1xDEP2 + - spec-decoding: "none" + conc-list: [ 256, 512, 1024, 2048, 2560 ] + prefill: + num-worker: 1 + tp: 1 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_NODES=1" + - "PREFILL_MAX_NUM_TOKENS=20000" + - "PREFILL_MAX_BATCH_SIZE=32" + decode: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "DECODE_NODES=1" + - "DECODE_MAX_NUM_TOKENS=20000" + - "DECODE_MAX_BATCH_SIZE=1536" + - "DECODE_GPU_MEM_FRACTION=0.9" + + # P: 1xTP1 D:2xDEP2 + - spec-decoding: "none" + conc-list: [ 512, 1024, 2048, 2560 ] + prefill: + num-worker: 1 + tp: 1 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_NODES=1" + - "PREFILL_MAX_NUM_TOKENS=20000" + - "PREFILL_MAX_BATCH_SIZE=32" + decode: + num-worker: 2 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "DECODE_NODES=1" + - "DECODE_MAX_NUM_TOKENS=20000" + - "DECODE_MAX_BATCH_SIZE=1536" + - "DECODE_GPU_MEM_FRACTION=0.9" + + # P: 1xTP1 D:1xDEP4 + - spec-decoding: "none" + conc-list: [ 256, 1024, 1536 ] + prefill: + num-worker: 1 + tp: 1 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_NODES=1" + - "PREFILL_MAX_NUM_TOKENS=20000" + - "PREFILL_MAX_BATCH_SIZE=32" + decode: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "DECODE_NODES=1" + - "DECODE_MAX_NUM_TOKENS=20000" + - "DECODE_MAX_BATCH_SIZE=512" + - "DECODE_GPU_MEM_FRACTION=0.9" + + # P: 1xTP1 D:3xDEP4 + - spec-decoding: "none" + conc-list: [ 3072 ] + prefill: + num-worker: 1 + tp: 1 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_NODES=1" + - "PREFILL_MAX_NUM_TOKENS=20000" + - "PREFILL_MAX_BATCH_SIZE=32" + decode: + num-worker: 3 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "DECODE_NODES=1" + - "DECODE_MAX_NUM_TOKENS=20000" + - "DECODE_MAX_BATCH_SIZE=1024" + - "DECODE_GPU_MEM_FRACTION=0.9" + + - isl: 8192 + osl: 1024 + search-space: + # Right side of pareto + - spec-decoding: "none" + conc-list: [1] + prefill: + num-worker: 1 + tp: 1 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_NODES=1" + - "PREFILL_MAX_NUM_TOKENS=20000" + - "PREFILL_MAX_BATCH_SIZE=32" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "DECODE_NODES=2" + - "DECODE_MAX_NUM_TOKENS=20000" + - "DECODE_MAX_BATCH_SIZE=4" + - "DECODE_GPU_MEM_FRACTION=0.9" + + - spec-decoding: "none" + conc-list: [2, 4, 8, 16, 32, 64] + prefill: + num-worker: 1 + tp: 1 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_NODES=1" + - "PREFILL_MAX_NUM_TOKENS=20000" + - "PREFILL_MAX_BATCH_SIZE=32" + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "DECODE_NODES=1" + - "DECODE_MAX_NUM_TOKENS=20000" + - "DECODE_MAX_BATCH_SIZE=128" + - "DECODE_GPU_MEM_FRACTION=0.9" + + # Middle of pareto + # P: 2xTP1 D:1xTP4 + - spec-decoding: "none" + conc-list: [128, 512] + prefill: + num-worker: 2 + tp: 1 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_NODES=1" + - "PREFILL_MAX_NUM_TOKENS=20000" + - "PREFILL_MAX_BATCH_SIZE=32" + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "DECODE_NODES=1" + - "DECODE_MAX_NUM_TOKENS=20000" + - "DECODE_MAX_BATCH_SIZE=1024" + - "DECODE_GPU_MEM_FRACTION=0.9" + + # P: 2xTP1 D:1xTP2 + - spec-decoding: "none" + conc-list: [256, 384] + prefill: + num-worker: 2 + tp: 1 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_NODES=1" + - "PREFILL_MAX_NUM_TOKENS=20000" + - "PREFILL_MAX_BATCH_SIZE=32" + decode: + num-worker: 1 + tp: 2 + ep: 1 + dp-attn: false + additional-settings: + - "DECODE_NODES=1" + - "DECODE_MAX_NUM_TOKENS=20000" + - "DECODE_MAX_BATCH_SIZE=512" + - "DECODE_GPU_MEM_FRACTION=0.9" + + # P: 2xTP1 D:1xDEP2 + - spec-decoding: "none" + conc-list: [128, 512] + prefill: + num-worker: 2 + tp: 1 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_NODES=1" + - "PREFILL_MAX_NUM_TOKENS=20000" + - "PREFILL_MAX_BATCH_SIZE=32" + decode: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "DECODE_NODES=1" + - "DECODE_MAX_NUM_TOKENS=20000" + - "DECODE_MAX_BATCH_SIZE=512" + - "DECODE_GPU_MEM_FRACTION=0.9" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 74eb03c530..8d84bd0dd7 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -384,7 +384,6 @@ dsr1-fp4-b200-dynamo-trt: ep: 8 dp-attn: true - dsr1-fp8-b200-dynamo-trt: image: nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:0.8.1.post2 model: deepseek-ai/DeepSeek-R1-0528 @@ -1783,7 +1782,6 @@ dsv4-fp4-b200-vllm-agentic: # Retain the external-cache transition and peak-throughput region. - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: mooncake, conc-list: [16, 38, 44, 56, 64, 66, 68] } - dsv4-fp4-b200-trt: image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066 model: deepseek-ai/DeepSeek-V4-Pro @@ -2123,7 +2121,6 @@ qwen3.5-fp8-b200-sglang-agentic: - search-space: - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32] } - qwen3.5-fp4-b200-sglang: image: lmsysorg/sglang:v0.5.14-cu130 model: nvidia/Qwen3.5-397B-A17B-NVFP4 @@ -2654,7 +2651,6 @@ qwen3.5-fp8-b200-sglang-mtp: - { tp: 8, ep: 1, conc-start: 4, conc-end: 4, spec-decoding: mtp } - { tp: 4, ep: 1, conc-start: 4, conc-end: 256, spec-decoding: mtp } - qwen3.5-fp8-b300-sglang-mtp: image: lmsysorg/sglang:v0.5.12-cu130 model: Qwen/Qwen3.5-397B-A17B-FP8 @@ -2815,7 +2811,6 @@ kimik2.5-int4-b200-vllm-agentic: - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32] } - { tp: 8, kv-offloading: dram, kv-offload-backend: native, conc-list: [32, 64, 96, 128] } - kimik2.5-int4-b300-vllm: image: vllm/vllm-openai:v0.24.0 model: moonshotai/Kimi-K2.5 @@ -2871,7 +2866,6 @@ kimik2.5-int4-h200-vllm-agentic: - { tp: 8, kv-offloading: none, conc-list: [1, 2, 3, 4, 5, 6, 7] } - { tp: 8, kv-offloading: dram, kv-offload-backend: native, conc-list: [6, 7, 8, 9, 10, 11, 12, 13, 14] } - # NOTE: At the time of submission, https://docs.vllm.ai/projects/recipes/en/latest/moonshotai/Kimi-K2.5.html # does not have a B300-specific recipe, so this config reuses the existing # Kimi-K2.5 FP4 B200 vLLM recipe as-is until B300-specific tuning is available. @@ -2984,7 +2978,6 @@ kimik2.5-fp4-b300-vllm-agentic: - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32, 40, 48, 56, 64] } - { tp: 8, ep: 1, kv-offloading: dram, kv-offload-backend: native, conc-list: [1, 2, 4, 8, 16, 32, 40, 48, 56, 64] } - dsr1-fp8-b200-trt: image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc14 model: deepseek-ai/DeepSeek-R1-0528 @@ -3135,7 +3128,6 @@ dsv4-fp8-h200-vllm-agentic: - search-space: - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [1, 2, 4, 8, 16] } - # MTP variant of dsv4-fp8-h200-sglang. Mirrors the non-MTP recipe (same image, # runner pool, search space) and adds EAGLE speculative decoding via # --speculative-algorithm EAGLE with the (3,1,4) chain matching dsv4-fp4-b300-sglang-mtp. @@ -3237,7 +3229,6 @@ dsv4-fp4-b300-vllm-agentic: # TP8 DEP retains representative low, peak, and transition points. - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [52, 72, 100, 128, 144] } - dsv4-fp4-b300-trt: image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066 model: deepseek-ai/DeepSeek-V4-Pro @@ -4420,91 +4411,6 @@ dsr1-fp8-h100-dynamo-trt: ep: 16 dp-attn: true -gptoss-fp4-b200-trt: - image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc14 - model: openai/gpt-oss-120b - model-prefix: gptoss - runner: b200 - precision: fp4 - framework: trt - multinode: false - scenarios: - fixed-seq-len: - # Low ==> high TP from Left to Right of pareto - - isl: 1024 - osl: 1024 - search-space: - - { tp: 1, conc-start: 256, conc-end: 256 } - - { tp: 2, ep: 2, dp-attn: true, conc-start: 256, conc-end: 256 } - - { tp: 2, conc-start: 4, conc-end: 256 } - - { tp: 4, ep: 4, dp-attn: true, conc-start: 64, conc-end: 64 } - - { tp: 4, conc-start: 4, conc-end: 4 } - - { tp: 8, conc-start: 4, conc-end: 4 } - # Low ==> high TP from Left to Right of pareto - - isl: 8192 - osl: 1024 - search-space: - - { tp: 1, conc-start: 4, conc-end: 256} - - { tp: 2, conc-start: 4, conc-end: 256} - - { tp: 4, conc-start: 4, conc-end: 32} - - { tp: 8, conc-start: 4, conc-end: 4} - -gptoss-fp4-b200-vllm: - image: vllm/vllm-openai:v0.22.0 - model: openai/gpt-oss-120b - model-prefix: gptoss - runner: b200 - precision: fp4 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 1, conc-start: 4, conc-end: 128 } - - { tp: 2, conc-start: 4, conc-end: 128 } - - { tp: 4, conc-start: 4, conc-end: 64 } - - { tp: 8, conc-start: 4, conc-end: 8 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 1, conc-start: 4, conc-end: 128 } - - { tp: 2, conc-start: 4, conc-end: 128 } - - { tp: 4, conc-start: 4, conc-end: 64 } - - { tp: 8, conc-start: 4, conc-end: 4 } - -gptoss-fp4-h100-vllm: - image: vllm/vllm-openai:v0.21.0 - model: openai/gpt-oss-120b - model-prefix: gptoss - runner: h100 - precision: fp4 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 2, conc-start: 4, conc-end: 64 } - - { tp: 4, conc-start: 4, conc-end: 64 } - - { tp: 8, conc-start: 4, conc-end: 64 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 2, conc-start: 4, conc-end: 64 } - - { tp: 4, conc-start: 4, conc-end: 64 } - - { tp: 8, conc-start: 4, conc-end: 16 } - -# Day-zero MiniMax-M3 recipe (https://recipes.vllm.ai/MiniMaxAI/MiniMax-M3). -# M3 support has not shipped in a stable vLLM release; the dedicated -# vllm/vllm-openai:minimax-m3 image is the supported path. MXFP8 variant -# (NVIDIA-quantized, ~427 GB weights) is the lowest precision available — -# BF16 (~854 GB) does not fit 8x H100 (640 GB) at all, so H100 is TP8-only: -# weights alone take ~56 GB of each 80 GB GPU, leaving no room below TP8. -# dp-attn: true maps to the recipe's "DP8 + Expert Parallel" serve mode -# (vLLM --data-parallel-size 8 --enable-expert-parallel). minimaxm3-fp8-h100-vllm: image: vllm/vllm-openai:minimax-m3 model: MiniMaxAI/MiniMax-M3-MXFP8 @@ -4665,63 +4571,6 @@ dsr1-fp8-h100-dynamo-sglang: ep: 16 dp-attn: true -gptoss-fp4-h200-trt: - image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc14 - model: openai/gpt-oss-120b - model-prefix: gptoss - runner: h200 - precision: fp4 - framework: trt - multinode: false - # For all sequence lengths, EP=TP, DP_ATTENTION=false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 1, ep: 1, dp-attn: false, conc-start: 4, conc-end: 64 } - - { tp: 2, ep: 2, dp-attn: false, conc-start: 4, conc-end: 64 } - - { tp: 4, ep: 4, dp-attn: false, conc-start: 4, conc-end: 32 } - - { tp: 8, ep: 8, dp-attn: false, conc-start: 4, conc-end: 8 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 1, ep: 1, dp-attn: false, conc-start: 4, conc-end: 64 } - - { tp: 2, ep: 2, dp-attn: false, conc-start: 4, conc-end: 64 } - - { tp: 4, ep: 4, dp-attn: false, conc-start: 4, conc-end: 64 } - - { tp: 8, ep: 8, dp-attn: false, conc-start: 4, conc-end: 8 } - -gptoss-fp4-h200-vllm: - image: vllm/vllm-openai:v0.22.0 - model: openai/gpt-oss-120b - model-prefix: gptoss - runner: h200 - precision: fp4 - framework: vllm - multinode: false - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - { tp: 1, conc-start: 4, conc-end: 4 } - - { tp: 2, conc-start: 4, conc-end: 64 } - - { tp: 4, conc-start: 4, conc-end: 64 } - - { tp: 8, conc-start: 4, conc-end: 64 } - - isl: 8192 - osl: 1024 - search-space: - - { tp: 1, conc-start: 4, conc-end: 64 } - - { tp: 2, conc-start: 4, conc-end: 64 } - - { tp: 4, conc-start: 4, conc-end: 64 } - - { tp: 8, conc-start: 4, conc-end: 32 } - -# Day-zero MiniMax-M3 recipe (https://recipes.vllm.ai/MiniMaxAI/MiniMax-M3). -# Dedicated vllm/vllm-openai:minimax-m3 image (no stable release has M3 yet). -# MXFP8 variant (~427 GB weights) is the lowest precision available; on -# 8x H200 (1128 GB) it leaves ample KV headroom where BF16 is a tight fit. -# TP4 (~112 GB weights/GPU) is memory-tight — swept only at low/mid conc. -# dp-attn: true maps to the recipe's "DP8 + Expert Parallel" serve mode. minimaxm3-fp8-h200-vllm: image: vllm/vllm-openai:minimax-m3 model: MiniMaxAI/MiniMax-M3-MXFP8 @@ -6845,277 +6694,6 @@ dsr1-fp8-gb300-dynamo-trt: tp: 8 ep: 8 dp-attn: true -gptoss-fp4-gb200-dynamo-trt: - image: nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:0.7.0.post2 - model: openai/gpt-oss-120b - model-prefix: gptoss - runner: gb200 - precision: fp4 - framework: dynamo-trt - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - #Right of pareto - #P: 1xTP1 D:1xTP4 - - spec-decoding: "none" - conc-list: [ 1, 2, 4, 16, 32, 64, 128 ] - prefill: - num-worker: 1 - tp: 1 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - "PREFILL_MAX_NUM_TOKENS=20000" - - "PREFILL_MAX_BATCH_SIZE=32" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MAX_NUM_TOKENS=20000" - - "DECODE_MAX_BATCH_SIZE=256" - - "DECODE_GPU_MEM_FRACTION=0.9" - - # P: 1xTP1 D:4xTP2 - - spec-decoding: "none" - conc-list: [ 16 ] - prefill: - num-worker: 1 - tp: 1 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - "PREFILL_MAX_NUM_TOKENS=20000" - - "PREFILL_MAX_BATCH_SIZE=32" - decode: - num-worker: 4 - tp: 2 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MAX_NUM_TOKENS=20000" - - "DECODE_MAX_BATCH_SIZE=32" - - "DECODE_GPU_MEM_FRACTION=0.9" - - # P: 1xTP1 D:1xDEP2 - - spec-decoding: "none" - conc-list: [ 256, 512, 1024, 2048, 2560 ] - prefill: - num-worker: 1 - tp: 1 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - "PREFILL_MAX_NUM_TOKENS=20000" - - "PREFILL_MAX_BATCH_SIZE=32" - decode: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MAX_NUM_TOKENS=20000" - - "DECODE_MAX_BATCH_SIZE=1536" - - "DECODE_GPU_MEM_FRACTION=0.9" - - # P: 1xTP1 D:2xDEP2 - - spec-decoding: "none" - conc-list: [ 512, 1024, 2048, 2560 ] - prefill: - num-worker: 1 - tp: 1 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - "PREFILL_MAX_NUM_TOKENS=20000" - - "PREFILL_MAX_BATCH_SIZE=32" - decode: - num-worker: 2 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MAX_NUM_TOKENS=20000" - - "DECODE_MAX_BATCH_SIZE=1536" - - "DECODE_GPU_MEM_FRACTION=0.9" - - # P: 1xTP1 D:1xDEP4 - - spec-decoding: "none" - conc-list: [ 256, 1024, 1536 ] - prefill: - num-worker: 1 - tp: 1 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - "PREFILL_MAX_NUM_TOKENS=20000" - - "PREFILL_MAX_BATCH_SIZE=32" - decode: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MAX_NUM_TOKENS=20000" - - "DECODE_MAX_BATCH_SIZE=512" - - "DECODE_GPU_MEM_FRACTION=0.9" - - # P: 1xTP1 D:3xDEP4 - - spec-decoding: "none" - conc-list: [ 3072 ] - prefill: - num-worker: 1 - tp: 1 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - "PREFILL_MAX_NUM_TOKENS=20000" - - "PREFILL_MAX_BATCH_SIZE=32" - decode: - num-worker: 3 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MAX_NUM_TOKENS=20000" - - "DECODE_MAX_BATCH_SIZE=1024" - - "DECODE_GPU_MEM_FRACTION=0.9" - - - isl: 8192 - osl: 1024 - search-space: - # Right side of pareto - - spec-decoding: "none" - conc-list: [1] - prefill: - num-worker: 1 - tp: 1 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - "PREFILL_MAX_NUM_TOKENS=20000" - - "PREFILL_MAX_BATCH_SIZE=32" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MAX_NUM_TOKENS=20000" - - "DECODE_MAX_BATCH_SIZE=4" - - "DECODE_GPU_MEM_FRACTION=0.9" - - - spec-decoding: "none" - conc-list: [2, 4, 8, 16, 32, 64] - prefill: - num-worker: 1 - tp: 1 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - "PREFILL_MAX_NUM_TOKENS=20000" - - "PREFILL_MAX_BATCH_SIZE=32" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MAX_NUM_TOKENS=20000" - - "DECODE_MAX_BATCH_SIZE=128" - - "DECODE_GPU_MEM_FRACTION=0.9" - - # Middle of pareto - # P: 2xTP1 D:1xTP4 - - spec-decoding: "none" - conc-list: [128, 512] - prefill: - num-worker: 2 - tp: 1 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - "PREFILL_MAX_NUM_TOKENS=20000" - - "PREFILL_MAX_BATCH_SIZE=32" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MAX_NUM_TOKENS=20000" - - "DECODE_MAX_BATCH_SIZE=1024" - - "DECODE_GPU_MEM_FRACTION=0.9" - - # P: 2xTP1 D:1xTP2 - - spec-decoding: "none" - conc-list: [256, 384] - prefill: - num-worker: 2 - tp: 1 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - "PREFILL_MAX_NUM_TOKENS=20000" - - "PREFILL_MAX_BATCH_SIZE=32" - decode: - num-worker: 1 - tp: 2 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MAX_NUM_TOKENS=20000" - - "DECODE_MAX_BATCH_SIZE=512" - - "DECODE_GPU_MEM_FRACTION=0.9" - - # P: 2xTP1 D:1xDEP2 - - spec-decoding: "none" - conc-list: [128, 512] - prefill: - num-worker: 2 - tp: 1 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - "PREFILL_MAX_NUM_TOKENS=20000" - - "PREFILL_MAX_BATCH_SIZE=32" - decode: - num-worker: 1 - tp: 2 - ep: 2 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MAX_NUM_TOKENS=20000" - - "DECODE_MAX_BATCH_SIZE=512" - - "DECODE_GPU_MEM_FRACTION=0.9" - dsr1-fp8-h200-dynamo-sglang: image: lmsysorg/sglang:v0.5.8.post1-cu130 model: deepseek-ai/DeepSeek-R1-0528 @@ -9378,7 +8956,6 @@ qwen3.5-fp8-gb200-dynamo-sglang: ep: 16 dp-attn: true - # MTP variant of dsv4-fp4-gb200-dynamo-sglang. dsv4-fp4-gb200-dynamo-sglang-mtp: image: lmsysorg/sglang:nightly-dev-cu13-20260528-0abe6a85 @@ -10947,7 +10524,6 @@ dsv4-fp4-gb300-dynamo-sglang-mtp: ep: 8 dp-attn: true - kimik2.5-int4-h100-vllm: image: vllm/vllm-openai:v0.22.0 model: moonshotai/Kimi-K2.5 @@ -10965,7 +10541,6 @@ kimik2.5-int4-h100-vllm: - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 8, 12, 16, 20] } - { tp: 8, kv-offloading: dram, kv-offload-backend: native, conc-list: [1, 2, 4, 8, 12, 16, 20] } - qwen3.5-fp8-h100-sglang: image: lmsysorg/sglang:v0.5.14-cu130 model: Qwen/Qwen3.5-397B-A17B-FP8 @@ -11528,7 +11103,6 @@ qwen3.5-fp8-b300-sglang-agentic-hicache: - { tp: 4, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32] } - { tp: 4, ep: 1, kv-offloading: dram, kv-offload-backend: hicache, conc-list: [16, 32, 48, 64] } - kimik2.5-fp4-b200-vllm-agentic-lmcache: image: vllm/vllm-openai:v0.22.0 model: nvidia/Kimi-K2.5-NVFP4 @@ -11546,7 +11120,6 @@ kimik2.5-fp4-b200-vllm-agentic-lmcache: - { tp: 4, ep: 1, kv-offloading: none, conc-list: [8, 12, 14, 16, 18, 20] } - { tp: 4, ep: 1, kv-offloading: dram, kv-offload-backend: lmcache, conc-list: [12, 14, 16, 18, 20, 22, 24, 32] } - # CONC range conservative for H100's 80 GB HBM3 under the long-ISL with- # subagents corpus. hicache arm capped at conc 16 since high-conc + hicache # tends to flake on first runs and conc 16 covers the cliff. The bench script @@ -11615,8 +11188,6 @@ dsv4-fp4-gb300-dynamo-vllm-agentic: ep: 8 dp-attn: true - - qwen3.5-fp8-h100-sglang-agentic: image: lmsysorg/sglang:v0.5.12-cu130 model: Qwen/Qwen3.5-397B-A17B-FP8 @@ -11632,7 +11203,6 @@ qwen3.5-fp8-h100-sglang-agentic: - { tp: 8, ep: 8, kv-offloading: none, conc-list: [1, 2, 4, 8, 12, 14, 16] } - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: hicache, conc-list: [12, 14, 16, 20, 24, 28, 32, 42] } - # MiniMax-M3 NVFP4 disagg sweep on the same B300 topology matrix as the MXFP8 # baseline above. The image includes vLLM PR #46380, so no runtime patch is # needed. @@ -13089,7 +12659,6 @@ minimaxm3-fp8-h100-vllm-agentic: - { tp: 8, kv-offloading: dram, kv-offload-backend: mooncake, conc-list: [3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16] } - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: mooncake, conc-list: [3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16] } - minimaxm3-fp8-h200-vllm-agentic: image: vllm/vllm-openai:nightly-04c2a8deac44fdb1ca3e2b5ec3e6bf16f3f6a914 model: MiniMaxAI/MiniMax-M3-MXFP8 @@ -13110,8 +12679,6 @@ minimaxm3-fp8-h200-vllm-agentic: - { tp: 8, ep: 8, kv-offloading: none, conc-list: [2, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 18, 20] } - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: mooncake, conc-list: [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 18, 20] } - - dsv4-fp4-b200-sglang-agentic-hicache: image: lmsysorg/sglang:v0.5.13-cu130 model: deepseek-ai/DeepSeek-V4-Pro @@ -13131,8 +12698,6 @@ dsv4-fp4-b200-sglang-agentic-hicache: - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [16, 24, 32, 38, 44, 48, 50, 52] } - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: hicache, conc-list: [16, 32, 38, 44, 50, 56, 64, 66, 68] } - - # GB200 DeepSeek-V4 disaggregated AgentX frontier. The 3P/2D TEP8/TP8 curve # covers the middle/high-interactivity range omitted by the one-decode DEP # throughput curves below. Each engine start carries at most four concurrencies. @@ -13154,7 +12719,6 @@ dsv4-fp4-b300-sglang-agentic-hicache: - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: hicache, conc-list: [32, 40, 48, 56, 64, 72, 80, 88, 96, 128] } - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [52, 72, 100, 128, 144, 196, 512] } - # DEP8 prefill uses an 8K batch because 16K OOMs in the FP4 MoE intermediate; # decode uses FULL_DECODE_ONLY after the controlled graph test restored decode # throughput. Dynamo KV routing and AIPerf conversation-aware routing remain