From f638951551a6288a4a44578c87a1dd404e45add8 Mon Sep 17 00:00:00 2001 From: Fangzhou Ai Date: Mon, 6 Jul 2026 14:44:49 -0500 Subject: [PATCH 1/2] [AMD] MiniMax-M3 FP4 MI355X vLLM STP: INT4 all-reduce + fp8 KV, TP4 sweep MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drop --hf-overrides indexer-skipping (disallowed per PR_REVIEW_CHECKLIST). Keep INT4 quick-reduce env knobs and --kv-cache-dtype fp8; bump to latest nightly and TP4-only conc 1-512 search space (#2003). 中文：MiniMax-M3 FP4 MI355X vLLM STP 保留 INT4 all-reduce 与 fp8 KV，去掉不允许的 indexer --hf-overrides；镜像升级至最新 nightly，扫描范围收窄为 TP4 conc 1-512。 Co-authored-by: Cursor --- .../fixed_seq_len/minimaxm3_fp4_mi355x_vllm.sh | 14 ++++++++++++++ configs/amd-master.yaml | 14 +++----------- perf-changelog.yaml | 8 ++++++++ 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_mi355x_vllm.sh b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_mi355x_vllm.sh index 4be977a806..6254674ecd 100755 --- a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_mi355x_vllm.sh +++ b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_mi355x_vllm.sh @@ -4,6 +4,12 @@ # https://huggingface.co/amd/MiniMax-M3-MXFP4#reproduction # Block size 128 is mandatory for MSA. This fixed-sequence benchmark uses the # text-only language-model path with AITER MoE (vllm-project/vllm#46419). +# +# High-concurrency tuning (no model-architecture overrides): +# * INT4 quantized all-reduce (env knobs below) -- reduces the all-reduce +# cost (the biggest decode kernel); measured ~-12% to -17% TPOT at conc +# 64/128/256. +# * fp8 KV cache (--kv-cache-dtype fp8). source "$(dirname "$0")/../../benchmark_lib.sh" @@ -35,6 +41,13 @@ export VLLM_USE_BREAKABLE_CUDAGRAPH=0 export VLLM_ROCM_USE_AITER=1 export VLLM_ROCM_USE_AITER_MOE=1 export VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 +# INT4 quantized all-reduce for the (~1.5 MB) decode all-reduces, which are the +# single biggest decode kernel at high concurrency. The MIN_SIZE_KB override is +# required: vLLM's default INT4 quick-reduce size gate for (bf16, TP4) is 16 MB, +# so it never fires for decode-sized tensors without it. +export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 +export VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16=0 +export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION_MIN_SIZE_KB=256 if [ "${EVAL_ONLY}" = "true" ]; then setup_eval_context @@ -63,6 +76,7 @@ vllm serve "$MODEL" --port "$PORT" \ --max-model-len "$MAX_MODEL_LEN" \ --attention-backend TRITON_ATTN \ --moe-backend aiter \ + --kv-cache-dtype fp8 \ --tool-call-parser minimax_m3 \ --enable-auto-tool-choice \ --reasoning-parser minimax_m3 > "$SERVER_LOG" 2>&1 & diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 9851e8d6d0..ac69d18b31 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -2472,7 +2472,7 @@ minimaxm3-fp4-mi355x-vllm-disagg: # language-model path and mirror the MXFP8 MI355X search space for a direct # precision comparison. minimaxm3-fp4-mi355x-vllm: - image: vllm/vllm-openai-rocm:nightly-4559c43a9526597c00cbcc4f59979496500268d1 + image: vllm/vllm-openai-rocm:nightly-69715823df89b11ee684b84066390cbb9092d5c1 model: amd/MiniMax-M3-MXFP4 model-prefix: minimaxm3 runner: mi355x @@ -2484,19 +2484,11 @@ minimaxm3-fp4-mi355x-vllm: - isl: 1024 osl: 1024 search-space: - - { tp: 8, conc-start: 1, conc-end: 64 } - - { tp: 8, ep: 8, conc-start: 1, conc-end: 512 } - - { tp: 4, conc-start: 1, conc-end: 64 } - - { tp: 4, ep: 4, conc-start: 64, conc-end: 512 } - - { tp: 2, ep: 2, conc-start: 16, conc-end: 128 } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 256, conc-end: 1024 } + - { tp: 4, conc-start: 1, conc-end: 512 } - isl: 8192 osl: 1024 search-space: - - { tp: 8, conc-start: 1, conc-end: 64 } - - { tp: 8, ep: 8, conc-start: 1, conc-end: 512 } - - { tp: 4, conc-start: 1, conc-end: 128 } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 128, conc-end: 512 } + - { tp: 4, conc-start: 1, conc-end: 512 } # EAGLE3 speculative-decoding variant of minimaxm3-fp4-mi355x-vllm. Pair the # amd/MiniMax-M3-MXFP4 target with Inferact/MiniMax-M3-EAGLE3 and three draft diff --git a/perf-changelog.yaml b/perf-changelog.yaml index b9981ef896..499f431848 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4574,3 +4574,11 @@ description: - "Add high concurrency configs" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1994 + +- config-keys: + - minimaxm3-fp4-mi355x-vllm + description: + - "Close the high-concurrency gap vs the ATOM recipe on MiniMax-M3 MXFP4 MI355X single-node vLLM STP. Bump image to latest nightly vllm/vllm-openai-rocm:nightly-69715823df89b11ee684b84066390cbb9092d5c1." + - "Add INT4 quantized all-reduce (VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4, VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16=0, VLLM_ROCM_QUICK_REDUCE_QUANTIZATION_MIN_SIZE_KB=256) and fp8 KV cache (--kv-cache-dtype fp8). No --hf-overrides indexer-skipping knobs (disallowed: reduces model architecture FLOPs per PR_REVIEW_CHECKLIST.md)." + - "Simplify search space to a single TP4 conc 1-512 sweep for 1k1k and 8k1k (drop TP8, TP4/EP4, TP2/EP2, and dp-attn variants), matching minimaxm3-fp8-mi355x-vllm (#2003)." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX From d22dc47462c3c48cca9df9664a301ee5f41e12cf Mon Sep 17 00:00:00 2001 From: Fangzhou Ai Date: Mon, 6 Jul 2026 14:45:10 -0500 Subject: [PATCH 2/2] fix(changelog): set pr-link to #2104 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：将 changelog pr-link 更新为 #2104。 Co-authored-by: Cursor --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 499f431848..dd80984380 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4581,4 +4581,4 @@ - "Close the high-concurrency gap vs the ATOM recipe on MiniMax-M3 MXFP4 MI355X single-node vLLM STP. Bump image to latest nightly vllm/vllm-openai-rocm:nightly-69715823df89b11ee684b84066390cbb9092d5c1." - "Add INT4 quantized all-reduce (VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4, VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16=0, VLLM_ROCM_QUICK_REDUCE_QUANTIZATION_MIN_SIZE_KB=256) and fp8 KV cache (--kv-cache-dtype fp8). No --hf-overrides indexer-skipping knobs (disallowed: reduces model architecture FLOPs per PR_REVIEW_CHECKLIST.md)." - "Simplify search space to a single TP4 conc 1-512 sweep for 1k1k and 8k1k (drop TP8, TP4/EP4, TP2/EP2, and dp-attn variants), matching minimaxm3-fp8-mi355x-vllm (#2003)." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2104