diff --git a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_mi355x_vllm.sh b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_mi355x_vllm.sh index 4be977a806..6254674ecd 100755 --- a/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_mi355x_vllm.sh +++ b/benchmarks/single_node/fixed_seq_len/minimaxm3_fp4_mi355x_vllm.sh @@ -4,6 +4,12 @@ # https://huggingface.co/amd/MiniMax-M3-MXFP4#reproduction # Block size 128 is mandatory for MSA. This fixed-sequence benchmark uses the # text-only language-model path with AITER MoE (vllm-project/vllm#46419). +# +# High-concurrency tuning (no model-architecture overrides): +# * INT4 quantized all-reduce (env knobs below) -- reduces the all-reduce +# cost (the biggest decode kernel); measured ~-12% to -17% TPOT at conc +# 64/128/256. +# * fp8 KV cache (--kv-cache-dtype fp8). source "$(dirname "$0")/../../benchmark_lib.sh" @@ -35,6 +41,13 @@ export VLLM_USE_BREAKABLE_CUDAGRAPH=0 export VLLM_ROCM_USE_AITER=1 export VLLM_ROCM_USE_AITER_MOE=1 export VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 +# INT4 quantized all-reduce for the (~1.5 MB) decode all-reduces, which are the +# single biggest decode kernel at high concurrency. The MIN_SIZE_KB override is +# required: vLLM's default INT4 quick-reduce size gate for (bf16, TP4) is 16 MB, +# so it never fires for decode-sized tensors without it. +export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 +export VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16=0 +export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION_MIN_SIZE_KB=256 if [ "${EVAL_ONLY}" = "true" ]; then setup_eval_context @@ -63,6 +76,7 @@ vllm serve "$MODEL" --port "$PORT" \ --max-model-len "$MAX_MODEL_LEN" \ --attention-backend TRITON_ATTN \ --moe-backend aiter \ + --kv-cache-dtype fp8 \ --tool-call-parser minimax_m3 \ --enable-auto-tool-choice \ --reasoning-parser minimax_m3 > "$SERVER_LOG" 2>&1 & diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 9851e8d6d0..ac69d18b31 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -2472,7 +2472,7 @@ minimaxm3-fp4-mi355x-vllm-disagg: # language-model path and mirror the MXFP8 MI355X search space for a direct # precision comparison. minimaxm3-fp4-mi355x-vllm: - image: vllm/vllm-openai-rocm:nightly-4559c43a9526597c00cbcc4f59979496500268d1 + image: vllm/vllm-openai-rocm:nightly-69715823df89b11ee684b84066390cbb9092d5c1 model: amd/MiniMax-M3-MXFP4 model-prefix: minimaxm3 runner: mi355x @@ -2484,19 +2484,11 @@ minimaxm3-fp4-mi355x-vllm: - isl: 1024 osl: 1024 search-space: - - { tp: 8, conc-start: 1, conc-end: 64 } - - { tp: 8, ep: 8, conc-start: 1, conc-end: 512 } - - { tp: 4, conc-start: 1, conc-end: 64 } - - { tp: 4, ep: 4, conc-start: 64, conc-end: 512 } - - { tp: 2, ep: 2, conc-start: 16, conc-end: 128 } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 256, conc-end: 1024 } + - { tp: 4, conc-start: 1, conc-end: 512 } - isl: 8192 osl: 1024 search-space: - - { tp: 8, conc-start: 1, conc-end: 64 } - - { tp: 8, ep: 8, conc-start: 1, conc-end: 512 } - - { tp: 4, conc-start: 1, conc-end: 128 } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 128, conc-end: 512 } + - { tp: 4, conc-start: 1, conc-end: 512 } # EAGLE3 speculative-decoding variant of minimaxm3-fp4-mi355x-vllm. Pair the # amd/MiniMax-M3-MXFP4 target with Inferact/MiniMax-M3-EAGLE3 and three draft diff --git a/perf-changelog.yaml b/perf-changelog.yaml index b9981ef896..dd80984380 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4574,3 +4574,11 @@ description: - "Add high concurrency configs" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1994 + +- config-keys: + - minimaxm3-fp4-mi355x-vllm + description: + - "Close the high-concurrency gap vs the ATOM recipe on MiniMax-M3 MXFP4 MI355X single-node vLLM STP. Bump image to latest nightly vllm/vllm-openai-rocm:nightly-69715823df89b11ee684b84066390cbb9092d5c1." + - "Add INT4 quantized all-reduce (VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4, VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16=0, VLLM_ROCM_QUICK_REDUCE_QUANTIZATION_MIN_SIZE_KB=256) and fp8 KV cache (--kv-cache-dtype fp8). No --hf-overrides indexer-skipping knobs (disallowed: reduces model architecture FLOPs per PR_REVIEW_CHECKLIST.md)." + - "Simplify search space to a single TP4 conc 1-512 sweep for 1k1k and 8k1k (drop TP8, TP4/EP4, TP2/EP2, and dp-attn variants), matching minimaxm3-fp8-mi355x-vllm (#2003)." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2104