From 710cb2d1c01435ef07a6097b266257eda708e616 Mon Sep 17 00:00:00 2001 From: wzhao18 Date: Mon, 6 Jul 2026 13:00:10 -0700 Subject: [PATCH 01/10] add --- .../single_node/agentic/kimik2.5_fp4_b200.sh | 197 ++++++++---------- configs/nvidia-master.yaml | 14 +- perf-changelog.yaml | 6 + 3 files changed, 95 insertions(+), 122 deletions(-) diff --git a/benchmarks/single_node/agentic/kimik2.5_fp4_b200.sh b/benchmarks/single_node/agentic/kimik2.5_fp4_b200.sh index b4ac77f2b..ddf30bbca 100755 --- a/benchmarks/single_node/agentic/kimik2.5_fp4_b200.sh +++ b/benchmarks/single_node/agentic/kimik2.5_fp4_b200.sh @@ -7,7 +7,7 @@ set -x # Required env vars: # MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR # -# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=lmcache. +# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=native or KV_OFFLOAD_BACKEND=mooncake. source "$(dirname "$0")/../../benchmark_lib.sh" @@ -37,149 +37,116 @@ install_agentic_deps # ---- Server config ---------------------------------------------------------- SERVER_LOG="$RESULT_DIR/server.log" -LMCACHE_LOG="$RESULT_DIR/lmcache_server.log" +MOONCAKE_MASTER_LOG="$RESULT_DIR/mooncake_master.log" mkdir -p "$RESULT_DIR" -OFFLOAD_ARGS=() -PREFIX_CACHE_ARGS=() -LMCACHE_PID="" +SERVER_PID="" +MOONCAKE_MASTER_PID="" -cleanup_lmcache_server() { - if [[ -n "$LMCACHE_PID" ]] && kill -0 "$LMCACHE_PID" 2>/dev/null; then - kill "$LMCACHE_PID" 2>/dev/null || true - wait "$LMCACHE_PID" 2>/dev/null || true - fi +cleanup_agentic_services() { + local exit_code=$? + trap - EXIT INT TERM + set +e + stop_background_process_tree "$SERVER_PID" "vLLM server" 60 + stop_background_process_tree "$MOONCAKE_MASTER_PID" "Mooncake master" + exit "$exit_code" } +trap cleanup_agentic_services EXIT +trap 'exit 130' INT +trap 'exit 143' TERM -trap cleanup_lmcache_server EXIT - -wait_for_lmcache_ready() { - { set +x; } 2>/dev/null - local attempts="${LMCACHE_READY_ATTEMPTS:-120}" - local tail_pid="" - - while [ ! -f "$LMCACHE_LOG" ]; do - if [[ -n "$LMCACHE_PID" ]] && ! kill -0 "$LMCACHE_PID" 2>/dev/null; then - echo "LMCache server died before creating log file. Exiting." >&2 - exit 1 - fi - sleep 1 - done - - tail -f -n +1 "$LMCACHE_LOG" & - tail_pid=$! - - for ((i = 1; i <= attempts; i++)); do - if curl --output /dev/null --silent --fail "http://127.0.0.1:${LMCACHE_HTTP_PORT}/healthcheck"; then - kill "$tail_pid" 2>/dev/null || true - wait "$tail_pid" 2>/dev/null || true - return 0 - fi - if [[ -n "$LMCACHE_PID" ]] && ! kill -0 "$LMCACHE_PID" 2>/dev/null; then - echo "LMCache server died before becoming healthy. Log follows:" >&2 - kill "$tail_pid" 2>/dev/null || true - wait "$tail_pid" 2>/dev/null || true - cat "$LMCACHE_LOG" >&2 || true - exit 1 - fi - sleep 1 - done - - echo "Timed out waiting for LMCache server healthcheck. Log follows:" >&2 - kill "$tail_pid" 2>/dev/null || true - wait "$tail_pid" 2>/dev/null || true - cat "$LMCACHE_LOG" >&2 || true - exit 1 -} +OFFLOAD_ARGS=() -if require_agentic_kv_offload_backend lmcache; then +if agentic_kv_offload_enabled; then + case "$KV_OFFLOAD_BACKEND" in + native) + export VLLM_USE_SIMPLE_KV_OFFLOAD=1 + OFFLOAD_ARGS=( + --kv_offloading_backend native + --kv_offloading_size "$TOTAL_CPU_DRAM_GB" + --disable-hybrid-kv-cache-manager + ) + ;; + mooncake) { set +x; } 2>/dev/null unset VLLM_USE_SIMPLE_KV_OFFLOAD - agentic_pip_install --quiet --no-cache-dir lmcache - python3 -c "import lmcache.integration.vllm.lmcache_mp_connector" >/dev/null - - # MP mode owns the configured CPU pool in the external LMCache - # server instead of passing - # --kv-offloading-size through vLLM's integrated LMCache convenience - # path, which divides the value by TP and then hits a large single-shot - # cudaHostAlloc in LMCache 0.4.5's single-process local CPU backend. - LMCACHE_HOST="${LMCACHE_HOST:-127.0.0.1}" - LMCACHE_PORT="${LMCACHE_PORT:-5555}" - LMCACHE_HTTP_PORT="${LMCACHE_HTTP_PORT:-8080}" - # LMCacheMPConnector builds its ZMQ endpoint by concatenating - # lmcache.mp.host and lmcache.mp.port, and its default host already - # includes the tcp:// scheme. Keep the server bind host raw, but pass - # a ZMQ-style host string to the connector. - LMCACHE_CONNECT_HOST="${LMCACHE_CONNECT_HOST:-tcp://$LMCACHE_HOST}" - LMCACHE_L1_SIZE_GB="${LMCACHE_L1_SIZE_GB:-$TOTAL_CPU_DRAM_GB}" - if [ "$LMCACHE_L1_SIZE_GB" -gt "$TOTAL_CPU_DRAM_GB" ]; then - echo "Error: LMCACHE_L1_SIZE_GB=$LMCACHE_L1_SIZE_GB exceeds configured capacity $TOTAL_CPU_DRAM_GB" >&2 + PER_RANK_GB=$((TOTAL_CPU_DRAM_GB / TP)) + + MOONCAKE_VERSION=0.3.11.post1 + agentic_pip_install --quiet --no-cache-dir --no-deps \ + --force-reinstall "mooncake-transfer-engine-cuda13==$MOONCAKE_VERSION" + + MOONCAKE_MASTER_PORT=$((PORT + 12000)) + MOONCAKE_CONFIG_PATH="$RESULT_DIR/mooncake_config.json" + cat > "$MOONCAKE_CONFIG_PATH" < "$MOONCAKE_MASTER_LOG" 2>&1 & + MOONCAKE_MASTER_PID=$! + sleep 2 + if ! kill -0 "$MOONCAKE_MASTER_PID" 2>/dev/null; then + echo "Mooncake master died during startup." >&2 + cat "$MOONCAKE_MASTER_LOG" >&2 exit 1 fi - # Initial allocation is deliberately small; --l1-size-gb above is the - # actual pool capacity and grows lazily as the run fills the cache. - LMCACHE_L1_INIT_SIZE_GB="${LMCACHE_L1_INIT_SIZE_GB:-20}" - LMCACHE_CHUNK_SIZE="${LMCACHE_CHUNK_SIZE:-256}" - LMCACHE_MAX_WORKERS="${LMCACHE_MAX_WORKERS:-$TP}" - export PYTHONHASHSEED="${PYTHONHASHSEED:-0}" - - echo "Starting LMCache MP server..." - LMCACHE_CMD=( - lmcache server - --host "$LMCACHE_HOST" - --port "$LMCACHE_PORT" - --http-host "$LMCACHE_HOST" - --http-port "$LMCACHE_HTTP_PORT" - --l1-size-gb "$LMCACHE_L1_SIZE_GB" - --l1-init-size-gb "$LMCACHE_L1_INIT_SIZE_GB" - --chunk-size "$LMCACHE_CHUNK_SIZE" - --max-workers "$LMCACHE_MAX_WORKERS" - --eviction-policy LRU - ) - printf '%q ' "${LMCACHE_CMD[@]}" > "$RESULT_DIR/lmcache_command.txt" - printf '\n' >> "$RESULT_DIR/lmcache_command.txt" - "${LMCACHE_CMD[@]}" > "$LMCACHE_LOG" 2>&1 & - LMCACHE_PID=$! - echo "LMCache server PID: $LMCACHE_PID" - wait_for_lmcache_ready - - PREFIX_CACHE_ARGS=(--enable-prefix-caching) OFFLOAD_ARGS=( --kv-transfer-config - "{\"kv_connector\":\"LMCacheMPConnector\",\"kv_connector_module_path\":\"lmcache.integration.vllm.lmcache_mp_connector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"lmcache.mp.host\":\"$LMCACHE_CONNECT_HOST\",\"lmcache.mp.port\":$LMCACHE_PORT}}" - --disable-hybrid-kv-cache-manager + '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both"}' ) + ;; + *) echo "Error: unsupported KV_OFFLOAD_BACKEND value '$KV_OFFLOAD_BACKEND' (expected one of: native, mooncake)" >&2; exit 1 ;; + esac fi echo "Starting vllm server..." export TORCH_CUDA_ARCH_LIST="10.0" export PYTHONNOUSERSITE=1 # Disable vLLM v0.21+ CUDA-graph memory estimator. Its pre-reservation -# eats ~32% of HBM upfront which, combined with FP4 weights at TP=4 -# (~62 GB/GPU), leaves no room for KV blocks -- _check_enough_kv_cache_memory -# trips before the engine starts. Our --gpu-memory-utilization=0.90 already -# leaves ~18 GB/GPU slack outside vLLM's budget, which is the same safety -# net the estimator provides, so disabling it is redundant rather than -# unsafe. +# can over-budget FP4 Kimi weights at TP=4 before KV sizing completes; the +# explicit CUDA graph settings below define the capture behavior for this +# recipe. export VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0 +DCP_ARGS=() +if [[ "$CONC" -ge 16 ]]; then + DCP_ARGS=(--decode-context-parallel-size "$TP") +fi + +export VLLM_FLASHINFER_ALLREDUCE_BACKEND=trtllm + { set +x; } 2>/dev/null VLLM_CMD=( vllm serve "$MODEL_PATH" --served-model-name "$MODEL" --host 0.0.0.0 --port "$PORT" - --tensor-parallel-size="$TP" - --gpu-memory-utilization 0.90 - --max-num-seqs "$CONC" - --reasoning-parser kimi_k2 - --tool-call-parser kimi_k2 - --compilation_config.pass_config.fuse_allreduce_rms true --kv-cache-dtype fp8 - --max-cudagraph-capture-size 2048 - --stream-interval 20 --trust-remote-code - "${PREFIX_CACHE_ARGS[@]}" + --block-size 64 + --language-model-only + --attention-config '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' + --compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' + --max-cudagraph-capture-size 2048 + --max-num-batched-tokens 16384 + --stream-interval 10 + --enable-prefix-caching + --tensor-parallel-size "$TP" + "${DCP_ARGS[@]}" "${OFFLOAD_ARGS[@]}" ) printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 8d84bd0dd..f7e4a7b00 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -11103,9 +11103,9 @@ qwen3.5-fp8-b300-sglang-agentic-hicache: - { tp: 4, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32] } - { tp: 4, ep: 1, kv-offloading: dram, kv-offload-backend: hicache, conc-list: [16, 32, 48, 64] } -kimik2.5-fp4-b200-vllm-agentic-lmcache: - image: vllm/vllm-openai:v0.22.0 - model: nvidia/Kimi-K2.5-NVFP4 +kimik2.5-fp4-b200-vllm-agentic: + image: vllm/vllm-openai:nightly-09663abde0f50944a8d5ea30120666024b503faa + model: Kimi-K2.6-NVFP4 model-prefix: kimik2.5 runner: cluster:b200-dgxc precision: fp4 @@ -11115,10 +11115,10 @@ kimik2.5-fp4-b200-vllm-agentic-lmcache: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 24] } - - { tp: 8, ep: 1, kv-offloading: dram, kv-offload-backend: lmcache, conc-list: [16, 24, 32, 36] } - - { tp: 4, ep: 1, kv-offloading: none, conc-list: [8, 12, 14, 16, 18, 20] } - - { tp: 4, ep: 1, kv-offloading: dram, kv-offload-backend: lmcache, conc-list: [12, 14, 16, 18, 20, 22, 24, 32] } + - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1] } + - { tp: 4, ep: 1, kv-offloading: none, conc-list: [1] } + - { tp: 4, ep: 1, kv-offloading: dram, kv-offload-backend: native, conc-list: [1] } + - { tp: 4, ep: 1, kv-offloading: dram, kv-offload-backend: mooncake, conc-list: [1] } # CONC range conservative for H100's 80 GB HBM3 under the long-ISL with- # subagents corpus. hicache arm capped at conc 16 since high-conc + hicache diff --git a/perf-changelog.yaml b/perf-changelog.yaml index b9981ef89..594412375 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4574,3 +4574,9 @@ description: - "Add high concurrency configs" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1994 + +- config-keys: + - kimik2.5-fp4-b200-vllm-agentic + description: + - "Update kimi k2.5 agentx B200" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX From f382bf39db16463cbcd001e19da9802466705b33 Mon Sep 17 00:00:00 2001 From: wzhao18 Date: Mon, 6 Jul 2026 13:00:46 -0700 Subject: [PATCH 02/10] fix --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 594412375..aee8eaa08 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4579,4 +4579,4 @@ - kimik2.5-fp4-b200-vllm-agentic description: - "Update kimi k2.5 agentx B200" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2106 From 075c080124555633418576dd43bdff5ca9e02464 Mon Sep 17 00:00:00 2001 From: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Date: Mon, 6 Jul 2026 16:04:42 -0400 Subject: [PATCH 03/10] Update nvidia-master.yaml --- configs/nvidia-master.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index f7e4a7b00..0d6efcfb7 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -11105,7 +11105,7 @@ qwen3.5-fp8-b300-sglang-agentic-hicache: kimik2.5-fp4-b200-vllm-agentic: image: vllm/vllm-openai:nightly-09663abde0f50944a8d5ea30120666024b503faa - model: Kimi-K2.6-NVFP4 + model: nvidia/Kimi-K2.5-NVFP4 model-prefix: kimik2.5 runner: cluster:b200-dgxc precision: fp4 From 90c99bc82fbfe6262cef563295486f2c7546ff99 Mon Sep 17 00:00:00 2001 From: wzhao18 Date: Mon, 6 Jul 2026 13:40:27 -0700 Subject: [PATCH 04/10] fx --- configs/nvidia-master.yaml | 4 ++-- runners/launch_b200-dgxc.sh | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 0d6efcfb7..4d11b4ca6 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -11105,8 +11105,8 @@ qwen3.5-fp8-b300-sglang-agentic-hicache: kimik2.5-fp4-b200-vllm-agentic: image: vllm/vllm-openai:nightly-09663abde0f50944a8d5ea30120666024b503faa - model: nvidia/Kimi-K2.5-NVFP4 - model-prefix: kimik2.5 + model: nvidia/Kimi-K2.6-NVFP4 + model-prefix: kimik2.6 runner: cluster:b200-dgxc precision: fp4 framework: vllm diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index 7fe590cb3..7e82cbcdb 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -54,6 +54,9 @@ elif [[ $MODEL_PREFIX == "kimik2.5" && $PRECISION == "int4" ]]; then elif [[ $MODEL_PREFIX == "kimik2.5" && $PRECISION == "fp4" ]]; then export MODEL_PATH="/lustre/fsw/models/Kimi-K2.5-NVFP4" export SRT_SLURM_MODEL_PREFIX="kimik2.5-fp4" +elif [[ $MODEL_PREFIX == "kimik2.6" && $PRECISION == "fp4" ]]; then + export MODEL_PATH="/lustre/fsw/models/Kimi-K2.6-NVFP4" + export SRT_SLURM_MODEL_PREFIX="kimik2.6-fp4" elif [[ $MODEL_PREFIX == "minimaxm2.5" && $PRECISION == "fp8" ]]; then export MODEL_PATH="/lustre/fsw/models/MiniMax-M2.5" export SRT_SLURM_MODEL_PREFIX="minimax-m2.5-fp8" From 91cd6290d05a1891eef32022888dcb2d18e9a1d0 Mon Sep 17 00:00:00 2001 From: wzhao18 Date: Mon, 6 Jul 2026 13:43:39 -0700 Subject: [PATCH 05/10] fix --- runners/launch_b200-dgxc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index 7e82cbcdb..348c1acfb 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -56,7 +56,7 @@ elif [[ $MODEL_PREFIX == "kimik2.5" && $PRECISION == "fp4" ]]; then export SRT_SLURM_MODEL_PREFIX="kimik2.5-fp4" elif [[ $MODEL_PREFIX == "kimik2.6" && $PRECISION == "fp4" ]]; then export MODEL_PATH="/lustre/fsw/models/Kimi-K2.6-NVFP4" - export SRT_SLURM_MODEL_PREFIX="kimik2.6-fp4" + export SRT_SLURM_MODEL_PREFIX="kimik2.5-fp4" elif [[ $MODEL_PREFIX == "minimaxm2.5" && $PRECISION == "fp8" ]]; then export MODEL_PATH="/lustre/fsw/models/MiniMax-M2.5" export SRT_SLURM_MODEL_PREFIX="minimax-m2.5-fp8" From 1ecbc9768184f8b300e4cd8cd624a4dc29bc3e33 Mon Sep 17 00:00:00 2001 From: wzhao18 Date: Mon, 6 Jul 2026 13:47:35 -0700 Subject: [PATCH 06/10] fix --- configs/nvidia-master.yaml | 4 ++-- runners/launch_b200-dgxc.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 4d11b4ca6..0d6efcfb7 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -11105,8 +11105,8 @@ qwen3.5-fp8-b300-sglang-agentic-hicache: kimik2.5-fp4-b200-vllm-agentic: image: vllm/vllm-openai:nightly-09663abde0f50944a8d5ea30120666024b503faa - model: nvidia/Kimi-K2.6-NVFP4 - model-prefix: kimik2.6 + model: nvidia/Kimi-K2.5-NVFP4 + model-prefix: kimik2.5 runner: cluster:b200-dgxc precision: fp4 framework: vllm diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index 348c1acfb..728094f40 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -54,7 +54,7 @@ elif [[ $MODEL_PREFIX == "kimik2.5" && $PRECISION == "int4" ]]; then elif [[ $MODEL_PREFIX == "kimik2.5" && $PRECISION == "fp4" ]]; then export MODEL_PATH="/lustre/fsw/models/Kimi-K2.5-NVFP4" export SRT_SLURM_MODEL_PREFIX="kimik2.5-fp4" -elif [[ $MODEL_PREFIX == "kimik2.6" && $PRECISION == "fp4" ]]; then +elif [[ $MODEL_PREFIX == "kimik2.5" && $PRECISION == "fp4" ]]; then export MODEL_PATH="/lustre/fsw/models/Kimi-K2.6-NVFP4" export SRT_SLURM_MODEL_PREFIX="kimik2.5-fp4" elif [[ $MODEL_PREFIX == "minimaxm2.5" && $PRECISION == "fp8" ]]; then From bdb4dfaea620296405326c43c1ea2d52723d256b Mon Sep 17 00:00:00 2001 From: wzhao18 Date: Mon, 6 Jul 2026 13:50:42 -0700 Subject: [PATCH 07/10] fix --- runners/launch_b200-dgxc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index 728094f40..3b691d3af 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -52,7 +52,7 @@ elif [[ $MODEL_PREFIX == "kimik2.5" && $PRECISION == "int4" ]]; then export MODEL_PATH="/lustre/fsw/models/Kimi-K2.5" export SRT_SLURM_MODEL_PREFIX="kimik2.5" elif [[ $MODEL_PREFIX == "kimik2.5" && $PRECISION == "fp4" ]]; then - export MODEL_PATH="/lustre/fsw/models/Kimi-K2.5-NVFP4" + export MODEL_PATH="/lustre/fsw/models/Kimi-K2.6-NVFP4" export SRT_SLURM_MODEL_PREFIX="kimik2.5-fp4" elif [[ $MODEL_PREFIX == "kimik2.5" && $PRECISION == "fp4" ]]; then export MODEL_PATH="/lustre/fsw/models/Kimi-K2.6-NVFP4" From 86d5b590839f980a094299b807e294aad5214298 Mon Sep 17 00:00:00 2001 From: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Date: Mon, 6 Jul 2026 18:53:09 -0400 Subject: [PATCH 08/10] Modify concurrency options in nvidia-master.yaml Updated the search space for agentic-coding scenarios to include additional concurrency options. --- configs/nvidia-master.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 0d6efcfb7..a18f09b79 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -11115,10 +11115,10 @@ kimik2.5-fp4-b200-vllm-agentic: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1] } - - { tp: 4, ep: 1, kv-offloading: none, conc-list: [1] } - - { tp: 4, ep: 1, kv-offloading: dram, kv-offload-backend: native, conc-list: [1] } - - { tp: 4, ep: 1, kv-offloading: dram, kv-offload-backend: mooncake, conc-list: [1] } + - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4] } + - { tp: 4, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32] } + - { tp: 4, ep: 1, kv-offloading: dram, kv-offload-backend: native, conc-list: [1, 2, 4, 8, 16, 32] } + - { tp: 4, ep: 1, kv-offloading: dram, kv-offload-backend: mooncake, conc-list: [1, 2, 4, 8, 16, 32] } # CONC range conservative for H100's 80 GB HBM3 under the long-ISL with- # subagents corpus. hicache arm capped at conc 16 since high-conc + hicache From 52603a962107415c1a4c8254358763293b6f14dd Mon Sep 17 00:00:00 2001 From: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Date: Mon, 6 Jul 2026 18:53:48 -0400 Subject: [PATCH 09/10] Remove duplicate model path condition for kimik2.5 Removed duplicate condition for 'kimik2.5' and 'fp4' in model path exports. --- runners/launch_b200-dgxc.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index 3b691d3af..d490c20e6 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -54,9 +54,6 @@ elif [[ $MODEL_PREFIX == "kimik2.5" && $PRECISION == "int4" ]]; then elif [[ $MODEL_PREFIX == "kimik2.5" && $PRECISION == "fp4" ]]; then export MODEL_PATH="/lustre/fsw/models/Kimi-K2.6-NVFP4" export SRT_SLURM_MODEL_PREFIX="kimik2.5-fp4" -elif [[ $MODEL_PREFIX == "kimik2.5" && $PRECISION == "fp4" ]]; then - export MODEL_PATH="/lustre/fsw/models/Kimi-K2.6-NVFP4" - export SRT_SLURM_MODEL_PREFIX="kimik2.5-fp4" elif [[ $MODEL_PREFIX == "minimaxm2.5" && $PRECISION == "fp8" ]]; then export MODEL_PATH="/lustre/fsw/models/MiniMax-M2.5" export SRT_SLURM_MODEL_PREFIX="minimax-m2.5-fp8" From 25256270367fc1a0a21fdc5bab0b100dfbc8f1d6 Mon Sep 17 00:00:00 2001 From: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Date: Tue, 7 Jul 2026 01:38:08 -0400 Subject: [PATCH 10/10] Update dram-utilization configurations in nvidia-master.yaml Commented out previous configurations for dram-utilization and updated the conc-list for mooncake backend. --- configs/nvidia-master.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index a18f09b79..2bd1ef764 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -11115,10 +11115,10 @@ kimik2.5-fp4-b200-vllm-agentic: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4] } - - { tp: 4, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32] } - - { tp: 4, ep: 1, kv-offloading: dram, kv-offload-backend: native, conc-list: [1, 2, 4, 8, 16, 32] } - - { tp: 4, ep: 1, kv-offloading: dram, kv-offload-backend: mooncake, conc-list: [1, 2, 4, 8, 16, 32] } + # - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4] } + # - { tp: 4, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32] } + # - { tp: 4, ep: 1, kv-offloading: dram, kv-offload-backend: native, conc-list: [1, 2, 4, 8, 16, 32] } + - { tp: 4, ep: 1, kv-offloading: dram, kv-offload-backend: mooncake, conc-list: [1, 2, 4, 8, 16, 32, 48, 64, 72, 80] } # CONC range conservative for H100's 80 GB HBM3 under the long-ISL with- # subagents corpus. hicache arm capped at conc 16 since high-conc + hicache