From 3033f976a6dea339840d8e2fdc353789a4153e1f Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 3 Jul 2026 15:13:56 -0500 Subject: [PATCH 1/6] chore(agentx): update B200 vLLM to v0.24.0 Signed-off-by: Cam Quilici --- configs/nvidia-master.yaml | 2 +- utils/matrix_logic/generate_sweep_configs.py | 14 +++++- .../test_generate_sweep_configs.py | 50 +++++++++++++++++++ 3 files changed, 64 insertions(+), 2 deletions(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index bdb6a4602..594d28ccb 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -1763,7 +1763,7 @@ dsv4-fp4-b200-vllm: - { tp: 8, ep: 8, dp-attn: true, conc-start: 64, conc-end: 1024 } dsv4-fp4-b200-vllm-agentic: - image: vllm/vllm-openai:v0.23.0 + image: vllm/vllm-openai:v0.24.0 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:b200-dgxc diff --git a/utils/matrix_logic/generate_sweep_configs.py b/utils/matrix_logic/generate_sweep_configs.py index 545eb9f96..b2fa80a1e 100644 --- a/utils/matrix_logic/generate_sweep_configs.py +++ b/utils/matrix_logic/generate_sweep_configs.py @@ -294,7 +294,7 @@ def generate_full_sweep(args, all_config_data, runner_data): """Generate full sweep configurations with optional filtering. Supports filtering by model prefix, precision, framework, runner type, sequence lengths, - and max concurrency. + KV offloading mode, and concurrency. All filters are optional - can generate sweeps for all configs or filter by specific criteria. @@ -597,6 +597,11 @@ def generate_full_sweep(args, all_config_data, runner_data): dp_attn = bmk.get(Fields.DP_ATTN.value) kv_offloading = bmk[Fields.KV_OFFLOADING.value] kv_offload_backend = bmk.get(Fields.KV_OFFLOAD_BACKEND.value) + if ( + getattr(args, "kv_offloading", None) + and kv_offloading not in args.kv_offloading + ): + continue total_cpu_dram_gb = ( 0 if is_multinode @@ -1081,6 +1086,13 @@ def main(): required=False, help='Runner type(s) to filter by (e.g., h200, h100) (optional, can specify multiple)' ) + full_sweep_parser.add_argument( + '--kv-offloading', + nargs='+', + choices=['none', 'dram'], + required=False, + help='Agentic KV offloading mode(s) to include (optional, can specify multiple)' + ) full_sweep_parser.add_argument( '--seq-lens', nargs='+', diff --git a/utils/matrix_logic/test_generate_sweep_configs.py b/utils/matrix_logic/test_generate_sweep_configs.py index 0e2c64d95..c73ccd1ae 100644 --- a/utils/matrix_logic/test_generate_sweep_configs.py +++ b/utils/matrix_logic/test_generate_sweep_configs.py @@ -744,6 +744,56 @@ def test_filter_by_runner_type(self, sample_single_node_config, sample_runner_co ) assert len(result) == 0 + @pytest.mark.parametrize("mode", ["none", "dram"]) + def test_filter_agentic_by_kv_offloading( + self, + mode, + sample_runner_config, + full_sweep_args_single_node, + ): + """KV offloading filter should retain only matching AgentX entries.""" + config = { + "dsv4-fp4-b200-vllm-agentic": { + "image": "vllm/vllm-openai:v0.24.0", + "model": "deepseek-ai/DeepSeek-V4-Pro", + "model-prefix": "dsv4", + "precision": "fp4", + "framework": "vllm", + "runner": "cluster:b200-dgxc", + "multinode": False, + "scenarios": { + "agentic-coding": [{ + "dram-utilization": 0.80, + "search-space": [ + { + "tp": 8, + "kv-offloading": "none", + "conc-list": [8], + }, + { + "tp": 8, + "kv-offloading": "dram", + "kv-offload-backend": "mooncake", + "conc-list": [8], + }, + ], + }], + }, + }, + } + full_sweep_args_single_node.scenario_type = ["agentic-coding"] + full_sweep_args_single_node.kv_offloading = [mode] + + result = generate_full_sweep( + full_sweep_args_single_node, + config, + sample_runner_config, + ) + + assert len(result) == 1 + assert result[0]["kv-offloading"] == mode + assert ("kv-offload-backend" in result[0]) is (mode == "dram") + def test_invalid_runner_type_raises_error(self, sample_single_node_config, sample_runner_config, full_sweep_args_single_node): """Invalid runner type should raise ValueError.""" full_sweep_args_single_node.runner_type = ["invalid_runner"] From 48fdff54bf8e41d573dcb149fd63d6a064f9426e Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 3 Jul 2026 15:36:58 -0500 Subject: [PATCH 2/6] chore(agentx): remove stale B200 setup Signed-off-by: Cam Quilici --- benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh | 7 ------- 1 file changed, 7 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh index fdebbc6f8..86797c020 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh @@ -50,12 +50,6 @@ install_agentic_deps export AIPERF_AGENTIC_CACHE_WARMUP_DURATION=600 -# vLLM v0.22.1 can ship CUTLASS DSL 4.5.2 with stale native MLIR bindings, -# which fails DSV4 indexer compilation with mlir_global_dtors(..., data). -# Reinstall the matching native wheel until NVIDIA/cutlass#3259 is resolved. -agentic_pip_install --quiet --force-reinstall --no-deps \ - 'nvidia-cutlass-dsl-libs-cu13==4.5.2' - # vllm-project/router expands the one HTTP backend into one logical worker per # DP rank and sends X-data-parallel-rank on forwarded requests. aiperf's # X-Correlation-ID is stable for every turn of a conversation; alias it to the @@ -147,7 +141,6 @@ EOF cat "$MOONCAKE_MASTER_LOG" >&2 exit 1 fi - unset VLLM_USE_SIMPLE_KV_OFFLOAD OFFLOAD_ARGS=( --kv-transfer-config '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}' From 876161f0612e492500a1e366ab05dba68a952c0b Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 3 Jul 2026 15:55:06 -0500 Subject: [PATCH 3/6] fix(agentx): avoid DEP worker RPC deadlock Signed-off-by: Cam Quilici --- benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh index 86797c020..9a165cfb6 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh @@ -149,7 +149,12 @@ fi PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) if [ "$DP_ATTENTION" = "true" ]; then - PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") + # Each DEP rank owns one GPU, so keep its worker in the EngineCore process. + PARALLEL_ARGS=( + --tensor-parallel-size 1 + --data-parallel-size "$TP" + --distributed-executor-backend uni + ) fi EP_ARGS=() From 73657f1c5d8e589d75e20e18f6751197db7522a6 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 3 Jul 2026 16:49:01 -0500 Subject: [PATCH 4/6] fix(agentx): avoid async Mooncake load deadlock Signed-off-by: Cam Quilici --- benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh index 9a165cfb6..2211920af 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh @@ -143,7 +143,7 @@ EOF fi OFFLOAD_ARGS=( --kv-transfer-config - '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}' + '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both"}' ) fi From f4a73c4fd68c8d77ae25b92f2127460c3230987c Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 3 Jul 2026 17:47:46 -0500 Subject: [PATCH 5/6] fix(agentic): disable async Mooncake loads on B200 Signed-off-by: Cam Quilici --- benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh index 2211920af..21d90458e 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh @@ -143,7 +143,7 @@ EOF fi OFFLOAD_ARGS=( --kv-transfer-config - '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both"}' + '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":false}}' ) fi From e8948df1231667319ebe9c09e5fe68ba7c459522 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 3 Jul 2026 18:23:52 -0500 Subject: [PATCH 6/6] fix(agentic): backport vllm async kv scheduler fix Signed-off-by: Cam Quilici --- .../single_node/agentic/dsv4_fp4_b200_vllm.sh | 4 +- .../patch_vllm_0_24_async_kv_deadlock.py | 63 +++++++++++++++++++ 2 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 benchmarks/single_node/agentic/patch_vllm_0_24_async_kv_deadlock.py diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh index 21d90458e..cd2af1c0f 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm.sh @@ -143,8 +143,10 @@ EOF fi OFFLOAD_ARGS=( --kv-transfer-config - '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":false}}' + '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both"}' ) + + python3 /workspace/benchmarks/single_node/agentic/patch_vllm_0_24_async_kv_deadlock.py fi PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) diff --git a/benchmarks/single_node/agentic/patch_vllm_0_24_async_kv_deadlock.py b/benchmarks/single_node/agentic/patch_vllm_0_24_async_kv_deadlock.py new file mode 100644 index 000000000..1c4170fe2 --- /dev/null +++ b/benchmarks/single_node/agentic/patch_vllm_0_24_async_kv_deadlock.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +"""Backport vLLM PR #45406 to the v0.24.0 scheduler.""" + +from importlib.metadata import distribution +from pathlib import Path + + +OLD_BLOCK = """\ + if request.has_encoder_inputs: + self.encoder_cache_manager.free(request) + break +""" + +NEW_BLOCK = """\ + if request.has_encoder_inputs: + self.encoder_cache_manager.free(request) + if self.running: + # Running requests will free blocks when they + # complete; stop here to preserve queue-order + # admission. + break + # Nothing is running, so no future event frees blocks and + # stopping at this request would freeze this state + # permanently. Requests behind this one may hold blocks + # while parked (async KV loads in WAITING_FOR_REMOTE_KVS) + # and are only promoted when this traversal reaches them. + # Keep scanning so they can be promoted, scheduled, and + # eventually free the blocks this request needs. + # See https://github.com/vllm-project/vllm/issues/45388 + request_queue.pop_request() + step_skipped_waiting.prepend_request(request) + continue +""" + + +def main() -> None: + vllm_distribution = distribution("vllm") + installed_version = vllm_distribution.version + if installed_version != "0.24.0": + raise RuntimeError( + f"Expected vLLM 0.24.0, found {installed_version}; " + "review whether PR #45406 is still needed" + ) + + scheduler_path = Path( + vllm_distribution.locate_file("vllm/v1/core/sched/scheduler.py") + ) + source = scheduler_path.read_text() + if NEW_BLOCK in source: + print("vLLM async-KV scheduler fix already present") + return + if source.count(OLD_BLOCK) != 1: + raise RuntimeError( + f"Could not uniquely locate the vLLM 0.24.0 scheduler block in " + f"{scheduler_path}" + ) + + scheduler_path.write_text(source.replace(OLD_BLOCK, NEW_BLOCK)) + print(f"Applied vLLM PR #45406 scheduler fix to {scheduler_path}") + + +if __name__ == "__main__": + main()