From 438b018f17b14cd9d4a1a2fe186e52f3b97b0e2f Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 6 Jul 2026 11:41:47 -0500 Subject: [PATCH] feat(config): add serving component metadata --- .../workflows/benchmark-multinode-tmpl.yml | 10 ++++ .github/workflows/benchmark-tmpl.yml | 10 ++++ .github/workflows/e2e-tests.yml | 12 +++++ .github/workflows/run-sweep.yml | 14 ++++++ configs/CONFIGS.md | 17 +++++++ .../aggregation/process_agentic_result.py | 26 +++++++++++ .../test_process_agentic_result.py | 23 ++++++++++ utils/matrix_logic/generate_sweep_configs.py | 18 ++++++++ .../test_generate_sweep_configs.py | 15 ++++++ utils/matrix_logic/test_validation.py | 32 +++++++++++++ utils/matrix_logic/validation.py | 46 +++++++++++++++++++ utils/process_result.py | 26 +++++++++++ utils/test_process_result.py | 30 ++++++++++++ 13 files changed, 279 insertions(+) diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index c20b7cc356..ae7c40292b 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -115,6 +115,14 @@ on: description: "KV offload backend for agentic scenarios when kv-offloading is not none" required: false type: string + router: + description: "Optional router name and version metadata as JSON" + required: false + type: string + kv-transfer: + description: "Optional KV transfer name and version metadata as JSON" + required: false + type: string total-cpu-dram-gb: description: "Total CPU DRAM in GB for KV offloading" required: false @@ -155,6 +163,8 @@ env: DURATION: ${{ inputs.duration }} KV_OFFLOADING: ${{ inputs.kv-offloading }} KV_OFFLOAD_BACKEND: ${{ inputs.kv-offload-backend }} + ROUTER_METADATA: ${{ inputs.router }} + KV_TRANSFER_METADATA: ${{ inputs.kv-transfer }} TOTAL_CPU_DRAM_GB: ${{ inputs.total-cpu-dram-gb }} PYTHONDONTWRITEBYTECODE: '1' PYTHONPYCACHEPREFIX: /tmp/inferencex-pycache diff --git a/.github/workflows/benchmark-tmpl.yml b/.github/workflows/benchmark-tmpl.yml index 4cfa9d7a78..d6c3c95e57 100644 --- a/.github/workflows/benchmark-tmpl.yml +++ b/.github/workflows/benchmark-tmpl.yml @@ -81,6 +81,14 @@ on: description: "KV offload backend for agentic scenarios when kv-offloading is not none" required: false type: string + router: + description: "Optional router name and version metadata as JSON" + required: false + type: string + kv-transfer: + description: "Optional KV transfer name and version metadata as JSON" + required: false + type: string total-cpu-dram-gb: description: "Configured CPU DRAM capacity in GB for KV offloading" required: false @@ -118,6 +126,8 @@ env: IS_AGENTIC: ${{ inputs.scenario-type == 'agentic-coding' && '1' || '0' }} KV_OFFLOADING: ${{ inputs.kv-offloading }} KV_OFFLOAD_BACKEND: ${{ inputs.kv-offload-backend }} + ROUTER_METADATA: ${{ inputs.router }} + KV_TRANSFER_METADATA: ${{ inputs.kv-transfer }} TOTAL_CPU_DRAM_GB: ${{ inputs.total-cpu-dram-gb }} DURATION: ${{ inputs.duration }} AIPERF_FAILED_REQUEST_THRESHOLD: '0.10' diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index f05c3ef99b..957e887a5a 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -102,6 +102,8 @@ jobs: model-prefix: ${{ matrix.config.model-prefix }} framework: ${{ matrix.config.framework }} precision: ${{ matrix.config.precision }} + router: ${{ matrix.config.router && toJson(matrix.config.router) || '' }} + kv-transfer: ${{ matrix.config['kv-transfer'] && toJson(matrix.config['kv-transfer']) || '' }} exp-name: ${{ matrix.config.exp-name }} conc-list: ${{ toJson(matrix.config.conc) }} spec-decoding: ${{ matrix.config.spec-decoding }} @@ -141,6 +143,8 @@ jobs: model-prefix: ${{ matrix.config.model-prefix }} framework: ${{ matrix.config.framework }} precision: ${{ matrix.config.precision }} + router: ${{ matrix.config.router && toJson(matrix.config.router) || '' }} + kv-transfer: ${{ matrix.config['kv-transfer'] && toJson(matrix.config['kv-transfer']) || '' }} exp-name: ${{ matrix.config.exp-name }} conc-list: ${{ toJson(matrix.config.conc) }} spec-decoding: ${{ matrix.config.spec-decoding }} @@ -180,6 +184,8 @@ jobs: model-prefix: ${{ matrix.config.model-prefix }} framework: ${{ matrix.config.framework }} precision: ${{ matrix.config.precision }} + router: ${{ matrix.config.router && toJson(matrix.config.router) || '' }} + kv-transfer: ${{ matrix.config['kv-transfer'] && toJson(matrix.config['kv-transfer']) || '' }} tp: ${{ matrix.config.tp }} ep: ${{ matrix.config.ep }} dp-attn: ${{ matrix.config.dp-attn }} @@ -218,6 +224,8 @@ jobs: model-prefix: ${{ matrix.config.model-prefix }} framework: ${{ matrix.config.framework }} precision: ${{ matrix.config.precision }} + router: ${{ matrix.config.router && toJson(matrix.config.router) || '' }} + kv-transfer: ${{ matrix.config['kv-transfer'] && toJson(matrix.config['kv-transfer']) || '' }} conc-list: ${{ toJson(matrix.config.conc) }} spec-decoding: ${{ matrix.config.spec-decoding }} disagg: ${{ matrix.config.disagg }} @@ -260,6 +268,8 @@ jobs: model-prefix: ${{ matrix.config.model-prefix }} framework: ${{ matrix.config.framework }} precision: ${{ matrix.config.precision }} + router: ${{ matrix.config.router && toJson(matrix.config.router) || '' }} + kv-transfer: ${{ matrix.config['kv-transfer'] && toJson(matrix.config['kv-transfer']) || '' }} tp: ${{ matrix.config.tp }} ep: ${{ matrix.config.ep }} dp-attn: ${{ matrix.config.dp-attn }} @@ -290,6 +300,8 @@ jobs: model-prefix: ${{ matrix.config.model-prefix }} framework: ${{ matrix.config.framework }} precision: ${{ matrix.config.precision }} + router: ${{ matrix.config.router && toJson(matrix.config.router) || '' }} + kv-transfer: ${{ matrix.config['kv-transfer'] && toJson(matrix.config['kv-transfer']) || '' }} tp: ${{ matrix.config.tp }} ep: ${{ matrix.config.ep }} dp-attn: ${{ matrix.config.dp-attn }} diff --git a/.github/workflows/run-sweep.yml b/.github/workflows/run-sweep.yml index 4ba50ef91d..1704dda98a 100644 --- a/.github/workflows/run-sweep.yml +++ b/.github/workflows/run-sweep.yml @@ -330,6 +330,8 @@ jobs: model-prefix: ${{ matrix.config.model-prefix }} framework: ${{ matrix.config.framework }} precision: ${{ matrix.config.precision }} + router: ${{ matrix.config.router && toJson(matrix.config.router) || '' }} + kv-transfer: ${{ matrix.config['kv-transfer'] && toJson(matrix.config['kv-transfer']) || '' }} tp: ${{ matrix.config.tp }} ep: ${{ matrix.config.ep }} dp-attn: ${{ matrix.config.dp-attn }} @@ -365,6 +367,8 @@ jobs: model-prefix: ${{ matrix.config.model-prefix }} framework: ${{ matrix.config.framework }} precision: ${{ matrix.config.precision }} + router: ${{ matrix.config.router && toJson(matrix.config.router) || '' }} + kv-transfer: ${{ matrix.config['kv-transfer'] && toJson(matrix.config['kv-transfer']) || '' }} exp-name: ${{ matrix.config.exp-name }} conc-list: ${{ toJson(matrix.config.conc) }} spec-decoding: ${{ matrix.config.spec-decoding }} @@ -431,6 +435,8 @@ jobs: model-prefix: ${{ matrix.config.model-prefix }} framework: ${{ matrix.config.framework }} precision: ${{ matrix.config.precision }} + router: ${{ matrix.config.router && toJson(matrix.config.router) || '' }} + kv-transfer: ${{ matrix.config['kv-transfer'] && toJson(matrix.config['kv-transfer']) || '' }} tp: ${{ matrix.config.tp }} ep: ${{ matrix.config.ep }} dp-attn: ${{ matrix.config.dp-attn }} @@ -484,6 +490,8 @@ jobs: model-prefix: ${{ matrix.config.model-prefix }} framework: ${{ matrix.config.framework }} precision: ${{ matrix.config.precision }} + router: ${{ matrix.config.router && toJson(matrix.config.router) || '' }} + kv-transfer: ${{ matrix.config['kv-transfer'] && toJson(matrix.config['kv-transfer']) || '' }} tp: ${{ matrix.config.tp }} ep: ${{ matrix.config.ep }} dp-attn: ${{ matrix.config.dp-attn }} @@ -528,6 +536,8 @@ jobs: model-prefix: ${{ matrix.config.model-prefix }} framework: ${{ matrix.config.framework }} precision: ${{ matrix.config.precision }} + router: ${{ matrix.config.router && toJson(matrix.config.router) || '' }} + kv-transfer: ${{ matrix.config['kv-transfer'] && toJson(matrix.config['kv-transfer']) || '' }} conc-list: '[${{ matrix.config.conc }}]' spec-decoding: ${{ matrix.config.spec-decoding }} disagg: ${{ matrix.config.disagg }} @@ -577,6 +587,8 @@ jobs: model-prefix: ${{ matrix.config.model-prefix }} framework: ${{ matrix.config.framework }} precision: ${{ matrix.config.precision }} + router: ${{ matrix.config.router && toJson(matrix.config.router) || '' }} + kv-transfer: ${{ matrix.config['kv-transfer'] && toJson(matrix.config['kv-transfer']) || '' }} tp: ${{ matrix.config.tp }} ep: ${{ matrix.config.ep }} dp-attn: ${{ matrix.config.dp-attn }} @@ -615,6 +627,8 @@ jobs: model-prefix: ${{ matrix.config.model-prefix }} framework: ${{ matrix.config.framework }} precision: ${{ matrix.config.precision }} + router: ${{ matrix.config.router && toJson(matrix.config.router) || '' }} + kv-transfer: ${{ matrix.config['kv-transfer'] && toJson(matrix.config['kv-transfer']) || '' }} conc-list: ${{ toJson(matrix.config.conc) }} spec-decoding: ${{ matrix.config.spec-decoding }} disagg: ${{ matrix.config.disagg }} diff --git a/configs/CONFIGS.md b/configs/CONFIGS.md index f46c4efda1..b1c6c45ff9 100644 --- a/configs/CONFIGS.md +++ b/configs/CONFIGS.md @@ -12,6 +12,9 @@ entry-name: runner: string precision: string framework: string + # Optional defaults for every search-space entry in this config. + router: { name: string, version: string } + kv-transfer: { name: string, version: string } scenarios: fixed-seq-len: - isl: int @@ -20,6 +23,12 @@ entry-name: - { tp: int, conc-start: int, conc-end: int } # Optionally, specify 'ep' (expert-parallelism) and 'dp-attn' (data parallel attention) - { tp: int, ep: int, dp-attn: bool, conc-start: int, conc-end: int } + # Optionally, declare router and KV transfer component metadata. + - tp: int + router: { name: string, version: string } + kv-transfer: { name: string, version: string } + conc-start: int + conc-end: int - ... - ... agentic-coding: # optional @@ -52,10 +61,18 @@ The below list describes what each field is: - Note: the step factor between `conc-start` and `conc-end` is 2, so if `conc-start` is 4 and `conc-end` is 128, all concurrencies `4, 8, 16, 32, ..., 128` will be run. - (Optional) `ep`: An integer representing the expert parallelism level that the configuration will be served at. Default is 1 (no expert parallelism) when not specified. - (Optional) `dp-attn`: A boolean representing whether or not to activate data parallel attention for the configuration. Default is false when not specified. + - (Optional) `router`: Router metadata containing exactly non-empty `name` and `version` strings. + - (Optional) `kv-transfer`: KV transfer metadata containing exactly non-empty `name` and `version` strings. - `agentic-coding`: Agentic trace replay benchmarks using real conversation traces. Each entry must have: - `trace-source`: Identifier for the trace dataset to use. - `search-space`: Same structure as `fixed-seq-len` search-space entries. +`router` and `kv-transfer` may be omitted independently. When either field is +present, both `name` and `version` are required and additional keys are rejected. +Top-level declarations apply to every scenario and search-space entry in that +master config. A search-space declaration overrides the corresponding top-level +value for that point. + Agentic duration is not a master YAML field. Matrix generation defaults agentic jobs to 3600 seconds; reusable workflow callers may override the `duration` input. diff --git a/utils/agentic/aggregation/process_agentic_result.py b/utils/agentic/aggregation/process_agentic_result.py index 83211f4f33..f41edc7836 100644 --- a/utils/agentic/aggregation/process_agentic_result.py +++ b/utils/agentic/aggregation/process_agentic_result.py @@ -36,6 +36,24 @@ def required_env(name: str) -> str: return value +def optional_component_metadata(env_name: str) -> dict[str, str] | None: + """Parse strict optional component metadata from a JSON environment value.""" + raw_value = os.environ.get(env_name) + if raw_value in (None, "", "null"): + return None + + try: + metadata = json.loads(raw_value) + except json.JSONDecodeError as exc: + raise SystemExit(f"{env_name} must contain valid JSON") from exc + + if not isinstance(metadata, dict) or set(metadata) != {"name", "version"}: + raise SystemExit(f"{env_name} must contain exactly 'name' and 'version'") + if not all(isinstance(metadata[key], str) and metadata[key] for key in metadata): + raise SystemExit(f"{env_name} name and version must be non-empty strings") + return metadata + + def _validate_kv_offload_env() -> tuple[str, str]: kv_offloading = required_env("KV_OFFLOADING") kv_offload_backend = os.environ.get("KV_OFFLOAD_BACKEND", "") @@ -139,6 +157,14 @@ def build_agg( } agg.update(multinode_fields) + for env_name, result_field in ( + ("ROUTER_METADATA", "router"), + ("KV_TRANSFER_METADATA", "kv_transfer"), + ): + metadata = optional_component_metadata(env_name) + if metadata is not None: + agg[result_field] = metadata + metadata = aggregate.get("metadata") if isinstance(metadata, dict): dataset = metadata.get("dataset") diff --git a/utils/agentic/aggregation/test_process_agentic_result.py b/utils/agentic/aggregation/test_process_agentic_result.py index a2bac8ea52..db33c074c1 100644 --- a/utils/agentic/aggregation/test_process_agentic_result.py +++ b/utils/agentic/aggregation/test_process_agentic_result.py @@ -375,6 +375,29 @@ def test_processor_preserves_dataset_provenance(tmp_path: Path): } +def test_processor_emits_component_metadata_when_present(tmp_path: Path): + result_dir = _write_fixture(tmp_path) + agg = _run_processor( + result_dir, + tmp_path / "out", + env_overrides={ + "ROUTER_METADATA": json.dumps({"name": "vllm-router", "version": "0.1.14"}), + "KV_TRANSFER_METADATA": json.dumps({"name": "mooncake", "version": "0.3.11.post1"}), + }, + ) + + assert agg["router"] == {"name": "vllm-router", "version": "0.1.14"} + assert agg["kv_transfer"] == {"name": "mooncake", "version": "0.3.11.post1"} + + +def test_processor_omits_component_metadata_when_absent(tmp_path: Path): + result_dir = _write_fixture(tmp_path) + agg = _run_processor(result_dir, tmp_path / "out") + + assert "router" not in agg + assert "kv_transfer" not in agg + + def test_processor_latency_units_are_seconds(tmp_path: Path): """aiperf reports ms; legacy schema is seconds. Verify conversion.""" result_dir = _write_fixture(tmp_path) diff --git a/utils/matrix_logic/generate_sweep_configs.py b/utils/matrix_logic/generate_sweep_configs.py index 545eb9f969..dc3bd07b02 100644 --- a/utils/matrix_logic/generate_sweep_configs.py +++ b/utils/matrix_logic/generate_sweep_configs.py @@ -114,6 +114,16 @@ def agentic_kv_offload_suffix(kv_offloading: str, kv_offload_backend: str | None return f"kv{kv_offloading}-{kv_offload_backend}" +def component_metadata(benchmark: dict, config: dict) -> dict: + """Resolve optional component metadata, preferring search-space overrides.""" + metadata = {} + for field in (Fields.ROUTER, Fields.KV_TRANSFER): + value = benchmark.get(field.value, config.get(field.value)) + if value is not None: + metadata[field.value] = value + return metadata + + def chunk_multinode_agentic_concurrencies(conc_values: list[int]) -> list[list[int]]: """Bound sequential agentic profiles sharing one server allocation.""" size = MAX_MULTINODE_AGENTIC_CONCURRENCIES_PER_ALLOCATION @@ -457,6 +467,7 @@ def generate_full_sweep(args, all_config_data, runner_data): Fields.DISAGG.value: disagg, Fields.RUN_EVAL.value: False, # Default, may be overridden by mark_eval_entries } + entry.update(component_metadata(bmk, val)) validate_matrix_entry(entry, is_multinode) matrix_values.append(entry) @@ -570,6 +581,7 @@ def generate_full_sweep(args, all_config_data, runner_data): if dp_attn is not None: entry[Fields.DP_ATTN.value] = dp_attn + entry.update(component_metadata(bmk, val)) validate_matrix_entry(entry, is_multinode) matrix_values.append(entry) @@ -654,6 +666,7 @@ def generate_full_sweep(args, all_config_data, runner_data): Fields.DISAGG.value: disagg, Fields.SCENARIO_TYPE.value: "agentic-coding", } + entry.update(component_metadata(bmk, val)) validate_agentic_matrix_entry(entry) matrix_values.append(entry) else: @@ -681,6 +694,7 @@ def generate_full_sweep(args, all_config_data, runner_data): } if kv_offload_backend is not None: entry[Fields.KV_OFFLOAD_BACKEND.value] = kv_offload_backend + entry.update(component_metadata(bmk, val)) validate_agentic_matrix_entry(entry) matrix_values.append(entry) @@ -797,6 +811,7 @@ def generate_test_config_sweep(args, all_config_data, runner_data=None): Fields.DISAGG.value: disagg, Fields.RUN_EVAL.value: False, } + entry.update(component_metadata(bmk, val)) matrix_values.append(validate_matrix_entry(entry, is_multinode=True)) else: # Single-node config @@ -849,6 +864,7 @@ def generate_test_config_sweep(args, all_config_data, runner_data=None): Fields.DISAGG.value: disagg, Fields.RUN_EVAL.value: False, } + entry.update(component_metadata(bmk, val)) matrix_values.append(validate_matrix_entry(entry, is_multinode=False)) # ---- Agentic-coding scenarios ---- @@ -921,6 +937,7 @@ def generate_test_config_sweep(args, all_config_data, runner_data=None): Fields.DISAGG.value: disagg, Fields.SCENARIO_TYPE.value: "agentic-coding", } + entry.update(component_metadata(bmk, val)) matrix_values.append(validate_agentic_matrix_entry(entry)) else: for conc in conc_values: @@ -947,6 +964,7 @@ def generate_test_config_sweep(args, all_config_data, runner_data=None): } if kv_offload_backend is not None: entry[Fields.KV_OFFLOAD_BACKEND.value] = kv_offload_backend + entry.update(component_metadata(bmk, val)) matrix_values.append(validate_agentic_matrix_entry(entry)) return matrix_values diff --git a/utils/matrix_logic/test_generate_sweep_configs.py b/utils/matrix_logic/test_generate_sweep_configs.py index 0e2c64d95e..f3f26adfae 100644 --- a/utils/matrix_logic/test_generate_sweep_configs.py +++ b/utils/matrix_logic/test_generate_sweep_configs.py @@ -1831,6 +1831,9 @@ class TestGenerateTestConfigSweep: def test_runner_node_filter_expands_config_runner(self, sample_multinode_config, sample_runner_config): """test-config should allow targeting one concrete runner node.""" + master_entry = sample_multinode_config["dsr1-fp4-gb200-dynamo-trt"] + master_entry["router"] = {"name": "trt-router", "version": "0.20.0"} + master_entry["kv-transfer"] = {"name": "nixl", "version": "0.7.0"} args = argparse.Namespace( config_keys=["dsr1-fp4-gb200-dynamo-trt"], seq_lens=None, @@ -1846,6 +1849,8 @@ def test_runner_node_filter_expands_config_runner(self, sample_multinode_config, assert len(result) == 1 assert result[0]["runner"] == "gb200-nv_0" + assert result[0]["router"] == {"name": "trt-router", "version": "0.20.0"} + assert result[0]["kv-transfer"] == {"name": "nixl", "version": "0.7.0"} def test_runner_node_filter_no_match_skips_config(self, sample_multinode_config, sample_runner_config): """Unmatched node filters should produce no entries.""" @@ -1875,6 +1880,8 @@ def test_runner_node_filter_expands_agentic_config_runner(self, sample_runner_co "framework": "sglang", "runner": "cluster:b300-nv", "multinode": False, + "router": {"name": "default-router", "version": "1.0.0"}, + "kv-transfer": {"name": "default-transfer", "version": "1.0.0"}, "scenarios": { "agentic-coding": [ { @@ -1885,6 +1892,8 @@ def test_runner_node_filter_expands_agentic_config_runner(self, sample_runner_co "ep": 1, "kv-offloading": "dram", "kv-offload-backend": "hicache", + "router": {"name": "sglang-router", "version": "0.3.0"}, + "kv-transfer": {"name": "moonstone", "version": "1.2.3"}, "conc-list": [64], } ], @@ -1908,6 +1917,8 @@ def test_runner_node_filter_expands_agentic_config_runner(self, sample_runner_co assert result[0]["scenario-type"] == "agentic-coding" assert result[0]["total-cpu-dram-gb"] == 2399 assert result[0]["duration"] == 3600 + assert result[0]["router"] == {"name": "sglang-router", "version": "0.3.0"} + assert result[0]["kv-transfer"] == {"name": "moonstone", "version": "1.2.3"} def test_agentic_node_dram_uses_explicit_gpu_count(self, sample_runner_config): config = { @@ -2004,6 +2015,8 @@ def test_multinode_agentic_groups_concurrencies_per_search_entry(self): "search-space": [ { "conc-list": [16, 32, 64, 128, 256], + "router": {"name": "dynamo-router", "version": "1.3.0"}, + "kv-transfer": {"name": "nixl", "version": "0.7.0"}, "prefill": {"num-worker": 2, "tp": 8, "ep": 8, "dp-attn": False}, "decode": {"num-worker": 1, "tp": 8, "ep": 1, "dp-attn": False}, } @@ -2028,6 +2041,8 @@ def test_multinode_agentic_groups_concurrencies_per_search_entry(self): assert result[0]["exp-name"] == "dsv4_p2x8_d1x8_conc16x32x64x128" assert result[1]["conc"] == [256] assert result[1]["exp-name"] == "dsv4_p2x8_d1x8_conc256" + assert all(entry["router"] == {"name": "dynamo-router", "version": "1.3.0"} for entry in result) + assert all(entry["kv-transfer"] == {"name": "nixl", "version": "0.7.0"} for entry in result) # ============================================================================= diff --git a/utils/matrix_logic/test_validation.py b/utils/matrix_logic/test_validation.py index 7e794331f5..82fc3348ee 100644 --- a/utils/matrix_logic/test_validation.py +++ b/utils/matrix_logic/test_validation.py @@ -353,10 +353,42 @@ def test_arbitrary_backend_is_valid_for_agentic_search_space(self): "tp": 8, "kv-offloading": "dram", "kv-offload-backend": "future-backend", + "router": {"name": "vllm-router", "version": "0.1.14"}, + "kv-transfer": {"name": "mooncake", "version": "0.3.11.post1"}, "conc-list": [1, 2], }) assert entry.kv_offloading == "dram" assert entry.kv_offload_backend == "future-backend" + assert entry.router.name == "vllm-router" + assert entry.router.version == "0.1.14" + assert entry.kv_transfer.name == "mooncake" + + def test_component_metadata_is_optional_for_fixed_sequence_search_space(self): + entry = SingleNodeSearchSpaceEntry(**{ + "tp": 8, + "router": {"name": "vllm-router", "version": "0.1.14"}, + "kv-transfer": {"name": "nixl", "version": "0.7.0"}, + "conc-list": [1, 2], + }) + assert entry.router.name == "vllm-router" + assert entry.kv_transfer.version == "0.7.0" + + @pytest.mark.parametrize("field", ["router", "kv-transfer"]) + @pytest.mark.parametrize("metadata", [ + {"name": "vllm-router"}, + {"version": "0.1.14"}, + {"name": "vllm-router", "version": "0.1.14", "mode": "round-robin"}, + {"name": "", "version": "0.1.14"}, + {"name": "vllm-router", "version": ""}, + ]) + def test_component_metadata_requires_exact_non_empty_fields(self, field, metadata): + with pytest.raises(Exception): + AgenticCodingSearchSpaceEntry(**{ + "tp": 8, + "kv-offloading": "none", + field: metadata, + "conc-list": [1, 2], + }) def test_kv_offload_backend_requires_dram_mode(self): with pytest.raises(Exception, match="kv-offload-backend"): diff --git a/utils/matrix_logic/validation.py b/utils/matrix_logic/validation.py index 1f4986272a..605f77fe19 100644 --- a/utils/matrix_logic/validation.py +++ b/utils/matrix_logic/validation.py @@ -55,6 +55,8 @@ class Fields(Enum): # Agentic coding fields KV_OFFLOADING = 'kv-offloading' KV_OFFLOAD_BACKEND = 'kv-offload-backend' + ROUTER = 'router' + KV_TRANSFER = 'kv-transfer' TOTAL_CPU_DRAM_GB = 'total-cpu-dram-gb' AVAILABLE_CPU_DRAM_MIB = 'available-cpu-dram-mib' DRAM_UTILIZATION = 'dram-utilization' @@ -85,6 +87,14 @@ class Fields(Enum): """ +class ComponentMetadata(BaseModel): + """Strict name and version metadata for an optional runtime component.""" + model_config = ConfigDict(extra='forbid') + + name: str = Field(min_length=1) + version: str = Field(min_length=1) + + class SingleNodeMatrixEntry(BaseModel): """Pydantic model for validating single node matrix entry structure. This validates the input that should be expected to .github/workflows/benchmark-tmpl.yml""" @@ -110,6 +120,10 @@ class SingleNodeMatrixEntry(BaseModel): disagg: bool run_eval: bool = Field(alias=Fields.RUN_EVAL.value) eval_only: bool = Field(alias=Fields.EVAL_ONLY.value, default=False) + router: Optional[ComponentMetadata] = None + kv_transfer: Optional[ComponentMetadata] = Field( + default=None, alias=Fields.KV_TRANSFER.value + ) class WorkerConfig(BaseModel): @@ -152,6 +166,10 @@ class MultiNodeMatrixEntry(BaseModel): eval_all_concs: bool = Field( default=False, alias=Fields.EVAL_ALL_CONCS.value ) + router: Optional[ComponentMetadata] = None + kv_transfer: Optional[ComponentMetadata] = Field( + default=None, alias=Fields.KV_TRANSFER.value + ) class SingleNodeAgenticMatrixEntry(BaseModel): @@ -174,6 +192,10 @@ class SingleNodeAgenticMatrixEntry(BaseModel): kv_offload_backend: Optional[str] = Field( default=None, alias=Fields.KV_OFFLOAD_BACKEND.value ) + router: Optional[ComponentMetadata] = None + kv_transfer: Optional[ComponentMetadata] = Field( + default=None, alias=Fields.KV_TRANSFER.value + ) total_cpu_dram_gb: int = Field(alias=Fields.TOTAL_CPU_DRAM_GB.value, ge=0) duration: int = Field(alias=Fields.DURATION.value) exp_name: str = Field(alias=Fields.EXP_NAME.value) @@ -201,6 +223,10 @@ class MultiNodeAgenticMatrixEntry(BaseModel): decode: WorkerConfig conc: list[int] kv_offloading: Literal["none"] = Field(alias=Fields.KV_OFFLOADING.value) + router: Optional[ComponentMetadata] = None + kv_transfer: Optional[ComponentMetadata] = Field( + default=None, alias=Fields.KV_TRANSFER.value + ) duration: int = Field(alias=Fields.DURATION.value) exp_name: str = Field(alias=Fields.EXP_NAME.value) disagg: bool @@ -336,6 +362,10 @@ class SingleNodeSearchSpaceEntry(BaseModel): default="none", alias=Fields.SPEC_DECODING.value) dp_attn: Optional[bool] = Field( default=None, alias=Fields.DP_ATTN.value) + router: Optional[ComponentMetadata] = None + kv_transfer: Optional[ComponentMetadata] = Field( + default=None, alias=Fields.KV_TRANSFER.value + ) conc_start: Optional[int] = Field( default=None, alias=Fields.CONC_START.value) conc_end: Optional[int] = Field( @@ -356,6 +386,10 @@ class MultiNodeSearchSpaceEntry(BaseModel): default="none", alias=Fields.SPEC_DECODING.value) prefill: WorkerConfig decode: WorkerConfig + router: Optional[ComponentMetadata] = None + kv_transfer: Optional[ComponentMetadata] = Field( + default=None, alias=Fields.KV_TRANSFER.value + ) conc_start: Optional[int] = Field( default=None, alias=Fields.CONC_START.value) conc_end: Optional[int] = Field( @@ -405,6 +439,10 @@ class AgenticCodingSearchSpaceEntry(BaseModel): kv_offload_backend: Optional[str] = Field( default=None, alias=Fields.KV_OFFLOAD_BACKEND.value ) + router: Optional[ComponentMetadata] = None + kv_transfer: Optional[ComponentMetadata] = Field( + default=None, alias=Fields.KV_TRANSFER.value + ) conc_start: Optional[int] = Field(default=None, alias=Fields.CONC_START.value) conc_end: Optional[int] = Field(default=None, alias=Fields.CONC_END.value) conc_list: Optional[List[int]] = Field(default=None, alias=Fields.CONC_LIST.value) @@ -501,6 +539,10 @@ class SingleNodeMasterConfigEntry(BaseModel): runner: str multinode: Literal[False] disagg: bool = Field(default=False) + router: Optional[ComponentMetadata] = None + kv_transfer: Optional[ComponentMetadata] = Field( + default=None, alias=Fields.KV_TRANSFER.value + ) scenarios: SingleNodeScenarios @model_validator(mode='after') @@ -521,6 +563,10 @@ class MultiNodeMasterConfigEntry(BaseModel): runner: str multinode: Literal[True] disagg: bool = Field(default=False) + router: Optional[ComponentMetadata] = None + kv_transfer: Optional[ComponentMetadata] = Field( + default=None, alias=Fields.KV_TRANSFER.value + ) scenarios: MultiNodeScenarios @model_validator(mode='after') diff --git a/utils/process_result.py b/utils/process_result.py index 5fb0594739..efdc163f59 100644 --- a/utils/process_result.py +++ b/utils/process_result.py @@ -22,6 +22,24 @@ def get_required_env_vars(required_vars): return env_values +def get_optional_component_metadata(env_var): + """Parse strict optional component metadata from a JSON environment value.""" + raw_value = os.environ.get(env_var) + if raw_value in (None, "", "null"): + return None + + try: + metadata = json.loads(raw_value) + except json.JSONDecodeError as exc: + raise ValueError(f"{env_var} must contain valid JSON") from exc + + if not isinstance(metadata, dict) or set(metadata) != {"name", "version"}: + raise ValueError(f"{env_var} must contain exactly 'name' and 'version'") + if not all(isinstance(metadata[key], str) and metadata[key] for key in metadata): + raise ValueError(f"{env_var} name and version must be non-empty strings") + return metadata + + # Base required env vars base_env = get_required_env_vars([ 'RUNNER_TYPE', 'FRAMEWORK', 'PRECISION', 'SPEC_DECODING', @@ -56,6 +74,14 @@ def get_required_env_vars(required_vars): 'osl': int(osl), } +for env_var, result_field in ( + ('ROUTER_METADATA', 'router'), + ('KV_TRANSFER_METADATA', 'kv_transfer'), +): + metadata = get_optional_component_metadata(env_var) + if metadata is not None: + data[result_field] = metadata + is_multinode = os.environ.get('IS_MULTINODE', 'false').lower() == 'true' if is_multinode: diff --git a/utils/test_process_result.py b/utils/test_process_result.py index 4037689eaf..ec13de3d4e 100644 --- a/utils/test_process_result.py +++ b/utils/test_process_result.py @@ -239,6 +239,36 @@ def test_multinode_processing(self, tmp_path, sample_benchmark_result, multinode assert output_data["output_tput_per_gpu"] == pytest.approx(12000.0 / 8) # decode gpus assert output_data["input_tput_per_gpu"] == pytest.approx((15000.5 - 12000.0) / 20) # prefill gpus + def test_component_metadata_is_emitted_when_present( + self, tmp_path, sample_benchmark_result, single_node_env_vars + ): + env = { + **single_node_env_vars, + "ROUTER_METADATA": json.dumps({"name": "vllm-router", "version": "0.1.14"}), + "KV_TRANSFER_METADATA": json.dumps({"name": "mooncake", "version": "0.3.11.post1"}), + } + + result = run_script(tmp_path, env, sample_benchmark_result) + + assert result.returncode == 0, f"Script failed: {result.stderr}" + output_data = json.loads(result.stdout) + assert output_data["router"] == {"name": "vllm-router", "version": "0.1.14"} + assert output_data["kv_transfer"] == {"name": "mooncake", "version": "0.3.11.post1"} + + @pytest.mark.parametrize("metadata", [ + {"name": "vllm-router"}, + {"name": "vllm-router", "version": "0.1.14", "mode": "round-robin"}, + ]) + def test_component_metadata_rejects_partial_or_extra_fields( + self, tmp_path, sample_benchmark_result, single_node_env_vars, metadata + ): + env = {**single_node_env_vars, "ROUTER_METADATA": json.dumps(metadata)} + + result = run_script(tmp_path, env, sample_benchmark_result) + + assert result.returncode != 0 + assert "must contain exactly 'name' and 'version'" in result.stderr + def test_missing_base_env_vars(self, tmp_path, sample_benchmark_result): """Test that missing base env vars causes failure.""" result_file = tmp_path / "benchmark_result.json"