From b2a5cd17734f81959f9e9e3bf4c4c2d21ea80043 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sat, 4 Jul 2026 06:59:16 +0800
Subject: [PATCH 001/210] feat(collectivex): gate and publish v1 artifacts
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Require an explicit V1 release tag and the locked full-matrix digest before a sweep can become publication input. Promote exactly three successful tagged runs from one source SHA in disposable runner storage, then retain only the verified sanitized NDJSON artifact for JIT delivery.
中文：为 CollectiveX V1 增加显式发布门禁与产物发布流程。只有显式选择 V1 release tag 且匹配固定完整矩阵摘要的扫描运行才能作为发布输入。发布工作流只接收来自同一 source SHA 的三个成功运行，在 runner 临时存储中完成 promotion，并仅保留通过验证和清理的 NDJSON 产物供前端即时读取。
---
.github/workflows/collectivex-publish.yml | 197 +
.github/workflows/collectivex-sweep.yml | 464 ++-
experimental/CollectiveX/.gitignore | 15 +
experimental/CollectiveX/README.md | 160 +
experimental/CollectiveX/README_zh.md | 154 +
experimental/CollectiveX/artifact_safety.py | 212 ++
experimental/CollectiveX/capability.py | 232 ++
experimental/CollectiveX/configs/suites.yaml | 36 +
.../CollectiveX/configs/workloads.yaml | 9 +
experimental/CollectiveX/contracts.py | 2823 ++++++++++++++
experimental/CollectiveX/docs/methodology.md | 305 ++
.../CollectiveX/docs/methodology_zh.md | 297 ++
experimental/CollectiveX/identity.py | 201 +
.../CollectiveX/launchers/launch_gb-nv.sh | 110 +
.../CollectiveX/launchers/launch_mi-amds.sh | 160 +
.../launchers/launch_single-slurm.sh | 154 +
experimental/CollectiveX/publisher.py | 3284 +++++++++++++++++
experimental/CollectiveX/requirements.txt | 8 +
experimental/CollectiveX/runtime/common.sh | 2339 ++++++++++++
.../CollectiveX/runtime/run_in_container.sh | 1087 ++++++
.../schemas/channel-v1.schema.json | 23 +
.../schemas/private-bundle-v1.schema.json | 162 +
.../schemas/public-dataset-v1.schema.json | 606 +++
.../schemas/raw-case-v1.schema.json | 1199 ++++++
.../schemas/samples-v1.schema.json | 80 +
.../schemas/terminal-outcome-v1.schema.json | 289 ++
experimental/CollectiveX/source_archive.py | 349 ++
experimental/CollectiveX/summarize.py | 105 +
experimental/CollectiveX/sweep_matrix.py | 1031 ++++++
experimental/CollectiveX/tests/ep_deepep.py | 294 ++
.../CollectiveX/tests/ep_deepep_hybrid.py | 457 +++
.../CollectiveX/tests/ep_deepep_v2.py | 528 +++
experimental/CollectiveX/tests/ep_harness.py | 1780 +++++++++
experimental/CollectiveX/tests/ep_mori.py | 359 ++
experimental/CollectiveX/tests/ep_nccl.py | 186 +
experimental/CollectiveX/tests/ep_uccl.py | 405 ++
experimental/CollectiveX/tests/eplb.py | 199 +
.../CollectiveX/tests/make_workloads.py | 119 +
experimental/CollectiveX/tests/routing.py | 191 +
experimental/CollectiveX/tests/run_ep.py | 427 +++
.../tests/test_deepep_v2_contract.py | 2151 +++++++++++
.../CollectiveX/tests/test_publisher.py | 2418 ++++++++++++
.../tests/test_sampling_contract.py | 3213 ++++++++++++++++
experimental/CollectiveX/tests/workload.py | 358 ++
44 files changed, 29085 insertions(+), 91 deletions(-)
create mode 100644 .github/workflows/collectivex-publish.yml
create mode 100644 experimental/CollectiveX/.gitignore
create mode 100644 experimental/CollectiveX/README.md
create mode 100644 experimental/CollectiveX/README_zh.md
create mode 100644 experimental/CollectiveX/artifact_safety.py
create mode 100644 experimental/CollectiveX/capability.py
create mode 100644 experimental/CollectiveX/configs/suites.yaml
create mode 100644 experimental/CollectiveX/configs/workloads.yaml
create mode 100644 experimental/CollectiveX/contracts.py
create mode 100644 experimental/CollectiveX/docs/methodology.md
create mode 100644 experimental/CollectiveX/docs/methodology_zh.md
create mode 100644 experimental/CollectiveX/identity.py
create mode 100644 experimental/CollectiveX/launchers/launch_gb-nv.sh
create mode 100644 experimental/CollectiveX/launchers/launch_mi-amds.sh
create mode 100644 experimental/CollectiveX/launchers/launch_single-slurm.sh
create mode 100644 experimental/CollectiveX/publisher.py
create mode 100644 experimental/CollectiveX/requirements.txt
create mode 100644 experimental/CollectiveX/runtime/common.sh
create mode 100644 experimental/CollectiveX/runtime/run_in_container.sh
create mode 100644 experimental/CollectiveX/schemas/channel-v1.schema.json
create mode 100644 experimental/CollectiveX/schemas/private-bundle-v1.schema.json
create mode 100644 experimental/CollectiveX/schemas/public-dataset-v1.schema.json
create mode 100644 experimental/CollectiveX/schemas/raw-case-v1.schema.json
create mode 100644 experimental/CollectiveX/schemas/samples-v1.schema.json
create mode 100644 experimental/CollectiveX/schemas/terminal-outcome-v1.schema.json
create mode 100644 experimental/CollectiveX/source_archive.py
create mode 100644 experimental/CollectiveX/summarize.py
create mode 100644 experimental/CollectiveX/sweep_matrix.py
create mode 100644 experimental/CollectiveX/tests/ep_deepep.py
create mode 100644 experimental/CollectiveX/tests/ep_deepep_hybrid.py
create mode 100644 experimental/CollectiveX/tests/ep_deepep_v2.py
create mode 100644 experimental/CollectiveX/tests/ep_harness.py
create mode 100644 experimental/CollectiveX/tests/ep_mori.py
create mode 100644 experimental/CollectiveX/tests/ep_nccl.py
create mode 100644 experimental/CollectiveX/tests/ep_uccl.py
create mode 100644 experimental/CollectiveX/tests/eplb.py
create mode 100644 experimental/CollectiveX/tests/make_workloads.py
create mode 100644 experimental/CollectiveX/tests/routing.py
create mode 100644 experimental/CollectiveX/tests/run_ep.py
create mode 100644 experimental/CollectiveX/tests/test_deepep_v2_contract.py
create mode 100644 experimental/CollectiveX/tests/test_publisher.py
create mode 100644 experimental/CollectiveX/tests/test_sampling_contract.py
create mode 100644 experimental/CollectiveX/tests/workload.py
diff --git a/.github/workflows/collectivex-publish.yml b/.github/workflows/collectivex-publish.yml
new file mode 100644
index 0000000000..38143123b9
--- /dev/null
+++ b/.github/workflows/collectivex-publish.yml
@@ -0,0 +1,197 @@
+name: CollectiveX Publish V1
+
+on:
+ workflow_dispatch:
+ inputs:
+ run_ids:
+ description: Three successful CollectiveX Sweep run IDs, comma-separated
+ required: true
+ type: string
+
+permissions:
+ actions: read
+ contents: read
+
+concurrency:
+ group: collectivex-publish-v1
+ cancel-in-progress: false
+
+jobs:
+ publish:
+ runs-on: ubuntu-latest
+ timeout-minutes: 30
+ env:
+ GH_TOKEN: ${{ github.token }}
+ RUN_IDS: ${{ inputs.run_ids }}
+ steps:
+ - name: Verify source runs
+ id: runs
+ env:
+ REPOSITORY: ${{ github.repository }}
+ run: |
+ set -euo pipefail
+ IFS=',' read -r -a run_ids <<< "$RUN_IDS"
+ [ "${#run_ids[@]}" -eq 3 ] || {
+ echo 'run_ids must contain exactly three IDs' >&2
+ exit 1
+ }
+ [ "$(printf '%s\n' "${run_ids[@]}" | sort -u | wc -l)" -eq 3 ] || {
+ echo 'run_ids must be unique' >&2
+ exit 1
+ }
+
+ : > "$RUNNER_TEMP/collectivex-runs.tsv"
+ source_sha=''
+ for run_id in "${run_ids[@]}"; do
+ [[ "$run_id" =~ ^[1-9][0-9]*$ ]] || {
+ echo 'run_ids contains a non-decimal ID' >&2
+ exit 1
+ }
+ metadata=$(gh api "repos/$REPOSITORY/actions/runs/$run_id")
+ name=$(jq -r '.name' <<< "$metadata")
+ path=$(jq -r '.path' <<< "$metadata")
+ branch=$(jq -r '.head_branch' <<< "$metadata")
+ status=$(jq -r '.status' <<< "$metadata")
+ conclusion=$(jq -r '.conclusion' <<< "$metadata")
+ sha=$(jq -r '.head_sha' <<< "$metadata")
+ attempt=$(jq -r '.run_attempt' <<< "$metadata")
+ [ "$name" = 'CollectiveX Sweep' ] \
+ && [ "$path" = '.github/workflows/collectivex-sweep.yml' ] \
+ && [ "$branch" = 'collectivex' ] \
+ && [ "$status" = 'completed' ] \
+ && [ "$conclusion" = 'success' ] \
+ && [[ "$sha" =~ ^[0-9a-f]{40}$ ]] \
+ && [[ "$attempt" =~ ^[1-9][0-9]*$ ]] || {
+ echo "run $run_id is not an eligible CollectiveX V1 sweep" >&2
+ exit 1
+ }
+ if [ -z "$source_sha" ]; then
+ source_sha="$sha"
+ else
+ [ "$sha" = "$source_sha" ] || {
+ echo 'source runs do not share one source SHA' >&2
+ exit 1
+ }
+ fi
+ printf '%s\t%s\t%s\n' "$run_id" "$attempt" "$sha" \
+ >> "$RUNNER_TEMP/collectivex-runs.tsv"
+ done
+ echo "source_sha=$source_sha" >> "$GITHUB_OUTPUT"
+
+ - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v5.0.0
+ with:
+ ref: ${{ steps.runs.outputs.source_sha }}
+ clean: true
+ persist-credentials: false
+
+ - name: Install publisher dependencies
+ run: python3 -m pip install --quiet -r experimental/CollectiveX/requirements.txt
+
+ - name: Build promoted publication
+ env:
+ REPOSITORY: ${{ github.repository }}
+ run: |
+ set -euo pipefail
+ store="$RUNNER_TEMP/collectivex-publisher"
+ downloads="$RUNNER_TEMP/collectivex-downloads"
+ output="$RUNNER_TEMP/collectivex-publication"
+ umask 027
+ mkdir -m 750 "$store"
+ mkdir -m 750 "$downloads" "$output"
+ : > "$RUNNER_TEMP/collectivex-bundles.txt"
+
+ while IFS=$'\t' read -r run_id attempt source_sha; do
+ run_dir="$downloads/$run_id"
+ mkdir -m 750 "$run_dir"
+ gh run download "$run_id" --repo "$REPOSITORY" --dir "$run_dir"
+ matrix="$run_dir/cxsweep-matrix-$run_id-$attempt/matrix_full.json"
+ marker="$run_dir/cxrelease-v1-$run_id-$attempt/release.json"
+ [ -f "$matrix" ] || {
+ echo "run $run_id is missing its exact matrix artifact" >&2
+ exit 1
+ }
+ [ -f "$marker" ] || {
+ echo "run $run_id is not tagged for V1 publication" >&2
+ exit 1
+ }
+ matrix_sha=$(sha256sum "$matrix" | cut -d' ' -f1)
+ jq -e \
+ --arg run_id "$run_id" \
+ --arg attempt "$attempt" \
+ --arg source_sha "$source_sha" \
+ --arg matrix_sha "$matrix_sha" \
+ 'keys == ["format","matrix_sha256","release_tag","run_attempt","run_id","source_sha"]
+ and .format == "collectivex.release-tag.v1"
+ and .release_tag == "v1"
+ and .run_id == $run_id
+ and .run_attempt == $attempt
+ and .source_sha == $source_sha
+ and .matrix_sha256 == $matrix_sha' \
+ "$marker" >/dev/null || {
+ echo "run $run_id has an invalid V1 release marker" >&2
+ exit 1
+ }
+
+ mapfile -t artifacts < <(
+ find "$run_dir" -mindepth 1 -maxdepth 1 -type d \
+ \( -name "cxshard-*-$run_id-$attempt" \
+ -o -name "cxunsupported-$run_id-$attempt" \) -print | sort
+ )
+ [ "${#artifacts[@]}" -gt 0 ] || {
+ echo "run $run_id has no result artifacts" >&2
+ exit 1
+ }
+ artifact_args=()
+ for artifact in "${artifacts[@]}"; do
+ artifact_args+=(--artifact "$artifact")
+ done
+ result=$(
+ python3 experimental/CollectiveX/publisher.py --store-root "$store" ingest \
+ --matrix "$matrix" \
+ "${artifact_args[@]}" \
+ --repository "$REPOSITORY" \
+ --run-id "$run_id" \
+ --run-attempt "$attempt" \
+ --source-sha "$source_sha"
+ )
+ bundle_id=$(jq -er '.bundle_id' <<< "$result")
+ printf '%s\n' "$bundle_id" >> "$RUNNER_TEMP/collectivex-bundles.txt"
+ done < "$RUNNER_TEMP/collectivex-runs.tsv"
+
+ mapfile -t bundle_ids < "$RUNNER_TEMP/collectivex-bundles.txt"
+ promote_args=()
+ for bundle_id in "${bundle_ids[@]}"; do
+ promote_args+=(--bundle "$bundle_id")
+ done
+ result=$(
+ python3 experimental/CollectiveX/publisher.py --store-root "$store" promote \
+ "${promote_args[@]}"
+ )
+ dataset_id=$(jq -er '.dataset_sha256' <<< "$result")
+ dataset="$store/public/datasets/$dataset_id/dataset.json"
+ [ -f "$dataset" ] || {
+ echo 'publisher did not install the promoted dataset' >&2
+ exit 1
+ }
+ publication="$output/collectivex_public_v1_$dataset_id.ndjson"
+ cp -- "$dataset" "$publication"
+ python3 experimental/CollectiveX/artifact_safety.py "$publication"
+ python3 experimental/CollectiveX/publisher.py --store-root "$store" verify \
+ --channel dev-latest "${promote_args[@]}"
+ sha256sum "$publication"
+ {
+ echo '## CollectiveX V1 publication'
+ echo
+ echo "Dataset: \`$dataset_id\`"
+ echo
+ echo 'Source runs:'
+ sed 's/^/- `/' "$RUNNER_TEMP/collectivex-runs.tsv" | sed 's/$/`/'
+ } >> "$GITHUB_STEP_SUMMARY"
+
+ - name: Upload JIT publication artifact
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: cxpublication-v1-${{ github.run_id }}-${{ github.run_attempt }}
+ path: ${{ runner.temp }}/collectivex-publication/*.ndjson
+ if-no-files-found: error
+ retention-days: 90
diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml
index b56bc285aa..2837a6da10 100644
--- a/.github/workflows/collectivex-sweep.yml
+++ b/.github/workflows/collectivex-sweep.yml
@@ -1,40 +1,46 @@
# CollectiveX Sweep — one structured run instead of thousands of dispatches.
#
-# Shape (mirrors the InferenceX CI tracker): setup -> sweep (a MATRIX job = "a job with other jobs
-# in it") -> aggregate (the collector "at the end"). The matrix unit is a SHARD = one allocation that
-# sweeps many cases sharing (sku, backend, mode, resource) — generate_matrix's own grouping, chunked
-# so no cell exceeds the job budget. Each cell emits a handful of per-case JSONs; the aggregate job
-# collects every shard into ONE line-delimited file (results/aggregate/*.ndjson) so there aren't
-# thousands of individual result files. Run once per backend (deepep / uccl / flashinfer /
-# deepep-hybrid / nccl-ep, + deepep_v2) for full parity.
+# Shape: setup -> sweep. The matrix unit is a shard: one allocation that sweeps
+# cases sharing (sku, backend, nodes). Each cell uploads its privacy-checked raw
+# result JSONs. The isolated v1 publisher consumes downloaded shards separately.
name: CollectiveX Sweep
+permissions:
+ contents: read
on:
workflow_dispatch:
inputs:
backend:
- description: EP library to sweep (deepep matrix is remapped onto the others, capability-filtered)
+ description: "EP library to sweep — 'all' runs every EP backend in one matrix"
type: choice
- default: deepep
- options: [deepep, uccl, flashinfer, deepep-hybrid, nccl-ep]
- deepep_v2:
- description: DeepEP V2 from-source kernels (kernel_gen=v2; deepep backend only)
- type: boolean
- default: false
+ default: all
+ options: [all, deepep, deepep-v2, uccl, deepep-hybrid, mori, nccl-ep]
suites:
description: "'all' or comma-list of suite names"
type: string
default: all
only_sku:
- description: Restrict to one SKU (h100-dgxc|h200|b300|b200-dgxc|gb200|gb300|mi355x); blank = all
+ description: Restrict to one GHA runner pool (h100-dgxc|h200-dgxc|b300|b200-dgxc|gb200|gb300|mi325x|mi355x); blank = all
+ type: string
+ default: ''
+ min_nodes:
+ description: Keep only shards with at least this node/tray count (2 keeps every EP16 and GB EP8; blank = all)
+ type: string
+ default: ''
+ max_nodes:
+ description: Keep only shards with at most this node/tray count (1 keeps non-GB EP8; blank = all)
type: string
default: ''
max_cases:
- description: Max cases per shard cell (chunk larger shards)
+ description: Max cases per shard cell before chunking into another GHA job (128 = no chunking for current suites)
type: string
- default: '14'
-
+ default: '128'
+ release_tag:
+ description: Publication gate; unversioned runs are diagnostic and cannot be published
+ type: choice
+ default: unversioned
+ options: [unversioned, v1]
concurrency:
- group: cx-sweep-${{ github.ref }}-${{ inputs.backend }}-${{ inputs.deepep_v2 }}-${{ inputs.only_sku }}
+ group: cx-sweep-${{ github.ref }}-${{ inputs.release_tag }}-${{ inputs.backend }}-${{ inputs.only_sku }}
cancel-in-progress: false
jobs:
@@ -46,26 +52,150 @@ jobs:
n: ${{ steps.gen.outputs.n }}
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v5.0.0
- with: { clean: true }
- - run: pip install --quiet pyyaml
+ with: { clean: true, persist-credentials: false }
+ - name: Install matrix dependencies
+ run: python3 -m pip install --quiet PyYAML==6.0.2
- id: gen
working-directory: experimental/CollectiveX
+ env:
+ INPUT_BACKEND: ${{ inputs.backend }}
+ INPUT_SUITES: ${{ inputs.suites }}
+ INPUT_ONLY_SKU: ${{ inputs.only_sku }}
+ INPUT_MIN_NODES: ${{ inputs.min_nodes }}
+ INPUT_MAX_NODES: ${{ inputs.max_nodes }}
+ INPUT_MAX_CASES: ${{ inputs.max_cases }}
+ COLLECTIVEX_SOURCE_SHA: ${{ github.sha }}
+ COLLECTIVEX_ARTIFACT_NAME: cxunsupported-${{ github.run_id }}-${{ github.run_attempt }}
+ COLLECTIVEX_EXECUTION_ID: ${{ github.run_id }}_${{ github.run_attempt }}_unsupported
+ run: |
+ set -euo pipefail
+ args=(--suites "$INPUT_SUITES" --max-cases "$INPUT_MAX_CASES")
+ case "$INPUT_BACKEND" in
+ all) args+=(--backends all) ;;
+ *) args+=(--backend "$INPUT_BACKEND") ;;
+ esac
+ [ -n "$INPUT_ONLY_SKU" ] && args+=(--only-sku "$INPUT_ONLY_SKU")
+ [ -n "$INPUT_MIN_NODES" ] && args+=(--min-nodes "$INPUT_MIN_NODES")
+ [ -n "$INPUT_MAX_NODES" ] && args+=(--max-nodes "$INPUT_MAX_NODES")
+ python3 sweep_matrix.py "${args[@]}" --out matrix_full.json >/dev/null
+ python3 artifact_safety.py matrix_full.json
+ SLIM=$(python3 -c "import json;m=json.load(open('matrix_full.json'));print(json.dumps({'include':[{k:v for k,v in x.items() if k!='case_ids'} for x in m['include']]}))")
+ {
+ echo "matrix=$SLIM"
+ echo "n=$(python3 -c "import json;print(len(json.load(open('matrix_full.json'))['include']))")"
+ echo "source_backends=$(python3 -c "import json;m=json.load(open('matrix_full.json'));print(' '.join(sorted({x['backend'] for x in m['include']} & {'deepep-v2','deepep-hybrid'})))")"
+ } >> "$GITHUB_OUTPUT"
+ unsupported_n=$(python3 -c "import json;m=json.load(open('matrix_full.json'));print(sum(x['disposition']=='unsupported' for x in m['requested_cases']))")
+ echo "unsupported_n=$unsupported_n" >> "$GITHUB_OUTPUT"
+ if [ "$unsupported_n" -gt 0 ]; then
+ python3 sweep_matrix.py --emit-unsupported-from matrix_full.json \
+ --out-dir unsupported
+ fi
+ python3 -c "import json;m=json.load(open('matrix_full.json'));r=m['requested_cases'];print('shard-cells:',len(m['include']),'runnable:',sum(x['disposition']=='runnable' for x in r),'unsupported:',sum(x['disposition']=='unsupported' for x in r))"
+ - name: Prepare pinned backend source archive
+ if: ${{ steps.gen.outputs.source_backends != '' }}
+ working-directory: experimental/CollectiveX
+ env:
+ SOURCE_BACKENDS: ${{ steps.gen.outputs.source_backends }}
+ COLLECTIVEX_EXECUTION_ID: ${{ github.run_id }}_${{ github.run_attempt }}_sources
run: |
set -euo pipefail
- ov=""; [ "${{ inputs.backend }}" != "deepep" ] && ov="--backend ${{ inputs.backend }}"
- v2=""; [ "${{ inputs.deepep_v2 }}" = "true" ] && v2="--deepep-v2"
- os=""; [ -n "${{ inputs.only_sku }}" ] && os="--only-sku ${{ inputs.only_sku }}"
- # full matrix (with cases) -> artifact for the cells; slim (no cases) -> the strategy output.
- python3 sweep_matrix.py --suites "${{ inputs.suites }}" --max-cases "${{ inputs.max_cases }}" $ov $v2 $os --out matrix_full.json >/dev/null
- SLIM=$(python3 -c "import json;m=json.load(open('matrix_full.json'));print(json.dumps({'include':[{k:v for k,v in x.items() if k!='cases'} for x in m['include']]}))")
- echo "matrix=$SLIM" >> "$GITHUB_OUTPUT"
- echo "n=$(python3 -c "import json;print(len(json.load(open('matrix_full.json'))['include']))")" >> "$GITHUB_OUTPUT"
- python3 -c "import json;m=json.load(open('matrix_full.json'));print('shard-cells:',len(m['include']),'cases:',sum(x['n'] for x in m['include']))"
+ source runtime/common.sh
+ work="$RUNNER_TEMP/collectivex-backend-sources"
+ archive="$RUNNER_TEMP/collectivex-backend-sources.tar"
+ rm -rf -- "$work" "$archive"
+ umask 077
+ mkdir -m 700 "$work"
+ mkdir -p "$work/experimental/CollectiveX"
+ read -r -a backends <<< "$SOURCE_BACKENDS"
+ [ "${#backends[@]}" -gt 0 ]
+ for backend in "${backends[@]}"; do
+ cx_prepare_backend_source "$work" "$backend"
+ done
+ cx_cleanup_private_logs 0
+ tar --sort=name --mtime='@1' --owner=0 --group=0 --numeric-owner \
+ -C "$work/experimental/CollectiveX" -cf "$archive" .cx_sources
+ sha256sum "$archive"
+ rm -rf -- "$work"
+ - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ if: ${{ steps.gen.outputs.source_backends != '' }}
+ with:
+ name: cxbackend-sources-${{ github.run_id }}-${{ github.run_attempt }}
+ path: ${{ runner.temp }}/collectivex-backend-sources.tar
+ if-no-files-found: error
+ retention-days: 3
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
- name: cxsweep-matrix-${{ github.run_id }}
+ name: cxsweep-matrix-${{ github.run_id }}-${{ github.run_attempt }}
path: experimental/CollectiveX/matrix_full.json
if-no-files-found: error
+ retention-days: 3
+ - name: Create V1 release marker
+ if: ${{ inputs.release_tag == 'v1' }}
+ env:
+ EXPECTED_MATRIX_SHA256: f1ca85f9689922b90edd5767b9ff2a902f6b896f32f68b2ca086dde3fd2157d0
+ RUN_ID: ${{ github.run_id }}
+ RUN_ATTEMPT: ${{ github.run_attempt }}
+ SOURCE_SHA: ${{ github.sha }}
+ run: |
+ set -euo pipefail
+ destination="$RUNNER_TEMP/collectivex-release"
+ install -d -m 700 "$destination"
+ python3 - "$destination/release.json" <<'PY'
+ import hashlib
+ import json
+ import os
+ import pathlib
+ import sys
+
+ matrix = pathlib.Path("experimental/CollectiveX/matrix_full.json").read_bytes()
+ matrix_sha256 = hashlib.sha256(matrix).hexdigest()
+ if matrix_sha256 != os.environ["EXPECTED_MATRIX_SHA256"]:
+ raise SystemExit("V1 release tag requires the locked full matrix")
+ marker = {
+ "format": "collectivex.release-tag.v1",
+ "matrix_sha256": matrix_sha256,
+ "release_tag": "v1",
+ "run_attempt": os.environ["RUN_ATTEMPT"],
+ "run_id": os.environ["RUN_ID"],
+ "source_sha": os.environ["SOURCE_SHA"],
+ }
+ pathlib.Path(sys.argv[1]).write_text(
+ json.dumps(marker, sort_keys=True, separators=(",", ":")) + "\n"
+ )
+ PY
+ python3 experimental/CollectiveX/artifact_safety.py "$destination/release.json"
+ - name: Upload V1 release marker
+ if: ${{ inputs.release_tag == 'v1' }}
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: cxrelease-v1-${{ github.run_id }}-${{ github.run_attempt }}
+ path: ${{ runner.temp }}/collectivex-release/release.json
+ if-no-files-found: error
+ retention-days: 3
+ - name: Validate unsupported artifact safety
+ id: unsupported_safety
+ if: ${{ always() && fromJSON(steps.gen.outputs.unsupported_n) > 0 }}
+ run: |
+ python3 experimental/CollectiveX/artifact_safety.py experimental/CollectiveX/unsupported/*.json
+ - name: Validate unsupported outcomes
+ id: unsupported_contracts
+ if: ${{ always() && fromJSON(steps.gen.outputs.unsupported_n) > 0 && steps.unsupported_safety.outcome == 'success' }}
+ env:
+ COLLECTIVEX_ARTIFACT_NAME: cxunsupported-${{ github.run_id }}-${{ github.run_attempt }}
+ COLLECTIVEX_EXECUTION_ID: ${{ github.run_id }}_${{ github.run_attempt }}_unsupported
+ run: |
+ python3 experimental/CollectiveX/contracts.py validate-delivery \
+ --source experimental/CollectiveX/matrix_full.json \
+ --disposition unsupported \
+ experimental/CollectiveX/unsupported/*.json
+ - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ if: ${{ always() && fromJSON(steps.gen.outputs.unsupported_n) > 0 && steps.unsupported_contracts.outcome == 'success' && steps.unsupported_safety.outcome == 'success' }}
+ with:
+ name: cxunsupported-${{ github.run_id }}-${{ github.run_attempt }}
+ path: experimental/CollectiveX/unsupported/*.json
+ if-no-files-found: error
+ retention-days: 3
# ---- sweep: ONE matrix cell per shard (the parent job with child jobs) ----
sweep:
@@ -73,82 +203,234 @@ jobs:
if: ${{ fromJSON(needs.setup.outputs.n) > 0 }}
strategy:
fail-fast: false
- max-parallel: 10 # don't saturate the ~20-runner fleet; cells queue as slots free
+ max-parallel: 10
matrix: ${{ fromJSON(needs.setup.outputs.matrix) }}
- # h200 label spans two clusters; pin to the validated dgxc pool (mirrors collectivex-experimental).
- runs-on: ${{ matrix.sku == 'h200' && 'h200-dgxc' || matrix.sku }}
+ runs-on: ${{ matrix.sku }}
timeout-minutes: 350
env:
CX_BENCH: ${{ matrix.backend }}
- CX_DEEPEP_V2: ${{ matrix.deepep_v2 && '1' || '' }}
CX_NODES: ${{ matrix.nodes }}
- CX_SHARD_FILE: results/.shard_${{ matrix.id }}.json
+ CX_GPUS_PER_NODE: ${{ matrix.gpus_per_node }}
+ CX_SCALE_UP_DOMAIN: ${{ matrix.scale_up_domain }}
+ CX_SHARD_FILE: .shards/${{ matrix.id }}.json
+ CX_SHARD_SKU: ${{ matrix.sku }}
+ COLLECTIVEX_CANONICAL_GHA: '1'
COLLECTIVEX_SOURCE_SHA: ${{ github.sha }}
- CX_NODELIST: ${{ matrix.sku == 'mi355x' && 'mia1-p01-g10,mia1-p01-g15' || '' }}
- CX_STAGE_DIR: ${{ matrix.sku == 'gb200' && '/mnt/lustre01/users-public/sa-shared/cx-stage' || '' }}
+ COLLECTIVEX_ARTIFACT_NAME: cxshard-${{ matrix.id }}-${{ github.run_id }}-${{ github.run_attempt }}
+ # Consolidated shards run one bounded build-group in one Slurm allocation, so
+ # the launcher's default 45-min --time is too short. 300 min covers a cold
+ # compute-node image import plus the shard. The allocation releases early
+ # when the shard finishes, so short shards don't waste it.
+ CX_TIME: '300'
+ COLLECTIVEX_EXECUTION_ID: ${{ github.run_id }}_${{ github.run_attempt }}_${{ matrix.id }}
+ CX_JOB_ROOT: /tmp/inferencex-collectivex-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.id }}
+ CX_SOURCE_ROOT: /tmp/inferencex-collectivex-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.id }}/source
+ HOME: /tmp/inferencex-collectivex-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.id }}/home
steps:
- - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v5.0.0
- with: { clean: true }
+ - name: Prepare isolated source
+ id: source
+ env:
+ COLLECTIVEX_REPOSITORY: ${{ github.repository }}
+ run: |
+ set -euo pipefail
+ python3 - <<'PY'
+ import os
+ import re
+ import shutil
+ import stat
+ import time
+
+ pattern = re.compile(r"inferencex-collectivex-[0-9]+-[0-9]+-[A-Za-z0-9._-]+")
+ cutoff = time.time() - 86400
+ for entry in os.scandir("/tmp"):
+ if not pattern.fullmatch(entry.name):
+ continue
+ try:
+ metadata = entry.stat(follow_symlinks=False)
+ except FileNotFoundError:
+ continue
+ if (
+ not stat.S_ISDIR(metadata.st_mode)
+ or metadata.st_uid != os.getuid()
+ or stat.S_IMODE(metadata.st_mode) != 0o700
+ or metadata.st_mtime >= cutoff
+ ):
+ continue
+ marked = False
+ for marker_name in ("cleanup-safe", "cleanup-unsafe"):
+ try:
+ marker = os.stat(
+ os.path.join(entry.path, marker_name), follow_symlinks=False
+ )
+ except FileNotFoundError:
+ continue
+ marked = (
+ stat.S_ISREG(marker.st_mode)
+ and marker.st_uid == os.getuid()
+ and stat.S_IMODE(marker.st_mode) == 0o600
+ )
+ if marked:
+ break
+ if marked:
+ shutil.rmtree(entry.path)
+ PY
+ [[ "$CX_JOB_ROOT" =~ ^/tmp/inferencex-collectivex-[0-9]+-[0-9]+-[A-Za-z0-9._-]+$ ]] \
+ || { echo "CollectiveX isolated root is invalid" >&2; exit 1; }
+ [ "$CX_SOURCE_ROOT" = "$CX_JOB_ROOT/source" ] \
+ || { echo "CollectiveX source root is invalid" >&2; exit 1; }
+ if [ -e "$CX_JOB_ROOT" ] || [ -L "$CX_JOB_ROOT" ]; then
+ echo "CollectiveX isolated root already exists" >&2
+ exit 1
+ fi
+ umask 077
+ mkdir -m 700 -- "$CX_JOB_ROOT"
+ trap 'rc=$?; [ "$rc" = 0 ] || rm -rf -- "$CX_JOB_ROOT"; exit "$rc"' EXIT
+ mkdir -m 700 -- "$HOME" "$CX_JOB_ROOT/control" "$CX_JOB_ROOT/artifact" "$CX_SOURCE_ROOT"
+ : > "$CX_JOB_ROOT/cleanup-safe"
+ if ! {
+ GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null git init -q "$CX_SOURCE_ROOT"
+ GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null \
+ git -C "$CX_SOURCE_ROOT" remote add origin \
+ "https://github.com/${COLLECTIVEX_REPOSITORY}.git"
+ GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null \
+ git -C "$CX_SOURCE_ROOT" -c credential.helper= -c protocol.version=2 \
+ fetch -q --no-tags --depth=1 origin "$COLLECTIVEX_SOURCE_SHA"
+ GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null \
+ git -C "$CX_SOURCE_ROOT" -c advice.detachedHead=false \
+ checkout -q --detach FETCH_HEAD
+ [ "$(git -C "$CX_SOURCE_ROOT" rev-parse HEAD)" = "$COLLECTIVEX_SOURCE_SHA" ]
+ } /dev/null 2>&1; then
+ echo "CollectiveX source preparation failed" >&2
+ exit 1
+ fi
+ [ "$(stat -c '%a' "$CX_JOB_ROOT")" = 700 ] \
+ || { echo "CollectiveX isolated root has unsafe permissions" >&2; exit 1; }
+ echo 'prepared=true' >> "$GITHUB_OUTPUT"
+ trap - EXIT
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
- name: cxsweep-matrix-${{ github.run_id }}
- path: experimental/CollectiveX
- - name: Extract this shard's cases (stdlib only — no runner deps)
- working-directory: experimental/CollectiveX
+ name: cxsweep-matrix-${{ github.run_id }}-${{ github.run_attempt }}
+ path: ${{ env.CX_JOB_ROOT }}/control
+ - name: Download pinned backend source archive
+ if: ${{ matrix.backend == 'deepep-v2' || matrix.backend == 'deepep-hybrid' }}
+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+ with:
+ name: cxbackend-sources-${{ github.run_id }}-${{ github.run_attempt }}
+ path: ${{ env.CX_JOB_ROOT }}/control
+ - name: Install pinned backend source seed
+ if: ${{ matrix.backend == 'deepep-v2' || matrix.backend == 'deepep-hybrid' }}
+ env:
+ EXPECTED_BACKEND: ${{ matrix.backend }}
run: |
set -euo pipefail
- python3 -c "
- import json
- m=json.load(open('matrix_full.json'))
- s=[x for x in m['include'] if x['id']=='${{ matrix.id }}']
- assert s, 'shard ${{ matrix.id }} not in matrix'
- s=s[0]
- json.dump({'id':s['id'],'sku':s['sku'],'backend':s['backend'],'nodes':s['nodes'],'deepep_v2':s['deepep_v2'],'cases':s['cases']}, open('results/.shard_${{ matrix.id }}.json','w'))
- print('shard ${{ matrix.id }}:', len(s['cases']), 'cases')
- "
+ archive="$CX_JOB_ROOT/control/collectivex-backend-sources.tar"
+ destination="$CX_SOURCE_ROOT/experimental/CollectiveX"
+ seed_root="$destination/.cx_sources"
+ [ -f "$archive" ] && [ ! -e "$seed_root" ] && [ ! -L "$seed_root" ]
+ source "$destination/runtime/common.sh"
+ source_path="$(cx_backend_source_path "$seed_root" "$EXPECTED_BACKEND")"
+ source_basename="${source_path#"$seed_root/"}"
+ [ -n "$source_basename" ] \
+ && [ "$source_path" = "$seed_root/$source_basename" ] \
+ && [[ "$source_basename" != */* ]]
+ python3 "$destination/source_archive.py" \
+ "$archive" "$destination" "$source_basename"
+ cx_backend_source_is_valid "$EXPECTED_BACKEND" "$source_path"
+ printf 'CX_BACKEND_SOURCE_SEED_ROOT=%s\n' "$seed_root" >> "$GITHUB_ENV"
+ - name: Extract and validate this shard's cases
+ run: |
+ set -euo pipefail
+ cd "$CX_SOURCE_ROOT/experimental/CollectiveX" 2>/dev/null \
+ || { echo "CollectiveX source is unavailable" >&2; exit 1; }
+ python3 sweep_matrix.py \
+ --extract-from "$CX_JOB_ROOT/control/matrix_full.json" \
+ --shard-id '${{ matrix.id }}' \
+ --expect-sku '${{ matrix.sku }}' \
+ --expect-backend '${{ matrix.backend }}' \
+ --expect-nodes '${{ matrix.nodes }}' \
+ --out '${{ env.CX_SHARD_FILE }}' >/dev/null
- name: Sweep shard ${{ matrix.id }} (${{ matrix.n }} cases, one allocation)
+ id: sweep_shard
env:
- RUNNER_NAME: ${{ runner.name }}
- run: bash "experimental/CollectiveX/launchers/launch_${RUNNER_NAME%%_*}.sh"
+ COLLECTIVEX_OPERATOR_CONFIG_CONTENT: ${{ secrets.COLLECTIVEX_OPERATOR_CONFIG_V1 }}
+ COLLECTIVEX_OPERATOR_CONFIG_REQUIRED: '1'
+ run: |
+ set -euo pipefail
+ umask 077
+ : > "$CX_JOB_ROOT/cleanup-unsafe"
+ rm -f -- "$CX_JOB_ROOT/cleanup-safe"
+ cd "$CX_SOURCE_ROOT" 2>/dev/null \
+ || { echo "CollectiveX source is unavailable" >&2; exit 1; }
+ bash "experimental/CollectiveX/launchers/launch_${{ matrix.launcher }}.sh"
+ - name: Confirm allocation cleanup
+ id: allocation_cleanup
+ if: ${{ always() && steps.source.outputs.prepared == 'true' }}
+ run: |
+ set -euo pipefail
+ [ -f "$CX_JOB_ROOT/cleanup-safe" ] && [ ! -e "$CX_JOB_ROOT/cleanup-unsafe" ] \
+ || { echo "CollectiveX allocation cleanup was not confirmed" >&2; exit 1; }
+ - name: Validate shard artifact safety
+ id: artifact_safety
+ if: ${{ always() && steps.allocation_cleanup.outcome == 'success' }}
+ run: |
+ cd "$CX_SOURCE_ROOT" 2>/dev/null \
+ || { echo "CollectiveX source is unavailable" >&2; exit 1; }
+ python3 experimental/CollectiveX/artifact_safety.py experimental/CollectiveX/results/*.json
+ - name: Validate shard delivery completeness
+ id: delivery_contracts
+ if: ${{ always() && steps.artifact_safety.outcome == 'success' }}
+ run: |
+ cd "$CX_SOURCE_ROOT" 2>/dev/null \
+ || { echo "CollectiveX source is unavailable" >&2; exit 1; }
+ python3 experimental/CollectiveX/contracts.py validate-delivery \
+ --source "experimental/CollectiveX/${CX_SHARD_FILE}" \
+ experimental/CollectiveX/results/*.json
- name: Shard summary
- if: always()
- run: python3 experimental/CollectiveX/summarize.py --results-dir experimental/CollectiveX/results --markdown >> "$GITHUB_STEP_SUMMARY" || true
+ if: ${{ always() && steps.artifact_safety.outcome == 'success' && steps.delivery_contracts.outcome == 'success' }}
+ run: |
+ cd "$CX_SOURCE_ROOT" 2>/dev/null \
+ || { echo "CollectiveX source is unavailable" >&2; exit 1; }
+ python3 experimental/CollectiveX/summarize.py \
+ --results-dir experimental/CollectiveX/results --markdown >> "$GITHUB_STEP_SUMMARY" || true
+ - name: Stage shard artifact
+ id: stage_artifact
+ if: ${{ always() && steps.delivery_contracts.outcome == 'success' && steps.artifact_safety.outcome == 'success' }}
+ run: |
+ set -euo pipefail
+ cd "$CX_SOURCE_ROOT" 2>/dev/null \
+ || { echo "CollectiveX source is unavailable" >&2; exit 1; }
+ cp -- experimental/CollectiveX/results/*.json "$CX_JOB_ROOT/artifact/"
- name: Upload shard results
- if: always()
+ id: upload_artifact
+ if: always() && steps.stage_artifact.outcome == 'success' && steps.delivery_contracts.outcome == 'success' && steps.artifact_safety.outcome == 'success'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
- name: cxshard-${{ matrix.id }}-${{ github.run_id }}
- path: experimental/CollectiveX/results/*.json # glob skips the hidden .shard_*.json
- if-no-files-found: warn
-
- # ---- aggregate: collect every shard into ONE ndjson (the "result aggregator at the end") ----
- aggregate:
- needs: sweep
- if: always()
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v5.0.0
- with: { clean: true }
- - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
- with:
- pattern: cxshard-*-${{ github.run_id }}
- path: _shards
- merge-multiple: true
- - name: Aggregate shards -> one ndjson
- working-directory: experimental/CollectiveX
+ name: cxshard-${{ matrix.id }}-${{ github.run_id }}-${{ github.run_attempt }}
+ path: |
+ ${{ env.CX_JOB_ROOT }}/artifact/*.json
+ if-no-files-found: error
+ retention-days: 3
+ - name: Cleanup isolated workspace
+ if: ${{ always() && steps.source.outputs.prepared == 'true' }}
run: |
set -euo pipefail
- tag="${{ inputs.backend }}${{ inputs.deepep_v2 && '-v2' || '' }}"
- python3 aggregate_results.py --in-dir ../../_shards --out "results/aggregate/collectivex_${tag}_${{ github.run_id }}.ndjson"
- {
- echo "## CollectiveX sweep aggregate (${tag})"
- echo '```'
- wc -l results/aggregate/*.ndjson 2>/dev/null || echo "no ndjson"
- echo '```'
- } >> "$GITHUB_STEP_SUMMARY"
- - name: Upload aggregate
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
- with:
- name: cxsweep-aggregate-${{ inputs.backend }}${{ inputs.deepep_v2 && '-v2' || '' }}-${{ github.run_id }}
- path: experimental/CollectiveX/results/aggregate/*.ndjson
- if-no-files-found: warn
+ [[ "$CX_JOB_ROOT" =~ ^/tmp/inferencex-collectivex-[0-9]+-[0-9]+-[A-Za-z0-9._-]+$ ]] \
+ || { echo "CollectiveX cleanup root is invalid" >&2; exit 1; }
+ [ "$CX_SOURCE_ROOT" = "$CX_JOB_ROOT/source" ] \
+ || { echo "CollectiveX cleanup source is invalid" >&2; exit 1; }
+ [ -f "$CX_JOB_ROOT/cleanup-safe" ] && [ ! -e "$CX_JOB_ROOT/cleanup-unsafe" ] \
+ || { echo "CollectiveX allocation cleanup was not confirmed; retaining isolated files" >&2; exit 1; }
+ if [ '${{ steps.sweep_shard.outcome }}' = success ] \
+ && [ '${{ steps.allocation_cleanup.outcome }}' = success ] \
+ && [ '${{ steps.artifact_safety.outcome }}' = success ] \
+ && [ '${{ steps.delivery_contracts.outcome }}' = success ] \
+ && [ '${{ steps.stage_artifact.outcome }}' = success ] \
+ && [ '${{ steps.upload_artifact.outcome }}' = success ] \
+ && [ -f "$CX_SOURCE_ROOT/experimental/CollectiveX/runtime/common.sh" ]; then
+ # shellcheck source=/dev/null
+ if source "$CX_SOURCE_ROOT/experimental/CollectiveX/runtime/common.sh" \
+ >/dev/null 2>&1; then
+ cx_cleanup_private_logs 0
+ fi
+ fi
+ rm -rf -- "$CX_JOB_ROOT"
diff --git a/experimental/CollectiveX/.gitignore b/experimental/CollectiveX/.gitignore
new file mode 100644
index 0000000000..56b307215b
--- /dev/null
+++ b/experimental/CollectiveX/.gitignore
@@ -0,0 +1,15 @@
+__pycache__/
+*.pyc
+results/
+unsupported/
+.shards/
+.cx_workloads/
+.cx_backend/
+/matrix_full.json
+gpucore.*
+
+# Local plans and infrastructure inventory.
+goal.md
+notes.md
+configs/platforms.yaml
+private-infra.md
diff --git a/experimental/CollectiveX/README.md b/experimental/CollectiveX/README.md
new file mode 100644
index 0000000000..91469935a0
--- /dev/null
+++ b/experimental/CollectiveX/README.md
@@ -0,0 +1,160 @@
+# CollectiveX
+
+
+
+**English** | [中文](./README_zh.md)
+
+
+
+CollectiveX is an experimental MoE expert-parallel communication benchmark. It measures dispatch,
+combine, and paired roundtrip latency across EP libraries and accelerator systems.
+
+> Publication hold: historical schema 3-5 data is diagnostic. No current dataset is approved for
+> rankings, recommendations, or regression baselines.
+
+## v1 Execution Profile
+
+Every scheduled case is BF16 with backend-tuned resources and packed placement. The explicit mode
+selects one of two contracts:
+
+- Normal mode uses `layout-and-dispatch-v1`, rank-deduplicated token payloads, and activation-only
+ combine. Uniform core coverage and one Zipf sensitivity remain; EPLB is measured only as the Zipf
+ remedy.
+- Low-latency mode uses `expert-packed-weighted-combine-v1`, token-expert payloads, and gate-weighted
+ combine through genuine DeepEP V1 or UCCL low-latency APIs. It is decode-only and never shares a
+ ranking cohort with normal mode. Other backends are explicitly unsupported for this suite.
+
+Both modes use `fixed-512-v1`: 64 trials x 8 timed iterations with 32 synchronized full roundtrip
+warmups before each measured component at every trial/point. Roundtrip is measured first; each
+iteration takes the cross-rank maximum before nearest-rank p50/p90/p95/p99, and roundtrip p99 is the
+headline latency. A stdlib integer counter produces byte-identical routing and gate weights.
+
+The canonical matrix covers H100, H200, B200, B300, GB200, GB300, MI325X, and MI355X. It requests
+608 cases / 1,600 token points: 364 runnable cases / 940 points, emitted as 58 executable workflow
+shards/allocation cells, plus 244 explicit unsupported cases / 660 points. `sweep_matrix.py`
+materializes every token ladder and rejects missing, stale, malformed, or altered shard controls.
+Shards are emitted round-robin by SKU so the bounded GHA matrix uses every runner pool early.
+
+| Systems | EP8 | EP16 |
+|---|---|---|
+| H100/H200/B200/B300 | 1x8 NVLink, scale-up | 2x8 NVLink + RDMA, scale-out |
+| MI325X/MI355X | 1x8 XGMI, scale-up | 2x8 XGMI + RDMA, scale-out |
+| GB200/GB300 | 2x4 MNNVL, scale-up | 4x4 MNNVL, scale-up |
+
+Physical host count does not determine scope: both GB topologies stay inside one 72-GPU MNNVL
+scale-up domain.
+
+| Backend | Current scope |
+|---|---|
+| DeepEP V1 | Image-pinned `deep_ep.Buffer`: normal and native low-latency APIs; upstream v1.2.1 on x86 and the image's GB fork on arm64 |
+| DeepEP V2 | PR #605 `ElasticBuffer` plus #630: LSA for scale-up and GIN for x86 EP16 scale-out; source/SASS-bound reproducible JIT |
+| DeepEP Hybrid | Pinned `HybridEPBuffer`: x86 EP16 multi-domain RDMA/DOCA; GB EP8/EP16 in one MNNVL communication domain |
+| UCCL | Pinned 0.1.1 wheel and wrapper with normal and native low-latency APIs on Hopper; Blackwell is explicitly unsupported |
+| NCCL/RCCL A2A | Portable rank-deduplicated payload plus expert/routing-metadata reference |
+| MoRI | EP8 uses MI325X AsyncLL or MI355X IntraNode; EP16 pins InterNodeV1 over 2x8 XGMI + RDMA |
+
+FlashInfer is outside v1 because its exercised EP path failed intermittently at runtime. It is not
+misreported as a platform capability limitation and can return after a stable pinned path is proven.
+
+DeepEP V2 means the `ElasticBuffer` implementation introduced by
+[DeepEP PR #605](https://github.com/deepseek-ai/DeepEP/pull/605), not a newer legacy `Buffer` build.
+The pinned source is the minimal upstream [PR #630](https://github.com/deepseek-ai/DeepEP/pull/630)
+follow-up: its parent is the #605 merge tree and its only source change fixes pure scale-up
+initialization when GIN is unavailable. Scale-up cases request NCCL Device API LSA and fail closed
+unless the realized LSA team covers the full EP world. x86 EP16 scale-out cases instead require the
+hybrid path with GIN, two logical scale-out domains represented by two physical RDMA ranks, and eight
+scale-up ranks per domain; GB EP16 remains MNNVL scale-up and therefore uses LSA. The isolated build
+records the API, source, loaded libraries, generated JIT source, executable SASS, and raw CUBIN
+diagnostics. The current H100 runner pool is explicitly unsupported for V2 because NCCL 2.30.4
+reports that its EP8 communicator lacks Device API symmetric-memory support; re-enabling that pool
+requires an all-rank CUDA P2P/LSA-capable runtime. Other NVIDIA SKUs remain unvalidated until their
+GPU outcomes pass the native correctness and publication gates.
+
+Removed v1 axes include cached-layout `[cl]`, runtime-visible `[rv]`, FP8, quantized combine,
+extra routing distributions, activation profiles, uneven allocation, placement permutations, model
+envelopes, and scaling studies.
+
+## Workflow And Artifacts
+
+`.github/workflows/collectivex-sweep.yml` generates a public-SKU matrix, extracts a strict ignored
+`.shards/.json` control, executes one allocation per shard, privacy-checks result JSON, and uploads
+raw GitHub artifacts. Runs default to `release_tag=unversioned` and are diagnostic-only. A V1 run
+must explicitly select `release_tag=v1`; setup then requires the locked full-matrix digest and emits
+a run/attempt/source-bound `cxrelease-v1-*` marker. Partial and filtered runs cannot receive it.
+
+`.github/workflows/collectivex-publish.yml` is an explicit V1 gate. It accepts exactly three
+successful tagged sweep run IDs from one source SHA, revalidates their GitHub metadata and release
+markers, and runs `publisher.py` in a disposable runner-local workspace. Only a fully promoted,
+privacy-checked, content-addressed dataset is uploaded as `cxpublication-v1-*`; raw artifacts and
+the private publisher workspace are never exposed to the frontend.
+
+There is no results server, attached store, Vercel storage, GCP, Neon, managed database, or managed
+object store. With the existing server-side `GITHUB_TOKEN`, the frontend discovers the latest
+successful version-scoped publication workflow, downloads its NDJSON artifact just in time, verifies
+the ZIP layout, UTF-8/NDJSON shape, schema, promotion state, and SHA-256, then serves versioned channel
+and immutable dataset URLs. The UI keeps an explicit benchmark-version selector; V2 and later
+versions must use separate release tags and publication identities. The full validation contract is
+in [docs/methodology.md](docs/methodology.md).
+
+## Runner Configuration
+
+Runner-local Slurm and storage values use a strict per-SKU JSON document at
+`$XDG_CONFIG_HOME/inferencex/collectivex.json` or `COLLECTIVEX_OPERATOR_CONFIG`. The mode-0600,
+same-owner, non-symlink file is outside the checkout and never uploaded. Unknown runners, fields,
+duplicate keys, endpoint literals, unsafe paths, and non-JSON input fail closed; configuration is
+never evaluated as shell. GHA passes encrypted `COLLECTIVEX_OPERATOR_CONFIG_V1` content only to the
+launcher, which validates it, exports the selected SKU's allowlisted values, and deletes the
+temporary copy before allocation. Required JSON fields are:
+
+| SKU | Variables |
+|---|---|
+| `h100-dgxc`, `b200-dgxc` | `partition`, `account`, `squash_dir`, `stage_dir` |
+| `h200-dgxc` | `partition`, `squash_dir`, `stage_dir` |
+| `b300` | `partition`, `account`, `squash_dir`, `stage_dir` |
+| `gb200` | `partition`, `account`, ordered `storage_roots` |
+| `gb300` | `partition`, `account`, `squash_dir`, `stage_dir`, `enroot_cache_path` |
+| `mi325x`, `mi355x` | `partition`, `squash_dir`, `stage_dir` |
+
+Every selected non-MNNVL EP16 placement additionally requires `socket_ifname` and `rdma_devices`
+for its operator-approved fabric; optional
+`ib_gid_index` and `rdma_service_level` are also allowlisted. CollectiveX does not heuristically
+select a management route or HCA. After allocation, every non-MNNVL scale-out node must prove that
+all configured interfaces and active HCA ports exist before backend setup. Scale-up and MNNVL jobs
+clear these overrides. Scale-out NCCL/RCCL is pinned to `IB` with exact-match HCA selectors so a
+socket fallback fails instead of being mislabeled as RDMA.
+
+`stage_dir` is a pre-existing, runner-owned, non-symlinked base outside the checkout and workflow
+workspace. It is not group- or world-writable and is visible at the same path on the runner and every
+allocated node. Jobs create only a marked mode-0700 execution child, prove cross-node read/write
+visibility, and remove that exact child after allocation teardown; they never mount the runner
+checkout or create a stage beneath image storage on AMD.
+
+Before import, each Docker Hub tag is resolved with bounded registry requests and must match its
+pinned digest; digest-qualified overrides are rejected. Enroot imports use a fixed filesystem epoch
+and a versioned, registry-digest-bound cache key. Every mounted squash is freshly hashed. The
+verified registry digest and local squash hash are both recorded. Image-provided DeepEP is checked
+against exact wheel and installed-file fingerprints; source-built backends use pinned commits and
+runtime-verified GPU targets. DeepEP V2's mode-0700 cluster-local build cache is keyed by a versioned
+build recipe, verified image, architecture, upstream trees, and dependency pins; only its fixed
+`/cx-cache` mount reaches the container, and it never enters result artifacts.
+Pinned V2 and Hybrid sources are fetched once per workflow. Each job validates the complete archive,
+extracts only its exact backend root, permits only contained relative leaf symlinks to archived
+regular files, and revalidates the Git tree and submodule pins before staging.
+Compute containers receive an explicit environment allowlist. Private host, address, device, NIC,
+credential, workspace, and path data stays in encrypted config, ignored operator notes, or bounded
+mode-0600 runner logs; it is never uploaded.
+
+## Local Checks
+
+```bash
+uv run --with-requirements experimental/CollectiveX/requirements.txt \
+ python -m unittest discover experimental/CollectiveX/tests -p 'test_*.py'
+uv run --with-requirements experimental/CollectiveX/requirements.txt \
+ python experimental/CollectiveX/sweep_matrix.py --backends all --out /tmp/cx-matrix.json >/dev/null
+uv run --with-requirements experimental/CollectiveX/requirements.txt \
+ python experimental/CollectiveX/publisher.py --store-root "$COLLECTIVEX_STORE_ROOT" verify
+bash -n experimental/CollectiveX/runtime/*.sh experimental/CollectiveX/launchers/*.sh
+```
+
+Core paths are `capability.py`, `configs/`, `contracts.py`, `schemas/`, `sweep_matrix.py`,
+`publisher.py`, `runtime/`, `launchers/`, and `tests/`.
diff --git a/experimental/CollectiveX/README_zh.md b/experimental/CollectiveX/README_zh.md
new file mode 100644
index 0000000000..fb32369b40
--- /dev/null
+++ b/experimental/CollectiveX/README_zh.md
@@ -0,0 +1,154 @@
+# CollectiveX
+
+
+
+[English](./README.md) | **中文**
+
+
+
+CollectiveX 是实验性的 MoE 专家并行通信基准，用于测量不同 EP 库和加速器系统的
+dispatch、combine 及配对 roundtrip 延迟。
+
+> 发布暂停：历史 schema 3-5 数据仅供诊断。目前没有数据集获准用于排名、推荐或回归基线。
+
+## v1 执行配置
+
+每个调度用例均采用 BF16、后端调优资源和 packed placement。显式指定的 mode 选择以下两个
+契约之一：
+
+- Normal mode 使用 `layout-and-dispatch-v1`、按 rank 去重的 token payload 和 activation-only
+ combine。核心覆盖使用 uniform routing，并保留一个 Zipf 敏感性场景；EPLB 只作为 Zipf
+ 的修正方案测量。
+- Low-latency mode 使用 `expert-packed-weighted-combine-v1`、token-expert payload 和
+ gate-weighted combine，并且只调用真正的 DeepEP V1 或 UCCL low-latency API。该模式仅覆盖
+ 解码，绝不与 normal mode 共用排名 cohort。其他后端在此 suite 中均显式标为 unsupported。
+
+两种模式统一使用 `fixed-512-v1`：64 trials x 8 timed iterations；每个 trial/point 的每个被测
+组件前执行 32 次同步完整 roundtrip warmup。先测 roundtrip；每次 iteration 先取跨 rank 最大值，
+再按 nearest-rank 计算 p50/p90/p95/p99，主要延迟指标为 roundtrip p99。stdlib 整数计数器
+生成逐字节一致的 routing 和 gate weights。
+
+规范矩阵覆盖 H100、H200、B200、B300、GB200、GB300、MI325X 和 MI355X。矩阵请求
+608 个 cases / 1,600 个 token points：364 个可运行 cases / 940 个 points，并形成 58 个可执行
+workflow shards/allocation cells；另有 244 个显式 unsupported cases / 660 个 points。
+`sweep_matrix.py` 物化每个 token ladder，并拒绝缺失、过期、格式错误或被修改的 shard controls。
+分片按 SKU round-robin 发出，使受限的 GHA matrix 尽早使用所有 runner pools。
+
+| 系统 | EP8 | EP16 |
+|---|---|---|
+| H100/H200/B200/B300 | 1x8 NVLink，scale-up | 2x8 NVLink + RDMA，scale-out |
+| MI325X/MI355X | 1x8 XGMI，scale-up | 2x8 XGMI + RDMA，scale-out |
+| GB200/GB300 | 2x4 MNNVL，scale-up | 4x4 MNNVL，scale-up |
+
+物理主机数量不能决定通信范围：两种 GB 拓扑都位于同一个 72-GPU MNNVL scale-up domain 内。
+
+| 后端 | 当前范围 |
+|---|---|
+| DeepEP V1 | 镜像固定的 `deep_ep.Buffer`：提供 normal 和原生 low-latency API；x86 使用 upstream v1.2.1，arm64 使用镜像内 GB fork |
+| DeepEP V2 | PR #605 `ElasticBuffer` 加 #630：scale-up 使用 LSA，x86 EP16 scale-out 使用 GIN；JIT 可复现并绑定 source/SASS |
+| DeepEP Hybrid | 固定的 `HybridEPBuffer`：x86 EP16 使用 multi-domain RDMA/DOCA；GB EP8/EP16 位于同一个 MNNVL communication domain |
+| UCCL | Hopper 上固定的 0.1.1 wheel 和 wrapper，提供 normal 和原生 low-latency API；Blackwell 显式标为 unsupported |
+| NCCL/RCCL A2A | 可移植的 rank-deduplicated payload 加 expert/routing-metadata reference |
+| MoRI | EP8 使用 MI325X AsyncLL 或 MI355X IntraNode；EP16 固定使用 2x8 XGMI + RDMA 上的 InterNodeV1 |
+
+FlashInfer 不在 v1 范围内，因为已测试的 EP path 在运行时存在间歇性失败。该问题不会被误报为
+平台能力限制；在证明有稳定的固定实现后可重新加入。
+
+DeepEP V2 指 [DeepEP PR #605](https://github.com/deepseek-ai/DeepEP/pull/605) 引入的
+`ElasticBuffer` 实现，而不是更新的 legacy `Buffer` build。固定 source 使用最小化的 upstream
+[PR #630](https://github.com/deepseek-ai/DeepEP/pull/630) 后续修复：其 parent 是 #605 merge
+tree，唯一 source 变更是修复 GIN 不可用时的纯 scale-up 初始化。Scale-up cases 请求 NCCL
+Device API LSA；若实际建立的 LSA team 未覆盖整个 EP world，则直接失败。x86 EP16 scale-out
+cases 必须使用启用 GIN 的 hybrid path，其精确拓扑为两个逻辑 scale-out domains（由两个物理
+RDMA ranks 表示）、每个 domain 八个 scale-up ranks；GB EP16 仍是 MNNVL scale-up，因此继续
+使用 LSA。隔离构建会记录 API、source、loaded libraries、generated JIT source、executable
+SASS 与 raw CUBIN diagnostics。当前 H100 runner pool 被明确标记为 V2 unsupported，因为 NCCL
+2.30.4 报告其 EP8 communicator 不具备 Device API symmetric-memory 支持；只有该 pool 的
+runtime 支持全 rank CUDA P2P/LSA 后才能重新启用。其他 NVIDIA SKU 在 GPU outcome 通过 native
+correctness 和 publication gates 前仍为 unvalidated。
+
+v1 已移除的轴包括 cached-layout `[cl]`、runtime-visible `[rv]`、FP8、quantized combine、
+额外 routing distributions、activation profiles、uneven allocation、placement permutations、
+model envelopes 和 scaling studies。
+
+## Workflow 与产物
+
+`.github/workflows/collectivex-sweep.yml` 生成 public-SKU matrix，提取严格且被忽略的
+`.shards/.json` control，每个 shard 执行一次 allocation，对结果 JSON 做隐私检查并上传
+raw GitHub artifacts。运行默认使用 `release_tag=unversioned`，仅供诊断。V1 运行必须显式选择
+`release_tag=v1`；setup 随后要求固定的完整 matrix digest，并生成绑定 run、attempt 与 source 的
+`cxrelease-v1-*` marker。Partial 或 filtered 运行无法获得该 marker。
+
+`.github/workflows/collectivex-publish.yml` 是显式的 V1 gate。它只接受三个来自同一 source SHA、
+成功且带 V1 tag 的 sweep run IDs，重新校验 GitHub metadata 与 release markers，并在 runner 本地
+可丢弃工作区中执行 `publisher.py`。只有完整通过 promotion、隐私检查和内容寻址的数据集才会以
+`cxpublication-v1-*` 上传；raw artifacts 与 publisher private workspace 永不暴露给前端。
+
+系统不需要 results server、attached store、Vercel storage、GCP、Neon、managed database 或
+managed object store。前端使用已有的 server-side `GITHUB_TOKEN`，即时发现最新成功且按版本隔离
+的 publication workflow，下载其 NDJSON artifact，校验 ZIP layout、UTF-8/NDJSON 结构、schema、
+promotion 状态与 SHA-256，随后提供带版本的 channel URL 和 immutable dataset URL。UI 保留显式
+benchmark-version selector；V2 及后续版本必须使用独立的 release tag 与 publication identity。
+完整 validation contract 见 [docs/methodology_zh.md](docs/methodology_zh.md)。
+
+## Runner 配置
+
+Runner 本地 Slurm 和 storage 值使用严格的 per-SKU JSON 文档，路径为
+`$XDG_CONFIG_HOME/inferencex/collectivex.json` 或 `COLLECTIVEX_OPERATOR_CONFIG`。该 mode-0600、
+同 owner、非 symlink 文件位于 checkout 外且永不上传。未知 runners、fields、duplicate keys、
+endpoint literals、unsafe paths 和非 JSON 输入均 fail closed；配置绝不作为 shell 执行。GHA
+仅将加密的 `COLLECTIVEX_OPERATOR_CONFIG_V1` 内容传给 launcher；launcher 验证后只导出所选
+SKU 的 allowlisted values，并在 allocation 前删除临时副本。必需 JSON fields 如下：
+
+| SKU | 变量 |
+|---|---|
+| `h100-dgxc`, `b200-dgxc` | `partition`, `account`, `squash_dir`, `stage_dir` |
+| `h200-dgxc` | `partition`, `squash_dir`, `stage_dir` |
+| `b300` | `partition`, `account`, `squash_dir`, `stage_dir` |
+| `gb200` | `partition`, `account`, 有序 `storage_roots` |
+| `gb300` | `partition`, `account`, `squash_dir`, `stage_dir`, `enroot_cache_path` |
+| `mi325x`, `mi355x` | `partition`, `squash_dir`, `stage_dir` |
+
+每个已选中的非 MNNVL EP16 placement 还必须提供 `socket_ifname` 和 `rdma_devices`，用来指定
+operator 审核过的 fabric；还可配置 allowlisted
+`ib_gid_index` 与 `rdma_service_level`。CollectiveX 不会通过启发式规则选择 management route 或
+HCA。Allocation 完成后，每个非 MNNVL scale-out 节点都必须证明所有已配置 interface 与 active
+HCA port 存在，之后才允许初始化 backend。Scale-up 和 MNNVL job 会清除这些 overrides。
+Scale-out NCCL/RCCL 固定使用 `IB` 与精确匹配的 HCA selectors；如果无法使用 RDMA，job 会失败，
+而不会回退到 socket 后仍被错误标记为 RDMA。
+
+`stage_dir` 必须是 checkout 与 workflow workspace 外预创建且由 runner owner 持有的 base，
+不能经过 symlink，group 和 world 都不能写入，并且 runner 与所有 allocation 节点必须以相同路径
+访问。Job 只创建带 marker 的 mode-0700 execution child，验证跨节点读写可见性，并在
+allocation teardown 后只删除该 child；不会挂载 runner checkout，也不会在 AMD image storage
+下创建 stage。
+
+导入前，每个 Docker Hub tag 都通过有界 registry requests 解析，并且必须匹配固定 digest；拒绝
+digest-qualified overrides。Enroot imports 使用固定 filesystem epoch 和带版本、绑定 registry
+digest 的 cache key。每个已挂载 squash 都重新计算 hash，同时记录 verified registry digest 和
+local squash hash。镜像提供的 DeepEP 会按精确 wheel 和 installed-file fingerprints 检查；
+source-built backends 使用固定 commits 和 runtime-verified GPU targets。DeepEP V2 的 mode-0700
+cluster-local build cache 由版本化 build recipe、verified image、architecture、upstream
+trees 和 dependency pins 共同寻址；container 只看到固定的 `/cx-cache` mount，且该 cache 永不
+进入 result artifacts。
+固定的 V2 与 Hybrid source 在每个 workflow 中只获取一次。每个 job 都会验证完整 archive，仅
+提取自身精确 backend root，只允许指向 archive 内 regular file 的受限相对 leaf symlink，并在
+staging 前重新核对 Git tree 与 submodule pins。
+Compute containers 仅接收显式 environment allowlist。Private host、address、device、NIC、
+credential、workspace 和 path 数据只保留在加密配置、忽略的 operator notes 或有界 mode-0600
+runner logs 中，永不上传。
+
+## 本地检查
+
+```bash
+uv run --with-requirements experimental/CollectiveX/requirements.txt \
+ python -m unittest discover experimental/CollectiveX/tests -p 'test_*.py'
+uv run --with-requirements experimental/CollectiveX/requirements.txt \
+ python experimental/CollectiveX/sweep_matrix.py --backends all --out /tmp/cx-matrix.json >/dev/null
+uv run --with-requirements experimental/CollectiveX/requirements.txt \
+ python experimental/CollectiveX/publisher.py --store-root "$COLLECTIVEX_STORE_ROOT" verify
+bash -n experimental/CollectiveX/runtime/*.sh experimental/CollectiveX/launchers/*.sh
+```
+
+核心路径为 `capability.py`、`configs/`、`contracts.py`、`schemas/`、`sweep_matrix.py`、
+`publisher.py`、`runtime/`、`launchers/` 和 `tests/`。
diff --git a/experimental/CollectiveX/artifact_safety.py b/experimental/CollectiveX/artifact_safety.py
new file mode 100644
index 0000000000..83d522fba8
--- /dev/null
+++ b/experimental/CollectiveX/artifact_safety.py
@@ -0,0 +1,212 @@
+#!/usr/bin/env python3
+"""Fail-closed privacy check for CollectiveX public result documents."""
+from __future__ import annotations
+
+import argparse
+import ipaddress
+import json
+import os
+import re
+import stat
+
+
+SENSITIVE_FIELDS = frozenset({
+ "environment", "env", "host", "hostname", "uuid", "gpu_uuid", "device_uuid",
+ "pci_bus_id", "ip_address", "ip_addresses", "master_addr", "ssh", "ssh_target",
+ "nodelist", "node_list", "nic_guid", "ib_guid", "topology_matrix", "rdma_devices",
+ "user", "username", "password", "passwd", "secret", "token", "access_token",
+ "api_token", "auth_token", "api_key", "private_key", "credential", "credentials",
+ "address", "addresses", "ip", "ips",
+})
+SENSITIVE_FIELDS_COMPACT = frozenset(item.replace("_", "") for item in SENSITIVE_FIELDS)
+SENSITIVE_FIELD_SUFFIXES = (
+ "_host", "_hostname", "_address", "_addresses", "_path", "_paths", "_ip", "_ips",
+ "_password", "_passwd", "_secret", "_token", "_credential", "_credentials",
+ "_uuid", "_guid", "_bus_id",
+)
+SENSITIVE_VALUE_PATTERNS = (
+ ("private-path", re.compile(
+ r"(? str:
+ normalized = re.sub(r"([A-Z]+)([A-Z][a-z])", r"\1_\2", str(value).strip())
+ normalized = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", normalized)
+ return normalized.lower().replace("-", "_")
+
+
+def _sensitive_value_rule(value: str, *, contextual: bool = True) -> str | None:
+ matched = next(
+ (
+ name for name, pattern in SENSITIVE_VALUE_PATTERNS
+ if (contextual or name not in CONTEXTUAL_VALUE_RULES) and pattern.search(value)
+ ),
+ None,
+ )
+ if matched:
+ return matched
+ for candidate in IPV6_CANDIDATE.findall(value):
+ try:
+ address = candidate.split("%", 1)[0]
+ if ipaddress.ip_address(address).version == 6:
+ return "ipv6-address"
+ except ValueError:
+ continue
+ return None
+
+
+def assert_publication_safe(docs: list[dict]) -> None:
+ """Reject private infrastructure fields and value shapes."""
+ def walk(value, doc_index: int, parent_field: str | None = None) -> None:
+ if isinstance(value, dict):
+ for key, child in value.items():
+ field = _normalized_field(key)
+ compact = field.replace("_", "")
+ if (
+ field in SENSITIVE_FIELDS
+ or compact in SENSITIVE_FIELDS_COMPACT
+ or field.endswith(SENSITIVE_FIELD_SUFFIXES)
+ ):
+ raise ArtifactSafetyError(
+ f"artifact safety: doc[{doc_index}] contains forbidden private field"
+ )
+ key_rule = _sensitive_value_rule(str(key))
+ if key_rule:
+ raise ArtifactSafetyError(
+ f"artifact safety: doc[{doc_index}] contains forbidden {key_rule} key"
+ )
+ walk(child, doc_index, field)
+ elif isinstance(value, list):
+ for child in value:
+ walk(child, doc_index, parent_field)
+ elif isinstance(value, str):
+ rule = _sensitive_value_rule(value, contextual=parent_field != "ref")
+ if rule:
+ raise ArtifactSafetyError(
+ f"artifact safety: doc[{doc_index}] contains forbidden {rule} value"
+ )
+
+ for index, doc in enumerate(docs):
+ if not isinstance(doc, dict):
+ raise ArtifactSafetyError(f"artifact safety: doc[{index}] is not a JSON object")
+ walk(doc, index)
+
+
+def load_documents(paths: list[str]) -> list[dict]:
+ docs: list[dict] = []
+ for path in paths:
+ try:
+ metadata = os.lstat(path)
+ except OSError as exc:
+ raise ArtifactSafetyError("artifact safety: result file is unavailable") from exc
+ if (
+ not stat.S_ISREG(metadata.st_mode)
+ or metadata.st_uid != os.getuid()
+ or metadata.st_size <= 0
+ or metadata.st_size > MAX_INPUT_BYTES
+ ):
+ raise ArtifactSafetyError("artifact safety: result file is unavailable")
+ descriptor = -1
+ try:
+ descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
+ opened = os.fstat(descriptor)
+ if (
+ not stat.S_ISREG(opened.st_mode)
+ or (opened.st_dev, opened.st_ino, opened.st_size)
+ != (metadata.st_dev, metadata.st_ino, metadata.st_size)
+ ):
+ raise ArtifactSafetyError("artifact safety: result file changed during open")
+ with os.fdopen(descriptor, encoding="utf-8") as fh:
+ descriptor = -1
+ if path.endswith(".ndjson"):
+ for line_number, line in enumerate(fh, 1):
+ if not line.strip():
+ continue
+ try:
+ docs.append(json.loads(line))
+ except json.JSONDecodeError as exc:
+ raise ArtifactSafetyError(
+ f"artifact safety: malformed NDJSON at input line {line_number}"
+ ) from exc
+ else:
+ docs.append(json.load(fh))
+ except json.JSONDecodeError as exc:
+ raise ArtifactSafetyError("artifact safety: malformed JSON input") from exc
+ except (OSError, UnicodeError) as exc:
+ raise ArtifactSafetyError("artifact safety: result file is unreadable") from exc
+ finally:
+ if descriptor >= 0:
+ os.close(descriptor)
+ if not docs:
+ raise ArtifactSafetyError("artifact safety: no public result documents found")
+ return docs
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description="Check CollectiveX result artifacts for private data")
+ parser.add_argument("paths", nargs="+")
+ args = parser.parse_args()
+ try:
+ docs = load_documents(args.paths)
+ assert_publication_safe(docs)
+ except ArtifactSafetyError as exc:
+ parser.error(str(exc))
+ print(f"artifact safety: {len(docs)} public document(s) passed")
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
new file mode 100644
index 0000000000..a9431a86b4
--- /dev/null
+++ b/experimental/CollectiveX/capability.py
@@ -0,0 +1,232 @@
+#!/usr/bin/env python3
+"""Public runner and backend capability registry for CollectiveX v1."""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+
+DEEPEP_V2_COMMIT = "fa8a9b16898204afd347c663b89e65ef87dc6ce6"
+DEEPEP_V2_SKU_CAPABILITIES = {
+ "h100-dgxc": {
+ "schedulable": False,
+ "basis": "current-runner-nccl-device-api-symmetric-memory-unavailable",
+ },
+ "h200-dgxc": {"schedulable": True, "basis": "upstream-sm90-requirement"},
+ "b200-dgxc": {"schedulable": True, "basis": "upstream-sm100-result"},
+ "gb200": {"schedulable": True, "basis": "upstream-sm100-result"},
+ "b300": {"schedulable": True, "basis": "pinned-pr605-pr630-sm103-maps-sm100f"},
+ "gb300": {"schedulable": True, "basis": "pinned-pr605-pr630-sm103-maps-sm100f"},
+ "mi325x": {"schedulable": False, "basis": "nvidia-only"},
+ "mi355x": {"schedulable": False, "basis": "nvidia-only"},
+}
+
+
+def _topologies(
+ product: str, *, gpus_per_node: int, scale_up_domain: int, scale_up_transport: str
+) -> dict[int, dict[str, Any]]:
+ scale_up_class = (
+ f"{product}-nvl72-mnnvl"
+ if scale_up_transport == "mnnvl"
+ else f"{product}-xgmi"
+ if scale_up_transport == "xgmi"
+ else f"{product}-{scale_up_transport}-island"
+ )
+ return {
+ 8: {
+ "nodes": 8 // gpus_per_node,
+ "gpus_per_node": gpus_per_node,
+ "scale_up_domain": scale_up_domain,
+ "scope": "scale-up",
+ "scale_up_transport": scale_up_transport,
+ "scale_out_transport": None,
+ "transport": scale_up_transport,
+ "topology_class": scale_up_class,
+ },
+ 16: {
+ "nodes": 16 // gpus_per_node,
+ "gpus_per_node": gpus_per_node,
+ "scale_up_domain": scale_up_domain,
+ "scope": "scale-up" if scale_up_domain >= 16 else "scale-out",
+ "scale_up_transport": scale_up_transport,
+ "scale_out_transport": None if scale_up_domain >= 16 else "rdma",
+ "transport": (
+ scale_up_transport
+ if scale_up_domain >= 16
+ else f"{scale_up_transport}-rdma"
+ ),
+ "topology_class": (
+ scale_up_class
+ if scale_up_domain >= 16
+ else f"{product}-{scale_up_transport}-rdma"
+ ),
+ },
+ }
+
+
+def _platform(
+ *, vendor: str, arch: str, machine: str, product: str, gpus_per_node: int,
+ scale_up_domain: int, scale_up_transport: str, launcher: str,
+) -> dict[str, Any]:
+ topologies = _topologies(
+ product,
+ gpus_per_node=gpus_per_node,
+ scale_up_domain=scale_up_domain,
+ scale_up_transport=scale_up_transport,
+ )
+ ep8 = topologies[8]
+ return {
+ "vendor": vendor,
+ "arch": arch,
+ "machine": machine,
+ "product": product,
+ # EP8 defaults remain while downstream readers migrate to per-EP records.
+ "transport": ep8["transport"],
+ "topology_class": ep8["topology_class"],
+ "gpus_per_node": gpus_per_node,
+ "scale_up_domain": scale_up_domain,
+ "ep_degrees": tuple(topologies),
+ "topologies": topologies,
+ "launcher": launcher,
+ }
+
+
+PLATFORMS = {
+ "h100-dgxc": _platform(
+ vendor="nvidia", arch="sm90", machine="amd64", product="h100",
+ gpus_per_node=8, scale_up_domain=8, scale_up_transport="nvlink",
+ launcher="single-slurm",
+ ),
+ "h200-dgxc": _platform(
+ vendor="nvidia", arch="sm90", machine="amd64", product="h200",
+ gpus_per_node=8, scale_up_domain=8, scale_up_transport="nvlink",
+ launcher="single-slurm",
+ ),
+ "b200-dgxc": _platform(
+ vendor="nvidia", arch="sm100", machine="amd64", product="b200",
+ gpus_per_node=8, scale_up_domain=8, scale_up_transport="nvlink",
+ launcher="single-slurm",
+ ),
+ "b300": _platform(
+ vendor="nvidia", arch="sm103", machine="amd64", product="b300",
+ gpus_per_node=8, scale_up_domain=8, scale_up_transport="nvlink",
+ launcher="single-slurm",
+ ),
+ "gb200": _platform(
+ vendor="nvidia", arch="sm100", machine="arm64", product="gb200",
+ gpus_per_node=4, scale_up_domain=72, scale_up_transport="mnnvl",
+ launcher="gb-nv",
+ ),
+ "gb300": _platform(
+ vendor="nvidia", arch="sm103", machine="arm64", product="gb300",
+ gpus_per_node=4, scale_up_domain=72, scale_up_transport="mnnvl",
+ launcher="gb-nv",
+ ),
+ "mi325x": _platform(
+ vendor="amd", arch="gfx942", machine="amd64", product="mi325x",
+ gpus_per_node=8, scale_up_domain=8, scale_up_transport="xgmi",
+ launcher="mi-amds",
+ ),
+ "mi355x": _platform(
+ vendor="amd", arch="gfx950", machine="amd64", product="mi355x",
+ gpus_per_node=8, scale_up_domain=8, scale_up_transport="xgmi",
+ launcher="mi-amds",
+ ),
+}
+
+BACKENDS = {
+ "deepep": {"vendors": {"nvidia"}},
+ "deepep-v2": {
+ "vendors": {"nvidia"},
+ "implementation": "deep_ep.ElasticBuffer",
+ "source": "deepseek-ai/DeepEP#605+#630",
+ "commit": DEEPEP_V2_COMMIT,
+ "communication_backend": "nccl-device-lsa",
+ "torch": "2.10.0+cu130",
+ "nccl": "2.30.4",
+ "sku_capabilities": DEEPEP_V2_SKU_CAPABILITIES,
+ },
+ "uccl": {
+ "vendors": {"nvidia"},
+ "machines": {"amd64"},
+ "excluded_skus": {"b200-dgxc", "b300"},
+ },
+ "deepep-hybrid": {"vendors": {"nvidia"}},
+ "mori": {"vendors": {"amd"}},
+ "nccl-ep": {"vendors": {"nvidia", "amd"}},
+}
+SWEEP_BACKENDS = tuple(BACKENDS)
+
+
+def runtime_identity_issues(
+ sku: str, *, vendor: str, arch: str, machine: str, device_name: str,
+ device_count: int, world_size: int,
+) -> list[str]:
+ """Validate public product identity on every rank without private device identifiers."""
+ platform = PLATFORMS.get(sku)
+ if platform is None:
+ return [f"unknown runner identity {sku!r}"]
+ issues = []
+ for field, observed in (("vendor", vendor), ("arch", arch), ("machine", machine)):
+ if observed != platform[field]:
+ issues.append(f"{field}={observed!r}, expected {platform[field]!r}")
+ products = set(re.findall(r"[a-z]+\d+[a-z]*", device_name.lower()))
+ if platform["product"] not in products:
+ issues.append(f"device product {device_name!r} does not identify {platform['product']}")
+ if device_count != platform["gpus_per_node"]:
+ issues.append(
+ f"visible GPUs={device_count}, expected {platform['gpus_per_node']} per node"
+ )
+ if world_size not in platform["ep_degrees"]:
+ issues.append(f"EP{world_size} is not registered for {sku}")
+ return issues
+
+
+def topology_for(sku: str, ep: int) -> dict[str, Any] | None:
+ """Return the exact public topology registered for one SKU/EP cell."""
+ platform = PLATFORMS.get(sku)
+ if platform is None:
+ return None
+ return platform["topologies"].get(ep)
+
+
+def resolve(sku: str, backend: str, *, ep: int | None = None, nodes: int | None = None,
+ routing: str = "uniform", eplb: bool = False,
+ mode: str = "normal") -> tuple[bool, str]:
+ """Return whether one fixed-v1 case can run on a public GHA runner label."""
+ platform, implementation = PLATFORMS.get(sku), BACKENDS.get(backend)
+ if platform is None:
+ return False, f"unknown GHA runner label {sku!r}"
+ if implementation is None:
+ return False, f"unknown backend {backend!r}"
+ if mode not in {"normal", "low-latency"}:
+ return False, f"unknown benchmark mode {mode!r}"
+ if mode == "low-latency" and backend not in {"deepep", "uccl"}:
+ return False, f"{backend} has no distinct low-latency API"
+ if ep is None:
+ if nodes is None:
+ ep = platform["ep_degrees"][0]
+ else:
+ matches = [
+ degree for degree, topology in platform["topologies"].items()
+ if topology["nodes"] == nodes
+ ]
+ if len(matches) != 1:
+ return False, f"{sku} does not register a unique {nodes}-node EP degree"
+ ep = matches[0]
+ topology = topology_for(sku, ep)
+ if topology is None or (nodes is not None and nodes != topology["nodes"]):
+ return False, f"{sku} does not register EP{ep} on {nodes} nodes"
+ if routing not in {"uniform", "zipf"} or (eplb and routing != "zipf"):
+ return False, "v1 routing is uniform or zipf, with EPLB only on zipf"
+ if platform["vendor"] not in implementation["vendors"]:
+ return False, f"{backend} does not support {platform['vendor']}"
+ sku_capability = implementation.get("sku_capabilities", {}).get(sku)
+ if sku_capability is not None and not sku_capability["schedulable"]:
+ return False, f"{backend} is unsupported on {sku}: {sku_capability['basis']}"
+ if platform["machine"] not in implementation.get("machines", {platform["machine"]}):
+ return False, f"{backend} does not support {platform['machine']}"
+ if sku in implementation.get("excluded_skus", set()):
+ return False, f"{backend} is unavailable on {sku}"
+ return True, "ok"
diff --git a/experimental/CollectiveX/configs/suites.yaml b/experimental/CollectiveX/configs/suites.yaml
new file mode 100644
index 0000000000..1508695637
--- /dev/null
+++ b/experimental/CollectiveX/configs/suites.yaml
@@ -0,0 +1,36 @@
+# CollectiveX v1 comparison suites.
+schema_version: 1
+
+suites:
+ ep-core-v1:
+ mode: normal
+ workloads: [deepseek-v3-v1]
+ platforms: [h100-dgxc, h200-dgxc, b300, b200-dgxc, gb300, gb200, mi355x, mi325x]
+ ep_degrees: [8, 16]
+ routings: [uniform]
+ phases: [decode, prefill]
+ token_points_prefill: [256, 512]
+ required_publication: official
+
+ ep-routing-v1:
+ mode: normal
+ workloads: [deepseek-v3-v1]
+ platforms: [h100-dgxc, h200-dgxc, b300, b200-dgxc, gb300, gb200, mi355x, mi325x]
+ ep_degrees: [8, 16]
+ routings: [zipf]
+ eplb: [false, true]
+ phases: [decode, prefill]
+ token_points_decode: [128]
+ token_points_prefill: [512]
+ required_publication: comparable-experimental
+
+ ep-low-latency-v1:
+ mode: low-latency
+ backends: [deepep, uccl]
+ workloads: [deepseek-v3-v1]
+ platforms: [h100-dgxc, h200-dgxc, b300, b200-dgxc, gb300, gb200, mi355x, mi325x]
+ ep_degrees: [8, 16]
+ routings: [uniform]
+ phases: [decode]
+ token_points_decode: [1, 2, 4, 8, 16, 32, 64, 128]
+ required_publication: official
diff --git a/experimental/CollectiveX/configs/workloads.yaml b/experimental/CollectiveX/configs/workloads.yaml
new file mode 100644
index 0000000000..b5b68334c4
--- /dev/null
+++ b/experimental/CollectiveX/configs/workloads.yaml
@@ -0,0 +1,9 @@
+# CollectiveX v1 canonical workload and phase metadata.
+schema_version: 1
+
+model_derived:
+ deepseek-v3-v1:
+ hidden: 7168
+ topk: 8
+ routed_experts: 256
+ verified_against: "deepseek-ai/DeepSeek-V3@e815299b0bcbac849fa540c768ef21845365c9eb/config.json"
diff --git a/experimental/CollectiveX/contracts.py b/experimental/CollectiveX/contracts.py
new file mode 100644
index 0000000000..fbebe070c8
--- /dev/null
+++ b/experimental/CollectiveX/contracts.py
@@ -0,0 +1,2823 @@
+#!/usr/bin/env python3
+"""Strict native attempt contracts and metric validation for CollectiveX v1."""
+from __future__ import annotations
+
+import argparse
+import datetime as dt
+from functools import lru_cache
+import hashlib
+import json
+import math
+import os
+from pathlib import Path, PurePosixPath
+import re
+import sys
+from typing import Any, Iterable
+
+import artifact_safety
+import capability
+import identity
+
+TESTS = Path(__file__).resolve().parent / "tests"
+sys.path.insert(0, str(TESTS))
+import eplb as eplb_contract # noqa: E402
+import workload as workload_contract # noqa: E402
+
+RAW_FORMAT = "collectivex.ep.v1"
+SAMPLES_FORMAT = "collectivex.samples.v1"
+TERMINAL_FORMAT = "collectivex.terminal.v1"
+TERMINAL_CASE_FIELDS = {
+ "backend", "canonical", "eplb", "ep", "experts", "gpus_per_node", "hidden",
+ "ladder", "mode", "nodes", "phase", "required_publication", "routing",
+ "samples_per_point", "scale_out_transport", "scale_up_domain", "scale_up_transport",
+ "scope", "suite", "timing", "topk", "topology_class", "transport",
+ "warmup_semantics", "workload",
+}
+ALLOCATION_FACTOR_FIELDS = {
+ "artifact", "execution_id", "job", "repo", "run_attempt", "run_id", "runner",
+ "source_sha",
+}
+GIT_RUN_FIELDS = {"artifact", "job", "ref", "repo", "run_attempt", "run_id", "source_sha"}
+PRE_EXECUTION_FAILURE_REASONS = {
+ "setup": "launcher-setup-failed",
+ "repository-stage": "repository-staging-failed",
+ "registry-verification": "container-registry-verification-failed",
+ "scheduler-allocation": "scheduler-allocation-failed",
+ "container-import": "container-image-preparation-failed",
+ "container-hash": "container-image-identity-failed",
+ "container-launch": "container-runtime-launch-failed",
+ "backend-setup": "backend-setup-failed",
+ "artifact-collection": "artifact-collection-failed",
+}
+RUNTIME_FAILURE_REASONS = {
+ **PRE_EXECUTION_FAILURE_REASONS,
+ "runtime-identity": "runtime-identity-mismatch",
+ "timeout": "execution-timeout",
+ "deadlock": "execution-deadlock",
+ "execution": "distributed-command-failed",
+}
+POST_EMIT_FAILURE_REASONS = {
+ mode: "post-emit-distributed-command-failed"
+ for mode in ("runtime-identity", "timeout", "deadlock", "execution")
+}
+CAPABILITY_FAILURE_REASONS = frozenset({
+ "backend-platform-unsupported",
+ "backend-token-capacity",
+})
+RETURN_CODE_FAILURE_MODES = {
+ 5: "runtime-identity",
+ 124: "timeout",
+ 137: "timeout",
+}
+PERCENTILES = ("p50", "p90", "p95", "p99")
+V1_CONDITIONING_LADDERS = {
+ "decode": (1, 2, 4, 8, 16, 32, 64, 128),
+ "prefill": (1, 2, 4, 8, 16, 32, 64, 128, 256, 512),
+}
+V1_CONDITIONING_ROUNDS_PER_SHAPE = 8
+DEEPEP_V2_JIT_KERNELS = frozenset({
+ "barrier", "combine", "combine_reduce_epilogue", "dispatch",
+ "dispatch_copy_epilogue",
+})
+DEEPEP_V2_V1_PROVENANCE = {
+ "deepep_version": "2.0.0",
+ "deepep_distribution_version": "2.0.0+fa8a9b1",
+ "deepep_commit": "fa8a9b16898204afd347c663b89e65ef87dc6ce6",
+ "deepep_tree": "29809e75c5874e6609dac4804e7b651d5226959f",
+ "deepep_pr": 605,
+ "deepep_fix_pr": 630,
+ "fmt_commit": "a4c7e17133ee9cb6a2f45545f6e974dd3c393efa",
+ "torch_version": "2.10.0+cu130",
+ "nccl_package_version": "2.30.4",
+ "nccl_version": "2.30.4",
+ "nvshmem_package_version": "3.3.9",
+}
+UCCL_DEPENDENCY_VERSIONS = {
+ "intervaltree": "3.1.0",
+ "nvidia-cuda-runtime-cu12": "12.9.79",
+ "sortedcontainers": "2.4.0",
+}
+SCHEMA_DIR = Path(__file__).resolve().parent / "schemas"
+_SCHEMA_CACHE: dict[str, dict[str, Any]] = {}
+REQUIRED_BACKEND_PROVENANCE = {
+ "deepep": (
+ "deepep_version", "deepep_commit", "backend_lineage", "allow_mnnvl",
+ "mnnvl_comm", "mode", "num_nvl_bytes", "num_rdma_bytes",
+ ),
+ "deepep-v2": (
+ *DEEPEP_V2_V1_PROVENANCE, "api_signature_sha256", "loaded_libraries",
+ "jit_cubins", "jit_random_seed", "deterministic", "num_experts",
+ "tuning_num_experts", "allow_hybrid_mode", "gin_enabled",
+ "communication_backend",
+ ),
+ "deepep-hybrid": (
+ "deepep_commit", "deepep_tree", "branch", "backend_lineage",
+ "loaded_libraries", "realized_config", "jit_kernel_keys", "jit_shared_objects",
+ ),
+ "uccl": (
+ "uccl_version", "uccl_commit", "uccl_wrapper_commit", "backend_lineage",
+ "loaded_libraries", "uccl_dependency_versions", "mode", "num_nvl_bytes",
+ "num_rdma_bytes",
+ ),
+ "mori": ("mori_commit",),
+ "nccl-ep": ("nccl_version", "collective_library", "backend_lineage"),
+}
+PROVENANCE_KEYS = {
+ "allocated_qps", "allow_hybrid_mode", "allow_mnnvl", "allow_multiple_reduction",
+ "api", "api_signature_sha256", "backend", "backend_lineage", "block_num",
+ "block_num_floored", "block_num_target", "branch", "collective_library",
+ "combine_dtype", "combine_warps", "communication_backend", "cuda_version",
+ "deepep_commit", "deepep_distribution_version", "deepep_fix_pr", "deepep_pr", "deepep_tree",
+ "deepep_version", "deterministic", "device_cus",
+ "device_sms", "dispatch_dtype", "dispatch_warps", "enable_sdma", "fmt_commit",
+ "gin_enabled",
+ "gpus_per_node", "heap_size",
+ "impl", "jit_cache_key", "jit_cubins", "jit_kernel_keys", "jit_random_seed",
+ "jit_shared_objects", "kernel_type",
+ "loaded_libraries", "local_experts",
+ "logical_scaleout_ranks",
+ "logical_scaleup_ranks", "mapping_variant", "max_num_inp_token_per_rank",
+ "max_num_tokens", "max_total_recv_tokens", "mnnvl_comm", "mode", "mori_commit",
+ "nccl_communicator", "nccl_package_version", "nccl_version", "num_experts",
+ "nvshmem_package_version",
+ "num_max_tokens_per_rank", "num_nvl_bytes", "num_qps", "num_qps_per_rank",
+ "num_rdma_bytes", "num_sms", "path",
+ "physical_nvlink_ranks", "physical_rdma_ranks", "prefer_overlap_with_compute",
+ "rdma_block_num",
+ "realized_config", "reference_semantics", "requested_num_sms", "resource_mode", "routing_factor",
+ "routing_metadata", "sm_fraction", "top_k",
+ "torch_git_version", "torch_version", "transport", "trtllm", "tuned_source",
+ "tuning_num_experts",
+ "uccl_commit", "uccl_dependency_versions", "uccl_version", "uccl_wrapper_commit",
+ "use_external_inp_buf",
+ "workspace",
+}
+
+
+class ContractError(ValueError):
+ """A document differs from the native v1 contract."""
+
+
+def scheduled_case_profile(case: dict[str, Any], path: str = "case") -> dict[str, Any]:
+ """Resolve an explicit scheduled mode to its immutable measurement profile."""
+ try:
+ return identity.profile_for_case(case)
+ except identity.IdentityError as exc:
+ raise ContractError(f"{path}: {exc}") from exc
+
+
+def resolve_deepep_mnnvl(
+ *, requested: bool, signature_parameters: Iterable[str], deepep_commit: str | None
+) -> tuple[dict[str, bool], str]:
+ """Resolve one explicit DeepEP MNNVL API mode without signature fallbacks."""
+ if not requested:
+ return {}, "not-requested"
+ if "allow_mnnvl" in set(signature_parameters):
+ return {"allow_mnnvl": True}, "explicit-allow-mnnvl"
+ raise ContractError(
+ f"requested DeepEP MNNVL is unsupported by commit {deepep_commit or 'unknown'}"
+ )
+
+
+def collective_kernel_generation(collective_library: Any) -> str:
+ """Return the public NCCL/RCCL implementation lineage."""
+ if collective_library not in {"nccl", "rccl"}:
+ raise ContractError("reference collective library must be nccl or rccl")
+ return collective_library
+
+
+def project_resource_profile(provenance: dict[str, Any]) -> dict[str, Any]:
+ """Project backend provenance into the canonical cross-backend resource vocabulary."""
+ device_units = provenance.get("device_sms") or provenance.get("device_cus")
+ if provenance.get("num_sms") is not None:
+ kind, configured = "sm", provenance["num_sms"]
+ elif (
+ provenance.get("block_num") is not None
+ and provenance.get("kernel_type") != "AsyncLL"
+ ):
+ kind, configured = "cu_block", provenance["block_num"]
+ else:
+ kind, configured = None, None
+ achieved = configured / device_units if configured and device_units else None
+ fixed = "fixed-kernel" in str(provenance.get("tuned_source", ""))
+ source = str(provenance.get("tuned_source", ""))
+ num_nvl_bytes = provenance.get("num_nvl_bytes")
+ num_rdma_bytes = provenance.get("num_rdma_bytes")
+ persistent_bytes = (
+ (num_nvl_bytes or 0) + (num_rdma_bytes or 0)
+ if num_nvl_bytes is not None or num_rdma_bytes is not None
+ else provenance.get("heap_size")
+ )
+ return {
+ "achieved_fraction": round(achieved, 4) if achieved else None,
+ "comm_units_kind": kind,
+ "configured_units": configured,
+ "conformance_class": (
+ "not-applicable" if fixed else "best-known" if "default" not in source
+ else "backend-default"
+ ),
+ "device_units": device_units,
+ "fixed_kernel": fixed,
+ "nonconforming": False,
+ "pareto_eligible": False,
+ "persistent_bytes": persistent_bytes,
+ "qps_per_rank": provenance.get("num_qps_per_rank"),
+ "requested_fraction": None,
+ "resource_class": "fixed-kernel" if fixed else "backend-tuned",
+ "target_achieved_within_tol": None,
+ "tolerance": 0.10,
+ "tuned_source": provenance.get("tuned_source"),
+ "warps_combine": provenance.get("combine_warps"),
+ "warps_dispatch": provenance.get("dispatch_warps"),
+ }
+
+
+def backend_version(provenance: dict[str, Any]) -> str | None:
+ """Return the canonical public backend version from implementation provenance."""
+ for field in (
+ "deepep_version", "uccl_version", "nccl_version",
+ "mori_commit", "deepep_commit",
+ ):
+ value = provenance.get(field)
+ if value is not None and str(value).strip():
+ return str(value)[:160]
+ return None
+
+
+def public_series_config(
+ *, kernel_generation: Any, provenance: dict[str, Any],
+ resource_profile: dict[str, Any], resource_mode: Any, device_product: Any,
+) -> dict[str, Any]:
+ """Project raw implementation facts into the exact public configuration fields."""
+ generation = None if kernel_generation == "n-a" else kernel_generation
+ profile = "profile-" + _sha256_json(resource_profile)[:16]
+ return {
+ "backend": {
+ "generation": generation,
+ "version": backend_version(provenance),
+ },
+ "resource": {
+ "mode": resource_mode,
+ "profile": profile,
+ "comm_units_kind": resource_profile.get("comm_units_kind"),
+ "configured_units": resource_profile.get("configured_units"),
+ },
+ "system": {"label": str(device_product)[:160]},
+ }
+
+
+def public_series_config_sha256(config: dict[str, Any]) -> str:
+ """Commit the canonical public configuration projection into series identity."""
+ return _sha256_json(config)
+
+
+SOURCE_BUILT_LIBRARY_ROLES = frozenset({
+ "deepep-extension", "deepep-hybrid-extension",
+})
+
+
+def series_provenance(provenance: dict[str, Any]) -> dict[str, Any]:
+ """Project stable semantic build identity while retaining raw binaries in private evidence."""
+ projected = {
+ key: value for key, value in provenance.items()
+ if key not in {"jit_cache_key", "jit_shared_objects", "path", "sm_fraction"}
+ }
+ libraries = provenance.get("loaded_libraries")
+ if isinstance(libraries, list):
+ projected["loaded_libraries"] = [
+ {
+ "name": item.get("name"),
+ "role": item.get("role"),
+ "source_tree": provenance.get("deepep_tree"),
+ }
+ if isinstance(item, dict) and item.get("role") in SOURCE_BUILT_LIBRARY_ROLES
+ else item
+ for item in libraries
+ ]
+ jit_cubins = provenance.get("jit_cubins")
+ if isinstance(jit_cubins, list):
+ projected["jit_cubins"] = [
+ {
+ "cache_key": item.get("cache_key"),
+ "sass_sha256": item.get("sass_sha256"),
+ "source_sha256": item.get("source_sha256"),
+ }
+ if isinstance(item, dict)
+ else item
+ for item in jit_cubins
+ ]
+ return projected
+
+
+def routing_implementation_control_sha256(implementation: dict[str, Any]) -> str:
+ """Bind routing cohorts to the same static build/generator and non-treatment configuration."""
+ provenance = implementation.get("provenance")
+ if not isinstance(provenance, dict):
+ raise ContractError("implementation provenance is unavailable")
+ semantic = series_provenance(provenance)
+ treatment_fields = {
+ "jit_cache_key", "jit_cubins", "jit_kernel_keys", "jit_shared_objects",
+ "local_experts", "num_experts", "path", "realized_config", "sm_fraction",
+ }
+ return _sha256_json({
+ "kernel_generation": implementation.get("kernel_generation"),
+ "name": implementation.get("name"),
+ "provenance": {
+ key: value for key, value in semantic.items()
+ if key not in treatment_fields
+ },
+ "resource_profile": implementation.get("resource_profile"),
+ })
+
+
+def _resolved_provenance_value(field: str, value: Any) -> bool:
+ if value is None or isinstance(value, (dict, list, tuple, set)) and not value:
+ return False
+ text = str(value).strip().lower()
+ if not text or text in {"unknown", "none", "null", "n/a", "?", "capture-failed"}:
+ return False
+ if "capture-failed" in text:
+ return False
+ if field.endswith("_commit") and (
+ text in {"main", "hybrid-ep", "uccl", "pkg-uccl"}
+ or text.endswith(("-unknown", "-none", "-main", "-hybrid-ep"))
+ ):
+ return False
+ return True
+
+
+def _content_evidence_is_valid(value: Any, required_roles: set[str]) -> bool:
+ if not isinstance(value, list) or not value:
+ return False
+ records: set[tuple[str, str]] = set()
+ roles: set[str] = set()
+ for item in value:
+ if not isinstance(item, dict) or set(item) != {"name", "role", "sha256"}:
+ return False
+ name, role, digest = item["name"], item["role"], item["sha256"]
+ if (
+ not isinstance(name, str)
+ or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.+-]{0,159}", name)
+ or not isinstance(role, str)
+ or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.+-]{0,127}", role)
+ or not isinstance(digest, str)
+ or not re.fullmatch(r"[0-9a-f]{64}", digest)
+ or (role, name) in records
+ ):
+ return False
+ records.add((role, name))
+ roles.add(role)
+ return required_roles <= roles
+
+
+def _deepep_v2_jit_cubins_are_valid(value: Any) -> bool:
+ if not isinstance(value, list) or len(value) != len(DEEPEP_V2_JIT_KERNELS):
+ return False
+ cache_keys = []
+ kernel_names = set()
+ for item in value:
+ if not isinstance(item, dict) or set(item) != {
+ "cache_key", "cubin_sha256", "sass_sha256", "source_sha256",
+ }:
+ return False
+ cache_key = item["cache_key"]
+ match = (
+ re.fullmatch(r"kernel\.([A-Za-z0-9_+-]+)\.[0-9a-f]{32}", cache_key)
+ if isinstance(cache_key, str)
+ else None
+ )
+ if (
+ match is None
+ or any(
+ not isinstance(item[field], str)
+ or not re.fullmatch(r"[0-9a-f]{64}", item[field])
+ for field in ("cubin_sha256", "sass_sha256", "source_sha256")
+ )
+ ):
+ return False
+ cache_keys.append(cache_key)
+ kernel_names.add(match.group(1))
+ return (
+ cache_keys == sorted(set(cache_keys))
+ and kernel_names == DEEPEP_V2_JIT_KERNELS
+ )
+
+
+HYBRID_REALIZED_CONFIG_FIELDS = {
+ "hidden_dim", "max_num_of_tokens_per_rank", "num_of_experts_per_rank",
+ "num_of_ranks_per_node", "num_of_nodes", "pad_multiple",
+ "num_of_tokens_per_chunk_preprocessing_api",
+ "num_of_threads_per_block_preprocessing_api", "num_of_blocks_preprocessing_api",
+ "num_of_blocks_permute", "num_of_blocks_unpermute", "token_data_type",
+ "num_of_stages_dispatch_api", "num_of_stages_permute_block_dispatch_api",
+ "num_of_in_flight_s2g_dispatch_api",
+ "num_of_in_flight_s2g_permute_block_dispatch_api",
+ "num_of_additional_in_flight_s2g_dispatch_api",
+ "num_of_tokens_per_chunk_dispatch_api", "num_of_blocks_dispatch_api",
+ "forward_dispatch_api", "device_side_sync_dispatch_api",
+ "num_of_stages_g2s_combine_api", "num_of_stages_s2g_combine_api",
+ "num_of_tokens_per_chunk_combine_api", "num_of_tokens_per_group_combine_api",
+ "num_of_blocks_combine_api", "num_of_additional_in_flight_s2g_combine_api",
+ "backward_combine_api", "device_side_sync_combine_api",
+}
+HYBRID_REALIZED_BOOL_FIELDS = {
+ "forward_dispatch_api", "device_side_sync_dispatch_api", "backward_combine_api",
+ "device_side_sync_combine_api",
+}
+
+
+def _hybrid_realized_config_is_valid(value: Any) -> bool:
+ if not isinstance(value, dict) or set(value) != HYBRID_REALIZED_CONFIG_FIELDS:
+ return False
+ for field, field_value in value.items():
+ if field in HYBRID_REALIZED_BOOL_FIELDS:
+ if type(field_value) is not bool:
+ return False
+ elif field == "token_data_type":
+ if field_value not in {"UINT8", "UINT16"}:
+ return False
+ elif type(field_value) is not int or field_value < 0:
+ return False
+ return all(value[field] > 0 for field in (
+ "hidden_dim", "max_num_of_tokens_per_rank", "num_of_experts_per_rank",
+ "num_of_ranks_per_node", "num_of_nodes",
+ ))
+
+
+def hybrid_communication_domains(ep_size: int, scale_up_domain: int) -> tuple[int, int]:
+ """Return active ranks per fabric domain and the number of such domains."""
+ if type(ep_size) is not int or type(scale_up_domain) is not int:
+ raise ContractError("hybrid communication topology must be integral")
+ if ep_size <= 0 or scale_up_domain <= 0:
+ raise ContractError("hybrid communication topology must be positive")
+ domain_ranks = min(ep_size, scale_up_domain)
+ if ep_size % domain_ranks:
+ raise ContractError("hybrid EP size does not divide into communication domains")
+ return domain_ranks, ep_size // domain_ranks
+
+
+def _hybrid_kernel_keys_are_valid(value: Any) -> bool:
+ return (
+ isinstance(value, list)
+ and len(value) == 3
+ and len(set(value)) == 3
+ and value == sorted(value)
+ and all(
+ isinstance(key, str)
+ and re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.+-]{0,511}", key)
+ for key in value
+ )
+ )
+
+
+def _hybrid_jit_evidence_is_valid(value: Any, kernel_keys: Any) -> bool:
+ if not _hybrid_kernel_keys_are_valid(kernel_keys) or not isinstance(value, list):
+ return False
+ if len(value) != len(kernel_keys):
+ return False
+ rank_sets = []
+ for expected_key, item in zip(kernel_keys, value):
+ if not isinstance(item, dict) or set(item) != {"kernel_key", "rank_artifacts"}:
+ return False
+ rank_artifacts = item["rank_artifacts"]
+ if item["kernel_key"] != expected_key or not isinstance(rank_artifacts, list):
+ return False
+ ranks = []
+ for artifact in rank_artifacts:
+ if not isinstance(artifact, dict) or set(artifact) != {"bytes", "rank", "sha256"}:
+ return False
+ rank, digest, size = artifact["rank"], artifact["sha256"], artifact["bytes"]
+ if (
+ type(rank) is not int
+ or rank < 0
+ or not isinstance(digest, str)
+ or not re.fullmatch(r"[0-9a-f]{64}", digest)
+ or type(size) is not int
+ or size <= 0
+ ):
+ return False
+ ranks.append(rank)
+ if not ranks or ranks != list(range(len(ranks))):
+ return False
+ rank_sets.append(ranks)
+ return all(ranks == rank_sets[0] for ranks in rank_sets)
+
+
+def backend_provenance_issues(backend: str, provenance: dict[str, Any]) -> list[str]:
+ unknown = [
+ field for field, value in provenance.items()
+ if isinstance(value, str) and value.strip().lower() == "unknown"
+ ]
+ unresolved = [
+ field for field in REQUIRED_BACKEND_PROVENANCE.get(backend, ())
+ if not _resolved_provenance_value(field, provenance.get(field))
+ ]
+ if backend == "deepep":
+ mode = provenance.get("mnnvl_comm")
+ allow = provenance.get("allow_mnnvl")
+ valid_modes = {
+ "not-requested": False,
+ "explicit-allow-mnnvl": True,
+ }
+ if type(allow) is not bool or valid_modes.get(mode) is not allow:
+ unresolved.append("mnnvl_comm")
+ if provenance.get("backend_lineage") != "deepep-v1":
+ unresolved.append("backend_lineage")
+ if backend in {"deepep", "uccl"}:
+ mode = provenance.get("mode")
+ num_nvl_bytes = provenance.get("num_nvl_bytes")
+ num_rdma_bytes = provenance.get("num_rdma_bytes")
+ if mode not in {"normal", "low-latency"}:
+ unresolved.append("mode")
+ if type(num_nvl_bytes) is not int or num_nvl_bytes < 0:
+ unresolved.append("num_nvl_bytes")
+ if type(num_rdma_bytes) is not int or num_rdma_bytes < 0:
+ unresolved.append("num_rdma_bytes")
+ if mode == "normal" and (type(num_nvl_bytes) is not int or num_nvl_bytes <= 0):
+ unresolved.append("num_nvl_bytes")
+ if mode == "low-latency":
+ if num_nvl_bytes != 0:
+ unresolved.append("num_nvl_bytes")
+ if type(num_rdma_bytes) is not int or num_rdma_bytes <= 0:
+ unresolved.append("num_rdma_bytes")
+ if (
+ type(provenance.get("num_max_tokens_per_rank")) is not int
+ or provenance["num_max_tokens_per_rank"] <= 0
+ ):
+ unresolved.append("num_max_tokens_per_rank")
+ if backend == "deepep" and (
+ type(provenance.get("num_qps_per_rank")) is not int
+ or provenance["num_qps_per_rank"] <= 0
+ ):
+ unresolved.append("num_qps_per_rank")
+ if backend == "deepep-v2":
+ for field in ("num_experts", "tuning_num_experts"):
+ if type(provenance.get(field)) is not int or provenance[field] <= 0:
+ unresolved.append(field)
+ if not _deepep_v2_jit_cubins_are_valid(provenance.get("jit_cubins")):
+ unresolved.append("jit_cubins")
+ if provenance.get("jit_random_seed") != "collectivex-deepep-v2-fa8a9b1":
+ unresolved.append("jit_random_seed")
+ unresolved.extend(
+ field for field, expected in DEEPEP_V2_V1_PROVENANCE.items()
+ if provenance.get(field) != expected
+ )
+ policy = (
+ provenance.get("allow_hybrid_mode"),
+ provenance.get("gin_enabled"),
+ provenance.get("communication_backend"),
+ )
+ if policy not in {
+ (False, False, "nccl-device-lsa"),
+ (True, True, "nccl-gin"),
+ }:
+ unresolved.extend(
+ ("allow_hybrid_mode", "gin_enabled", "communication_backend")
+ )
+ content_roles = {
+ "deepep-v2": {"deepep-extension", "nccl", "nvshmem"},
+ "deepep-hybrid": {"deepep-extension", "deepep-hybrid-extension"},
+ "uccl": {
+ "uccl-distribution", "uccl-wrapper", "intervaltree-distribution",
+ "sortedcontainers-distribution", "cuda-runtime",
+ },
+ }.get(backend)
+ if content_roles is not None and not _content_evidence_is_valid(
+ provenance.get("loaded_libraries"), content_roles
+ ):
+ unresolved.append("loaded_libraries")
+ if backend in {"deepep-v2", "deepep-hybrid"} and not re.fullmatch(
+ r"[0-9a-f]{40}", str(provenance.get("deepep_tree", ""))
+ ):
+ unresolved.append("deepep_tree")
+ if backend == "deepep-hybrid" and provenance.get("backend_lineage") != "deepep-hybrid":
+ unresolved.append("backend_lineage")
+ if backend == "deepep-hybrid":
+ if not _hybrid_realized_config_is_valid(provenance.get("realized_config")):
+ unresolved.append("realized_config")
+ if not _hybrid_kernel_keys_are_valid(provenance.get("jit_kernel_keys")):
+ unresolved.append("jit_kernel_keys")
+ if not _hybrid_jit_evidence_is_valid(
+ provenance.get("jit_shared_objects"), provenance.get("jit_kernel_keys")
+ ):
+ unresolved.append("jit_shared_objects")
+ if backend == "uccl" and provenance.get("backend_lineage") != "uccl":
+ unresolved.append("backend_lineage")
+ if backend == "uccl" and provenance.get("uccl_dependency_versions") != (
+ UCCL_DEPENDENCY_VERSIONS
+ ):
+ unresolved.append("uccl_dependency_versions")
+ if backend == "nccl-ep":
+ collective = provenance.get("collective_library")
+ if collective not in {"nccl", "rccl"}:
+ unresolved.append("collective_library")
+ if provenance.get("backend_lineage") != collective:
+ unresolved.append("backend_lineage")
+ if backend == "mori" and provenance.get("kernel_type") == "InterNodeV1":
+ expected = {
+ "block_num": 96,
+ "rdma_block_num": 64,
+ "dispatch_warps": 8,
+ "combine_warps": 8,
+ "num_qps": 1,
+ "use_external_inp_buf": True,
+ "gpus_per_node": 8,
+ }
+ unresolved.extend(
+ field for field, value in expected.items()
+ if provenance.get(field) != value
+ )
+ for field, minimum in (
+ ("num_nvl_bytes", 0), ("num_rdma_bytes", 0),
+ ("num_qps_per_rank", 1),
+ ):
+ if field in provenance and (
+ type(provenance[field]) is not int or provenance[field] < minimum
+ ):
+ unresolved.append(field)
+ if "rdma_block_num" in provenance and (
+ type(provenance["rdma_block_num"]) is not int
+ or provenance["rdma_block_num"] < 0
+ ):
+ unresolved.append("rdma_block_num")
+ if "use_external_inp_buf" in provenance and type(
+ provenance["use_external_inp_buf"]
+ ) is not bool:
+ unresolved.append("use_external_inp_buf")
+ return sorted(set(unknown + unresolved))
+
+
+def provenance_complete(
+ provenance: dict[str, Any], backend: str, git_run: dict[str, Any] | None,
+ *, image_digest: Any, image_verified: Any, squash_sha256: Any,
+) -> bool:
+ image = str(image_digest or "")
+ squash = str(squash_sha256 or "")
+ return (
+ not backend_provenance_issues(backend, provenance)
+ and image_verified is True
+ and bool(re.fullmatch(r"sha256:[0-9a-f]{64}", image))
+ and bool(re.fullmatch(r"[0-9a-f]{64}", squash))
+ and isinstance(git_run, dict)
+ and all(git_run.get(field) for field in GIT_RUN_FIELDS)
+ )
+
+
+def strict_load(path: str | os.PathLike[str]) -> Any:
+ """Load JSON while rejecting duplicate keys and non-finite constants."""
+ def pairs(items):
+ result = {}
+ for key, value in items:
+ if key in result:
+ raise ContractError(f"duplicate JSON key {key!r}")
+ result[key] = value
+ return result
+
+ def constant(value):
+ raise ContractError(f"non-finite JSON number {value}")
+
+ try:
+ with open(path) as handle:
+ return json.load(handle, object_pairs_hook=pairs, parse_constant=constant)
+ except (OSError, json.JSONDecodeError) as exc:
+ raise ContractError(f"invalid JSON {path}: {exc}") from exc
+
+
+def canonical_json_bytes(value: Any) -> bytes:
+ """Canonical finite JSON bytes for checksums and immutable artifacts."""
+ _finite_tree(value)
+ try:
+ return json.dumps(
+ value, allow_nan=False, ensure_ascii=False, sort_keys=True,
+ separators=(",", ":"),
+ ).encode("utf-8")
+ except (TypeError, ValueError) as exc:
+ raise ContractError(f"value is not canonical JSON: {exc}") from exc
+
+
+def content_manifest_evidence(
+ *, role: str, name: str, files: Iterable[tuple[str, str | os.PathLike[str]]]
+) -> dict[str, str]:
+ """Hash a labeled file set without exposing any host path in provenance."""
+ if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.+-]{0,127}", role):
+ raise ContractError("content evidence role is invalid")
+ if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.+-]{0,159}", name):
+ raise ContractError("content evidence name is invalid")
+ manifest: list[dict[str, Any]] = []
+ labels: set[str] = set()
+ for label, raw_path in files:
+ logical = PurePosixPath(label)
+ if (
+ not label
+ or logical.is_absolute()
+ or ".." in logical.parts
+ or label in labels
+ or any(ord(character) < 0x20 or ord(character) > 0x7E for character in label)
+ ):
+ raise ContractError("content evidence label is invalid or duplicated")
+ path = Path(raw_path)
+ if not path.is_file():
+ raise ContractError("content evidence source is not a file")
+ digest = hashlib.sha256()
+ size = 0
+ with path.open("rb") as handle:
+ for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+ digest.update(chunk)
+ size += len(chunk)
+ labels.add(label)
+ manifest.append({"bytes": size, "label": label, "sha256": digest.hexdigest()})
+ if not manifest:
+ raise ContractError("content evidence cannot be empty")
+ digest = hashlib.sha256(
+ canonical_json_bytes(sorted(manifest, key=lambda item: item["label"]))
+ ).hexdigest()
+ return {"name": name, "role": role, "sha256": digest}
+
+
+def _obj(value: Any, path: str) -> dict[str, Any]:
+ if not isinstance(value, dict):
+ raise ContractError(f"{path} must be an object")
+ return value
+
+
+def _keys(value: Any, expected: set[str], path: str) -> dict[str, Any]:
+ obj = _obj(value, path)
+ actual = set(obj)
+ if actual != expected:
+ raise ContractError(
+ f"{path} fields differ: missing={sorted(expected - actual)}, "
+ f"extra={sorted(actual - expected)}"
+ )
+ return obj
+
+
+def _text(value: Any, path: str, *, nullable: bool = False) -> str | None:
+ if nullable and value is None:
+ return None
+ if not isinstance(value, str) or not value:
+ raise ContractError(f"{path} must be a non-empty string")
+ return value
+
+
+def _integer(value: Any, path: str, *, minimum: int = 0) -> int:
+ if type(value) is not int or value < minimum:
+ raise ContractError(f"{path} must be an integer >= {minimum}")
+ return value
+
+
+def validate_conditioning_contract(value: Any, phase: str) -> dict[str, Any]:
+ """Validate the exact phase-specific v1 conditioning schedule."""
+ if phase not in V1_CONDITIONING_LADDERS:
+ raise ContractError("raw conditioning phase is invalid")
+ conditioning = _keys(
+ value, {"contract", "ladder", "roundtrips_per_shape"},
+ "raw.measurement.conditioning",
+ )
+ ladder = conditioning["ladder"]
+ if (
+ conditioning["contract"] != identity.V1_CASE_PROFILE["conditioning_contract"]
+ or type(ladder) is not list
+ or any(type(point) is not int for point in ladder)
+ or ladder != list(V1_CONDITIONING_LADDERS[phase])
+ or _integer(
+ conditioning["roundtrips_per_shape"],
+ "raw.measurement.conditioning.roundtrips_per_shape",
+ minimum=1,
+ ) != V1_CONDITIONING_ROUNDS_PER_SHAPE
+ ):
+ raise ContractError(f"raw {phase} conditioning contract differs")
+ return conditioning
+
+
+def _number(value: Any, path: str, *, minimum: float | None = None) -> float:
+ if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value):
+ raise ContractError(f"{path} must be finite")
+ result = float(value)
+ if minimum is not None and result < minimum:
+ raise ContractError(f"{path} must be >= {minimum}")
+ return result
+
+
+def _finite_tree(value: Any, path: str = "$") -> None:
+ if isinstance(value, float) and not math.isfinite(value):
+ raise ContractError(f"{path} contains a non-finite number")
+ if isinstance(value, list):
+ for index, item in enumerate(value):
+ _finite_tree(item, f"{path}[{index}]")
+ elif isinstance(value, dict):
+ for key, item in value.items():
+ _finite_tree(item, f"{path}.{key}")
+
+
+def _typed(value: Any, kind: str, path: str) -> str:
+ if not identity.is_typed_id(value, kind):
+ raise ContractError(f"{path} is not a {kind} ID")
+ return value
+
+
+def _sha256_json(value: Any) -> str:
+ payload = json.dumps(
+ value, allow_nan=False, ensure_ascii=False, sort_keys=True, separators=(",", ":")
+ ).encode()
+ return hashlib.sha256(payload).hexdigest()
+
+
+@lru_cache(maxsize=None)
+def _expected_eplb_plan(
+ routing: str,
+ topk: int,
+ logical_experts: int,
+ physical_experts: int,
+ ep_size: int,
+ seed: int,
+ reference_tokens_per_rank: int,
+) -> dict[str, Any]:
+ indices, _ = workload_contract.canonical_routing_rows(
+ reference_tokens_per_rank * ep_size,
+ logical_experts,
+ topk,
+ routing,
+ seed,
+ )
+ load = [0] * logical_experts
+ for row in indices:
+ for expert in row:
+ load[expert] += 1
+ return eplb_contract.build_plan(load, physical_experts, ep_size)
+
+
+@lru_cache(maxsize=None)
+def _expected_canonical_trace(
+ routing: str,
+ hidden: int,
+ topk: int,
+ logical_experts: int,
+ physical_experts: int,
+ ep_size: int,
+ tokens_per_rank: int,
+ seed: int,
+ eplb_enabled: bool,
+ reference_tokens_per_rank: int,
+) -> tuple[str, dict[str, str], str, list[list[int]], list[list[float]]]:
+ member, checksums, indices, weights = workload_contract.canonical_member(
+ routing,
+ hidden,
+ topk,
+ logical_experts,
+ ep_size,
+ tokens_per_rank,
+ seed,
+ )
+ if eplb_enabled:
+ plan = _expected_eplb_plan(
+ routing,
+ topk,
+ logical_experts,
+ physical_experts,
+ ep_size,
+ seed,
+ reference_tokens_per_rank,
+ )
+ indices = eplb_contract.remap_rows(indices, plan)
+ routing_hash = workload_contract.trace_checksums(indices, weights)["trace"]
+ return member, checksums, routing_hash, indices, weights
+
+
+def _coefficient_of_variation(values: list[int]) -> float:
+ mean = sum(values) / len(values)
+ if mean == 0:
+ return 0.0
+ variance = sum((value - mean) ** 2 for value in values) / len(values)
+ return variance**0.5 / mean
+
+
+def _expected_routing_summary(
+ indices: list[list[int]],
+ weights: list[list[float]],
+ *,
+ physical_experts: int,
+ ep_size: int,
+ tokens_per_rank: int,
+ gpus_per_node: int,
+ scale_up_domain: int,
+) -> dict[str, Any]:
+ """Recompute every published routing/load statistic without torch."""
+ experts_per_rank = physical_experts // ep_size
+ expert_load = [0] * physical_experts
+ assignment_load = [0] * ep_size
+ payload_load = [0] * ep_size
+ fanouts: list[int] = []
+ local = same_node = same_domain = copies = 0
+ for token, row in enumerate(indices):
+ destinations = {expert // experts_per_rank for expert in row}
+ source = token // tokens_per_rank
+ fanouts.append(len(destinations))
+ for expert in row:
+ expert_load[expert] += 1
+ assignment_load[expert // experts_per_rank] += 1
+ for destination in destinations:
+ payload_load[destination] += 1
+ copies += 1
+ local += destination == source
+ same_node += destination // gpus_per_node == source // gpus_per_node
+ same_domain += destination // scale_up_domain == source // scale_up_domain
+ fanout_histogram = [fanouts.count(value) for value in range(1, ep_size + 1)]
+ expert_mean = sum(expert_load) / len(expert_load)
+ return {
+ "empty_expert_count": expert_load.count(0),
+ "empty_rank_count": payload_load.count(0),
+ "expert_assignment_rank_cv": _coefficient_of_variation(assignment_load),
+ "expert_assignments_per_rank": assignment_load,
+ "expert_load_cv": _coefficient_of_variation(expert_load),
+ "expert_load_max": max(expert_load),
+ "expert_load_mean": expert_mean,
+ "expert_load_min": min(expert_load),
+ "fanout_histogram": fanout_histogram,
+ "fanout_max": max(fanouts),
+ "fanout_mean": sum(fanouts) / len(fanouts),
+ "fanout_min": min(fanouts),
+ "hash": workload_contract.trace_checksums(indices, weights)["trace"],
+ "hotspot_ratio": max(expert_load) / expert_mean if expert_mean else 0.0,
+ "locality": {
+ "placement": "packed",
+ "local_rank_fraction": local / copies,
+ "same_node_fraction": same_node / copies,
+ "same_scaleup_domain_fraction": same_domain / copies,
+ "cross_node_fraction": 1 - same_node / copies,
+ "cross_domain_fraction": 1 - same_domain / copies,
+ "gpus_per_node": gpus_per_node,
+ "scale_up_domain": scale_up_domain,
+ "copies": copies,
+ },
+ "payload_copies_per_rank": payload_load,
+ "payload_rank_cv": _coefficient_of_variation(payload_load),
+ "routed_copies": copies,
+ "source_token_stats": {
+ "min": tokens_per_rank,
+ "mean": float(tokens_per_rank),
+ "max": tokens_per_rank,
+ "cv": 0.0,
+ "empty_ranks": 0,
+ "total": tokens_per_rank * ep_size,
+ "ranks": ep_size,
+ },
+ }
+
+
+def _expected_histogram(samples: list[float], bins: int = 40) -> dict[str, Any]:
+ low, high = min(samples), max(samples)
+ if high <= low:
+ return {"n": len(samples), "min": low, "max": high, "bins": bins, "counts": [len(samples)]}
+ counts = [0] * bins
+ span = high - low
+ for sample in samples:
+ index = min(bins - 1, int((sample - low) / span * bins))
+ counts[index] += 1
+ return {
+ "n": len(samples),
+ "min": round(low, 3),
+ "max": round(high, 3),
+ "bins": bins,
+ "counts": counts,
+ }
+
+
+def _expected_anomalies(
+ tokens: int, components: dict[str, Any]
+) -> list[dict[str, Any]]:
+ dispatch = components["dispatch"]["percentiles_us"]
+ combine = components["combine"]["percentiles_us"]
+ roundtrip = components["roundtrip"]["percentiles_us"]
+ isolated = components["isolated_sum"]["percentiles_us"]
+ anomalies: list[dict[str, Any]] = []
+ if isolated is not None and roundtrip["p99"] > 3.0 * isolated["p99"]:
+ anomalies.append({
+ "type": "roundtrip_gt_isolated_sum",
+ "T": tokens,
+ "roundtrip_p99": round(roundtrip["p99"], 2),
+ "isolated_sum_p99": round(isolated["p99"], 2),
+ "ratio": round(roundtrip["p99"] / isolated["p99"], 2),
+ "threshold": 3.0,
+ })
+ floor = max(dispatch["p50"], combine["p50"]) if dispatch and combine else None
+ if floor and roundtrip["p50"] < 0.95 * floor:
+ anomalies.append({
+ "type": "roundtrip_lt_component_floor",
+ "T": tokens,
+ "roundtrip_p50": round(roundtrip["p50"], 2),
+ "component_floor_p50": round(floor, 2),
+ })
+ return anomalies
+
+
+def _validate_canonical_workload(
+ workload: dict[str, Any],
+ scheduled_case: dict[str, Any],
+ rows: list[dict[str, Any]],
+ eplb: dict[str, Any],
+) -> None:
+ """Bind every canonical member and measured routing hash to its scheduled token row."""
+ profile = identity.V1_CASE_PROFILE
+ if eplb["enabled"]:
+ plan = _expected_eplb_plan(
+ scheduled_case["routing"],
+ scheduled_case["topk"],
+ scheduled_case["experts"],
+ eplb["num_physical_experts"],
+ scheduled_case["ep"],
+ profile["seed"],
+ profile["eplb_reference_tokens_per_rank"],
+ )
+ if eplb["mapping_hash"] != eplb_contract.mapping_hash(plan):
+ raise ContractError("raw EPLB mapping differs from the frozen canonical plan")
+
+ expected: dict[str, dict[str, str]] = {}
+ for index, row in enumerate(rows):
+ member, checksums, routing_hash, _, _ = _expected_canonical_trace(
+ scheduled_case["routing"],
+ scheduled_case["hidden"],
+ scheduled_case["topk"],
+ scheduled_case["experts"],
+ eplb["num_physical_experts"],
+ scheduled_case["ep"],
+ row["tokens_per_rank"],
+ profile["seed"],
+ eplb["enabled"],
+ profile["eplb_reference_tokens_per_rank"],
+ )
+ if row["routing"]["hash"] != routing_hash:
+ raise ContractError(
+ f"raw.measurement.rows[{index}].routing.hash differs from its canonical member"
+ )
+ expected[member] = checksums
+ if (
+ len(expected) != len(rows)
+ or workload["members"] != sorted(expected)
+ or workload["manifest_checksums"] != expected
+ ):
+ raise ContractError("raw canonical member set/checksums differ from scheduled rows")
+ expected_workload_id = identity.workload_id({
+ "members": [
+ {"checksums": expected[member], "workload_id": member}
+ for member in sorted(expected)
+ ]
+ })
+ if workload["workload_id"] != expected_workload_id:
+ raise ContractError("raw composite workload identity differs from scheduled rows")
+
+
+def _nearest_rank(samples: list[float], q: int) -> float:
+ ordered = sorted(samples)
+ return ordered[max(0, min(len(ordered) - 1, math.ceil(q / 100 * len(ordered)) - 1))]
+
+
+def _close(observed: Any, expected: float, path: str, tolerance: float = 1e-6) -> None:
+ value = _number(observed, path)
+ if not math.isclose(value, expected, rel_tol=tolerance, abs_tol=tolerance):
+ raise ContractError(f"{path}={value} differs from recomputed {expected}")
+
+
+def _equivalent(
+ observed: Any, expected: Any, path: str, *, tolerance: float = 1e-6
+) -> None:
+ """Compare a recomputed JSON subtree while allowing only float roundoff."""
+ if isinstance(expected, dict):
+ value = _keys(observed, set(expected), path)
+ for key, child in expected.items():
+ _equivalent(value[key], child, f"{path}.{key}", tolerance=tolerance)
+ return
+ if isinstance(expected, list):
+ if not isinstance(observed, list) or len(observed) != len(expected):
+ raise ContractError(f"{path} differs from recomputed evidence")
+ for index, child in enumerate(expected):
+ _equivalent(observed[index], child, f"{path}[{index}]", tolerance=tolerance)
+ return
+ if isinstance(expected, float):
+ _close(observed, expected, path, tolerance)
+ return
+ if type(observed) is not type(expected) or observed != expected:
+ raise ContractError(f"{path} differs from recomputed evidence")
+
+
+def _schema_equal(left: Any, right: Any) -> bool:
+ """JSON Schema equality: booleans are distinct from numbers."""
+ if isinstance(left, bool) or isinstance(right, bool):
+ return type(left) is type(right) and left == right
+ if isinstance(left, dict) and isinstance(right, dict):
+ return set(left) == set(right) and all(
+ _schema_equal(left[key], right[key]) for key in left
+ )
+ if isinstance(left, list) and isinstance(right, list):
+ return len(left) == len(right) and all(
+ _schema_equal(a, b) for a, b in zip(left, right, strict=True)
+ )
+ return left == right
+
+
+def _schema_ref(root: dict[str, Any], reference: str) -> dict[str, Any]:
+ if not reference.startswith("#/"):
+ raise ContractError("native artifact schema contains a non-local reference")
+ value: Any = root
+ for part in reference[2:].split("/"):
+ part = part.replace("~1", "/").replace("~0", "~")
+ if not isinstance(value, dict) or part not in value:
+ raise ContractError("native artifact schema contains a broken reference")
+ value = value[part]
+ if not isinstance(value, dict):
+ raise ContractError("native artifact schema reference is not an object")
+ return value
+
+
+def _schema_type_matches(value: Any, expected: str) -> bool:
+ if expected == "null":
+ return value is None
+ if expected == "boolean":
+ return type(value) is bool
+ if expected == "object":
+ return isinstance(value, dict)
+ if expected == "array":
+ return isinstance(value, list)
+ if expected == "string":
+ return isinstance(value, str)
+ if expected == "number":
+ return (
+ not isinstance(value, bool)
+ and isinstance(value, (int, float))
+ and math.isfinite(value)
+ )
+ if expected == "integer":
+ return (
+ not isinstance(value, bool)
+ and isinstance(value, (int, float))
+ and math.isfinite(value)
+ and float(value).is_integer()
+ )
+ raise ContractError(f"native artifact schema uses unsupported type {expected!r}")
+
+
+def _validate_schema_value(
+ value: Any, schema: dict[str, Any], root: dict[str, Any], path: str
+) -> None:
+ """Validate the bounded JSON Schema subset used by native artifact contracts."""
+ if "$ref" in schema:
+ _validate_schema_value(value, _schema_ref(root, schema["$ref"]), root, path)
+ return
+ if "oneOf" in schema:
+ matches = 0
+ for candidate in schema["oneOf"]:
+ try:
+ _validate_schema_value(value, candidate, root, path)
+ except ContractError:
+ continue
+ matches += 1
+ if matches != 1:
+ raise ContractError(f"{path} must match exactly one native schema alternative")
+ return
+ expected_type = schema.get("type")
+ if expected_type is not None and not _schema_type_matches(value, expected_type):
+ raise ContractError(f"{path} is not a schema {expected_type}")
+ if "const" in schema and not _schema_equal(value, schema["const"]):
+ raise ContractError(f"{path} differs from its schema constant")
+ if "enum" in schema and not any(_schema_equal(value, item) for item in schema["enum"]):
+ raise ContractError(f"{path} is outside its schema enum")
+
+ if isinstance(value, dict):
+ required = set(schema.get("required", ()))
+ properties = schema.get("properties", {})
+ missing = required - set(value)
+ if missing:
+ raise ContractError(f"{path} lacks schema fields {sorted(missing)}")
+ additional = schema.get("additionalProperties", True)
+ extra = set(value) - set(properties)
+ if additional is False and extra:
+ raise ContractError(f"{path} has extra schema fields {sorted(extra)}")
+ for key, item in value.items():
+ if key in properties:
+ _validate_schema_value(item, properties[key], root, f"{path}.{key}")
+ elif isinstance(additional, dict):
+ _validate_schema_value(item, additional, root, f"{path}.{key}")
+ property_names = schema.get("propertyNames")
+ if property_names is not None:
+ for key in value:
+ _validate_schema_value(key, property_names, root, f"{path}.")
+
+ if isinstance(value, list):
+ if len(value) < schema.get("minItems", 0):
+ raise ContractError(f"{path} has too few schema items")
+ maximum = schema.get("maxItems")
+ if maximum is not None and len(value) > maximum:
+ raise ContractError(f"{path} has too many schema items")
+ if schema.get("uniqueItems") and any(
+ _schema_equal(item, prior)
+ for index, item in enumerate(value)
+ for prior in value[:index]
+ ):
+ raise ContractError(f"{path} schema items are not unique")
+ if "items" in schema:
+ for index, item in enumerate(value):
+ _validate_schema_value(item, schema["items"], root, f"{path}[{index}]")
+
+ if isinstance(value, str):
+ if len(value) < schema.get("minLength", 0):
+ raise ContractError(f"{path} is shorter than its schema minimum")
+ maximum = schema.get("maxLength")
+ if maximum is not None and len(value) > maximum:
+ raise ContractError(f"{path} is longer than its schema maximum")
+ if "pattern" in schema and re.search(schema["pattern"], value) is None:
+ raise ContractError(f"{path} does not match its schema pattern")
+ if schema.get("format") == "date-time":
+ try:
+ parsed = dt.datetime.fromisoformat(value.replace("Z", "+00:00"))
+ except ValueError as exc:
+ raise ContractError(f"{path} is not a schema date-time") from exc
+ if parsed.tzinfo is None:
+ raise ContractError(f"{path} schema date-time lacks a timezone")
+
+ if (
+ not isinstance(value, bool)
+ and isinstance(value, (int, float))
+ and math.isfinite(value)
+ ):
+ if "minimum" in schema and value < schema["minimum"]:
+ raise ContractError(f"{path} is below its schema minimum")
+ if "maximum" in schema and value > schema["maximum"]:
+ raise ContractError(f"{path} is above its schema maximum")
+
+
+def _validate_native_schema(name: str, value: Any) -> None:
+ schema = _SCHEMA_CACHE.get(name)
+ if schema is None:
+ loaded = strict_load(SCHEMA_DIR / name)
+ if not isinstance(loaded, dict):
+ raise ContractError(f"native artifact schema {name} is not an object")
+ schema = loaded
+ _SCHEMA_CACHE[name] = schema
+ _validate_schema_value(value, schema, schema, "$")
+
+
+def validate_samples_document(document: Any) -> dict[str, Any]:
+ _validate_native_schema("samples-v1.schema.json", document)
+ doc = _keys(
+ document,
+ {"allocation_id", "attempt_id", "case_id", "format", "points", "sampling",
+ "schema_version", "series_id"},
+ "samples",
+ )
+ if doc["format"] != SAMPLES_FORMAT or doc["schema_version"] != 1:
+ raise ContractError("samples format/schema differs from v1")
+ for field, kind in (
+ ("allocation_id", "allocation"), ("attempt_id", "attempt"),
+ ("case_id", "case"), ("series_id", "series"),
+ ):
+ _typed(doc[field], kind, f"samples.{field}")
+ sampling = _keys(
+ doc["sampling"], {"iterations_per_trial", "reduction", "trials"}, "samples.sampling"
+ )
+ if (
+ _integer(sampling["iterations_per_trial"], "samples.sampling.iterations_per_trial", minimum=1) != 8
+ or _integer(sampling["trials"], "samples.sampling.trials", minimum=1) != 64
+ or sampling["reduction"] != identity.V1_CASE_PROFILE["rank_reduction"]
+ ):
+ raise ContractError("samples must use the fixed 8x64 cross-rank-max contract")
+ points = doc["points"]
+ if not isinstance(points, list) or not points:
+ raise ContractError("samples.points must be non-empty")
+ seen = set()
+ for index, point_value in enumerate(points):
+ path = f"samples.points[{index}]"
+ point = _keys(
+ point_value,
+ {"components", "evidence_id", "point_id", "sample_sha256", "tokens_per_rank"},
+ path,
+ )
+ tokens = _integer(point["tokens_per_rank"], f"{path}.tokens_per_rank", minimum=1)
+ if tokens in seen:
+ raise ContractError(f"duplicate sample token point {tokens}")
+ seen.add(tokens)
+ _typed(point["point_id"], "point", f"{path}.point_id")
+ _typed(point["evidence_id"], "evidence", f"{path}.evidence_id")
+ components = _keys(point["components"], {"combine", "dispatch", "roundtrip"}, f"{path}.components")
+ for name, component_value in components.items():
+ component = _keys(
+ component_value, {"availability", "sample_count", "trials"},
+ f"{path}.components.{name}",
+ )
+ availability = component["availability"]
+ count = _integer(component["sample_count"], f"{path}.components.{name}.sample_count")
+ trials = component["trials"]
+ if availability == "unavailable":
+ if count != 0 or trials is not None or name == "roundtrip":
+ raise ContractError(f"{path}.components.{name} has invalid unavailability")
+ continue
+ if availability != "measured" or not isinstance(trials, list) or len(trials) != 64:
+ raise ContractError(f"{path}.components.{name} must contain 64 measured trials")
+ if any(not isinstance(trial, list) or len(trial) != 8 for trial in trials):
+ raise ContractError(f"{path}.components.{name} trials must each contain 8 samples")
+ flattened = [
+ _number(sample, f"{path}.components.{name}.trials", minimum=0.0)
+ for trial in trials for sample in trial
+ ]
+ if count != 512 or len(flattened) != 512:
+ raise ContractError(f"{path}.components.{name} must contain 512 samples")
+ sample_base = {"components": components, "tokens_per_rank": tokens}
+ if point["sample_sha256"] != _sha256_json(sample_base):
+ raise ContractError(f"{path}.sample_sha256 differs")
+ return doc
+
+
+def _validate_component(
+ component_value: Any,
+ sample_component: dict[str, Any] | None,
+ path: str,
+ *,
+ derived: bool = False,
+) -> None:
+ component = _keys(
+ component_value, {"availability", "origin", "percentiles_us", "sample_count"}, path
+ )
+ availability = component["availability"]
+ if availability == "unavailable":
+ if component != {
+ "availability": "unavailable", "origin": None,
+ "percentiles_us": None, "sample_count": 0,
+ }:
+ raise ContractError(f"{path} has invalid unavailable representation")
+ if sample_component and sample_component["availability"] != "unavailable":
+ raise ContractError(f"{path} disagrees with samples")
+ return
+ expected_availability = "derived" if derived else "measured"
+ expected_origin = "derived-percentile-sum" if derived else "measured"
+ if availability != expected_availability or component["origin"] != expected_origin:
+ raise ContractError(f"{path} has invalid availability/origin")
+ percentiles = _keys(component["percentiles_us"], set(PERCENTILES), f"{path}.percentiles_us")
+ if derived:
+ if component["sample_count"] != 0:
+ raise ContractError(f"{path}.sample_count must be zero for a derived value")
+ return
+ if sample_component is None or sample_component["availability"] != "measured":
+ raise ContractError(f"{path} lacks measured sample evidence")
+ flattened = [sample for trial in sample_component["trials"] for sample in trial]
+ if component["sample_count"] != len(flattened):
+ raise ContractError(f"{path}.sample_count differs from exact samples")
+ for name, percentile in zip(PERCENTILES, (50, 90, 95, 99), strict=True):
+ _close(percentiles[name], _nearest_rank(flattened, percentile), f"{path}.{name}")
+
+
+def _validate_oracle(
+ value: Any, path: str, profile: dict[str, Any] | None = None
+) -> dict[str, Any]:
+ profile = profile or identity.V1_NORMAL_CASE_PROFILE
+ oracle = _keys(
+ value,
+ {"atol", "checks", "combine_weight_semantics", "contract", "dispatch_sha256",
+ "max_absolute_error", "max_elementwise_relative_error", "max_relative_error",
+ "max_weight_error", "order_sha256", "ordering_contract", "passed", "receive_count",
+ "rtol"},
+ path,
+ )
+ if oracle["contract"] != profile["oracle_contract"]:
+ raise ContractError(f"{path}.contract differs")
+ checks = _keys(
+ oracle["checks"],
+ {"combine_values", "counts", "metadata", "multiplicity", "payload", "source_set",
+ "weights"},
+ f"{path}.checks",
+ )
+ if any(type(value) is not bool for value in checks.values()):
+ raise ContractError(f"{path}.checks must be boolean")
+ if type(oracle["passed"]) is not bool:
+ raise ContractError(f"{path}.passed must be boolean")
+ _integer(oracle["receive_count"], f"{path}.receive_count")
+ _text(oracle["ordering_contract"], f"{path}.ordering_contract")
+ expected_weight_semantics = (
+ "gate-weighted-sum"
+ if profile["combine_semantics"] == "gate-weighted"
+ else "unweighted-rank-sum"
+ )
+ if oracle["combine_weight_semantics"] != expected_weight_semantics:
+ raise ContractError(f"{path}.combine_weight_semantics differs from v1")
+ _close(oracle["rtol"], 5e-2, f"{path}.rtol")
+ _close(oracle["atol"], 2e-2, f"{path}.atol")
+ for field in ("dispatch_sha256", "order_sha256"):
+ digest = oracle[field]
+ if digest is not None and (
+ not isinstance(digest, str) or len(digest) != 64
+ or any(character not in "0123456789abcdef" for character in digest)
+ ):
+ raise ContractError(f"{path}.{field} is not a SHA-256 digest")
+ for field in (
+ "max_absolute_error", "max_elementwise_relative_error", "max_relative_error",
+ "max_weight_error",
+ ):
+ if oracle[field] is not None:
+ _number(oracle[field], f"{path}.{field}", minimum=0.0)
+ expected_pass = (
+ all(checks.values())
+ and oracle["max_relative_error"] is not None
+ and oracle["max_relative_error"] < 5e-2
+ )
+ if oracle["passed"] != expected_pass:
+ raise ContractError(f"{path}.passed differs from its evidence")
+ return oracle
+
+
+def validate_raw_document(document: Any, samples_document: Any) -> dict[str, Any]:
+ """Validate identities, exact samples, formulas, privacy, and the native raw shape."""
+ _validate_native_schema("raw-case-v1.schema.json", document)
+ doc = _keys(
+ document,
+ {"case", "format", "generated_at", "identity", "implementation", "measurement",
+ "outcome", "provenance", "record_type", "runtime_fingerprint", "sample_artifact",
+ "schema_version", "topology", "workload"},
+ "raw",
+ )
+ _finite_tree(doc)
+ if doc["format"] != RAW_FORMAT or doc["schema_version"] != 1 or doc["record_type"] != "case-attempt":
+ raise ContractError("raw format/schema/record type differs from v1")
+ _text(doc["generated_at"], "raw.generated_at")
+ identifiers = _keys(
+ doc["identity"],
+ {"allocation_factors", "allocation_id", "attempt_id", "attempt_ordinal", "case_factors",
+ "case_id", "series_factors", "series_id"},
+ "raw.identity",
+ )
+ for field, kind in (
+ ("allocation_id", "allocation"), ("attempt_id", "attempt"),
+ ("case_id", "case"), ("series_id", "series"),
+ ):
+ _typed(identifiers[field], kind, f"raw.identity.{field}")
+ ordinal = _integer(identifiers["attempt_ordinal"], "raw.identity.attempt_ordinal", minimum=1)
+ allocation_factors = _keys(
+ identifiers["allocation_factors"], ALLOCATION_FACTOR_FIELDS,
+ "raw.identity.allocation_factors",
+ )
+ case_factors = _keys(
+ identifiers["case_factors"], {"case", "profile", "sku"},
+ "raw.identity.case_factors",
+ )
+ scheduled_case = _keys(
+ case_factors["case"], TERMINAL_CASE_FIELDS, "raw.identity.case_factors.case"
+ )
+ profile = scheduled_case_profile(scheduled_case, "raw.identity.case_factors.case")
+ if case_factors["profile"] != profile:
+ raise ContractError("raw case profile differs from CollectiveX v1")
+ _text(case_factors["sku"], "raw.identity.case_factors.sku")
+ series_factors = _keys(
+ identifiers["series_factors"],
+ {"backend", "case_id", "image_digest", "implementation_contract_sha256",
+ "public_config_sha256", "routing_control_sha256",
+ "runtime_fingerprint_sha256", "source_sha", "squash_sha256", "workload_id"},
+ "raw.identity.series_factors",
+ )
+ if identity.allocation_id(identifiers["allocation_factors"]) != identifiers["allocation_id"]:
+ raise ContractError("allocation identity differs")
+ if identity.digest("case", identifiers["case_factors"]) != identifiers["case_id"]:
+ raise ContractError("case identity differs")
+ if identity.series_id(identifiers["series_factors"]) != identifiers["series_id"]:
+ raise ContractError("series identity differs")
+ if identity.attempt_id(
+ allocation=identifiers["allocation_id"], case=identifiers["case_id"], ordinal=ordinal
+ ) != identifiers["attempt_id"]:
+ raise ContractError("attempt identity differs")
+
+ samples = validate_samples_document(samples_document)
+ for field in ("allocation_id", "attempt_id", "case_id", "series_id"):
+ if samples[field] != identifiers[field]:
+ raise ContractError(f"samples.{field} differs from raw identity")
+ sample_by_token = {point["tokens_per_rank"]: point for point in samples["points"]}
+
+ case = _keys(
+ doc["case"],
+ {"attempt_ordinal", "backend", "eplb", "ep_size", "mode", "phase",
+ "required_publication", "resource_mode", "runner", "shape", "suite", "workload_name"},
+ "raw.case",
+ )
+ ep_size = _integer(case["ep_size"], "raw.case.ep_size", minimum=1)
+ if case["attempt_ordinal"] != ordinal:
+ raise ContractError("case attempt ordinal differs")
+ for field in ("backend", "mode", "phase", "required_publication", "resource_mode", "runner",
+ "suite", "workload_name"):
+ _text(case[field], f"raw.case.{field}")
+ shape = _keys(
+ case["shape"],
+ {"activation_profile", "dispatch_dtype", "eplb", "experts", "experts_per_rank",
+ "hidden", "kernel_gen", "num_logical_experts", "quant", "routing", "topk"},
+ "raw.case.shape",
+ )
+ hidden = _integer(shape["hidden"], "raw.case.shape.hidden", minimum=1)
+ topk = _integer(shape["topk"], "raw.case.shape.topk", minimum=1)
+ physical_experts = _integer(
+ shape["experts"], "raw.case.shape.experts", minimum=1
+ )
+ logical_experts = _integer(
+ shape["num_logical_experts"],
+ "raw.case.shape.num_logical_experts",
+ minimum=1,
+ )
+ experts_per_rank = _integer(
+ shape["experts_per_rank"], "raw.case.shape.experts_per_rank", minimum=1
+ )
+ quant = _keys(
+ shape["quant"],
+ {"combine_accum_dtype", "combine_input_dtype", "combine_output_dtype",
+ "combine_quant_mode", "scale_layout"},
+ "raw.case.shape.quant",
+ )
+ eplb = _keys(
+ case["eplb"],
+ {"enabled", "imbalance_after", "imbalance_before", "mapping_hash", "max_replicas",
+ "num_logical_experts", "num_physical_experts", "num_redundant", "planner",
+ "reference_tokens_per_rank", "replicated_experts"},
+ "raw.case.eplb",
+ )
+ if not isinstance(eplb["enabled"], bool):
+ raise ContractError("raw.case.eplb.enabled must be boolean")
+ expected_redundant = (
+ profile["eplb_redundant_experts"] if eplb["enabled"] else 0
+ )
+ expected_physical = eplb_contract.physical_count(
+ scheduled_case["experts"], expected_redundant, ep_size
+ )
+ if (
+ shape["eplb"] != eplb["enabled"]
+ or logical_experts != scheduled_case["experts"]
+ or physical_experts != expected_physical
+ or experts_per_rank * ep_size != physical_experts
+ or eplb["num_logical_experts"] != logical_experts
+ or eplb["num_physical_experts"] != physical_experts
+ or eplb["num_redundant"] != expected_redundant
+ ):
+ raise ContractError("raw EPLB/shape dimensions differ from the frozen profile")
+ if eplb["enabled"]:
+ expected_plan = _expected_eplb_plan(
+ scheduled_case["routing"],
+ topk,
+ logical_experts,
+ physical_experts,
+ ep_size,
+ profile["seed"],
+ profile["eplb_reference_tokens_per_rank"],
+ )
+ expected_eplb = {
+ "enabled": True,
+ "imbalance_after": expected_plan["imbalance_after"],
+ "imbalance_before": expected_plan["imbalance_before"],
+ "mapping_hash": eplb_contract.mapping_hash(expected_plan),
+ "max_replicas": expected_plan["max_replicas"],
+ "num_logical_experts": logical_experts,
+ "num_physical_experts": physical_experts,
+ "num_redundant": expected_redundant,
+ "planner": profile["eplb_planner"],
+ "reference_tokens_per_rank": profile[
+ "eplb_reference_tokens_per_rank"
+ ],
+ "replicated_experts": expected_plan["replicated_experts"],
+ }
+ else:
+ expected_eplb = {
+ "enabled": False,
+ "imbalance_after": None,
+ "imbalance_before": None,
+ "mapping_hash": None,
+ "max_replicas": None,
+ "num_logical_experts": logical_experts,
+ "num_physical_experts": physical_experts,
+ "num_redundant": 0,
+ "planner": None,
+ "reference_tokens_per_rank": None,
+ "replicated_experts": 0,
+ }
+ _equivalent(eplb, expected_eplb, "raw.case.eplb", tolerance=1e-9)
+ if case_factors["sku"] != case["runner"]:
+ raise ContractError("raw case runner differs from case identity")
+
+ workload = _keys(
+ doc["workload"],
+ {"activation_generator", "activation_identity", "activation_profile",
+ "cross_rank_consistent", "manifest_checksums", "members", "routing_generator", "source",
+ "trace_hashes", "trace_signature", "workload_id"},
+ "raw.workload",
+ )
+ if workload["source"] not in {"canonical-serialized", "seeded-runtime"}:
+ raise ContractError("raw workload source is invalid")
+ if workload["source"] == "canonical-serialized":
+ _typed(workload["workload_id"], "workload", "raw.workload.workload_id")
+ members = workload["members"]
+ checksums = workload["manifest_checksums"]
+ if (
+ not isinstance(members, list)
+ or not members
+ or members != sorted(set(members))
+ or not all(identity.is_typed_id(member, "workload") for member in members)
+ or not isinstance(checksums, dict)
+ or set(checksums) != set(members)
+ ):
+ raise ContractError("raw canonical workload members/checksums are invalid")
+ for member, values in checksums.items():
+ if (
+ not isinstance(values, dict)
+ or set(values) != {"topk_idx", "topk_weights", "trace"}
+ or any(not re.fullmatch(r"[0-9a-f]{64}", str(value)) for value in values.values())
+ ):
+ raise ContractError(f"raw canonical workload checksums differ for {member}")
+ expected_workload_id = identity.workload_id({
+ "members": [
+ {"checksums": checksums[member], "workload_id": member}
+ for member in members
+ ]
+ })
+ if workload["workload_id"] != expected_workload_id:
+ raise ContractError("raw composite workload identity differs from its members")
+ elif any(workload[field] is not None for field in ("members", "manifest_checksums", "workload_id")):
+ raise ContractError("raw seeded workload cannot claim serialized members")
+ if workload["cross_rank_consistent"] is not True:
+ raise ContractError("raw workload is not consistent across ranks")
+
+ measurement = _keys(
+ doc["measurement"],
+ {"component_order_contract", "conditioning", "contract", "rows",
+ "sampling", "source_allocation"},
+ "raw.measurement",
+ )
+ validate_conditioning_contract(measurement["conditioning"], case["phase"])
+ sampling = _keys(
+ measurement["sampling"],
+ {"contract", "iterations_per_trial", "percentile_method", "reduction",
+ "samples_per_component", "trials", "warmup_iterations", "warmup_semantics"},
+ "raw.measurement.sampling",
+ )
+ expected_sampling = {
+ "contract": profile["sampling_contract"], "iterations_per_trial": 8,
+ "percentile_method": profile["percentile_method"],
+ "reduction": profile["rank_reduction"],
+ "samples_per_component": 512, "trials": 64, "warmup_iterations": 32,
+ "warmup_semantics": "full-roundtrip-before-each-component-trial-point-v1",
+ }
+ if sampling != expected_sampling:
+ raise ContractError("raw sampling contract differs from fixed-512-v1")
+ if (
+ case["mode"] != profile["mode"]
+ or case["resource_mode"] != profile["resource_mode"]
+ or measurement["contract"] != profile["contract"]
+ or measurement["component_order_contract"] != profile["component_order_contract"]
+ or measurement["source_allocation"] != "even"
+ or shape["activation_profile"] != profile["activation_profile"]
+ or shape["dispatch_dtype"] != profile["dtype"]
+ or quant["combine_input_dtype"] != profile["combine_dtype"]
+ or quant["combine_output_dtype"] != profile["combine_dtype"]
+ or quant["combine_quant_mode"] != profile["combine_quant_mode"]
+ or quant["scale_layout"] is not None
+ or workload["activation_generator"] != profile["activation_generator"]
+ or workload["activation_profile"] != profile["activation_profile"]
+ or workload["routing_generator"] != profile["routing_generator"]
+ ):
+ raise ContractError("raw case differs from the frozen v1 profile")
+ expected_activation = hashlib.sha256(
+ (
+ f"counter|seed={profile['seed']}|hidden={hidden}|"
+ f"gen={profile['activation_generator']}"
+ ).encode()
+ ).hexdigest()
+ if workload["activation_identity"] != expected_activation:
+ raise ContractError("raw activation identity differs from the frozen seed/profile")
+ rows = measurement["rows"]
+ if not isinstance(rows, list) or not rows:
+ raise ContractError("raw.measurement.rows must be non-empty")
+ seen_points = set()
+ row_tokens = []
+ recomputed_anomalies = 0
+ for index, row_value in enumerate(rows):
+ path = f"raw.measurement.rows[{index}]"
+ row = _keys(
+ row_value,
+ {"anomalies", "components", "correctness", "evidence_id", "global_tokens",
+ "logical_bytes", "point_id", "receive", "routing",
+ "sample_histograms", "sample_sha256", "token_rate_at_latency_percentile",
+ "tokens_per_rank"},
+ path,
+ )
+ tokens = _integer(row["tokens_per_rank"], f"{path}.tokens_per_rank", minimum=1)
+ row_tokens.append(tokens)
+ if tokens in seen_points or tokens not in sample_by_token:
+ raise ContractError(f"{path} token point is duplicate or missing samples")
+ seen_points.add(tokens)
+ if row["global_tokens"] != tokens * ep_size:
+ raise ContractError(f"{path}.global_tokens formula differs")
+ sample_point = sample_by_token[tokens]
+ expected_point = identity.point_id(series=identifiers["series_id"], tokens_per_rank=tokens)
+ if row["point_id"] != expected_point or sample_point["point_id"] != expected_point:
+ raise ContractError(f"{path}.point_id differs")
+ expected_evidence = identity.evidence_id(
+ point=expected_point, allocation=identifiers["allocation_id"],
+ attempt=identifiers["attempt_id"], sample_sha256=sample_point["sample_sha256"],
+ )
+ if row["evidence_id"] != expected_evidence or sample_point["evidence_id"] != expected_evidence:
+ raise ContractError(f"{path}.evidence_id differs")
+ if row["sample_sha256"] != sample_point["sample_sha256"]:
+ raise ContractError(f"{path}.sample_sha256 differs")
+ components = _keys(
+ row["components"], {"combine", "dispatch", "isolated_sum", "roundtrip"},
+ f"{path}.components",
+ )
+ for name in ("combine", "dispatch", "roundtrip"):
+ _validate_component(
+ components[name], sample_point["components"][name], f"{path}.components.{name}"
+ )
+ _validate_component(
+ components["isolated_sum"], None, f"{path}.components.isolated_sum", derived=True
+ )
+ _, _, _, expected_indices, expected_weights = _expected_canonical_trace(
+ scheduled_case["routing"],
+ hidden,
+ topk,
+ logical_experts,
+ physical_experts,
+ ep_size,
+ tokens,
+ profile["seed"],
+ eplb["enabled"],
+ profile["eplb_reference_tokens_per_rank"],
+ )
+ expected_routing = _expected_routing_summary(
+ expected_indices,
+ expected_weights,
+ physical_experts=physical_experts,
+ ep_size=ep_size,
+ tokens_per_rank=tokens,
+ gpus_per_node=scheduled_case["gpus_per_node"],
+ scale_up_domain=scheduled_case["scale_up_domain"],
+ )
+ _equivalent(
+ row["routing"], expected_routing, f"{path}.routing", tolerance=1e-5
+ )
+ expected_payload_counts = (
+ expected_routing["expert_assignments_per_rank"]
+ if profile["payload_unit"] == "token-expert"
+ else expected_routing["payload_copies_per_rank"]
+ )
+ throughput = _keys(
+ row["token_rate_at_latency_percentile"], set(PERCENTILES),
+ f"{path}.token_rate_at_latency_percentile",
+ )
+ for percentile in PERCENTILES:
+ latency = components["roundtrip"]["percentiles_us"][percentile]
+ if latency <= 0:
+ raise ContractError(f"{path} roundtrip latency must be positive")
+ _close(
+ throughput[percentile], row["global_tokens"] / (latency * 1e-6),
+ f"{path}.token_rate_at_latency_percentile.{percentile}", 1e-9,
+ )
+ correctness = _keys(
+ row["correctness"],
+ {"contract", "max_relative_error", "passed", "rank_evidence", "scope"},
+ f"{path}.correctness",
+ )
+ if (
+ correctness["contract"] != profile["oracle_contract"]
+ or correctness["scope"] != profile["correctness_scope"]
+ or type(correctness["passed"]) is not bool
+ ):
+ raise ContractError(f"{path}.correctness contract differs")
+ _number(
+ correctness["max_relative_error"],
+ f"{path}.correctness.max_relative_error",
+ minimum=0.0,
+ )
+ rank_evidence = correctness["rank_evidence"]
+ if not isinstance(rank_evidence, list) or len(rank_evidence) != ep_size:
+ raise ContractError(f"{path}.correctness.rank_evidence must cover every rank")
+ ranks = set()
+ observed_max_error = 0.0
+ evidence_passed = True
+ for evidence_index, evidence_value in enumerate(rank_evidence):
+ evidence_path = f"{path}.correctness.rank_evidence[{evidence_index}]"
+ evidence = _keys(
+ evidence_value,
+ {"input_unchanged", "order_stable", "post_timing", "pre_timing", "rank"},
+ evidence_path,
+ )
+ evidence_rank = _integer(evidence["rank"], f"{evidence_path}.rank")
+ if evidence_rank >= ep_size:
+ raise ContractError(f"{evidence_path}.rank is outside the EP group")
+ ranks.add(evidence_rank)
+ if type(evidence["input_unchanged"]) is not bool or type(evidence["order_stable"]) is not bool:
+ raise ContractError(f"{evidence_path} stability fields must be boolean")
+ pre = _validate_oracle(
+ evidence["pre_timing"], f"{evidence_path}.pre_timing", profile
+ )
+ post = _validate_oracle(
+ evidence["post_timing"], f"{evidence_path}.post_timing", profile
+ )
+ if (
+ pre["receive_count"] != expected_payload_counts[evidence_rank]
+ or post["receive_count"] != expected_payload_counts[evidence_rank]
+ ):
+ raise ContractError(
+ f"{evidence_path}.receive_count differs from canonical routing"
+ )
+ expected_stability = all(
+ pre[field] == post[field]
+ for field in ("ordering_contract", "order_sha256", "dispatch_sha256")
+ )
+ if evidence["order_stable"] != expected_stability:
+ raise ContractError(f"{evidence_path}.order_stable differs from the evidence")
+ errors = [
+ oracle["max_relative_error"]
+ for oracle in (pre, post)
+ if oracle["max_relative_error"] is not None
+ ]
+ observed_max_error = max([observed_max_error, *errors])
+ evidence_passed = evidence_passed and all(
+ (evidence["input_unchanged"], evidence["order_stable"], pre["passed"], post["passed"])
+ )
+ if ranks != set(range(ep_size)) or correctness["passed"] != evidence_passed:
+ raise ContractError(f"{path}.correctness rank coverage or outcome differs")
+ _close(
+ correctness["max_relative_error"], observed_max_error,
+ f"{path}.correctness.max_relative_error",
+ )
+ if components["dispatch"]["availability"] == "measured":
+ for percentile in PERCENTILES:
+ expected = (
+ components["dispatch"]["percentiles_us"][percentile]
+ + components["combine"]["percentiles_us"][percentile]
+ )
+ _close(
+ components["isolated_sum"]["percentiles_us"][percentile], expected,
+ f"{path}.components.isolated_sum.{percentile}",
+ )
+ logical_copies = (
+ sum(expected_routing["expert_assignments_per_rank"])
+ if profile["payload_unit"] == "token-expert"
+ else expected_routing["routed_copies"]
+ )
+ expected_bytes = logical_copies * hidden * 2
+ expected_logical = {
+ "combine": expected_bytes,
+ "dispatch": expected_bytes,
+ "roundtrip": expected_bytes * 2,
+ }
+ _equivalent(row["logical_bytes"], expected_logical, f"{path}.logical_bytes")
+
+ max_receive = max(expected_payload_counts)
+ expected_receive = {
+ "max": max_receive,
+ "mean": sum(expected_payload_counts) / ep_size,
+ "min": min(expected_payload_counts),
+ "total": sum(expected_payload_counts),
+ }
+ _equivalent(row["receive"], expected_receive, f"{path}.receive")
+ expected_histograms = {
+ name: (
+ _expected_histogram([
+ sample
+ for trial in sample_point["components"][name]["trials"]
+ for sample in trial
+ ])
+ if sample_point["components"][name]["availability"] == "measured"
+ else None
+ )
+ for name in ("dispatch", "combine", "roundtrip")
+ }
+ _equivalent(
+ row["sample_histograms"], expected_histograms, f"{path}.sample_histograms"
+ )
+ expected_anomalies = _expected_anomalies(tokens, components)
+ _equivalent(row["anomalies"], expected_anomalies, f"{path}.anomalies")
+ recomputed_anomalies += len(expected_anomalies)
+ if seen_points != set(sample_by_token):
+ raise ContractError("raw rows and sample points differ")
+ if row_tokens != sorted(row_tokens):
+ raise ContractError("raw rows must follow the scheduled token ladder")
+ expected_trace_hashes = sorted(row["routing"]["hash"] for row in rows)
+ if workload["trace_hashes"] != expected_trace_hashes:
+ raise ContractError("raw workload trace hashes differ from measured rows")
+ expected_trace_signature = hashlib.sha256(
+ "|".join(expected_trace_hashes).encode()
+ ).hexdigest()
+ if workload["trace_signature"] != expected_trace_signature:
+ raise ContractError("raw workload trace signature differs from measured rows")
+
+ implementation = _keys(
+ doc["implementation"], {"kernel_generation", "name", "provenance", "resource_profile"},
+ "raw.implementation",
+ )
+ if (
+ implementation["name"] != case["backend"]
+ or implementation["kernel_generation"] != shape["kernel_gen"]
+ ):
+ raise ContractError("raw implementation identity differs from the case")
+ provenance_fields = _obj(implementation["provenance"], "raw.implementation.provenance")
+ unknown = set(provenance_fields) - PROVENANCE_KEYS
+ if unknown:
+ raise ContractError(f"raw implementation provenance has unknown fields {sorted(unknown)}")
+ if (
+ implementation["name"] == "deepep-v2"
+ and provenance_fields.get("deterministic") is not False
+ ):
+ raise ContractError("DeepEP V2 deterministic mode differs from the v1 kernel contract")
+ if implementation["name"] == "deepep-v2" and (
+ _integer(
+ provenance_fields.get("tuning_num_experts"),
+ "raw.implementation.provenance.tuning_num_experts",
+ minimum=1,
+ ) != logical_experts
+ or _integer(
+ provenance_fields.get("num_experts"),
+ "raw.implementation.provenance.num_experts",
+ minimum=1,
+ ) != physical_experts
+ ):
+ raise ContractError("DeepEP V2 expert-count provenance differs from the case")
+ if implementation["name"] == "deepep-hybrid":
+ realized_config = provenance_fields.get("realized_config")
+ jit_kernel_keys = provenance_fields.get("jit_kernel_keys")
+ jit_shared_objects = provenance_fields.get("jit_shared_objects")
+ domain_ranks, communication_domains = hybrid_communication_domains(
+ ep_size, scheduled_case["scale_up_domain"]
+ )
+ if (
+ not _hybrid_realized_config_is_valid(realized_config)
+ or not _hybrid_jit_evidence_is_valid(jit_shared_objects, jit_kernel_keys)
+ or realized_config["hidden_dim"] != shape["hidden"]
+ or realized_config["num_of_experts_per_rank"] * ep_size != physical_experts
+ or realized_config["num_of_ranks_per_node"] != domain_ranks
+ or realized_config["num_of_nodes"] != communication_domains
+ or realized_config["token_data_type"] != "UINT16"
+ or any(
+ len(artifact["rank_artifacts"]) != ep_size
+ for artifact in jit_shared_objects
+ )
+ ):
+ raise ContractError("DeepEP Hybrid realized config/JIT evidence differs from the case")
+ if implementation["name"] == "nccl-ep" and implementation["kernel_generation"] != (
+ collective_kernel_generation(provenance_fields.get("collective_library"))
+ ):
+ raise ContractError("NCCL/RCCL kernel generation differs from collective lineage")
+ resource_profile = _obj(
+ implementation["resource_profile"], "raw.implementation.resource_profile"
+ )
+ expected_resource_profile = project_resource_profile(provenance_fields)
+ if resource_profile != expected_resource_profile:
+ raise ContractError("raw resource profile differs from implementation provenance")
+ topology = _keys(
+ doc["topology"],
+ {"device_count", "device_product", "gpus_per_node", "nodes", "placement",
+ "realized_placement", "scale_out_transport", "scale_up_domain",
+ "scale_up_transport", "scope", "topology_class", "transport", "world_size"},
+ "raw.topology",
+ )
+ for field in ("device_count", "gpus_per_node", "nodes", "scale_up_domain", "world_size"):
+ _integer(topology[field], f"raw.topology.{field}", minimum=1)
+ for field in ("scale_up_transport", "scope", "topology_class", "transport"):
+ _text(topology[field], f"raw.topology.{field}")
+ if topology["scale_out_transport"] is not None:
+ _text(topology["scale_out_transport"], "raw.topology.scale_out_transport")
+ realized = _keys(
+ topology["realized_placement"],
+ {"gpus_per_node", "nodes", "ranks_per_node", "unique_local_ranks", "valid"},
+ "raw.topology.realized_placement",
+ )
+ if realized != {
+ "gpus_per_node": topology["gpus_per_node"],
+ "nodes": topology["nodes"],
+ "ranks_per_node": topology["gpus_per_node"],
+ "unique_local_ranks": True,
+ "valid": True,
+ }:
+ raise ContractError("raw realized placement differs from requested topology")
+ if (
+ topology["world_size"] != ep_size
+ or topology["nodes"] * topology["gpus_per_node"] != ep_size
+ or topology["device_count"] != topology["gpus_per_node"]
+ or topology["placement"] != profile["placement"]
+ or (
+ topology["scope"] == "scale-up"
+ and (
+ ep_size > topology["scale_up_domain"]
+ or topology["scale_out_transport"] is not None
+ )
+ )
+ or (
+ topology["scope"] == "scale-out"
+ and (
+ ep_size <= topology["scale_up_domain"]
+ or ep_size % topology["scale_up_domain"] != 0
+ or topology["scale_out_transport"] is None
+ )
+ )
+ or topology["scope"] not in {"scale-up", "scale-out"}
+ ):
+ raise ContractError("raw topology dimensions differ from the case")
+ if implementation["name"] == "deepep-v2":
+ scale_out = scheduled_case["scope"] == "scale-out"
+ expected_policy = (
+ (True, True, "nccl-gin")
+ if scale_out
+ else (False, False, "nccl-device-lsa")
+ )
+ if (
+ provenance_fields.get("allow_hybrid_mode"),
+ provenance_fields.get("gin_enabled"),
+ provenance_fields.get("communication_backend"),
+ ) != expected_policy:
+ raise ContractError("DeepEP V2 communication policy differs from the v1 contract")
+ lsa_topology = tuple(
+ _integer(
+ provenance_fields.get(field),
+ f"raw.implementation.provenance.{field}",
+ minimum=1,
+ )
+ for field in (
+ "physical_rdma_ranks", "physical_nvlink_ranks",
+ "logical_scaleout_ranks", "logical_scaleup_ranks",
+ )
+ )
+ domains = ep_size // scheduled_case["scale_up_domain"]
+ expected_v2_topology = (
+ (
+ domains,
+ scheduled_case["scale_up_domain"],
+ domains,
+ scheduled_case["scale_up_domain"],
+ )
+ if scale_out
+ else (1, ep_size, 1, ep_size)
+ )
+ if lsa_topology != expected_v2_topology:
+ raise ContractError("DeepEP V2 realized communication domains differ from topology")
+ runtime = _keys(
+ doc["runtime_fingerprint"],
+ {"accelerator_runtime", "collective_library", "device", "driver_version", "framework",
+ "machine", "python_version", "vendor"},
+ "raw.runtime_fingerprint",
+ )
+ for field in ("machine", "python_version", "vendor"):
+ _text(runtime[field], f"raw.runtime_fingerprint.{field}")
+ runtime_device = _keys(
+ runtime["device"], {"arch", "compute_units", "memory_bytes", "product", "warp_size"},
+ "raw.runtime_fingerprint.device",
+ )
+ if topology["device_product"] != runtime_device["product"]:
+ raise ContractError("raw topology and runtime device products differ")
+ platform = capability.PLATFORMS.get(case["runner"])
+ if platform is not None:
+ identity_issues = capability.runtime_identity_issues(
+ case["runner"], vendor=runtime["vendor"], arch=runtime_device["arch"],
+ machine=runtime["machine"], device_name=runtime_device["product"],
+ device_count=topology["device_count"], world_size=topology["world_size"],
+ )
+ registered_topology = capability.topology_for(case["runner"], ep_size)
+ if identity_issues or (
+ registered_topology is None
+ or topology["gpus_per_node"] != platform["gpus_per_node"]
+ or topology["scale_up_domain"] != platform["scale_up_domain"]
+ or any(
+ topology[field] != registered_topology[field]
+ for field in (
+ "nodes", "scope", "scale_up_transport", "scale_out_transport",
+ "topology_class", "transport",
+ )
+ )
+ ):
+ raise ContractError(
+ "raw runtime/topology differs from the scheduled SKU: "
+ + "; ".join(identity_issues)
+ )
+ raw_provenance = _keys(
+ doc["provenance"], {"command", "distributed_launcher", "git_run", "image", "redaction"},
+ "raw.provenance",
+ )
+ image = _keys(
+ raw_provenance["image"],
+ {"arch", "digest", "digest_verified", "reference", "squash_sha256"},
+ "raw.provenance.image",
+ )
+ if (
+ image["digest_verified"] is not True
+ or not isinstance(image["digest"], str)
+ or not re.fullmatch(r"sha256:[0-9a-f]{64}", image["digest"])
+ ):
+ raise ContractError("raw image digest was not registry-verified")
+ if raw_provenance["redaction"] != "sanitized-v1":
+ raise ContractError("raw provenance redaction contract differs")
+ git_run = raw_provenance["git_run"]
+ if git_run is not None:
+ git_run = _keys(git_run, GIT_RUN_FIELDS, "raw.provenance.git_run")
+ expected_provenance_complete = provenance_complete(
+ provenance_fields,
+ case["backend"],
+ git_run,
+ image_digest=image["digest"],
+ image_verified=image["digest_verified"],
+ squash_sha256=image["squash_sha256"],
+ )
+
+ actual_scheduled_case = {
+ "backend": case["backend"],
+ "canonical": workload["source"] == "canonical-serialized",
+ "eplb": eplb["enabled"],
+ "ep": ep_size,
+ "experts": shape["num_logical_experts"],
+ "gpus_per_node": topology["gpus_per_node"],
+ "hidden": hidden,
+ "ladder": " ".join(map(str, row_tokens)),
+ "mode": case["mode"],
+ "nodes": topology["nodes"],
+ "phase": case["phase"],
+ "required_publication": case["required_publication"],
+ "routing": shape["routing"],
+ "samples_per_point": sampling["samples_per_component"],
+ "scale_out_transport": topology["scale_out_transport"],
+ "scale_up_domain": topology["scale_up_domain"],
+ "scale_up_transport": topology["scale_up_transport"],
+ "scope": topology["scope"],
+ "suite": case["suite"],
+ "timing": (
+ f"{sampling['iterations_per_trial']}:{sampling['trials']}:"
+ f"{sampling['warmup_iterations']}"
+ ),
+ "topk": shape["topk"],
+ "topology_class": topology["topology_class"],
+ "transport": topology["transport"],
+ "warmup_semantics": sampling["warmup_semantics"],
+ "workload": case["workload_name"],
+ }
+ if scheduled_case != actual_scheduled_case:
+ mismatches = sorted(
+ field for field in scheduled_case
+ if scheduled_case[field] != actual_scheduled_case[field]
+ )
+ raise ContractError(f"raw data differs from scheduled case fields {mismatches}")
+
+ if workload["source"] == "canonical-serialized":
+ _validate_canonical_workload(workload, scheduled_case, rows, eplb)
+
+ expected_series = {
+ "backend": case["backend"],
+ "case_id": identifiers["case_id"],
+ "image_digest": image["digest"],
+ "implementation_contract_sha256": _sha256_json({
+ "kernel_generation": implementation["kernel_generation"],
+ "name": implementation["name"],
+ "provenance": series_provenance(provenance_fields),
+ "resource_profile": resource_profile,
+ }),
+ "public_config_sha256": public_series_config_sha256(public_series_config(
+ kernel_generation=implementation["kernel_generation"],
+ provenance=provenance_fields,
+ resource_profile=resource_profile,
+ resource_mode=case["resource_mode"],
+ device_product=topology["device_product"],
+ )),
+ "routing_control_sha256": routing_implementation_control_sha256(implementation),
+ "runtime_fingerprint_sha256": _sha256_json(runtime),
+ "source_sha": git_run["source_sha"] if git_run is not None else None,
+ "squash_sha256": image["squash_sha256"],
+ "workload_id": workload["workload_id"] or workload["trace_signature"],
+ }
+ if series_factors != expected_series:
+ raise ContractError("raw series factors differ from measured implementation/runtime")
+ expected_allocation = {
+ "artifact": git_run["artifact"] if git_run is not None else None,
+ "execution_id": allocation_factors["execution_id"],
+ "job": git_run["job"] if git_run is not None else None,
+ "repo": git_run["repo"] if git_run is not None else None,
+ "run_attempt": git_run["run_attempt"] if git_run is not None else None,
+ "run_id": git_run["run_id"] if git_run is not None else None,
+ "runner": case["runner"],
+ "source_sha": git_run["source_sha"] if git_run is not None else None,
+ }
+ if allocation_factors != expected_allocation:
+ raise ContractError("raw allocation factors differ from provenance")
+ artifact = _keys(doc["sample_artifact"], {"bytes", "format", "path", "sha256"}, "raw.sample_artifact")
+ if artifact["format"] != SAMPLES_FORMAT or Path(artifact["path"]).name != artifact["path"]:
+ raise ContractError("raw.sample_artifact format/path is invalid")
+ if not isinstance(artifact["sha256"], str) or len(artifact["sha256"]) != 64:
+ raise ContractError("raw.sample_artifact.sha256 is invalid")
+ _integer(artifact["bytes"], "raw.sample_artifact.bytes", minimum=1)
+ outcome = _keys(doc["outcome"], {"publication_status", "reasons", "status", "validity"}, "raw.outcome")
+ if outcome["status"] not in {"success", "invalid"} or outcome["publication_status"] not in {"diagnostic", "invalid"}:
+ raise ContractError("raw outcome status is invalid")
+ if not isinstance(outcome["reasons"], list) or not all(isinstance(x, str) for x in outcome["reasons"]):
+ raise ContractError("raw outcome reasons must be strings")
+ validity = _keys(
+ outcome["validity"],
+ {"anomaly_free", "execution_status", "measurement_conformance", "provenance_complete",
+ "resource_conformance", "sampling_conformance", "semantic_correctness",
+ "workload_identity", "workload_source"},
+ "raw.outcome.validity",
+ )
+ correctness_passed = all(row["correctness"]["passed"] for row in rows)
+ workload_consistent = workload["cross_rank_consistent"] is True
+ expected_status = "success" if correctness_passed and workload_consistent else "invalid"
+ expected_publication = "diagnostic" if expected_status == "success" else "invalid"
+ if (
+ outcome["status"] != expected_status
+ or outcome["publication_status"] != expected_publication
+ or bool(outcome["reasons"]) == (expected_status == "success")
+ or validity["execution_status"] != "complete"
+ or validity["semantic_correctness"] != ("pass" if correctness_passed else "fail")
+ or validity["workload_identity"] != (
+ "consistent-across-ranks" if workload_consistent else "inconsistent"
+ )
+ or validity["workload_source"] != workload["source"]
+ or validity["measurement_conformance"] != "conformant"
+ or validity["sampling_conformance"] != "conformant"
+ or validity["resource_conformance"] != resource_profile["conformance_class"]
+ or validity["anomaly_free"] != (recomputed_anomalies == 0)
+ or validity["provenance_complete"] is not expected_provenance_complete
+ ):
+ raise ContractError("raw outcome differs from its measurement evidence")
+ artifact_safety.assert_publication_safe([doc])
+ return doc
+
+
+def make_terminal_document(
+ *,
+ allocation_factors: dict[str, Any],
+ attempt_ordinal: int,
+ case: dict[str, Any],
+ case_factors: dict[str, Any],
+ control_sha256: str | None,
+ failure_mode: str,
+ generated_at: str,
+ git_run: dict[str, Any] | None,
+ reason: str,
+ return_code: int,
+ source: str,
+ status: str,
+ expected_case_id: str | None = None,
+) -> dict[str, Any]:
+ """Build and self-validate one attributable non-success attempt."""
+ case_id = identity.digest("case", case_factors)
+ if expected_case_id is not None and expected_case_id != case_id:
+ raise ContractError(
+ f"scheduled case ID differs from terminal factors: {expected_case_id} != {case_id}"
+ )
+ allocation_id = identity.allocation_id(allocation_factors)
+ attempt_id = identity.attempt_id(
+ allocation=allocation_id, case=case_id, ordinal=attempt_ordinal
+ )
+ document = {
+ "format": TERMINAL_FORMAT,
+ "schema_version": 1,
+ "record_type": "terminal-outcome",
+ "generated_at": generated_at,
+ "identity": {
+ "allocation_factors": allocation_factors,
+ "allocation_id": allocation_id,
+ "attempt_id": attempt_id,
+ "attempt_ordinal": attempt_ordinal,
+ "case_factors": case_factors,
+ "case_id": case_id,
+ },
+ "case": case,
+ "provenance": {
+ "git_run": git_run,
+ "control_sha256": control_sha256,
+ "redaction": "sanitized-v1",
+ "source": source,
+ },
+ "outcome": {
+ "status": status,
+ "failure_mode": failure_mode,
+ "reason": reason,
+ "return_code": return_code,
+ },
+ }
+ return validate_terminal_document(document)
+
+
+def validate_terminal_document(document: Any) -> dict[str, Any]:
+ _validate_native_schema("terminal-outcome-v1.schema.json", document)
+ doc = _keys(
+ document,
+ {"case", "format", "generated_at", "identity", "outcome", "provenance", "record_type",
+ "schema_version"},
+ "terminal",
+ )
+ if doc["format"] != TERMINAL_FORMAT or doc["schema_version"] != 1 or doc["record_type"] != "terminal-outcome":
+ raise ContractError("terminal format/schema/record type differs from v1")
+ ids = _keys(doc["identity"], {
+ "allocation_factors", "allocation_id", "attempt_id", "attempt_ordinal",
+ "case_factors", "case_id",
+ }, "terminal.identity")
+ for field, kind in (("allocation_id", "allocation"), ("attempt_id", "attempt"), ("case_id", "case")):
+ _typed(ids[field], kind, f"terminal.identity.{field}")
+ ordinal = _integer(ids["attempt_ordinal"], "terminal.identity.attempt_ordinal", minimum=1)
+ case = _keys(doc["case"], TERMINAL_CASE_FIELDS, "terminal.case")
+ factors = _keys(ids["case_factors"], {"case", "profile", "sku"}, "terminal.identity.case_factors")
+ if factors["case"] != case or factors["profile"] != scheduled_case_profile(
+ case, "terminal.case"
+ ):
+ raise ContractError("terminal case factors differ from the scheduled case/profile")
+ _text(factors["sku"], "terminal.identity.case_factors.sku")
+ allocation = _keys(
+ ids["allocation_factors"], ALLOCATION_FACTOR_FIELDS,
+ "terminal.identity.allocation_factors",
+ )
+ expected_case = identity.digest("case", factors)
+ expected_allocation = identity.allocation_id(allocation)
+ expected_attempt = identity.attempt_id(
+ allocation=expected_allocation, case=expected_case, ordinal=ordinal
+ )
+ if (ids["case_id"], ids["allocation_id"], ids["attempt_id"]) != (
+ expected_case, expected_allocation, expected_attempt
+ ):
+ raise ContractError("terminal typed identities do not match their factors")
+ provenance = _keys(
+ doc["provenance"], {"git_run", "control_sha256", "redaction", "source"},
+ "terminal.provenance",
+ )
+ git_run = provenance["git_run"]
+ if git_run is not None:
+ git_run = _keys(git_run, GIT_RUN_FIELDS, "terminal.provenance.git_run")
+ control = provenance["control_sha256"]
+ if control is not None and (
+ not isinstance(control, str) or len(control) != 64
+ or any(char not in "0123456789abcdef" for char in control)
+ ):
+ raise ContractError("terminal control_sha256 is invalid")
+ if provenance["redaction"] != "sanitized-v1":
+ raise ContractError("terminal redaction contract differs")
+ source = _text(provenance["source"], "terminal.provenance.source")
+ outcome = _keys(
+ doc["outcome"], {"failure_mode", "reason", "return_code", "status"}, "terminal.outcome"
+ )
+ if outcome["status"] not in {"failed", "invalid", "unsupported"}:
+ raise ContractError("terminal outcome status is invalid")
+ failure_mode = _text(outcome["failure_mode"], "terminal.outcome.failure_mode")
+ reason = _text(outcome["reason"], "terminal.outcome.reason")
+ _integer(outcome["return_code"], "terminal.outcome.return_code")
+ if source == "runtime-emitter":
+ expected_runner = factors["sku"]
+ expected_reason = RUNTIME_FAILURE_REASONS.get(failure_mode)
+ valid_outcome = outcome["status"] == "failed" and reason == expected_reason
+ elif source == "post-emit-command":
+ expected_runner = factors["sku"]
+ expected_reason = POST_EMIT_FAILURE_REASONS.get(failure_mode)
+ valid_outcome = outcome["status"] == "failed" and reason == expected_reason
+ elif source == "matrix-capability-resolver":
+ expected_runner = "capability-resolver"
+ valid_outcome = (
+ outcome["status"] == "unsupported"
+ and failure_mode == "capability"
+ and reason in CAPABILITY_FAILURE_REASONS
+ )
+ else:
+ raise ContractError("terminal provenance source is not registered")
+ if not valid_outcome:
+ raise ContractError("terminal source and outcome are not registered")
+ expected_allocation = {
+ "artifact": git_run["artifact"] if git_run is not None else None,
+ "execution_id": allocation["execution_id"],
+ "job": git_run["job"] if git_run is not None else None,
+ "repo": git_run["repo"] if git_run is not None else None,
+ "run_attempt": git_run["run_attempt"] if git_run is not None else None,
+ "run_id": git_run["run_id"] if git_run is not None else None,
+ "runner": expected_runner,
+ "source_sha": git_run["source_sha"] if git_run is not None else None,
+ }
+ if allocation != expected_allocation:
+ raise ContractError("terminal allocation factors differ from provenance or source")
+ artifact_safety.assert_publication_safe([doc])
+ return doc
+
+
+def load_raw_attempt(path: str | os.PathLike[str]) -> dict[str, Any]:
+ document = strict_load(path)
+ artifact = _obj(document, "raw").get("sample_artifact")
+ artifact = _obj(artifact, "raw.sample_artifact")
+ sample_path = Path(path).with_name(_text(artifact.get("path"), "raw.sample_artifact.path"))
+ payload = sample_path.read_bytes()
+ if len(payload) != artifact.get("bytes") or hashlib.sha256(payload).hexdigest() != artifact.get("sha256"):
+ raise ContractError("sample artifact bytes or digest differ")
+ samples = strict_load(sample_path)
+ return validate_raw_document(document, samples)
+
+
+def load_attempt(path: str | os.PathLike[str]) -> dict[str, Any]:
+ """Fully validate and return one native raw or terminal attempt."""
+ document = strict_load(path)
+ if isinstance(document, dict) and document.get("format") == RAW_FORMAT:
+ return load_raw_attempt(path)
+ if isinstance(document, dict) and document.get("format") == TERMINAL_FORMAT:
+ return validate_terminal_document(document)
+ raise ContractError("unknown native attempt format")
+
+
+def quarantine_invalid_attempt(path: str | os.PathLike[str]) -> bool:
+ """Move an invalid attempt and its basename-safe sample outside JSON upload globs."""
+ destination = Path(path)
+ if not destination.is_file():
+ return False
+ try:
+ load_attempt(destination)
+ return False
+ except (ContractError, OSError, ValueError):
+ try:
+ document = json.loads(destination.read_bytes())
+ except (OSError, json.JSONDecodeError):
+ document = {}
+ artifact = document.get("sample_artifact") if isinstance(document, dict) else None
+ sample_name = artifact.get("path") if isinstance(artifact, dict) else None
+ if isinstance(sample_name, str) and Path(sample_name).name == sample_name:
+ sample_path = destination.with_name(sample_name)
+ if sample_path.is_file():
+ os.replace(sample_path, sample_path.with_name(sample_path.name + ".quarantine"))
+ os.replace(destination, destination.with_name(destination.name + ".quarantine"))
+ return True
+
+
+def normalize_attempt(document: dict[str, Any]) -> dict[str, Any]:
+ """Return the publisher-facing projection after native validation."""
+ if document.get("format") == RAW_FORMAT:
+ ids = document["identity"]
+ return {
+ "allocation_id": ids["allocation_id"],
+ "attempt_id": ids["attempt_id"],
+ "case": document["case"],
+ "case_id": ids["case_id"],
+ "generated_at": document["generated_at"],
+ "outcome": document["outcome"],
+ "points": document["measurement"]["rows"],
+ "runtime_fingerprint": document["runtime_fingerprint"],
+ "series_id": ids["series_id"],
+ }
+ if document.get("format") == TERMINAL_FORMAT:
+ ids = document["identity"]
+ return {
+ "allocation_id": ids["allocation_id"],
+ "attempt_id": ids["attempt_id"],
+ "case": document["case"],
+ "case_id": ids["case_id"],
+ "generated_at": document["generated_at"],
+ "outcome": document["outcome"],
+ "points": [],
+ "runtime_fingerprint": None,
+ "series_id": None,
+ }
+ raise ContractError("unknown attempt format")
+
+
+def _env_integer(name: str, default: int) -> int:
+ try:
+ return int(os.environ.get(name, str(default)))
+ except ValueError:
+ return default
+
+
+def _env_enabled(name: str) -> bool:
+ return os.environ.get(name, "").lower() in {"1", "true", "yes"}
+
+
+def _terminal_case_from_environment(backend: str, phase: str) -> dict[str, Any]:
+ ep = _env_integer("CX_EP", _env_integer("CX_NGPUS", 1))
+ gpus_per_node = _env_integer("CX_GPUS_PER_NODE", ep)
+ ladder = os.environ.get("CX_TOKENS_LADDER", "") or (
+ "1 2 4 8 16 32 64 128"
+ if phase == "decode"
+ else "128 256 512 1024 2048 4096"
+ )
+ return {
+ "suite": os.environ.get("CX_SUITE") or "manual",
+ "workload": os.environ.get("CX_WORKLOAD_NAME") or "manual",
+ "required_publication": os.environ.get("CX_REQUIRED_PUBLICATION") or "diagnostic",
+ "backend": backend,
+ "mode": os.environ.get("CX_MODE", "normal"),
+ "routing": os.environ.get("CX_ROUTING", "uniform"),
+ "phase": phase,
+ "ep": ep,
+ "eplb": _env_enabled("CX_EPLB"),
+ "hidden": _env_integer("CX_HIDDEN", 7168),
+ "topk": _env_integer("CX_TOPK", 8),
+ "experts": _env_integer("CX_EXPERTS", 256),
+ "samples_per_point": _env_integer("CX_SAMPLES_PER_POINT", 512),
+ "warmup_semantics": os.environ.get(
+ "CX_WARMUP_SEMANTICS",
+ "full-roundtrip-before-each-component-trial-point-v1",
+ ),
+ "ladder": ladder,
+ "timing": (
+ f'{_env_integer("CX_ITERS", 8)}:{_env_integer("CX_TRIALS", 64)}:'
+ f'{_env_integer("CX_WARMUP", 32)}'
+ ),
+ "canonical": _env_enabled("CX_CANONICAL"),
+ "nodes": _env_integer("CX_NODES", _env_integer("SLURM_NNODES", 1)),
+ "gpus_per_node": gpus_per_node,
+ "scale_up_domain": _env_integer("CX_SCALE_UP_DOMAIN", gpus_per_node),
+ "scope": os.environ.get("CX_SCOPE", "scale-up"),
+ "topology_class": os.environ.get("CX_TOPO", "manual"),
+ "transport": os.environ.get("CX_TRANSPORT", "unknown"),
+ "scale_up_transport": os.environ.get("CX_SCALE_UP_TRANSPORT", "unknown"),
+ "scale_out_transport": os.environ.get("CX_SCALE_OUT_TRANSPORT") or None,
+ }
+
+
+def _git_run_from_environment() -> dict[str, Any] | None:
+ def value(name: str) -> str | None:
+ return os.environ.get(name) or None
+
+ git_run = {
+ "run_id": value("GITHUB_RUN_ID"),
+ "run_attempt": value("GITHUB_RUN_ATTEMPT"),
+ "ref": value("GITHUB_REF_NAME") or value("GITHUB_REF"),
+ "source_sha": value("COLLECTIVEX_SOURCE_SHA") or value("GITHUB_SHA"),
+ "repo": value("GITHUB_REPOSITORY"),
+ "job": value("GITHUB_JOB"),
+ "artifact": value("COLLECTIVEX_ARTIFACT_NAME"),
+ }
+ return git_run if any(value is not None for value in git_run.values()) else None
+
+
+def _allocation_factors_from_environment(
+ runner: str, git_run: dict[str, Any] | None
+) -> dict[str, Any]:
+ return {
+ "artifact": git_run["artifact"] if git_run is not None else None,
+ "execution_id": os.environ.get("COLLECTIVEX_EXECUTION_ID") or None,
+ "job": git_run["job"] if git_run is not None else None,
+ "repo": git_run["repo"] if git_run is not None else None,
+ "run_attempt": git_run["run_attempt"] if git_run is not None else None,
+ "run_id": git_run["run_id"] if git_run is not None else None,
+ "runner": runner,
+ "source_sha": git_run["source_sha"] if git_run is not None else None,
+ }
+
+
+def make_terminal_from_environment(
+ *, backend: str, phase: str, return_code: int, failure_mode: str | None = None
+) -> dict[str, Any]:
+ """Build a terminal document from the same exported case coordinates as run_ep."""
+ mode = failure_mode or RETURN_CODE_FAILURE_MODES.get(return_code, "execution")
+ reason = RUNTIME_FAILURE_REASONS.get(mode)
+ if reason is None:
+ raise ContractError("runtime failure mode is not registered")
+ runner = os.environ.get("CX_RUNNER", "")
+ case = _terminal_case_from_environment(backend, phase)
+ case_factors = {
+ "case": case,
+ "profile": scheduled_case_profile(case, "runtime case"),
+ "sku": runner,
+ }
+ git_run = _git_run_from_environment()
+ control = os.environ.get("COLLECTIVEX_CONTROL_SHA256") or None
+ return make_terminal_document(
+ allocation_factors=_allocation_factors_from_environment(runner, git_run),
+ attempt_ordinal=_env_integer("CX_ATTEMPT_ID", 1),
+ case=case,
+ case_factors=case_factors,
+ control_sha256=control,
+ failure_mode=mode,
+ generated_at=dt.datetime.now(dt.timezone.utc).isoformat(),
+ git_run=git_run,
+ reason=reason,
+ return_code=return_code,
+ source="runtime-emitter",
+ status="failed",
+ expected_case_id=os.environ.get("CX_CASE_ID") or None,
+ )
+
+
+def _write_document(path: str | os.PathLike[str], document: dict[str, Any]) -> None:
+ destination = Path(path)
+ destination.parent.mkdir(parents=True, exist_ok=True)
+ temporary = destination.with_name(destination.name + ".tmp")
+ temporary.write_text(json.dumps(document, indent=2, sort_keys=True) + "\n")
+ os.replace(temporary, destination)
+
+
+def demote_raw_attempt(path: str | os.PathLike[str], return_code: int) -> dict[str, Any]:
+ """Replace a rank-zero raw result when the distributed command later fails."""
+ destination = Path(path)
+ raw = strict_load(destination)
+ if not isinstance(raw, dict) or raw.get("format") != RAW_FORMAT:
+ raise ContractError("only a native raw attempt can be demoted")
+ ids = _obj(raw.get("identity"), "raw.identity")
+ required = {
+ "allocation_factors", "allocation_id", "attempt_id", "attempt_ordinal",
+ "case_factors", "case_id",
+ }
+ if not required.issubset(ids):
+ raise ContractError("raw identity lacks terminal factors")
+ mode = RETURN_CODE_FAILURE_MODES.get(return_code, "execution")
+ git_run = _obj(raw.get("provenance"), "raw.provenance").get("git_run")
+ if git_run is not None:
+ git_run = _keys(git_run, GIT_RUN_FIELDS, "raw.provenance.git_run")
+ terminal = make_terminal_document(
+ allocation_factors=ids["allocation_factors"],
+ attempt_ordinal=ids["attempt_ordinal"],
+ case=ids["case_factors"]["case"],
+ case_factors=ids["case_factors"],
+ control_sha256=os.environ.get("COLLECTIVEX_CONTROL_SHA256") or None,
+ failure_mode=mode,
+ generated_at=dt.datetime.now(dt.timezone.utc).isoformat(),
+ git_run=git_run,
+ reason=POST_EMIT_FAILURE_REASONS[mode],
+ return_code=return_code,
+ source="post-emit-command",
+ status="failed",
+ expected_case_id=ids["case_id"],
+ )
+ artifact = raw.get("sample_artifact") or {}
+ sample_name = artifact.get("path")
+ if isinstance(sample_name, str) and Path(sample_name).name == sample_name:
+ destination.with_name(sample_name).unlink(missing_ok=True)
+ _write_document(destination, terminal)
+ return terminal
+
+
+def validate_attempt_paths(paths: list[str]) -> int:
+ """Fully validate a result directory's attempts and paired sample artifacts."""
+ if not paths or len(paths) != len(set(paths)):
+ raise ContractError("validate-many requires unique result paths")
+ sample_paths: set[Path] = set()
+ referenced_samples: set[Path] = set()
+ attempt_count = 0
+ for raw_path in paths:
+ path = Path(raw_path).resolve()
+ document = strict_load(path)
+ if isinstance(document, dict) and document.get("format") == RAW_FORMAT:
+ document = load_raw_attempt(path)
+ referenced_samples.add(path.with_name(document["sample_artifact"]["path"]))
+ attempt_count += 1
+ elif isinstance(document, dict) and document.get("format") == TERMINAL_FORMAT:
+ validate_terminal_document(document)
+ attempt_count += 1
+ elif isinstance(document, dict) and document.get("format") == SAMPLES_FORMAT:
+ validate_samples_document(document)
+ sample_paths.add(path)
+ else:
+ raise ContractError(f"unknown result artifact {path.name}")
+ if sample_paths != referenced_samples:
+ raise ContractError("sample artifacts are missing, orphaned, or outside the validated set")
+ if attempt_count == 0:
+ raise ContractError("result set contains no native attempts")
+ return attempt_count
+
+
+def validate_delivery(
+ paths: list[str], source_path: str, *, disposition: str | None = None
+) -> int:
+ """Reconcile a shard or matrix disposition with its complete native attempt set."""
+ source_file = Path(source_path).resolve()
+ source = strict_load(source_file)
+ if isinstance(source, dict) and source.get("format") == "collectivex.matrix.v1":
+ if disposition is None:
+ raise ContractError("matrix delivery validation requires a disposition")
+ wrappers = [
+ item for item in source.get("requested_cases", [])
+ if isinstance(item, dict) and item.get("disposition") == disposition
+ ]
+ expected = {
+ item["case"]["case_id"]: (item["sku"], item["case"])
+ for item in wrappers
+ }
+ expected_count = len(wrappers)
+ require_one_allocation = disposition == "unsupported"
+ elif isinstance(source, dict) and isinstance(source.get("cases"), list):
+ expected = {
+ case["case_id"]: (source.get("sku"), case)
+ for case in source["cases"]
+ }
+ expected_count = len(source["cases"])
+ require_one_allocation = True
+ else:
+ raise ContractError("delivery source is not a matrix or shard control")
+ if not expected or len(expected) != expected_count:
+ raise ContractError("delivery source has empty or duplicate case coverage")
+
+ validate_attempt_paths(paths)
+ attempts = []
+ for raw_path in paths:
+ document = strict_load(raw_path)
+ if isinstance(document, dict) and document.get("format") in {RAW_FORMAT, TERMINAL_FORMAT}:
+ attempts.append(load_attempt(raw_path))
+ by_case: dict[str, list[dict[str, Any]]] = {}
+ attempt_ids = set()
+ allocation_ids = set()
+ source_sha256 = hashlib.sha256(source_file.read_bytes()).hexdigest()
+ for document in attempts:
+ ids = document["identity"]
+ case_id = ids["case_id"]
+ if case_id not in expected or ids["attempt_id"] in attempt_ids:
+ raise ContractError("delivery contains an extra case or duplicate attempt")
+ attempt_ids.add(ids["attempt_id"])
+ allocation_ids.add(ids["allocation_id"])
+ sku, scheduled = expected[case_id]
+ scheduled_case = {key: value for key, value in scheduled.items() if key != "case_id"}
+ if ids["case_factors"] != {
+ "case": scheduled_case,
+ "profile": scheduled_case_profile(scheduled_case, "delivery case"),
+ "sku": sku,
+ }:
+ raise ContractError("delivery attempt differs from its scheduled case")
+ factors = ids["allocation_factors"]
+ expected_environment = {
+ "artifact": os.environ.get("COLLECTIVEX_ARTIFACT_NAME"),
+ "execution_id": os.environ.get("COLLECTIVEX_EXECUTION_ID"),
+ "job": os.environ.get("GITHUB_JOB"),
+ "repo": os.environ.get("GITHUB_REPOSITORY"),
+ "run_attempt": os.environ.get("GITHUB_RUN_ATTEMPT"),
+ "run_id": os.environ.get("GITHUB_RUN_ID"),
+ "source_sha": os.environ.get("COLLECTIVEX_SOURCE_SHA") or os.environ.get("GITHUB_SHA"),
+ }
+ expected_runner = (
+ "capability-resolver"
+ if document["format"] == TERMINAL_FORMAT
+ and document["provenance"]["source"] == "matrix-capability-resolver"
+ else sku
+ )
+ if any(
+ value is not None and factors[field] != value
+ for field, value in expected_environment.items()
+ ) or factors["runner"] != expected_runner:
+ raise ContractError("delivery allocation factors differ from the workflow")
+ if document["format"] == TERMINAL_FORMAT:
+ control = document["provenance"]["control_sha256"]
+ if control != source_sha256:
+ raise ContractError("terminal outcome does not reference its exact control document")
+ by_case.setdefault(case_id, []).append(document)
+ if set(by_case) != set(expected):
+ raise ContractError("delivery case coverage is incomplete")
+ for case_id, documents in by_case.items():
+ ordinals = sorted(document["identity"]["attempt_ordinal"] for document in documents)
+ if ordinals != list(range(1, len(ordinals) + 1)):
+ raise ContractError(f"delivery attempt ordinals are not contiguous for {case_id}")
+ if require_one_allocation and len(allocation_ids) != 1:
+ raise ContractError("one shard must use exactly one allocation identity")
+ return len(attempts)
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description="CollectiveX native attempt contracts")
+ subparsers = parser.add_subparsers(dest="command", required=True)
+ probe = subparsers.add_parser("probe")
+ probe.add_argument("path")
+ probe.add_argument("--status", choices=("success", "invalid"))
+ emit = subparsers.add_parser("emit-terminal")
+ emit.add_argument("--out", required=True)
+ emit.add_argument("--backend", required=True)
+ emit.add_argument("--phase", required=True, choices=("decode", "prefill"))
+ emit.add_argument("--return-code", required=True, type=int)
+ emit.add_argument("--failure-mode")
+ demote = subparsers.add_parser("demote")
+ demote.add_argument("path")
+ demote.add_argument("--return-code", required=True, type=int)
+ validate_many = subparsers.add_parser("validate-many")
+ validate_many.add_argument("paths", nargs="+")
+ quarantine = subparsers.add_parser("quarantine-invalid")
+ quarantine.add_argument("path")
+ delivery = subparsers.add_parser("validate-delivery")
+ delivery.add_argument("--source", required=True)
+ delivery.add_argument("--disposition")
+ delivery.add_argument("paths", nargs="+")
+ args = parser.parse_args()
+ try:
+ if args.command == "probe":
+ document = load_attempt(args.path)
+ if args.status is None:
+ return 0
+ if document.get("format") != RAW_FORMAT:
+ return 1
+ outcome = document["outcome"]
+ validity = outcome.get("validity")
+ return int(
+ not (
+ isinstance(validity, dict)
+ and validity.get("execution_status") == "complete"
+ and outcome.get("status") == args.status
+ )
+ )
+ if args.command == "emit-terminal":
+ document = make_terminal_from_environment(
+ backend=args.backend,
+ phase=args.phase,
+ return_code=args.return_code,
+ failure_mode=args.failure_mode,
+ )
+ _write_document(args.out, document)
+ print(f"preserved terminal outcome ({document['outcome']['failure_mode']})")
+ return 0
+ if args.command == "validate-many":
+ print(f"validated {validate_attempt_paths(args.paths)} native attempts")
+ return 0
+ if args.command == "quarantine-invalid":
+ quarantine_invalid_attempt(args.path)
+ return 0
+ if args.command == "validate-delivery":
+ print(
+ f"validated {validate_delivery(args.paths, args.source, disposition=args.disposition)} "
+ "delivery attempts"
+ )
+ return 0
+ demote_raw_attempt(args.path, args.return_code)
+ return 0
+ except (ContractError, identity.IdentityError, OSError, ValueError) as exc:
+ print(f"terminal contract error: {exc}", file=sys.stderr)
+ return 1
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/experimental/CollectiveX/docs/methodology.md b/experimental/CollectiveX/docs/methodology.md
new file mode 100644
index 0000000000..606e7c8995
--- /dev/null
+++ b/experimental/CollectiveX/docs/methodology.md
@@ -0,0 +1,305 @@
+# CollectiveX EP v1 Contract
+
+
+
+**English** | [中文](./methodology_zh.md)
+
+
+
+This document defines new CollectiveX results. Historical run notes are evidence, not contract.
+
+## Product Boundary
+
+CollectiveX is a communication microbenchmark for:
+
+- comparing EP libraries on one chip/topology;
+- comparing EP latency and logical payload bandwidth across systems under the same workload; and
+- exposing unsupported, failed, invalid, and unstable evidence without contaminating decisions.
+
+It does not predict serving throughput without a separate correlation study.
+
+## Matrix
+
+The promoted workload is `deepseek-v3-v1`: hidden 7168, top-k 8, 256 routed experts, BF16 dispatch
+and combine, packed placement, and backend-tuned resources. Each case explicitly selects normal
+`layout-and-dispatch-v1` or low-latency `expert-packed-weighted-combine-v1` semantics.
+
+- `ep-core-v1`: uniform routing; decode T=1..128 powers of two; prefill T=256/512.
+- `ep-routing-v1`: Zipf with EPLB off/on; decode T=128; prefill T=512.
+- `ep-low-latency-v1`: DeepEP V1/UCCL native low-latency APIs; uniform decode T=1..128 powers of
+ two; the capability contract rejects every other backend instead of fabricating a low-latency path.
+- Canonical surface: 608 requested cases / 1,600 token points; 364 runnable cases / 940 points in
+ 58 executable workflow shards/allocation cells; 244 unsupported cases / 660 points.
+
+| Systems | EP8 | EP16 |
+|---|---|---|
+| H100/H200/B200/B300 | 1x8 NVLink, scale-up | 2x8 NVLink + RDMA, scale-out |
+| MI325X/MI355X | 1x8 XGMI, scale-up | 2x8 XGMI + RDMA, scale-out |
+| GB200/GB300 | 2x4 MNNVL, scale-up | 4x4 MNNVL, scale-up |
+
+Physical host count does not define scope. Both GB cells remain inside one 72-GPU MNNVL scale-up
+domain.
+
+Unsupported combinations are terminal outcomes, not silently skipped coverage. DeepEP V2 is the
+`ElasticBuffer` introduced by PR #605, pinned with upstream PR #630's minimal pure-scale-up fix.
+Scale-up cases request NCCL Device API LSA and fail closed unless the realized LSA team covers the
+full EP world. x86 EP16 scale-out uses the hybrid path with GIN and requires two logical scale-out
+domains represented by two physical RDMA ranks, with eight scale-up ranks per domain. GB EP16
+remains MNNVL scale-up and uses LSA. NVIDIA capabilities declared in source remain unvalidated until
+GPU outcomes pass the native oracle and publisher gates. H100 V2 on the current runner pool is a
+declared unsupported combination in v1 because NCCL 2.30.4 reports no Device API symmetric-memory
+support for its EP8 communicator; that pool can return only after all-rank CUDA P2P/LSA support is
+restored. Removed axes include `[cl]`, `[rv]`, quantization, alternate activation/routing profiles,
+uneven allocation, placement permutations, model envelopes, and scaling.
+FlashInfer is excluded from v1 after repeatable intermittent execution failures; those failures are
+not converted into planned-unsupported coverage.
+MoRI EP8 uses MI325X AsyncLL or MI355X IntraNode in normal mode. EP16 uses pinned InterNodeV1 over
+2x8 XGMI + RDMA with 96 blocks, 64 RDMA blocks, 8 warps, one QP per PE, and external input. MoRI's
+AsyncLL transport is not the genuine low-latency suite contract and is never labeled as such.
+
+## Workload Identity
+
+One canonical workload is generated over the global token batch and sliced by source rank. Expert
+indices and gate weights are serialized. Activations use a versioned integer counter formula whose
+BF16 values are exact across runtimes; its full identity is bound into the manifest. The manifest
+also binds shape/EP coordinates and oracle version. SHA-256 covers canonical bytes and parameters;
+library RNG regeneration is not proof of identity.
+
+Routing traffic distinguishes:
+
+- token-expert assignments, which determine expert compute load; and
+- rank-deduplicated token payload copies, which determine EP activation traffic.
+
+Adapters may not generate routing or reinterpret one quantity as the other.
+
+## Measurement
+
+Normal mode uses `layout-and-dispatch-v1`: dispatch timing includes layout plus communication, and
+combine returns activation payload through an unweighted rank-sum path. Low-latency mode uses
+`expert-packed-weighted-combine-v1`: native DeepEP V1/UCCL APIs dispatch token-expert assignments and
+perform gate-weighted combine. Expert-output staging is outside isolated combine timing and inside
+measured paired roundtrip. Each component declares availability, origin, start/end states, stage
+scope, and sample count. A paired-only API reports null isolated components. `isolated_sum` is
+derived and never used for throughput or recommendations. Mode is series identity, and normal and
+low-latency evidence cannot share a ranking cohort.
+
+Every measured component uses `fixed-512-v1`:
+
+- 64 trials x 8 timed iterations = 512 observations;
+- 32 synchronized full dispatch-stage-combine warmups before each available measured component at
+ every trial/point;
+- roundtrip first, then isolated dispatch and combine, with a fixed per-phase conditioning ladder; and
+- per-iteration maximum latency across ranks before nearest-rank p50/p90/p95/p99.
+
+Measured roundtrip p99 is the headline latency. Retries remain separate attempts; a later success
+does not erase earlier failures. Decode and prefill identify the serving regime represented by one
+MoE-layer collective; they do not change the timed primitive at an otherwise identical shape.
+
+The NCCL/RCCL reference is an end-to-end Python adapter, not a bare fabric primitive. Its dispatch
+boundary includes layout, count exchange, a device-to-host split synchronization, fresh receive
+allocation, and four payload/metadata all-to-all calls; activation-only combine adds one all-to-all plus
+scatter/reduction. Its p99 therefore measures the complete reference-adapter boundary and can be
+host/scheduler-sensitive. It is useful for portable system controls but must not be labeled fabric,
+link, bus, or single-collective latency.
+
+The versioned conditioning and EPLB planner contracts (reference trace, redundant count, and
+placement/remap version) are part of scheduled and evidence identity.
+
+Logical payload bandwidth is:
+
+`logical_payload_bytes / measured_latency_seconds`
+
+Normal-mode payload bytes use rank-deduplicated token-rank activations; low-latency bytes use
+token-expert assignments. Both add required scale bytes at the named boundary and exclude expert
+metadata, padding, and backend buffer capacity. Algorithm bandwidth, bus bandwidth, wire utilization,
+and physical-link utilization are not published without a defined primitive model or transport
+counters. Logical bandwidth must never be labeled physical bandwidth. Published payload and token
+rates are named `rate_at_latency_percentile`: bytes or tokens divided by the matching latency
+percentile. They are lower-tail service rates at p99 latency, not p99 percentiles of an inverted rate
+distribution.
+
+## Correctness
+
+An implementation-independent oracle uses an expert-specific deterministic transform so wrong
+expert routing cannot pass an identity roundtrip. For every rank and point it verifies:
+
+1. destination rank/expert, source token, multiplicity, gate weight, and receive counts;
+2. dispatched payload and metadata before timing;
+3. combined output before timing;
+4. unchanged semantic inputs through all timed samples; and
+5. dispatched payload/metadata and combined output again after timing.
+
+Normal-mode adapters use activation-only, unweighted rank-sum combine. The oracle builds each rank's
+gate-weighted expert aggregate before combine, independently derives `sum(gate * expert(token))`,
+and checks the dispatch metadata and transformed output. Low-latency adapters separately verify the
+expert-packed source/expert assignment, native gate weights, and gate-weighted combined output. Both
+contracts check every element with recorded `rtol=0.05` and `atol=0.02`. Any failed rank or point
+makes the case ineligible.
+Pre/post dispatch evidence is hashed in canonical source-token order. Native receive slots may be
+assigned nondeterministically, so physical receive order is not treated as a correctness property.
+
+## Native Result
+
+One raw case document uses `format: "collectivex.ep.v1"`, rejects unknown fields, and contains:
+
+- `case`: stable case ID, suite, required tier, and coordinate;
+- `workload`: canonical identity and logical MoE shape;
+- `measurement`: sampling, component states, timing, and byte accounting;
+- `implementation`: instantiated class/API, pinned source, loaded libraries, and resources;
+- `topology`: requested and realized SKU, devices, placement, scale-up domain, and transport;
+- `provenance`: source SHA, image/squash hashes, allocation, run, and attempt;
+- `rows`: point latency, byte accounting, token rate, correctness, load, fanout, and anomaly evidence; and
+- `outcome`: `success`, `failed`, `invalid`, `diagnostic`, or `unsupported`, with reasons.
+
+Raw result documents and exact samples pass through transient GitHub delivery artifacts before the
+publisher archives them in the private bundle; they never enter the public tree. Private environment
+details remain in local mode-0600 logs and ignored operator notes; they are never archived or
+published. Every expected case has one terminal selected outcome while every attempt remains retained.
+
+## Identity And Comparisons
+
+Canonical JSON produces three full SHA-256 IDs:
+
+- `series_id`: all locked factors except token coordinate and repeat allocation;
+- `point_id`: `series_id` plus token coordinate; and
+- `evidence_id`: `point_id` plus allocation/run/attempt/sample checksum.
+
+Locked factors include workload bytes, measurement and sampling contract, resources, realized
+topology, implementation/build, loaded libraries, image/squash, runtime, and source SHA.
+Deferred code generation is captured before measurement and recaptured afterward. DeepEP V2 uses a
+fixed NVCC random seed and binds final cache keys plus generated-source and executable-SASS hashes;
+raw CUBIN bytes remain private diagnostics. Hybrid binds its realized auto-tuned config and complete
+kernel-key set while retaining rank-local shared-object hashes as private diagnostics. Locally built
+extension hashes are diagnostic; their pinned source trees, build recipe, runtime, and dependencies
+remain series-bound.
+The series identity includes the case ID, which binds the complete scheduled token ladder and the
+frozen percentile, rank-reduction, conditioning, warmup, and correctness semantics.
+
+A controlled comparison declares one contrast:
+
+- `library`: backend implementation and its tuned resource profile may differ; the realized system,
+ workload, EP, resource policy, source, and measurement remain matched;
+- `chip`: a controlled platform contrast. The full realized system/topology and tuned resource
+ profile may differ while workload, EP, placement class, resource policy, backend lineage, source,
+ and measurement remain matched. It is not a silicon-only comparison;
+- `system`: all hardware/backend differences stay visible while workload, EP, and measurement match;
+- `routing`: routing distribution/EPLB differs while the static implementation build/generator,
+ system, model shape, resource profile, and measurement remain matched. Uniform and Zipf without
+ EPLB reuse the same generated implementation; EPLB's physical-expert/JIT configuration remains an
+ explicit treatment difference.
+
+Any undeclared mismatch rejects the overlay. Chip/system results describe measured systems, not
+silicon alone.
+
+## Evidence Policy
+
+Capability declarations say what may be attempted; artifacts determine evidence status. Promotion
+requires exact expected coverage with no missing, extra, duplicate, malformed, or heterogeneous
+case. Public coverage preserves each matrix disposition; promotion requires every runnable case to
+succeed and every planned-unsupported case to remain unsupported in every selected run. Only the
+pinned canonical full-v1 matrix, with a decision-grade library, chip, system, and routing cohort,
+may advance `dev-latest`; partial matrices remain diagnostic. The full-matrix digest intentionally
+pins the exact workflow shard grouping as well as the requested cases, so changing `--max-cases`
+or the SKU round-robin scheduling order produces diagnostic-only runs even when case coverage is
+unchanged. Superseded retries,
+planned-unsupported outcomes, and unstable comparison cohorts may render diagnostically but cannot
+rank or recommend; every successful required series in a promoted dataset remains decision-grade.
+Any failed, invalid, or diagnostic retry of a runnable case blocks promotion even if a later retry
+succeeds. Routing cohorts are comparable-experimental sensitivities and never produce configuration
+recommendations; official library/platform/system cohorts own actionable recommendations.
+
+A point becomes decision-grade only after three independent workflow runs and allocation IDs pass
+correctness, identity, provenance, tail gates, p50/p99 repeat-stability thresholds, and stable ordering. The
+publisher, not the frontend, computes eligibility, controlled cohorts, sensitivity pairs, and
+recommendations.
+
+## Execution Isolation
+
+Every non-MNNVL scale-out case uses operator-pinned socket and RDMA selectors. The launcher rejects
+missing or partial profiles, then probes every allocated node for the configured interface, active
+HCA port, and configured GID before backend initialization. It never substitutes a default route,
+inherited runner environment, or transport fallback. Scale-up and MNNVL cases clear the profile;
+scale-out NCCL/RCCL forces `NCCL_NET=IB` and exact HCA matching. Selector values remain in encrypted
+config and mode-0600 private logs.
+
+Repository staging uses a pre-existing, runner-owned, group/world non-writable shared base outside
+the checkout and workflow workspace. The parent process resolves the exact execution child before
+copying, claims it with a runner-owned marker, and verifies that all allocated nodes can read and
+write the same bytes. Cleanup waits for confirmed allocation teardown and removes only that child,
+including a safely identified partial claim. The same-run V2/Hybrid source archive is fully validated
+under fixed member and expanded-size bounds, and only the selected pinned root is extracted; a
+symlink is accepted only when it is a relative leaf pointing to a regular member inside the same
+backend root, followed by exact Git tree/submodule validation.
+
+## Artifact Validation And JIT Delivery
+
+There is no self-hosted service, Vercel storage, GCP, Neon, managed database, or managed object
+store. The publication workflow uses runner-local temporary storage only as a disposable validation
+and promotion workspace:
+
+```text
+$COLLECTIVEX_STORE_ROOT/
+ private/incoming/ # write-once downloaded GHA attempts
+ private/bundles// # immutable source archives, native results/samples, matrix, checksums
+ private/quarantine/ # rejected attempts plus machine-readable reasons
+ public/datasets// # immutable sanitized frontend datasets
+ public/channels/ # small atomic pointers: latest-attempt, dev-latest
+ locks/
+```
+
+Private and public trees use separate permissions. JSON manifests and checksums are authoritative;
+a rebuildable catalog is only an index. Raw sweep artifacts are transient publisher input; only the
+sanitized promoted NDJSON is retained as a frontend publication artifact.
+
+Container tags are checked against pinned registry digests. Enroot imports use a fixed
+`SOURCE_DATE_EPOCH` and versioned cache generation; every mounted squash is freshly hashed into
+series identity. Image-provided DeepEP is also checked against exact per-architecture wheel and
+installed-file fingerprints, so a stale cache cannot inherit the pinned source identity.
+Source-built DeepEP V2 uses a separate mode-0700 cluster-local cache mounted only as `/cx-cache`.
+Its content key binds a versioned build recipe, verified image digest, CPU/GPU architecture,
+upstream source trees, and pinned build dependencies. The cache is never an artifact or publisher
+input; per-execution source/results stages remain isolated and disposable, and marker plus runtime
+probes fail closed before reuse. The runner UID is inside the trusted cluster boundary: this cache
+guards against stale or accidental mutation, not hostile same-UID jobs. Only an unpublished partial
+build may be reset automatically; a published cache that fails integrity or runtime checks is left
+intact and rejected so a concurrent allocation cannot lose files it is using.
+
+Publication is fail-closed:
+
+1. acquire an exclusive filesystem lock and stage on the destination filesystem;
+2. archive source bytes before parsing;
+3. require the exact matrix-declared artifact set and reject every unconsumed archive member;
+4. validate strict schemas, privacy, checksums, identities, timing, and exact matrix outcomes;
+5. write checksums and `COMPLETE`, fsync, then atomically rename the private bundle;
+6. build and validate the sanitized content-addressed dataset, fsync, then atomically rename it;
+7. atomically replace `dev-latest.json` only when every promotion gate passes.
+
+Rejected attempts may update workspace `latest-attempt` but never `dev-latest`. The workspace is
+destroyed with the publication runner and is never attached to the frontend. No artifact is emitted
+unless all three selected bundles advance `dev-latest`.
+
+`publisher.py ingest` accepts the exact matrix plus one `--artifact` directory or ZIP per GitHub
+artifact. `promote` accepts explicit immutable bundle IDs. Default `verify` requires
+`latest-attempt`; it also verifies `dev-latest` when present, while an explicit
+`--channel dev-latest` requires it. The workflow copies only the verified sanitized dataset to a
+one-record `collectivex_public_v1_.ndjson` artifact. Raw artifacts and private workspace
+content are never bundled into the application.
+
+Sweeps default to `release_tag=unversioned`. Selecting `v1` requires the locked full-matrix digest
+and emits a marker bound to the run ID, attempt, source SHA, and matrix SHA-256. The manual
+publication workflow accepts exactly three unique successful `CollectiveX Sweep` run IDs from one
+source SHA, revalidates their metadata and exact markers, downloads their immutable artifacts, and
+passes the same provenance assertions to `publisher.py ingest`. Partial, filtered, untagged,
+cross-source, failed, or expired inputs fail closed.
+
+Using a server-side GitHub read token, the frontend discovers the latest successful version-scoped
+publication run and downloads the publication artifact just in time. It requires exactly one root
+NDJSON entry, validates UTF-8, schema, promotion status, and filename/body SHA-256, then exposes a
+short-lived versioned channel pointer and immutable versioned dataset URL. The benchmark-version
+selector currently exposes V1; later versions require separate release and publication identities.
+The frontend never invents missing values, selects retries, or recomputes decision eligibility.
+
+## Legacy Data
+
+Numeric schemas 3-5 are outside the v1 publisher and frontend reader. They remain historical
+diagnostic evidence and cannot seed `dev-latest` or drive v1 decisions.
diff --git a/experimental/CollectiveX/docs/methodology_zh.md b/experimental/CollectiveX/docs/methodology_zh.md
new file mode 100644
index 0000000000..7f6dcb67af
--- /dev/null
+++ b/experimental/CollectiveX/docs/methodology_zh.md
@@ -0,0 +1,297 @@
+# CollectiveX EP v1 契约
+
+
+
+[English](./methodology.md) | **中文**
+
+
+
+本文档定义新的 CollectiveX 结果。历史运行笔记是 evidence，不是 contract。
+
+## 产品边界
+
+CollectiveX 是通信 microbenchmark，用于：
+
+- 在同一 chip/topology 上比较 EP libraries；
+- 在相同 workload 下比较不同系统的 EP latency 和 logical payload bandwidth；
+- 展示 unsupported、failed、invalid 和 unstable evidence，同时避免污染决策。
+
+若没有单独的 correlation study，它不能预测 serving throughput。
+
+## 矩阵
+
+提升后的 workload 为 `deepseek-v3-v1`：hidden 7168、top-k 8、256 routed experts、BF16
+dispatch 和 combine、packed placement，以及 backend-tuned resources。每个 case 都显式选择
+normal `layout-and-dispatch-v1` 或 low-latency `expert-packed-weighted-combine-v1` 语义。
+
+- `ep-core-v1`：uniform routing；decode T=1..128 的 2 次幂；prefill T=256/512。
+- `ep-routing-v1`：Zipf，EPLB off/on；decode T=128；prefill T=512。
+- `ep-low-latency-v1`：使用 DeepEP V1/UCCL 原生 low-latency API；uniform decode T=1..128 的
+ 2 次幂；capability contract 会拒绝其他后端，不会伪造 low-latency path。
+- 规范矩阵范围：请求 608 个 cases / 1,600 个 token points；364 个可运行 cases / 940 个
+ points，分布在 58 个可执行 workflow shards/allocation cells；244 个 unsupported cases / 660 个
+ points。
+
+| 系统 | EP8 | EP16 |
+|---|---|---|
+| H100/H200/B200/B300 | 1x8 NVLink，scale-up | 2x8 NVLink + RDMA，scale-out |
+| MI325X/MI355X | 1x8 XGMI，scale-up | 2x8 XGMI + RDMA，scale-out |
+| GB200/GB300 | 2x4 MNNVL，scale-up | 4x4 MNNVL，scale-up |
+
+物理主机数量不能定义通信范围。两个 GB 配置都位于同一个 72-GPU MNNVL scale-up domain 内。
+
+Unsupported combinations 是 terminal outcomes，不会被静默跳过。DeepEP V2 指 PR #605
+引入的 `ElasticBuffer`，并固定使用 upstream PR #630 的最小纯 scale-up 修复。V2 的 scale-up
+cases 请求 NCCL Device API LSA；若实际建立的 LSA team 未覆盖整个 EP world，则直接失败。x86
+EP16 scale-out 使用启用 GIN 的 hybrid path，并要求两个逻辑 scale-out domains（由两个物理 RDMA
+ranks 表示）、每个 domain 八个 scale-up ranks。GB EP16 仍是 MNNVL scale-up，因此使用 LSA。
+Source 中声明的 NVIDIA capabilities 在 GPU outcomes 通过 native oracle 和 publisher gates 前仍为
+unvalidated。当前 runner pool 上的 H100 V2 在 v1 中被声明为 unsupported，因为 NCCL 2.30.4
+报告其 EP8 communicator 不具备 Device API symmetric-memory 支持；只有该 pool 恢复全 rank
+CUDA P2P/LSA 支持后才能重新加入。已移除的轴包括 `[cl]`、`[rv]`、quantization、alternate
+activation/routing profiles、uneven allocation、placement
+permutations、model envelopes 和 scaling。
+FlashInfer 因可重复出现的间歇性执行失败而排除在 v1 外；这些失败不会转为 planned-unsupported
+coverage。
+MoRI EP8 在 normal mode 下使用 MI325X AsyncLL 或 MI355X IntraNode。EP16 固定使用 2x8 XGMI +
+RDMA 上的 InterNodeV1，配置为 96 blocks、64 RDMA blocks、8 warps、每个 PE 一个 QP，以及
+external input。MoRI 的 AsyncLL transport 不属于真正的 low-latency suite contract，也绝不会
+以该模式标注。
+
+## Workload 身份
+
+一个 canonical workload 在 global token batch 上生成，再按 source rank 切分。Expert indices
+和 gate weights 会序列化。Activations 使用带版本的整数计数器公式，其 BF16 值在不同 runtime
+中精确一致；完整身份绑定到 manifest。Manifest 还绑定 shape/EP coordinates 和 oracle version。
+SHA-256 覆盖 canonical bytes 和 parameters；重新生成 library RNG 不能证明身份一致。
+
+Routing traffic 区分：
+
+- token-expert assignments，决定 expert compute load；
+- rank-deduplicated token payload copies，决定 EP activation traffic。
+
+Adapters 不得生成 routing，也不得将两种量相互解释。
+
+## 测量
+
+Normal mode 使用 `layout-and-dispatch-v1`：dispatch timing 包括 layout 和 communication，combine
+通过 unweighted rank-sum path 返回 activation payload。Low-latency mode 使用
+`expert-packed-weighted-combine-v1`：DeepEP V1/UCCL 原生 API dispatch token-expert assignments，
+并执行 gate-weighted combine。Expert-output staging 不计入 isolated combine timing，但计入被测
+paired roundtrip。每个 component 声明 availability、origin、start/end states、stage scope 和 sample
+count。仅有 paired API 时，isolated components 报 null。`isolated_sum` 为派生值，不用于
+throughput 或 recommendations。Mode 属于 series identity；normal 和 low-latency evidence 不能
+共用排名 cohort。
+
+每个被测 component 均使用 `fixed-512-v1`：
+
+- 64 trials x 8 timed iterations = 512 observations；
+- 每个 trial/point 的每个可用被测 component 前，执行 32 次同步完整
+ dispatch-stage-combine warmups；
+- 先测 roundtrip，再测 isolated dispatch 和 combine，并使用固定的 per-phase conditioning ladder；
+- 每次 iteration 先取跨 rank 最大 latency，再以 nearest-rank 计算 p50/p90/p95/p99。
+
+被测 roundtrip p99 是 headline latency。Retries 保持为独立 attempts；后续成功不会抹除早期失败。
+Decode 和 prefill 表示一个 MoE-layer collective 所代表的 serving regime；在其他 shape 相同时，
+它们不会改变 timed primitive。
+
+NCCL/RCCL reference 是 end-to-end Python adapter，而不是 bare fabric primitive。其 dispatch
+boundary 包含 layout、count exchange、device-to-host split synchronization、fresh receive
+allocation，以及四次 payload/metadata all-to-all；activation-only combine 还包含一次 all-to-all 和
+scatter/reduction。因此其 p99 测量完整 reference-adapter boundary，可能对 host/scheduler 敏感。
+它可作为 portable system control，但不得标记为 fabric、link、bus 或 single-collective latency。
+
+带版本的 conditioning 和 EPLB planner contracts（reference trace、redundant count 和
+placement/remap version）属于 scheduled 和 evidence identity。
+
+Logical payload bandwidth 为：
+
+`logical_payload_bytes / measured_latency_seconds`
+
+Normal-mode payload bytes 使用按 rank 去重的 token-rank activations；low-latency bytes 使用
+token-expert assignments。两种模式都在命名边界上加入必需 scale bytes，并排除 expert metadata、
+padding 和 backend buffer capacity。若没有定义 primitive model 或 transport counters，不发布
+algorithm bandwidth、bus bandwidth、wire utilization 或 physical-link utilization。Logical
+bandwidth 绝不能标为 physical bandwidth。已发布 payload 和 token rates 命名为
+`rate_at_latency_percentile`：bytes 或 tokens 除以对应 latency percentile。它们是 p99 latency
+下的 lower-tail service rates，不是 inverted rate distribution 的 p99 percentiles。
+
+## 正确性
+
+与实现无关的 oracle 使用 expert-specific deterministic transform，使错误 expert routing 无法
+通过 identity roundtrip。它对每个 rank 和 point 验证：
+
+1. destination rank/expert、source token、multiplicity、gate weight 和 receive counts；
+2. timing 前的 dispatched payload 和 metadata；
+3. timing 前的 combined output；
+4. 所有 timed samples 期间 semantic inputs 不变；
+5. timing 后再次验证 dispatched payload/metadata 和 combined output。
+
+Normal-mode adapters 使用 activation-only、unweighted rank-sum combine。Oracle 在 combine 前
+构造每个 rank 的 gate-weighted expert aggregate，独立计算 `sum(gate * expert(token))`，并检查
+dispatch metadata 和 transformed output。Low-latency adapters 单独验证 expert-packed
+source/expert assignment、原生 gate weights 和 gate-weighted combined output。两个契约都使用
+已记录的 `rtol=0.05` 和 `atol=0.02` 检查每个 element。任一 rank 或 point 失败都会使 case
+不合格。Pre/post dispatch evidence 按
+canonical source-token order 计算 hash。Native receive slots 可能非确定性分配，因此 physical
+receive order 不作为 correctness property。
+
+## Native 结果
+
+单个 raw case document 使用 `format: "collectivex.ep.v1"`，拒绝未知 fields，并包含：
+
+- `case`：稳定 case ID、suite、required tier 和 coordinate；
+- `workload`：canonical identity 和 logical MoE shape；
+- `measurement`：sampling、component states、timing 和 byte accounting；
+- `implementation`：实例化 class/API、固定 source、loaded libraries 和 resources；
+- `topology`：requested 和 realized SKU、devices、placement、scale-up domain 和 transport；
+- `provenance`：source SHA、image/squash hashes、allocation、run 和 attempt；
+- `rows`：point latency、byte accounting、token rate、correctness、load、fanout 和 anomaly evidence；
+- `outcome`：`success`、`failed`、`invalid`、`diagnostic` 或 `unsupported`，以及 reasons。
+
+Raw result documents 和 exact samples 会先经过临时 GitHub delivery artifacts，再由 publisher
+归档到 private bundle；它们不会进入 public tree。Private environment details 只保留在本地
+mode-0600 logs 和忽略的 operator notes 中；不会归档或发布。每个 expected case 有一个 terminal
+selected outcome，同时保留每次 attempt。
+
+## 身份与比较
+
+Canonical JSON 生成三个完整 SHA-256 IDs：
+
+- `series_id`：除 token coordinate 和 repeat allocation 外的所有 locked factors；
+- `point_id`：`series_id` 加 token coordinate；
+- `evidence_id`：`point_id` 加 allocation/run/attempt/sample checksum。
+
+Locked factors 包括 workload bytes、measurement 和 sampling contract、resources、realized
+topology、implementation/build、loaded libraries、image/squash、runtime 和 source SHA。
+Deferred code generation 会在 measurement 前捕获，并在之后再次捕获。DeepEP V2 使用固定的
+NVCC random seed，并绑定最终 cache keys、generated-source hashes 与 executable-SASS hashes；
+raw CUBIN bytes 仅保留为 private diagnostics。Hybrid 绑定实际自动调优配置与完整 kernel-key
+set，同时将各 rank 的 shared-object hashes 仅保留为 private diagnostics。本地构建的 extension
+hashes 属于 diagnostic；其固定 source trees、build recipe、runtime 与 dependencies 仍绑定到
+series。
+Series identity 包含 case ID；case ID 绑定完整 scheduled token ladder，以及固定的 percentile、
+rank-reduction、conditioning、warmup 和 correctness semantics。
+
+Controlled comparison 只声明一个 contrast：
+
+- `library`：backend implementation 及其 tuned resource profile 可以不同；realized system、
+ workload、EP、resource policy、source 和 measurement 必须匹配；
+- `chip`：受控 platform contrast。完整 realized system/topology 和 tuned resource profile 可以不同，
+ 但 workload、EP、placement class、resource policy、backend lineage、source 和 measurement 必须
+ 匹配。它不是 silicon-only comparison；
+- `system`：保留所有 hardware/backend 差异，同时匹配 workload、EP 和 measurement；
+- `routing`：routing distribution/EPLB 可以不同，但 static implementation build/generator、system、
+ model shape、resource profile 和 measurement 必须匹配。未启用 EPLB 的 Uniform 和 Zipf 复用
+ 同一 generated implementation；EPLB 的 physical-expert/JIT configuration 是显式 treatment
+ difference。
+
+任何未声明的 mismatch 都会拒绝 overlay。Chip/system results 描述 measured systems，而非仅描述
+silicon。
+
+## Evidence 策略
+
+Capability declarations 说明可以尝试什么；artifacts 决定 evidence status。Promotion 要求完整的
+expected coverage，不能有 missing、extra、duplicate、malformed 或 heterogeneous case。Public
+coverage 保留每个 matrix disposition；promotion 要求每个 runnable case 在所有 selected runs 中
+成功，且每个 planned-unsupported case 始终为 unsupported。只有固定 canonical full-v1 matrix，
+且具有 decision-grade library、chip、system 和 routing cohort，才能推进 `dev-latest`；partial
+matrices 仍为 diagnostic。Full-matrix digest 有意绑定精确 workflow shard grouping 和 requested
+cases，因此即使 case coverage 不变，修改 `--max-cases` 或 SKU round-robin scheduling order 也只
+会产生 diagnostic-only runs。Superseded retries、planned-unsupported outcomes 和 unstable
+comparison cohorts 可以用于诊断展示，但不能排名或推荐；promoted dataset 中每个成功的 required
+series 都必须保持 decision-grade。Runnable case 的任何 failed、invalid 或 diagnostic retry 都会
+阻止 promotion，即使后续 retry 成功。Routing cohorts 是 comparable-experimental sensitivities，
+不会产生 configuration recommendations；official library/platform/system cohorts 才能产生可执行
+recommendations。
+
+一个 point 只有在三个独立 workflow runs 和 allocation IDs 均通过 correctness、identity、
+provenance、tail gates、p50/p99 repeat-stability thresholds 和 stable ordering 后才成为
+decision-grade。Eligibility、controlled cohorts、sensitivity pairs 和 recommendations 由
+publisher 而非 frontend 计算。
+
+## 执行隔离
+
+每个非 MNNVL scale-out case 都使用 operator 固定的 socket 与 RDMA selectors。Launcher 会拒绝
+缺失或不完整的 profile，并在 backend 初始化前逐个 allocation 节点检查已配置 interface、active
+HCA port 与指定 GID。它不会改用 default route、继承的 runner environment 或 transport
+fallback。Scale-up 和 MNNVL case 会清除该 profile；scale-out NCCL/RCCL 强制设置
+`NCCL_NET=IB` 并精确匹配 HCA。Selector values 只保留在加密配置和 mode-0600 private logs 中。
+
+Repository staging 使用 checkout 与 workflow workspace 外预创建的 shared base；该 base 由
+runner owner 持有，group/world 均不可写。父进程在复制前解析精确 execution child，以
+runner-owned marker 声明所有权，并验证所有 allocation 节点读写的是同一份 bytes。Cleanup 会
+等待 allocation teardown 得到确认，并只删除该 child，包括可安全识别的未完成 claim。同一 run
+的 V2/Hybrid source archive 会在固定 member 数和解压大小上限内完整验证，并且只提取所选 fixed
+root；仅当相对 leaf symlink 指向同一 backend root 内的 regular member 时才允许创建，之后还要
+通过精确 Git tree/submodule 校验。
+
+## 产物验证与即时交付
+
+不使用 self-hosted service、Vercel storage、GCP、Neon、managed database 或 managed object
+store。Publication workflow 仅将 runner 本地临时存储用作可丢弃的 validation 与 promotion
+工作区：
+
+```text
+$COLLECTIVEX_STORE_ROOT/
+ private/incoming/ # write-once downloaded GHA attempts
+ private/bundles// # immutable source archives, native results/samples, matrix, checksums
+ private/quarantine/ # rejected attempts plus machine-readable reasons
+ public/datasets// # immutable sanitized frontend datasets
+ public/channels/ # small atomic pointers: latest-attempt, dev-latest
+ locks/
+```
+
+Private 和 public trees 使用不同 permissions。JSON manifests 和 checksums 是权威记录；可重建
+catalog 仅为 index。Raw sweep artifacts 只是 publisher 的临时输入；只有清理并完成 promotion
+的 NDJSON 会保留为前端 publication artifact。
+
+Container tags 会与固定 registry digests 核对。Enroot imports 使用固定
+`SOURCE_DATE_EPOCH` 和 versioned cache generation；每个 mounted squash 都重新计算 hash 并纳入
+series identity。Image-provided DeepEP 也按精确 per-architecture wheel 和 installed-file
+fingerprints 检查，因此 stale cache 不能继承固定 source identity。
+Source-built DeepEP V2 使用独立的 mode-0700 cluster-local cache，并且只以 `/cx-cache` 挂载。
+其 content key 绑定版本化 build recipe、verified image digest、CPU/GPU architecture、
+upstream source trees 和固定 build dependencies。该 cache 既不是 artifact，也不是 publisher
+input；每次执行的 source/results stage 仍然隔离且可丢弃，并在复用前以 marker 和 runtime probe
+fail closed。Runner UID 属于受信任的 cluster boundary：该 cache 用于防止 stale 或意外修改，
+不防御恶意的同 UID job。只有从未发布的 partial build 才能自动重置；已发布 cache 一旦未通过
+integrity 或 runtime 检查，将保持原样并被拒绝，避免并发 allocation 正在使用的文件被删除。
+
+Publication 采用 fail-closed：
+
+1. 获取 exclusive filesystem lock，并在 destination filesystem 上 stage；
+2. 解析前归档 source bytes；
+3. 要求精确 matrix-declared artifact set，并拒绝每个未消费 archive member；
+4. 验证 strict schemas、privacy、checksums、identities、timing 和精确 matrix outcomes；
+5. 写入 checksums 和 `COMPLETE`，fsync，然后原子 rename private bundle；
+6. 构建并验证 sanitized content-addressed dataset，fsync，然后原子 rename；
+7. 仅在全部 promotion gates 通过后原子替换 `dev-latest.json`。
+
+Rejected attempts 可以更新工作区中的 `latest-attempt`，但不能更新 `dev-latest`。工作区会随
+publication runner 销毁，且绝不连接到前端。只有三个选定 bundles 全部推进 `dev-latest` 后才会
+生成 artifact。
+
+`publisher.py ingest` 接受精确 matrix，并为每个 GitHub artifact 接受一个 `--artifact` directory
+或 ZIP。`promote` 接受显式 immutable bundle IDs。默认 `verify` 要求 `latest-attempt`；若存在
+`dev-latest` 也会验证，而显式 `--channel dev-latest` 则要求其存在。Workflow 只会将通过验证并
+清理后的 dataset 复制到单记录 `collectivex_public_v1_.ndjson` artifact。Raw artifacts 和
+private workspace 内容绝不打包进应用。
+
+Sweeps 默认使用 `release_tag=unversioned`。选择 `v1` 时必须匹配固定的完整 matrix digest，并
+生成绑定 run ID、attempt、source SHA 与 matrix SHA-256 的 marker。手动 publication workflow
+只接受三个唯一、成功、来自同一 source SHA 的 `CollectiveX Sweep` run IDs；它会重新校验
+metadata 与精确 markers，下载 immutable artifacts，并将相同 provenance assertions 传给
+`publisher.py ingest`。Partial、filtered、untagged、跨 source、失败或过期的输入都会 fail closed。
+
+前端使用 server-side GitHub read token，即时发现最新成功且按版本隔离的 publication run，并
+下载 publication artifact。它要求 ZIP 根目录只有一个 NDJSON entry，校验 UTF-8、schema、
+promotion 状态及 filename/body SHA-256，随后提供短期缓存的带版本 channel pointer 和 immutable
+带版本 dataset URL。Benchmark-version selector 当前只显示 V1；后续版本必须使用独立的 release
+与 publication identity。前端不会虚构 missing values、选择 retries，或重新计算 decision
+eligibility。
+
+## 历史数据
+
+Numeric schemas 3-5 不在 v1 publisher 和 frontend reader 范围内。它们仍是 historical
+diagnostic evidence，不能作为 `dev-latest` 初始数据或驱动 v1 decisions。
diff --git a/experimental/CollectiveX/identity.py b/experimental/CollectiveX/identity.py
new file mode 100644
index 0000000000..ff5b116996
--- /dev/null
+++ b/experimental/CollectiveX/identity.py
@@ -0,0 +1,201 @@
+#!/usr/bin/env python3
+"""Canonical, cross-runtime identities for CollectiveX v1."""
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+from typing import Any
+
+IDENTITY_VERSION = 1
+MAX_SAFE_INTEGER = (1 << 53) - 1
+PREFIXES = {
+ "case": "cxcase-v1-",
+ "workload": "cxwork-v1-",
+ "series": "cxseries-v1-",
+ "point": "cxpoint-v1-",
+ "evidence": "cxevidence-v1-",
+ "allocation": "cxallocation-v1-",
+ "attempt": "cxattempt-v1-",
+}
+V1_NORMAL_CASE_PROFILE = {
+ "activation_generator": "collectivex-activation-counter-v3",
+ "activation_profile": "canonical-counter-source-v3",
+ "combine_dtype": "bf16",
+ "combine_quant_mode": "none",
+ "combine_semantics": "activation-only",
+ "component_order_contract": "roundtrip-dispatch-activation-only-combine-v2",
+ "conditioning_contract": "fixed-phase-ramp-8-roundtrips-v1",
+ "contract": "layout-and-dispatch-v1",
+ "correctness_scope": "dispatch-metadata-and-transformed-combine",
+ "dtype": "bf16",
+ "eplb_planner": "greedy-rank-major-v1",
+ "eplb_redundant_experts": 32,
+ "eplb_reference_tokens_per_rank": 2048,
+ "mode": "normal",
+ "oracle_contract": "expert-specific-transform-v1",
+ "oracle_tolerances": "rtol=0.05,atol=0.02",
+ "payload_unit": "token-rank",
+ "placement": "packed",
+ "percentile_method": "nearest-rank",
+ "rank_reduction": "cross-rank-max-per-iteration",
+ "resource_mode": "tuned",
+ "routing_generator": "collectivex-routing-counter-v3",
+ "sampling_contract": "fixed-512-v1",
+ "seed": 67,
+}
+
+V1_LOW_LATENCY_CASE_PROFILE = {
+ **V1_NORMAL_CASE_PROFILE,
+ "component_order_contract": "roundtrip-dispatch-gate-weighted-combine-v1",
+ "combine_semantics": "gate-weighted",
+ "contract": "expert-packed-weighted-combine-v1",
+ "correctness_scope": "expert-assignment-and-weighted-combine",
+ "mode": "low-latency",
+ "oracle_contract": "expert-assignment-transform-v1",
+ "payload_unit": "token-expert",
+}
+
+# Compatibility alias for normal-mode callers. New scheduling and validation
+# must select a profile from the explicit case mode.
+V1_CASE_PROFILE = V1_NORMAL_CASE_PROFILE
+V1_CASE_PROFILES = {
+ "normal": V1_NORMAL_CASE_PROFILE,
+ "low-latency": V1_LOW_LATENCY_CASE_PROFILE,
+}
+
+
+def case_profile(mode: str) -> dict[str, Any]:
+ """Return the immutable measurement profile for one scheduled mode."""
+ try:
+ return V1_CASE_PROFILES[mode]
+ except KeyError as exc:
+ raise IdentityError(f"unknown CollectiveX case mode {mode!r}") from exc
+
+
+def profile_for_case(case: dict[str, Any]) -> dict[str, Any]:
+ """Resolve a scheduled case's explicit mode to its identity profile."""
+ mode = case.get("mode")
+ if not isinstance(mode, str):
+ raise IdentityError("scheduled case mode is missing")
+ return case_profile(mode)
+
+
+class IdentityError(ValueError):
+ """An identity payload cannot be represented consistently across runtimes."""
+
+
+def _validate(value: Any, path: str = "$") -> None:
+ if value is None or isinstance(value, bool):
+ return
+ if isinstance(value, str):
+ if any(ord(character) < 0x20 or ord(character) > 0x7E for character in value):
+ raise IdentityError(f"{path}: string must contain printable ASCII only")
+ return
+ if type(value) is int:
+ if abs(value) > MAX_SAFE_INTEGER:
+ raise IdentityError(f"{path}: integer exceeds the cross-runtime safe range")
+ return
+ if isinstance(value, list):
+ for index, item in enumerate(value):
+ _validate(item, f"{path}[{index}]")
+ return
+ if isinstance(value, dict):
+ for key, item in value.items():
+ if not isinstance(key, str):
+ raise IdentityError(f"{path}: object key is not a string")
+ if any(ord(character) < 0x20 or ord(character) > 0x7E for character in key):
+ raise IdentityError(f"{path}: object key must contain printable ASCII only")
+ _validate(item, f"{path}.{key}")
+ return
+ raise IdentityError(f"{path}: unsupported identity value {type(value).__name__}")
+
+
+def canonical_bytes(value: Any) -> bytes:
+ """Return compact UTF-8 JSON after enforcing the portable value subset."""
+ _validate(value)
+ return json.dumps(
+ value,
+ ensure_ascii=False,
+ allow_nan=False,
+ sort_keys=True,
+ separators=(",", ":"),
+ ).encode("utf-8")
+
+
+def digest(kind: str, value: Any) -> str:
+ """Hash a typed v1 identity payload and return its typed identifier."""
+ try:
+ prefix = PREFIXES[kind]
+ except KeyError as exc:
+ raise IdentityError(f"unknown identity kind {kind!r}") from exc
+ body = {"kind": kind, "value": value, "version": IDENTITY_VERSION}
+ return prefix + hashlib.sha256(canonical_bytes(body)).hexdigest()
+
+
+def is_typed_id(value: Any, kind: str) -> bool:
+ prefix = PREFIXES.get(kind)
+ return bool(
+ isinstance(value, str)
+ and prefix
+ and re.fullmatch(re.escape(prefix) + r"[0-9a-f]{64}", value)
+ )
+
+
+def case_id(*, sku: str, profile: dict[str, Any], case: dict[str, Any]) -> str:
+ return digest("case", {"case": case, "profile": profile, "sku": sku})
+
+
+def workload_id(value: dict[str, Any]) -> str:
+ return digest("workload", value)
+
+
+def series_id(value: dict[str, Any]) -> str:
+ return digest("series", value)
+
+
+def point_id(*, series: str, tokens_per_rank: int) -> str:
+ return digest("point", {"series_id": series, "tokens_per_rank": tokens_per_rank})
+
+
+def allocation_id(value: dict[str, Any]) -> str:
+ return digest("allocation", value)
+
+
+def attempt_id(*, allocation: str, case: str, ordinal: int) -> str:
+ return digest(
+ "attempt", {"allocation_id": allocation, "case_id": case, "ordinal": ordinal}
+ )
+
+
+def evidence_id(
+ *, point: str, allocation: str, attempt: str, sample_sha256: str
+) -> str:
+ return digest(
+ "evidence",
+ {
+ "allocation_id": allocation,
+ "attempt_id": attempt,
+ "point_id": point,
+ "sample_sha256": sample_sha256,
+ },
+ )
+
+
+IDENTITY_TEST_VECTOR = {
+ "payload": {"backend": "deepep", "ep": 8, "shape": [7168, 8, 256]},
+ "series_id": "cxseries-v1-a79bf758488e3edd50f5531f3af825f371bf42aae7c4097e461fd2a32615af81",
+}
+
+
+def verify_test_vector() -> None:
+ observed = series_id(IDENTITY_TEST_VECTOR["payload"])
+ if observed != IDENTITY_TEST_VECTOR["series_id"]:
+ raise IdentityError(
+ f"identity implementation differs: {observed} != {IDENTITY_TEST_VECTOR['series_id']}"
+ )
+
+
+if __name__ == "__main__":
+ verify_test_vector()
+ print(IDENTITY_TEST_VECTOR["series_id"])
diff --git a/experimental/CollectiveX/launchers/launch_gb-nv.sh b/experimental/CollectiveX/launchers/launch_gb-nv.sh
new file mode 100644
index 0000000000..21aae4c13c
--- /dev/null
+++ b/experimental/CollectiveX/launchers/launch_gb-nv.sh
@@ -0,0 +1,110 @@
+#!/usr/bin/env bash
+# CollectiveX shared GB200/GB300 NVL72 (aarch64) launcher.
+# shellcheck disable=SC2016,SC2034
+#
+# EP8/EP16 use one Slurm task per GPU across two or four trays in the same
+# MNNVL scale-up domain.
+set -euo pipefail
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+CX_DIR="$(cd "$HERE/.." && pwd)"; REPO_ROOT="$(cd "$CX_DIR/../.." && pwd)"
+# shellcheck source=../runtime/common.sh
+source "$HERE/../runtime/common.sh"
+
+PRODUCT="${CX_SHARD_SKU:-${CX_GB_PRODUCT:-${CX_PUBLIC_RUNNER:-}}}"
+case "$PRODUCT" in
+ gb200|gb300) ;;
+ *) cx_die "set CX_SHARD_SKU or CX_PUBLIC_RUNNER to gb200 or gb300" ;;
+esac
+RUNNER="$PRODUCT"
+export CX_RUNNER="$RUNNER" CX_BENCH="${CX_BENCH:-deepep}"
+export CX_IMAGE_PLATFORM=linux/arm64
+JOB_ID=""
+cx_install_launcher_fail_safe
+cx_set_failure_stage setup
+cx_load_operator_config
+cx_lock_canonical_gha_env "$RUNNER"
+NODES="${CX_NODES:-2}"; GPN="${CX_GPUS_PER_NODE:-4}"
+SCALE_UP_DOMAIN="${CX_SCALE_UP_DOMAIN:-72}"
+EXPECTED_WORLD=$((NODES * GPN))
+NGPUS="${CX_NGPUS:-$EXPECTED_WORLD}"
+if [ "$PRODUCT" = gb200 ]; then default_time=30; else default_time=90; fi
+TIME_MIN="${CX_TIME:-$default_time}"
+[ "$NODES" = 2 ] || [ "$NODES" = 4 ] \
+ || cx_die "$PRODUCT v1 supports two or four four-GPU trays"
+[ "$GPN" = 4 ] || cx_die "$PRODUCT requires four GPUs per tray"
+[ "$SCALE_UP_DOMAIN" = 72 ] || cx_die "$PRODUCT requires the NVL72 scale-up domain"
+[ "$NGPUS" = "$EXPECTED_WORLD" ] \
+ || cx_die "$PRODUCT world size must equal nodes x GPUs per tray"
+cx_apply_timing_profile
+IMAGE="${CX_IMAGE:-$(cx_default_image "$PRODUCT")}"
+TS="$(date -u +%Y-%m-%dT%H-%M-%SZ)"
+export CX_RUNNER="$RUNNER" CX_TS="$TS" CX_TOPO="${PRODUCT}-nvl72-mnnvl"
+export CX_SCOPE=scale-up CX_TRANSPORT=mnnvl CX_SCALE_UP_TRANSPORT=mnnvl
+export CX_NODES="$NODES" CX_GPUS_PER_NODE="$GPN" CX_SCALE_UP_DOMAIN="$SCALE_UP_DOMAIN"
+export CX_NGPUS="$NGPUS"
+unset CX_SCALE_OUT_TRANSPORT
+case "$CX_BENCH" in
+ deepep|deepep-v2|deepep-hybrid|nccl-ep) ;;
+ *) cx_die "unsupported $PRODUCT EP backend: $CX_BENCH" ;;
+esac
+cx_validate_shard_control "$CX_DIR"
+cx_require_vars CX_PARTITION CX_ACCOUNT CX_SQUASH_DIR CX_STAGE_DIR
+[ "$PRODUCT" != gb300 ] || cx_require_vars CX_ENROOT_CACHE_PATH
+PARTITION="$CX_PARTITION"; ACCOUNT="$CX_ACCOUNT"; SQUASH_DIR="$CX_SQUASH_DIR"
+[ -z "${CX_ENROOT_CACHE_PATH:-}" ] || export ENROOT_CACHE_PATH="$CX_ENROOT_CACHE_PATH"
+export NCCL_CUMEM_ENABLE=1 NCCL_MNNVL_ENABLE=1 MC_FORCE_MNNVL=1
+cx_apply_network_profile "$NODES" "$CX_TRANSPORT"
+
+cx_log "$PRODUCT nodes=$NODES x ${GPN}gpu world=$NGPUS bench=$CX_BENCH"
+[ "${CX_DRYRUN:-0}" = 1 ] && { cx_log "DRYRUN"; exit 0; }
+cx_set_failure_stage registry-verification
+cx_verify_registry_image "$IMAGE"
+cx_set_failure_stage repository-stage
+MOUNT_SRC="$(cx_stage_path "$REPO_ROOT" "$CX_STAGE_DIR")"
+cx_stage_repo "$REPO_ROOT" "$MOUNT_SRC"
+cx_prepare_runtime_marker "$MOUNT_SRC"
+CONTAINER_MOUNTS="$MOUNT_SRC:/ix"
+if [ "$CX_BENCH" = deepep-v2 ] || [ "$CX_BENCH" = deepep-hybrid ]; then
+ cx_set_failure_stage backend-setup
+ cx_prepare_backend_source "$MOUNT_SRC" "$CX_BENCH" \
+ || cx_die "cannot stage the pinned backend source"
+ export CX_BACKEND_SOURCE_ROOT=/ix/experimental/CollectiveX/.cx_sources
+fi
+if [ "$CX_BENCH" = deepep-v2 ]; then
+ cx_prepare_backend_cache "$CX_SQUASH_DIR" \
+ || cx_die "cannot prepare the isolated backend cache"
+ CONTAINER_MOUNTS="$CONTAINER_MOUNTS,$CX_PREPARED_BACKEND_CACHE:/cx-cache"
+ export CX_BACKEND_CACHE_ROOT=/cx-cache
+fi
+
+cx_set_failure_stage scheduler-allocation
+command -v salloc >/dev/null || cx_die "salloc not found"
+cx_salloc_jobid --partition="$PARTITION" --account="$ACCOUNT" --nodes="$NODES" \
+ --gres=gpu:"$GPN" --ntasks-per-node="$GPN" --exclusive --mem=0 --cpus-per-task=35 \
+ --time="$TIME_MIN"
+[ -n "$JOB_ID" ] || cx_die "no JOB_ID from salloc"
+cx_set_failure_stage container-import
+SQUASH_FILE="$(cx_ensure_squash_on_job "$JOB_ID" "$SQUASH_DIR" "$IMAGE")"
+cx_set_failure_stage container-hash
+cx_export_squash_identity "$SQUASH_FILE"
+cx_preflight_allocation "$JOB_ID" "$NODES" "$MOUNT_SRC" "$SQUASH_FILE" \
+ "${CX_SHARD_FILE:-}"
+
+# Keep the loader policy here because it is platform/container specific and
+# security tests evaluate this literal independently.
+SOURCE_BACKEND_ENV='case "${SLURM_NODEID:-}" in ""|*[!0-9]*) exit 66;; esac; env_file="/ix/experimental/CollectiveX/.cx_backend/env/node-${SLURM_NODEID}.sh"; env_root="${env_file%/*}"; [ -d "$env_root" ] && [ ! -L "$env_root" ] || exit 66; case "$(stat -c "%a" "$env_root")" in 700|[1-7]700) ;; *) exit 66;; esac; [ -f "$env_file" ] && [ -r "$env_file" ] && [ ! -L "$env_file" ] && [ "$(stat -c "%u:%a" "$env_file")" = "$(stat -c "%u" "$env_root"):600" ] || exit 66; . "$env_file" || exit 66'
+BACKEND_PROBE="$SOURCE_BACKEND_ENV"'; case "$CX_BENCH" in deepep) python3 -c "from deep_ep import Buffer";; deepep-v2) python3 -c "import deep_ep; assert hasattr(deep_ep, '\''ElasticBuffer'\'')";; deepep-hybrid) python3 -c "import deep_ep; assert hasattr(deep_ep, '\''HybridEPBuffer'\'')";; nccl-ep) python3 -c "import torch";; esac'
+WRAP="${SOURCE_BACKEND_ENV}"$'\n'"$(cx_slurm_rank_wrapper)"
+CX_DISTRIBUTED_CONTAINER_ARGS=(--container-writable --container-remap-root)
+[ "$CX_BENCH" != deepep ] || export CX_ALLOW_MNNVL=1
+run_rc=0
+cx_set_failure_stage container-launch
+cx_run_distributed_shard || run_rc=$?
+
+cx_adopt_runtime_stage "$MOUNT_SRC"
+collect_rc=0
+cx_collect_results "$MOUNT_SRC" "$REPO_ROOT" || collect_rc=$?
+[ "$run_rc" != 0 ] || [ "$collect_rc" = 0 ] || cx_set_failure_stage artifact-collection
+final_rc="$run_rc"
+[ "$final_rc" != 0 ] || final_rc="$collect_rc"
+exit "$final_rc"
diff --git a/experimental/CollectiveX/launchers/launch_mi-amds.sh b/experimental/CollectiveX/launchers/launch_mi-amds.sh
new file mode 100644
index 0000000000..f66f820f54
--- /dev/null
+++ b/experimental/CollectiveX/launchers/launch_mi-amds.sh
@@ -0,0 +1,160 @@
+#!/usr/bin/env bash
+# CollectiveX shared MI325X/MI355X AMD Slurm launcher (one or two nodes).
+# shellcheck disable=SC2016,SC2034
+set -euo pipefail
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+CX_DIR="$(cd "$HERE/.." && pwd)"
+REPO_ROOT="$(cd "$CX_DIR/../.." && pwd)"
+# shellcheck source=../runtime/common.sh
+source "$HERE/../runtime/common.sh"
+
+RUNNER="${CX_SHARD_SKU:-${CX_PUBLIC_RUNNER:-}}"
+case "$RUNNER" in
+ mi325x) CPUS_PER_TASK=256; DEVICE_MOUNTS=",/dev/kfd:/dev/kfd,/dev/dri:/dev/dri" ;;
+ mi355x) CPUS_PER_TASK=128; DEVICE_MOUNTS="" ;;
+ *) cx_die "set CX_SHARD_SKU or CX_PUBLIC_RUNNER to mi325x or mi355x" ;;
+esac
+export CX_RUNNER="$RUNNER" CX_BENCH="${CX_BENCH:-mori}"
+export CX_IMAGE_PLATFORM=linux/amd64
+JOB_ID=""
+cx_install_launcher_fail_safe
+cx_set_failure_stage setup
+cx_load_operator_config
+cx_lock_canonical_gha_env "$RUNNER"
+
+NODES="${CX_NODES:-1}"; GPN="${CX_GPUS_PER_NODE:-8}"
+SCALE_UP_DOMAIN="${CX_SCALE_UP_DOMAIN:-8}"
+EXPECTED_WORLD=$((NODES * GPN))
+NGPUS="${CX_NGPUS:-$EXPECTED_WORLD}"
+TIME_MIN="${CX_TIME:-60}"
+EXCLUDE_NODES="${CX_EXCLUDE_NODES:-}"
+NODELIST="${CX_NODELIST:-}"
+MOUNT_DIR=/ix
+TS="$(date -u +%Y-%m-%dT%H-%M-%SZ)"
+[ "$NODES" = 1 ] || [ "$NODES" = 2 ] \
+ || cx_die "$RUNNER supports one or two nodes"
+[ "$GPN" = 8 ] || cx_die "$RUNNER requires eight GPUs per node"
+[ "$SCALE_UP_DOMAIN" = 8 ] || cx_die "$RUNNER requires an eight-GPU scale-up domain"
+[ "$NGPUS" = "$EXPECTED_WORLD" ] \
+ || cx_die "$RUNNER world size must equal nodes x GPUs per node"
+case "$CX_BENCH" in
+ mori|nccl-ep) ;;
+ *) cx_die "unsupported AMD EP backend: $CX_BENCH" ;;
+esac
+
+if [ "$RUNNER" = mi325x ]; then
+ export MORI_DISABLE_AUTO_XGMI="${MORI_DISABLE_AUTO_XGMI:-0}"
+ export MORI_ENABLE_SDMA="${MORI_ENABLE_SDMA:-1}"
+ export MORI_APP_LOG_LEVEL="${MORI_APP_LOG_LEVEL:-info}"
+ export MORI_SHMEM_LOG_LEVEL="${MORI_SHMEM_LOG_LEVEL:-info}"
+ export MORI_IO_LOG_LEVEL="${MORI_IO_LOG_LEVEL:-info}"
+ [ "$CX_BENCH" != mori ] \
+ || export CX_IMAGE="${CX_IMAGE:-$CX_IMAGE_AMD_MORI_MI325}"
+fi
+if [ "$CX_BENCH" = mori ]; then
+ if [ "$NODES" -gt 1 ]; then
+ export CX_MORI_KERNEL_TYPE=internode-v1
+ elif [ "$RUNNER" = mi325x ]; then
+ export CX_MORI_KERNEL_TYPE="${CX_MORI_KERNEL_TYPE:-asyncll}"
+ else
+ export CX_MORI_KERNEL_TYPE="${CX_MORI_KERNEL_TYPE:-intranode}"
+ fi
+fi
+IMAGE="${CX_IMAGE:-$(cx_default_image "$RUNNER")}"
+export CX_RUNNER="$RUNNER" CX_NGPUS="$NGPUS" CX_NODES="$NODES"
+export CX_GPUS_PER_NODE="$GPN" CX_SCALE_UP_DOMAIN="$SCALE_UP_DOMAIN" CX_TS="$TS"
+export CX_SCALE_UP_TRANSPORT=xgmi
+if [ "$NODES" -gt 1 ]; then
+ export CX_SCOPE=scale-out CX_SCALE_OUT_TRANSPORT=rdma
+ export CX_TRANSPORT=xgmi-rdma CX_TOPO="${RUNNER}-xgmi-rdma"
+else
+ export CX_SCOPE=scale-up CX_TRANSPORT=xgmi CX_TOPO="${RUNNER}-xgmi"
+ unset CX_SCALE_OUT_TRANSPORT
+fi
+export CX_RUN_TIMEOUT="${CX_RUN_TIMEOUT:-1800}"
+cx_apply_network_profile "$NODES" "$CX_TRANSPORT"
+cx_validate_shard_control "$CX_DIR"
+cx_require_vars CX_PARTITION CX_SQUASH_DIR CX_STAGE_DIR
+PARTITION="$CX_PARTITION"; SQUASH_DIR="$CX_SQUASH_DIR"
+
+cx_log "runner=$RUNNER nodes=$NODES x ${GPN}gpu world=$NGPUS bench=$CX_BENCH"
+cx_set_failure_stage repository-stage
+MOUNT_SRC="$(cx_stage_path "$REPO_ROOT" "$CX_STAGE_DIR")"
+cx_stage_repo "$REPO_ROOT" "$MOUNT_SRC"
+cx_prepare_runtime_marker "$MOUNT_SRC"
+[ "${CX_DRYRUN:-0}" != 1 ] || { cx_log "CX_DRYRUN=1 - not allocating"; exit 0; }
+cx_set_failure_stage registry-verification
+cx_verify_registry_image "$IMAGE"
+cx_set_failure_stage scheduler-allocation
+command -v salloc >/dev/null || cx_die "salloc not found on this runner"
+
+allocation=(--partition="$PARTITION" --nodes="$NODES" --gres=gpu:"$GPN" --exclusive
+ --time="$TIME_MIN")
+if [ "$NODES" = 1 ]; then
+ allocation+=(--cpus-per-task="$CPUS_PER_TASK")
+else
+ allocation+=(--ntasks-per-node="$GPN" --cpus-per-task="$((CPUS_PER_TASK / GPN))")
+fi
+if [ -n "$NODELIST" ]; then
+ cx_log "using configured node pin"
+ allocation+=(--nodelist="$NODELIST")
+elif [ -n "$EXCLUDE_NODES" ]; then
+ allocation+=(--exclude="$EXCLUDE_NODES")
+fi
+cx_salloc_jobid "${allocation[@]}"
+[ -n "$JOB_ID" ] || cx_die "could not resolve allocated JOB_ID from salloc"
+cx_set_failure_stage setup
+cx_validate_network_profile_on_job "$JOB_ID" "$NODES" "$CX_TRANSPORT"
+
+cx_set_failure_stage container-import
+SQUASH_FILE="$(cx_ensure_squash_on_job \
+ "$JOB_ID" "$SQUASH_DIR" "$IMAGE" "${CX_LOCK_DIR:-}")"
+cx_set_failure_stage container-hash
+import_log="$(cx_private_log_path image-hash)"
+if ! COLLECTIVEX_SQUASH_SHA256="$(
+ srun --jobid="$JOB_ID" --nodes=1 --ntasks=1 --chdir=/tmp \
+ --export="$(cx_host_exports)" sha256sum "$SQUASH_FILE" \
+ 2>>"$import_log" | awk 'NR==1 {print $1}'
+)"; then
+ cx_fail_stage container-hash "$import_log"
+fi
+[[ "$COLLECTIVEX_SQUASH_SHA256" =~ ^[0-9a-f]{64}$ ]] \
+ || cx_fail_stage container-hash "$import_log"
+export COLLECTIVEX_SQUASH_SHA256
+cx_preflight_allocation "$JOB_ID" "$NODES" "$MOUNT_SRC" "$SQUASH_FILE" \
+ "${CX_SHARD_FILE:-}"
+CONTAINER_MOUNTS="$MOUNT_SRC:$MOUNT_DIR$DEVICE_MOUNTS"
+
+if [ "$NODES" = 1 ]; then
+ run_rc=0
+ cx_set_failure_stage container-launch
+ runtime_log="$(cx_private_log_path runtime)"
+ srun --jobid="$JOB_ID" --chdir=/tmp --container-image="$SQUASH_FILE" \
+ --container-mounts="$CONTAINER_MOUNTS" --container-writable --container-remap-root \
+ --no-container-mount-home --container-workdir="$MOUNT_DIR/experimental/CollectiveX" \
+ --no-container-entrypoint --export="$(cx_container_exports)" \
+ bash "$MOUNT_DIR/experimental/CollectiveX/runtime/run_in_container.sh" \
+ >"$runtime_log" 2>&1 || run_rc=$?
+else
+ SOURCE_BACKEND_ENV='case "${SLURM_NODEID:-}" in ""|*[!0-9]*) exit 66;; esac; env_file="/ix/experimental/CollectiveX/.cx_backend/env/node-${SLURM_NODEID}.sh"; env_root="${env_file%/*}"; [ -d "$env_root" ] && [ ! -L "$env_root" ] || exit 66; case "$(stat -c "%a" "$env_root")" in 700|[1-7]700) ;; *) exit 66;; esac; [ -f "$env_file" ] && [ -r "$env_file" ] && [ ! -L "$env_file" ] && [ "$(stat -c "%u:%a" "$env_file")" = "$(stat -c "%u" "$env_root"):600" ] || exit 66; . "$env_file" || exit 66'
+ BACKEND_PROBE="$SOURCE_BACKEND_ENV"'; case "$CX_BENCH" in mori) python3 -c "import mori";; nccl-ep) python3 -c "import torch";; esac'
+ WRAP="${SOURCE_BACKEND_ENV}"$'\n'"$(cx_slurm_rank_wrapper)"
+ CX_DISTRIBUTED_CONTAINER_ARGS=(--container-writable --container-remap-root)
+ run_rc=0
+ cx_set_failure_stage container-launch
+ cx_run_distributed_shard || run_rc=$?
+fi
+
+cx_adopt_runtime_stage "$MOUNT_SRC"
+if [ "$NODES" = 1 ] && [ "$run_rc" != 0 ]; then
+ cx_fail_stage "$CX_FAILSAFE_MODE" "$runtime_log" || true
+fi
+collect_rc=0
+cx_collect_results "$MOUNT_SRC" "$REPO_ROOT" || collect_rc=$?
+[ "$run_rc" != 0 ] || [ "$collect_rc" = 0 ] || cx_set_failure_stage artifact-collection
+final_rc="$run_rc"
+[ "$final_rc" != 0 ] || final_rc="$collect_rc"
+rm -f "$MOUNT_SRC"/experimental/CollectiveX/gpucore.* 2>/dev/null || true
+cx_log "done - result artifacts collected"
+exit "$final_rc"
diff --git a/experimental/CollectiveX/launchers/launch_single-slurm.sh b/experimental/CollectiveX/launchers/launch_single-slurm.sh
new file mode 100644
index 0000000000..eade8fb751
--- /dev/null
+++ b/experimental/CollectiveX/launchers/launch_single-slurm.sh
@@ -0,0 +1,154 @@
+#!/usr/bin/env bash
+# CollectiveX shared standard NVIDIA Slurm launcher (one or two nodes).
+# shellcheck disable=SC2016,SC2034
+set -euo pipefail
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+CX_DIR="$(cd "$HERE/.." && pwd)"
+REPO_ROOT="$(cd "$CX_DIR/../.." && pwd)"
+# shellcheck source=../runtime/common.sh
+source "$HERE/../runtime/common.sh"
+
+RUNNER="${CX_SHARD_SKU:-${CX_PUBLIC_RUNNER:-}}"
+ALLOC_EXTRA=(); SRUN_EXTRA=(); LOCAL_IMPORT=0
+case "$RUNNER" in
+ h100-dgxc) PRODUCT=h100; TOPO=h100-nvlink-island; DEFAULT_TIME=45; REQUIRE_ACCOUNT=1 ;;
+ h200-dgxc)
+ PRODUCT=h200; TOPO=h200-nvlink-island; DEFAULT_TIME=45; REQUIRE_ACCOUNT=0
+ SRUN_EXTRA=(--container-remap-root)
+ ;;
+ b200-dgxc)
+ PRODUCT=b200; TOPO=b200-nvlink-island; DEFAULT_TIME=30; REQUIRE_ACCOUNT=1
+ ALLOC_EXTRA=(--mem=0)
+ ;;
+ b300)
+ PRODUCT=b300; TOPO=b300-nvlink-island; DEFAULT_TIME=45; REQUIRE_ACCOUNT=1
+ # Do not restore ALLOC_EXTRA=(-N 1 --mem=0); it blocks two-node B300 jobs.
+ ALLOC_EXTRA=(--mem=0)
+ SRUN_EXTRA=(--mpi=none --container-remap-root)
+ LOCAL_IMPORT=1
+ ;;
+ *) cx_die "set CX_SHARD_SKU or CX_PUBLIC_RUNNER to a registered NVIDIA SKU" ;;
+esac
+export CX_RUNNER="$RUNNER" CX_BENCH="${CX_BENCH:-deepep}"
+export CX_IMAGE_PLATFORM=linux/amd64
+JOB_ID=""
+cx_install_launcher_fail_safe
+cx_set_failure_stage setup
+cx_load_operator_config
+cx_lock_canonical_gha_env "$RUNNER"
+
+NODES="${CX_NODES:-1}"; GPN="${CX_GPUS_PER_NODE:-8}"
+SCALE_UP_DOMAIN="${CX_SCALE_UP_DOMAIN:-8}"
+EXPECTED_WORLD=$((NODES * GPN))
+NGPUS="${CX_NGPUS:-$EXPECTED_WORLD}"
+TIME_MIN="${CX_TIME:-$DEFAULT_TIME}"
+IMAGE="${CX_IMAGE:-$(cx_default_image "$PRODUCT")}"
+TS="$(date -u +%Y-%m-%dT%H-%M-%SZ)"
+[ "$NODES" = 1 ] || [ "$NODES" = 2 ] \
+ || cx_die "$RUNNER supports one or two nodes"
+[ "$GPN" = 8 ] || cx_die "$RUNNER requires eight GPUs per node"
+[ "$SCALE_UP_DOMAIN" = 8 ] || cx_die "$RUNNER requires an eight-GPU scale-up domain"
+[ "$NGPUS" = "$EXPECTED_WORLD" ] \
+ || cx_die "$RUNNER world size must equal nodes x GPUs per node"
+case "$CX_BENCH" in
+ deepep|deepep-v2|deepep-hybrid|uccl|nccl-ep) ;;
+ *) cx_die "unsupported $RUNNER EP backend: $CX_BENCH" ;;
+esac
+
+export CX_RUNNER="$RUNNER" CX_NGPUS="$NGPUS" CX_NODES="$NODES"
+export CX_GPUS_PER_NODE="$GPN" CX_SCALE_UP_DOMAIN="$SCALE_UP_DOMAIN"
+export CX_TS="$TS" CX_SCALE_UP_TRANSPORT=nvlink
+if [ "$NODES" -gt 1 ]; then
+ export CX_SCOPE=scale-out CX_SCALE_OUT_TRANSPORT=rdma
+ export CX_TRANSPORT=nvlink-rdma CX_TOPO="${PRODUCT}-nvlink-rdma"
+else
+ export CX_SCOPE=scale-up CX_TRANSPORT=nvlink CX_TOPO="$TOPO"
+ unset CX_SCALE_OUT_TRANSPORT
+fi
+export CX_NCCL_HOME="${CX_NCCL_HOME:-/usr}" NCCL_CUMEM_ENABLE=1
+cx_apply_network_profile "$NODES" "$CX_TRANSPORT"
+cx_validate_shard_control "$CX_DIR"
+cx_require_vars CX_PARTITION CX_SQUASH_DIR
+[ "$REQUIRE_ACCOUNT" = 0 ] || cx_require_vars CX_ACCOUNT
+[ "$RUNNER" != b300 ] || cx_require_vars CX_STAGE_DIR
+
+cx_log "runner=$RUNNER nodes=$NODES x ${GPN}gpu world=$NGPUS bench=$CX_BENCH"
+[ "${CX_DRYRUN:-0}" != 1 ] || { cx_log "CX_DRYRUN=1 - not allocating"; exit 0; }
+cx_set_failure_stage registry-verification
+cx_verify_registry_image "$IMAGE"
+SQUASH_FILE=""
+cx_set_failure_stage repository-stage
+MOUNT_SRC="$(cx_stage_path "$REPO_ROOT" "${CX_STAGE_DIR:-}")"
+cx_stage_repo "$REPO_ROOT" "$MOUNT_SRC"
+cx_prepare_runtime_marker "$MOUNT_SRC"
+CONTAINER_MOUNTS="$MOUNT_SRC:/ix"
+if [ "$CX_BENCH" = deepep-v2 ] || [ "$CX_BENCH" = deepep-hybrid ]; then
+ cx_set_failure_stage backend-setup
+ cx_prepare_backend_source "$MOUNT_SRC" "$CX_BENCH" \
+ || cx_die "cannot stage the pinned backend source"
+ export CX_BACKEND_SOURCE_ROOT=/ix/experimental/CollectiveX/.cx_sources
+fi
+if [ "$CX_BENCH" = deepep-v2 ]; then
+ cx_prepare_backend_cache "$CX_SQUASH_DIR" \
+ || cx_die "cannot prepare the isolated backend cache"
+ CONTAINER_MOUNTS="$CONTAINER_MOUNTS,$CX_PREPARED_BACKEND_CACHE:/cx-cache"
+ export CX_BACKEND_CACHE_ROOT=/cx-cache
+fi
+
+cx_set_failure_stage scheduler-allocation
+command -v salloc >/dev/null || cx_die "salloc not found on this runner"
+allocation=(--partition="$CX_PARTITION" --nodes="$NODES" --gres=gpu:"$GPN" --exclusive
+ --time="$TIME_MIN" "${ALLOC_EXTRA[@]}")
+[ "$NODES" = 1 ] || allocation+=(--ntasks-per-node="$GPN")
+[ -z "${CX_ACCOUNT:-}" ] || allocation+=(--account="$CX_ACCOUNT")
+[ -z "${CX_EXCLUDE_NODES:-}" ] || allocation+=(--exclude="$CX_EXCLUDE_NODES")
+cx_salloc_jobid "${allocation[@]}"
+[ -n "$JOB_ID" ] || cx_die "could not resolve allocated JOB_ID from salloc"
+cx_set_failure_stage setup
+cx_validate_network_profile_on_job "$JOB_ID" "$NODES" "$CX_TRANSPORT"
+if [ "$LOCAL_IMPORT" = 1 ]; then
+ cx_set_failure_stage container-import
+ SQUASH_FILE="$(CX_ENROOT_LOCAL_IMPORT=1 cx_ensure_squash "$CX_SQUASH_DIR" "$IMAGE")"
+ cx_set_failure_stage container-hash
+ cx_export_squash_identity "$SQUASH_FILE"
+else
+ cx_set_failure_stage container-import
+ SQUASH_FILE="$(cx_ensure_squash_on_job "$JOB_ID" "$CX_SQUASH_DIR" "$IMAGE")"
+ cx_set_failure_stage container-hash
+ cx_export_squash_identity "$SQUASH_FILE"
+fi
+cx_preflight_allocation "$JOB_ID" "$NODES" "$MOUNT_SRC" "$SQUASH_FILE" \
+ "${CX_SHARD_FILE:-}"
+
+if [ "$NODES" = 1 ]; then
+ run_rc=0
+ cx_set_failure_stage container-launch
+ runtime_log="$(cx_private_log_path runtime)"
+ srun --jobid="$JOB_ID" --container-image="$SQUASH_FILE" \
+ --container-mounts="$CONTAINER_MOUNTS" --no-container-mount-home \
+ --container-workdir=/ix/experimental/CollectiveX --no-container-entrypoint \
+ "${SRUN_EXTRA[@]}" --export="$(cx_container_exports)" \
+ bash /ix/experimental/CollectiveX/runtime/run_in_container.sh \
+ >"$runtime_log" 2>&1 || run_rc=$?
+else
+ SOURCE_BACKEND_ENV='case "${SLURM_NODEID:-}" in ""|*[!0-9]*) exit 66;; esac; env_file="/ix/experimental/CollectiveX/.cx_backend/env/node-${SLURM_NODEID}.sh"; env_root="${env_file%/*}"; [ -d "$env_root" ] && [ ! -L "$env_root" ] || exit 66; case "$(stat -c "%a" "$env_root")" in 700|[1-7]700) ;; *) exit 66;; esac; [ -f "$env_file" ] && [ -r "$env_file" ] && [ ! -L "$env_file" ] && [ "$(stat -c "%u:%a" "$env_file")" = "$(stat -c "%u" "$env_root"):600" ] || exit 66; . "$env_file" || exit 66'
+ BACKEND_PROBE="$SOURCE_BACKEND_ENV"'; case "$CX_BENCH" in deepep) python3 -c "from deep_ep import Buffer";; deepep-v2) python3 -c "import deep_ep; assert hasattr(deep_ep, '\''ElasticBuffer'\'')";; deepep-hybrid) python3 -c "import deep_ep; assert hasattr(deep_ep, '\''HybridEPBuffer'\'')";; uccl) python3 -c "import torch; from uccl_deepep import Buffer";; nccl-ep) python3 -c "import torch";; esac'
+ WRAP="${SOURCE_BACKEND_ENV}"$'\n'"$(cx_slurm_rank_wrapper)"
+ CX_DISTRIBUTED_CONTAINER_ARGS=(--container-writable "${SRUN_EXTRA[@]}")
+ run_rc=0
+ cx_set_failure_stage container-launch
+ cx_run_distributed_shard || run_rc=$?
+fi
+
+cx_adopt_runtime_stage "$MOUNT_SRC"
+if [ "$NODES" = 1 ] && [ "$run_rc" != 0 ]; then
+ cx_fail_stage "$CX_FAILSAFE_MODE" "$runtime_log" || true
+fi
+collect_rc=0
+cx_collect_results "$MOUNT_SRC" "$REPO_ROOT" || collect_rc=$?
+[ "$run_rc" != 0 ] || [ "$collect_rc" = 0 ] || cx_set_failure_stage artifact-collection
+final_rc="$run_rc"
+[ "$final_rc" != 0 ] || final_rc="$collect_rc"
+cx_log "done - result artifacts collected"
+exit "$final_rc"
diff --git a/experimental/CollectiveX/publisher.py b/experimental/CollectiveX/publisher.py
new file mode 100644
index 0000000000..e04ba1f514
--- /dev/null
+++ b/experimental/CollectiveX/publisher.py
@@ -0,0 +1,3284 @@
+#!/usr/bin/env python3
+"""Fail-closed filesystem publisher for CollectiveX EP v1 artifacts."""
+from __future__ import annotations
+
+import argparse
+import contextlib
+import datetime as dt
+import fcntl
+from functools import lru_cache
+import hashlib
+import json
+import math
+import os
+from pathlib import Path, PurePosixPath
+import re
+import shutil
+import stat
+import statistics
+import sys
+import tempfile
+from typing import Any, Iterator, Sequence
+import zipfile
+
+import jsonschema
+
+HERE = Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE))
+
+import artifact_safety # noqa: E402
+import capability # noqa: E402
+import contracts # noqa: E402
+import identity # noqa: E402
+import sweep_matrix # noqa: E402
+
+FORMAT_BUNDLE = "collectivex.private.bundle.v1"
+FORMAT_PUBLIC = "collectivex.public.v1"
+FORMAT_CHANNEL = "collectivex.channel.v1"
+POLICY = "collectivex-decision-grade-v1"
+PUBLISHER_POLICY = "collectivex-publisher-v1"
+OUTCOMES = ("success", "unsupported", "failed", "invalid", "diagnostic")
+REQUIRED_ALLOCATIONS = 3
+REQUIRED_COHORT_KINDS = ("library", "chip", "system", "routing")
+REQUIRED_PROMOTION_COHORT_COUNTS = {"library": 76, "system": 12, "routing": 116}
+CANONICAL_FULL_V1_MATRIX_SHA256 = (
+ "f1ca85f9689922b90edd5767b9ff2a902f6b896f32f68b2ca086dde3fd2157d0"
+)
+CANONICAL_FULL_V1_CASE_CATALOG_SHA256 = (
+ "8e262178f770b0cdde12b7ec71604afd87251fa55685d4594f29717153ad6bbd"
+)
+P50_STABILITY_LIMIT = 1.10
+P99_STABILITY_LIMIT = 1.25
+MAX_ARCHIVE_MEMBERS = 20_000
+MAX_ARCHIVE_MEMBER_BYTES = 2 * 1024**3
+MAX_ARCHIVE_TOTAL_BYTES = 16 * 1024**3
+MAX_PUBLIC_DATASET_BYTES = 32 * 1024**2
+HEX64 = re.compile(r"[0-9a-f]{64}")
+SAFE_ID = re.compile(r"[a-z0-9][a-z0-9_.-]{0,127}")
+REASON = re.compile(r"[a-z0-9][a-z0-9.-]{0,95}")
+ARTIFACT_NAME = re.compile(
+ r"cx(?:unsupported|shard-[a-z0-9][a-z0-9_.-]{0,127})-[1-9][0-9]*-[1-9][0-9]*"
+)
+COVERAGE_TOPOLOGY_FIELDS = (
+ "ep_size", "nodes", "gpus_per_node", "scale_up_domain", "scope",
+ "scale_up_transport", "scale_out_transport", "transport", "topology_class",
+)
+CHANNEL_PATH = re.compile(r"datasets/([0-9a-f]{64})/dataset\.json")
+SCHEMA_DIR = HERE / "schemas"
+_SCHEMAS: dict[str, jsonschema.protocols.Validator] = {}
+
+
+class PublisherError(ValueError):
+ """Input or stored state violates the publication contract."""
+
+
+strict_load = contracts.strict_load
+_canonical = contracts.canonical_json_bytes
+
+
+def _sha_bytes(data: bytes) -> str:
+ return hashlib.sha256(data).hexdigest()
+
+
+def _sha_file(path: Path) -> str:
+ digest = hashlib.sha256()
+ with path.open("rb") as handle:
+ for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+ digest.update(chunk)
+ return digest.hexdigest()
+
+
+def _latest_timestamp(values: Sequence[str]) -> str:
+ """Return the latest evidence timestamp without introducing publisher wall time."""
+ if not values:
+ raise PublisherError("cannot derive a timestamp without evidence")
+
+ def parsed(value: str) -> dt.datetime:
+ try:
+ timestamp = dt.datetime.fromisoformat(value.replace("Z", "+00:00"))
+ except ValueError as exc:
+ raise PublisherError("evidence timestamp is not ISO-8601") from exc
+ if timestamp.tzinfo is None:
+ raise PublisherError("evidence timestamp must include a timezone")
+ return timestamp.astimezone(dt.timezone.utc)
+
+ return max(values, key=lambda value: (parsed(value), value))
+
+
+def _schema(name: str, value: Any) -> None:
+ validator = _SCHEMAS.get(name)
+ if validator is None:
+ schema = strict_load(SCHEMA_DIR / name)
+ jsonschema.Draft202012Validator.check_schema(schema)
+ validator = jsonschema.Draft202012Validator(
+ schema, format_checker=jsonschema.FormatChecker()
+ )
+ _SCHEMAS[name] = validator
+ errors = sorted(validator.iter_errors(value), key=lambda error: list(error.absolute_path))
+ if errors:
+ error = errors[0]
+ location = ".".join(map(str, error.absolute_path)) or "$"
+ raise PublisherError(f"{name}:{location}: {error.message}")
+def _exact(obj: Any, fields: set[str], path: str) -> dict[str, Any]:
+ if not isinstance(obj, dict):
+ raise PublisherError(f"{path} must be an object")
+ actual = set(obj)
+ if actual != fields:
+ raise PublisherError(
+ f"{path} fields differ: missing={sorted(fields - actual)}, "
+ f"extra={sorted(actual - fields)}"
+ )
+ return obj
+def _array(value: Any, path: str, *, nonempty: bool = False) -> list[Any]:
+ if not isinstance(value, list) or (nonempty and not value):
+ qualifier = "a nonempty" if nonempty else "an"
+ raise PublisherError(f"{path} must be {qualifier} array")
+ return value
+
+
+def _integer(value: Any, path: str, *, minimum: int = 0) -> int:
+ if type(value) is not int or value < minimum:
+ raise PublisherError(f"{path} must be an integer >= {minimum}")
+ return value
+
+
+def _unique(values: Sequence[Any], path: str) -> None:
+ serialized = [_canonical(value) for value in values]
+ if len(serialized) != len(set(serialized)):
+ raise PublisherError(f"{path} contains duplicates")
+
+def _eligibility(value: dict[str, Any], path: str) -> dict[str, Any]:
+ allocations = value["allocation_ids"]
+ p50 = value["p50_max_min_ratio"]
+ p99 = value["p99_max_min_ratio"]
+ gates = (
+ len(allocations) >= REQUIRED_ALLOCATIONS,
+ value["complete"], value["correct"], value["measured_roundtrip_p99"],
+ value["stable_p50"], value["stable_p99"], value["stable_ordering"],
+ p50 is not None and p50 <= P50_STABILITY_LIMIT,
+ p99 is not None and p99 <= P99_STABILITY_LIMIT,
+ )
+ if value["decision_grade"] != (all(gates) and not value["reasons"]):
+ raise PublisherError(f"{path}.decision_grade does not match promotion gates")
+ if value["decision_grade"] == bool(value["reasons"]):
+ raise PublisherError(f"{path}.reasons does not match decision status")
+ return value
+
+
+def validate_channel(doc: Any, *, expected_channel: str | None = None) -> dict[str, Any]:
+ _schema("channel-v1.schema.json", doc)
+ if expected_channel and doc["channel"] != expected_channel:
+ raise PublisherError("channel name does not match its file")
+ target = doc["dataset"]
+ match = CHANNEL_PATH.fullmatch(target["path"]) if isinstance(target["path"], str) else None
+ if not match or match.group(1) != target["sha256"]:
+ raise PublisherError("channel dataset path and sha256 do not agree")
+ return doc
+
+
+def _metric_value(series: dict[str, Any], metric: dict[str, Any]) -> tuple[str, float, str]:
+ point = next(
+ (point for point in series["points"] if point["tokens_per_rank"] == metric["tokens_per_rank"]),
+ None,
+ )
+ if point is None or series["phase"] != metric["phase"]:
+ raise PublisherError("decision metric references an unavailable point")
+ component = point["components"]["roundtrip"]
+ if metric["measure"] == "latency_us":
+ value = component["latency_us"][metric["statistic"]]
+ unit = "us"
+ else:
+ rates = component["logical_payload_rate_gbps_at_latency_percentile"]
+ if rates is None:
+ raise PublisherError("logical bandwidth decision has no logical byte contract")
+ value = rates[metric["statistic"]]
+ unit = "GB/s"
+ return point["point_id"], value, unit
+
+
+def _validate_metric(metric: dict[str, Any]) -> None:
+ expected = "min" if metric["measure"] == "latency_us" else "max"
+ if metric["objective"] != expected:
+ raise PublisherError(f"{metric['measure']} objective must be {expected}")
+
+
+def _metric_label(measure: str, statistic: str) -> str:
+ return (
+ f"{statistic} latency"
+ if measure == "latency_us"
+ else f"payload rate at {statistic} latency"
+ )
+
+
+def _routing_build_control(build: dict[str, Any]) -> dict[str, Any]:
+ return {
+ key: build[key]
+ for key in (
+ "routing_control_sha256", "image_digest", "source_sha", "squash_sha256",
+ )
+ }
+
+
+def _routing_implementation_mismatch(members: Sequence[dict[str, Any]]) -> bool:
+ off_eplb_hashes = {
+ member["build"]["implementation_contract_sha256"]
+ for member in members if not member["workload"]["eplb"]
+ }
+ return len(off_eplb_hashes) > 1
+
+
+def _public_case_factors(series: dict[str, Any]) -> dict[str, Any]:
+ workload = series["workload"]
+ system = series["system"]
+ measurement = series["measurement"]
+ ep_size = system["ep_size"]
+ case = {
+ "backend": series["backend"]["id"],
+ "canonical": True,
+ "eplb": workload["eplb"],
+ "ep": ep_size,
+ "experts": workload["experts"],
+ "gpus_per_node": system["gpus_per_node"],
+ "hidden": workload["hidden"],
+ "ladder": " ".join(str(point["tokens_per_rank"]) for point in series["points"]),
+ "mode": series["mode"],
+ "nodes": system["nodes"],
+ "phase": series["phase"],
+ "required_publication": series["publication_tier"],
+ "routing": workload["routing"],
+ "samples_per_point": measurement["samples_per_component"],
+ "scale_out_transport": system["scale_out_transport"],
+ "scale_up_domain": system["scale_up_domain"],
+ "scale_up_transport": system["scale_up_transport"],
+ "scope": system["scope"],
+ "suite": series["suite"],
+ "timing": (
+ f"{measurement['iters']}:{measurement['trials']}:"
+ f"{measurement['warmups']}"
+ ),
+ "topk": workload["top_k"],
+ "topology_class": system["topology_class"],
+ "transport": system["transport"],
+ "warmup_semantics": sweep_matrix.ep_harness.WARMUP_SEMANTICS,
+ "workload": series["model"],
+ }
+ return {
+ "case": case,
+ "profile": identity.profile_for_case(case),
+ "sku": system["sku"],
+ }
+
+
+def _coverage_topology(case: dict[str, Any]) -> dict[str, Any]:
+ """Project exact fabric placement without exposing private runner details."""
+ return {
+ "ep_size": case.get("ep_size", case.get("ep")),
+ **{field: case[field] for field in COVERAGE_TOPOLOGY_FIELDS if field != "ep_size"},
+ }
+
+
+def _coverage_coordinates(case: dict[str, Any]) -> dict[str, Any]:
+ return {
+ "sku": case["sku"], "backend": case["backend"],
+ "mode": case["mode"], "phase": case["phase"],
+ "topology": _coverage_topology(case),
+ }
+
+
+@lru_cache(maxsize=1)
+def _canonical_coverage_cases() -> dict[str, dict[str, Any]]:
+ matrix = sweep_matrix.resolve_matrix(suites="all", max_cases=128, backends="all")
+ return {
+ item["case"]["case_id"]: _coverage_coordinates({
+ "sku": item["sku"], **item["case"],
+ })
+ for item in matrix["requested_cases"]
+ }
+
+
+def _public_series_config(series: dict[str, Any]) -> dict[str, Any]:
+ return {
+ "backend": {
+ "generation": series["backend"]["generation"],
+ "version": series["backend"]["version"],
+ },
+ "resource": series["resource"],
+ "system": {"label": series["system"]["label"]},
+ }
+
+
+def _public_cohort_factors(kind: str, item: dict[str, Any]) -> tuple[Any, Any]:
+ workload = item["workload"]
+ build = item["build"]
+ shape = {
+ key: workload[key]
+ for key in (
+ "hidden", "top_k", "experts", "dispatch_dtype", "combine_dtype",
+ "activation_profile",
+ )
+ }
+ common = {
+ "model": item["model"], "mode": item["mode"], "phase": item["phase"],
+ "shape": shape, "measurement": item["measurement"],
+ "ep_size": item["system"]["ep_size"],
+ }
+ if kind == "library":
+ return (
+ {**common, "system": item["system"], "workload": workload,
+ "resource_mode": item["resource"]["mode"], "source": build["source_sha"]},
+ item["backend"]["id"],
+ )
+ if kind == "chip":
+ return (
+ {**common, "backend": item["backend"], "workload": workload,
+ "resource_mode": item["resource"]["mode"], "source": build["source_sha"]},
+ item["system"],
+ )
+ if kind == "system":
+ return {**common, "workload": workload, "source": build["source_sha"]}, [
+ item["system"]["sku"], item["backend"]["id"], item["resource"]["profile"]
+ ]
+ if kind == "routing":
+ return (
+ {**common, "backend": item["backend"], "system": item["system"],
+ "resource": item["resource"], "build": _routing_build_control(build)},
+ [workload["routing"], workload["eplb"],
+ build["implementation_contract_sha256"]],
+ )
+ raise PublisherError(f"unknown cohort kind {kind}")
+
+
+def _case_disposition_catalog_sha256(coverage: Sequence[dict[str, Any]]) -> str:
+ catalog = [
+ {"case_id": item["case_id"], "disposition": item["disposition"]}
+ for item in sorted(coverage, key=lambda item: item["case_id"])
+ ]
+ return _sha_bytes(_canonical(catalog))
+
+
+def validate_public_dataset(doc: Any) -> dict[str, Any]:
+ _schema("public-dataset-v1.schema.json", doc)
+ if len(_canonical(doc)) + 1 > MAX_PUBLIC_DATASET_BYTES:
+ raise PublisherError("public dataset exceeds the serving size limit")
+ try:
+ artifact_safety.assert_publication_safe([doc])
+ except artifact_safety.ArtifactSafetyError as exc:
+ raise PublisherError(str(exc)) from exc
+ if doc["source_bundle_ids"] != sorted(doc["source_bundle_ids"]):
+ raise PublisherError("source bundle IDs are not canonical")
+ for field, key in (
+ ("coverage", "case_id"), ("attempts", "attempt_id"),
+ ("series", "series_id"), ("cohorts", "cohort_id"),
+ ("rankings", "ranking_id"), ("recommendations", "recommendation_id"),
+ ("sensitivities", "sensitivity_id"),
+ ):
+ if doc[field] != sorted(doc[field], key=lambda item: item[key]):
+ raise PublisherError(f"{field} are not in canonical identity order")
+ promotion = doc["promotion"]
+ quarantined = promotion["status"] == "quarantined"
+ if quarantined != (promotion["reason"] is not None) or quarantined != (
+ promotion["matrix_id"] is None
+ ):
+ raise PublisherError("promotion reason/matrix identity differs from status")
+ attempts = {item["attempt_id"]: item for item in doc["attempts"]}
+ if len(attempts) != len(doc["attempts"]):
+ raise PublisherError("dataset has duplicate attempt IDs")
+ evidence = [
+ value["evidence_id"] for item in doc["attempts"] for value in item["evidence"]
+ ]
+ _unique(evidence, "dataset attempt evidence")
+ series = {item["series_id"]: item for item in doc["series"]}
+ if len(series) != len(doc["series"]):
+ raise PublisherError("dataset has duplicate series IDs")
+ allocation_ids = set(promotion["allocation_ids"])
+ case_ids = {item["case_id"] for item in doc["coverage"]}
+ if len(case_ids) != len(doc["coverage"]):
+ raise PublisherError("dataset has duplicate case coverage")
+ coverage_by_case = {item["case_id"]: item for item in doc["coverage"]}
+ series_case_ids = {
+ case_id for item in doc["series"] for case_id in item["case_ids"]
+ }
+ canonical_cases = _canonical_coverage_cases()
+ for item in doc["coverage"]:
+ topology = item["topology"]
+ registered = capability.topology_for(item["sku"], topology["ep_size"])
+ if (
+ item["sku"] not in capability.PLATFORMS
+ or item["backend"] not in capability.BACKENDS
+ or registered is None
+ or any(
+ topology[field] != registered[field]
+ for field in COVERAGE_TOPOLOGY_FIELDS if field != "ep_size"
+ )
+ ):
+ raise PublisherError("coverage topology differs from the capability registry")
+ coordinates = {
+ "sku": item["sku"], "backend": item["backend"],
+ "mode": item["mode"], "phase": item["phase"], "topology": topology,
+ }
+ canonical = canonical_cases.get(item["case_id"])
+ if canonical is not None and coordinates != canonical:
+ raise PublisherError("coverage coordinates differ from its case identity")
+ if canonical is None and item["case_id"] not in series_case_ids:
+ raise PublisherError("coverage case identity is outside the v1 catalog")
+ for item in doc["attempts"]:
+ if item["case_id"] not in case_ids or item["allocation_id"] not in allocation_ids:
+ raise PublisherError("attempt references undeclared coverage or allocation")
+ if item["series_id"] is not None and item["series_id"] not in series:
+ raise PublisherError("attempt references unknown series")
+ if (item["outcome"] == "success") != (item["reason"] is None):
+ raise PublisherError("attempt reason must be null exactly for success")
+ if item["outcome"] == "success" and item["failure_mode"] is not None:
+ raise PublisherError("successful attempt cannot have a failure mode")
+ if (item["outcome"] == "success" and item["selected"]) != (
+ item["series_id"] is not None
+ ):
+ raise PublisherError("attempt series must be present exactly for selected success")
+ if {item["allocation_id"] for item in doc["attempts"]} != allocation_ids:
+ raise PublisherError("promotion allocation catalog differs from attempts")
+ attempt_groups: dict[tuple[str, str], list[dict[str, Any]]] = {}
+ for item in doc["attempts"]:
+ attempt_groups.setdefault((item["case_id"], item["allocation_id"]), []).append(item)
+ for (case_id, allocation_id), group in attempt_groups.items():
+ ordinals = sorted(item["attempt_index"] for item in group)
+ if ordinals != list(range(1, len(group) + 1)):
+ raise PublisherError("public retries must retain contiguous attempt indexes")
+ if any(
+ item["attempt_id"] != identity.attempt_id(
+ allocation=allocation_id, case=case_id, ordinal=item["attempt_index"]
+ )
+ for item in group
+ ):
+ raise PublisherError("public retry identity differs from its case/allocation/index")
+ selected = [item for item in group if item["selected"]]
+ if len(selected) != 1 or selected[0]["attempt_index"] != ordinals[-1]:
+ raise PublisherError("publisher must select the latest retry per case/allocation")
+ selected_by_series: dict[str, list[dict[str, Any]]] = {}
+ for item in doc["attempts"]:
+ if item["selected"] and item["outcome"] == "success":
+ selected_by_series.setdefault(item["series_id"], []).append(item)
+ terminal = 0
+ for item in doc["coverage"]:
+ listed = set(item["attempt_ids"])
+ selected = item["selected_attempt_id"]
+ expected_attempts = {
+ attempt_id for attempt_id, attempt in attempts.items()
+ if attempt["case_id"] == item["case_id"]
+ }
+ if listed != expected_attempts:
+ raise PublisherError("coverage references attempts from another case")
+ if selected is not None:
+ terminal += 1
+ if (selected not in listed or not attempts[selected]["selected"]
+ or any(attempts[selected][field] != item[field]
+ for field in ("outcome", "failure_mode", "reason"))):
+ raise PublisherError("coverage selected outcome differs")
+ selected_candidates = [attempts[value] for value in listed if attempts[value]["selected"]]
+ latest = max(
+ selected_candidates,
+ key=lambda attempt: (
+ int(attempt["run_id"]), attempt["run_attempt"],
+ attempt["attempt_index"], attempt["attempt_id"]
+ ),
+ )
+ if selected != latest["attempt_id"]:
+ raise PublisherError("coverage does not select the latest canonical allocation")
+ if promotion["requested_cases"] != len(doc["coverage"]) or promotion["terminal_cases"] != terminal:
+ raise PublisherError("promotion coverage counts differ")
+ selected_evidence: dict[tuple[str, str], set[str]] = {}
+ for attempt in doc["attempts"]:
+ if attempt["selected"] and attempt["series_id"] is not None:
+ for value in attempt["evidence"]:
+ selected_evidence.setdefault(
+ (attempt["series_id"], value["point_id"]), set()
+ ).add(value["evidence_id"])
+ for item in doc["series"]:
+ eligibility = _eligibility(item["eligibility"], f"series {item['series_id']}")
+ workload = item["workload"]
+ model, hidden, top_k, experts = sweep_matrix.V1_WORKLOAD
+ suite_contract = sweep_matrix.V1_SUITE_CONTRACTS.get(item["suite"])
+ coordinate = (
+ item["mode"], item["phase"], workload["routing"], workload["eplb"]
+ )
+ profile = identity.case_profile(item["mode"])
+ if (
+ item["model"] != model
+ or (workload["hidden"], workload["top_k"], workload["experts"])
+ != (hidden, top_k, experts)
+ or suite_contract is None
+ or coordinate not in suite_contract["coordinates"]
+ or (
+ suite_contract.get("backends") is not None
+ and item["backend"]["id"] not in suite_contract["backends"]
+ )
+ or item["publication_tier"] != suite_contract["publication"]
+ or item["measurement"]["contract"] != profile["contract"]
+ or item["measurement"]["component_order_contract"]
+ != profile["component_order_contract"]
+ or item["measurement"]["combine_semantics"] != profile["combine_semantics"]
+ or item["measurement"]["payload_unit"] != profile["payload_unit"]
+ ):
+ raise PublisherError("series differs from the frozen v1 workload/suite profile")
+ backend_id = item["backend"]["id"]
+ expected_role = "reference" if backend_id == "nccl-ep" else "library"
+ if (
+ backend_id not in capability.BACKENDS
+ or item["backend"]["label"] != BACKEND_LABELS[backend_id]
+ or item["backend"]["role"] != expected_role
+ or item["backend"]["version"] is None
+ ):
+ raise PublisherError("series backend projection differs from v1")
+ sku = item["system"]["sku"]
+ platform = capability.PLATFORMS.get(sku)
+ ep_size = item["system"]["ep_size"]
+ registered_topology = capability.topology_for(sku, ep_size)
+ if platform is None or registered_topology is None:
+ raise PublisherError("series system projection differs from v1")
+ supported, _ = capability.resolve(
+ sku, backend_id, ep=ep_size, nodes=item["system"]["nodes"],
+ routing=workload["routing"], eplb=workload["eplb"],
+ mode=item["mode"],
+ )
+ if (
+ not supported
+ or item["system"]["vendor"] != platform["vendor"]
+ or any(
+ item["system"][field] != registered_topology[field]
+ for field in (
+ "nodes", "gpus_per_node", "scale_up_domain", "scope",
+ "scale_up_transport", "scale_out_transport", "transport",
+ "topology_class",
+ )
+ )
+ or item["system"]["world_size"] != ep_size
+ or platform["product"] not in set(
+ re.findall(r"[a-z]+\d+[a-z]*", item["system"]["label"].lower())
+ )
+ ):
+ raise PublisherError("series system projection differs from v1")
+ if contracts.public_series_config_sha256(_public_series_config(item)) != item[
+ "build"
+ ]["public_config_sha256"]:
+ raise PublisherError("public series configuration differs from its commitment")
+ covered = [coverage_by_case.get(case_id) for case_id in item["case_ids"]]
+ if not covered or any(
+ case is None
+ or {
+ "sku": case["sku"], "backend": case["backend"],
+ "mode": case["mode"], "phase": case["phase"],
+ "topology": case["topology"],
+ }
+ != {
+ "sku": sku, "backend": backend_id,
+ "mode": item["mode"], "phase": item["phase"],
+ "topology": _coverage_topology(item["system"]),
+ }
+ for case in covered
+ ):
+ raise PublisherError("series projection differs from its case coverage")
+ if (
+ item["eplb"]["enabled"] != item["workload"]["eplb"]
+ or item["eplb"]["logical_experts"] != item["workload"]["experts"]
+ ):
+ raise PublisherError("series EPLB descriptor differs from its workload")
+ eplb = item["eplb"]
+ expected_physical = eplb["logical_experts"] + eplb["redundant_experts"]
+ nullable_eplb = (
+ "planner", "mapping_sha256", "reference_tokens_per_rank", "max_replicas",
+ "imbalance_before", "imbalance_after",
+ )
+ if eplb["enabled"]:
+ if (
+ item["workload"]["routing"] != "zipf"
+ or any(eplb[field] is None for field in nullable_eplb)
+ or eplb["planner"] != "greedy-rank-major-v1"
+ or eplb["reference_tokens_per_rank"] != 2048
+ or eplb["redundant_experts"] != 32
+ or eplb["redundant_experts"] % ep_size != 0
+ or eplb["physical_experts"] != expected_physical
+ or eplb["logical_experts"] % ep_size != 0
+ or eplb["physical_experts"] % ep_size != 0
+ or not 1 <= eplb["replicated_experts"] <= min(
+ eplb["logical_experts"], eplb["redundant_experts"]
+ )
+ or not 2 <= eplb["max_replicas"] <= 1 + eplb["redundant_experts"]
+ or not 1 <= eplb["imbalance_after"] <= eplb["imbalance_before"] <= ep_size
+ ):
+ raise PublisherError("enabled EPLB descriptor is incomplete")
+ expected_plan = contracts._expected_eplb_plan(
+ workload["routing"], workload["top_k"],
+ eplb["logical_experts"], eplb["physical_experts"], ep_size,
+ identity.V1_CASE_PROFILE["seed"],
+ identity.V1_CASE_PROFILE["eplb_reference_tokens_per_rank"],
+ )
+ expected_eplb = {
+ "enabled": True,
+ "planner": identity.V1_CASE_PROFILE["eplb_planner"],
+ "mapping_sha256": contracts.eplb_contract.mapping_hash(expected_plan),
+ "logical_experts": eplb["logical_experts"],
+ "physical_experts": eplb["physical_experts"],
+ "redundant_experts": identity.V1_CASE_PROFILE["eplb_redundant_experts"],
+ "reference_tokens_per_rank": identity.V1_CASE_PROFILE[
+ "eplb_reference_tokens_per_rank"
+ ],
+ "replicated_experts": expected_plan["replicated_experts"],
+ "max_replicas": expected_plan["max_replicas"],
+ "imbalance_before": expected_plan["imbalance_before"],
+ "imbalance_after": expected_plan["imbalance_after"],
+ }
+ if eplb != expected_eplb:
+ raise PublisherError("enabled EPLB descriptor differs from deterministic plan")
+ elif (
+ any(eplb[field] is not None for field in nullable_eplb)
+ or eplb["physical_experts"] != expected_physical
+ or eplb["redundant_experts"] != 0
+ or eplb["replicated_experts"] != 0
+ ):
+ raise PublisherError("disabled EPLB descriptor claims a plan")
+ if item["backend"]["id"] == "nccl-ep":
+ expected_generation = (
+ "nccl" if item["system"]["vendor"] == "nvidia" else "rccl"
+ )
+ if item["backend"]["generation"] != expected_generation:
+ raise PublisherError("NCCL/RCCL reference generation differs from system vendor")
+ if (item["status"] == "decision-grade") != eligibility["decision_grade"]:
+ raise PublisherError("series status differs from eligibility")
+ if (
+ set(eligibility["allocation_ids"]) != set(item["allocation_ids"])
+ or eligibility["correct"] != all(point["correct"] for point in item["points"])
+ ):
+ raise PublisherError("series eligibility differs from its evidence")
+ selected_attempts = selected_by_series.get(item["series_id"], [])
+ if (
+ set(item["case_ids"]) != {attempt["case_id"] for attempt in selected_attempts}
+ or set(item["allocation_ids"])
+ != {attempt["allocation_id"] for attempt in selected_attempts}
+ ):
+ raise PublisherError("series case/allocation catalog differs from selected attempts")
+ if item["eligibility"]["decision_grade"] and len(
+ {attempt["run_id"] for attempt in selected_attempts}
+ ) < REQUIRED_ALLOCATIONS:
+ raise PublisherError("decision-grade series lacks independent workflow runs")
+ tokens = [point["tokens_per_rank"] for point in item["points"]]
+ if tokens != sorted(set(tokens)):
+ raise PublisherError("series points are not in unique ascending token order")
+ if len(item["case_ids"]) != 1:
+ raise PublisherError("public series must represent exactly one v1 case")
+ case_id = item["case_ids"][0]
+ if identity.digest("case", _public_case_factors(item)) != case_id:
+ raise PublisherError("public series projection differs from its case identity")
+ build = item["build"]
+ expected_series_id = identity.series_id({
+ "backend": backend_id,
+ "case_id": case_id,
+ "image_digest": build["image_digest"],
+ "implementation_contract_sha256": build[
+ "implementation_contract_sha256"
+ ],
+ "public_config_sha256": build["public_config_sha256"],
+ "routing_control_sha256": build["routing_control_sha256"],
+ "runtime_fingerprint_sha256": build["runtime_fingerprint_sha256"],
+ "source_sha": build["source_sha"],
+ "squash_sha256": build["squash_sha256"],
+ "workload_id": workload["workload_id"],
+ })
+ if item["series_id"] != expected_series_id:
+ raise PublisherError("public series identity differs from its committed factors")
+ for point in item["points"]:
+ if point["point_id"] != identity.point_id(series=item["series_id"], tokens_per_rank=point["tokens_per_rank"]):
+ raise PublisherError("point identity differs")
+ if point["global_tokens"] != point["tokens_per_rank"] * item["system"]["ep_size"]:
+ raise PublisherError("global_tokens must use EP size")
+ routing = point["routing"]
+ max_fanout = min(item["workload"]["top_k"], item["system"]["ep_size"])
+ if (
+ routing["routed_copies"] < point["global_tokens"]
+ or routing["routed_copies"] > point["global_tokens"] * max_fanout
+ or routing["recv_tokens_max"] > routing["routed_copies"]
+ or routing["recv_tokens_max"] * item["system"]["ep_size"]
+ < routing["routed_copies"]
+ or not math.isclose(
+ routing["fanout_mean"],
+ routing["routed_copies"] / point["global_tokens"],
+ rel_tol=1e-12,
+ )
+ or routing["hotspot_ratio"] < 1
+ or routing["empty_expert_count"] >= eplb["physical_experts"]
+ or routing["empty_rank_count"] >= item["system"]["ep_size"]
+ ):
+ raise PublisherError("point routing/load facts are internally inconsistent")
+ expected_evidence = selected_evidence.get(
+ (item["series_id"], point["point_id"]), set()
+ )
+ if set(point["evidence_ids"]) != expected_evidence:
+ raise PublisherError("point evidence differs from selected series attempts")
+ components = point["components"]
+ if (components["dispatch"] is None) != (components["combine"] is None):
+ raise PublisherError("dispatch/combine availability differs")
+ for name, component in components.items():
+ if component is None:
+ continue
+ expected_origin = "derived" if name == "isolated_sum" else "measured"
+ expected_samples = None if name == "isolated_sum" else 512
+ if component["origin"] != expected_origin or component["sample_count"] != expected_samples:
+ raise PublisherError(f"{name} origin or sample count differs")
+ if name == "isolated_sum" and (
+ component["logical_bytes"] is not None
+ or component["logical_payload_rate_gbps_at_latency_percentile"] is not None
+ ):
+ raise PublisherError("isolated_sum cannot publish logical bandwidth")
+ if name != "isolated_sum" and (
+ component["logical_bytes"] is None
+ or component["logical_payload_rate_gbps_at_latency_percentile"] is None
+ ):
+ raise PublisherError(f"{name} measured logical bandwidth is missing")
+ latency = component["latency_us"]
+ if list(latency.values()) != sorted(latency.values()):
+ raise PublisherError("latency percentiles are not ordered")
+ if component["logical_payload_rate_gbps_at_latency_percentile"] is not None:
+ for statistic, rate in component["logical_payload_rate_gbps_at_latency_percentile"].items():
+ expected = component["logical_bytes"] / (latency[statistic] * 1000.0)
+ if not math.isclose(rate, expected, rel_tol=1e-9, abs_tol=1e-12):
+ raise PublisherError("logical GB/s formula differs")
+ if components["roundtrip"] is None or components["roundtrip"]["origin"] != "measured":
+ raise PublisherError("roundtrip must be measured")
+ for statistic, throughput in point["roundtrip_token_rate_at_latency_percentile"].items():
+ expected = point["global_tokens"] / (
+ components["roundtrip"]["latency_us"][statistic] * 1e-6
+ )
+ if not math.isclose(throughput, expected, rel_tol=1e-9):
+ raise PublisherError("roundtrip token throughput formula differs")
+ if components["dispatch"] is not None:
+ derived = components["isolated_sum"]
+ if derived is None or any(not math.isclose(
+ derived["latency_us"][statistic],
+ components["dispatch"]["latency_us"][statistic]
+ + components["combine"]["latency_us"][statistic], rel_tol=1e-12
+ ) for statistic in ("p50", "p90", "p95", "p99")):
+ raise PublisherError("isolated_sum is not the component percentile sum")
+ elif components["isolated_sum"] is not None:
+ raise PublisherError("isolated_sum requires measured dispatch/combine components")
+ cohorts = {item["cohort_id"]: item for item in doc["cohorts"]}
+ if len(cohorts) != len(doc["cohorts"]):
+ raise PublisherError("dataset has duplicate cohort IDs")
+ for item in doc["cohorts"]:
+ if not set(item["series_ids"]).issubset(series):
+ raise PublisherError("cohort references unknown series")
+ members = [series[series_id] for series_id in item["series_ids"]]
+ expected_tier = (
+ "comparable-experimental"
+ if any(member["publication_tier"] == "comparable-experimental" for member in members)
+ else "official"
+ )
+ if item["publication_tier"] != expected_tier:
+ raise PublisherError("cohort publication tier differs from its members")
+ if f"/ {members[0]['mode']} /" not in item["label"]:
+ raise PublisherError("cohort label omits its controlled mode")
+ roles = {member["backend"]["role"] for member in members}
+ if item["kind"] == "library" and roles != {"library"}:
+ raise PublisherError("library cohort contains non-library evidence")
+ if item["kind"] == "system" and roles != {"reference"}:
+ raise PublisherError("system cohort is not a portable reference comparison")
+ if item["kind"] in {"chip", "routing"} and len(
+ {_canonical(member["backend"]) for member in members}
+ ) != 1:
+ raise PublisherError(f"{item['kind']} cohort mixes backend implementations")
+ public_factors = [_public_cohort_factors(item["kind"], member) for member in members]
+ if len({_canonical(value[0]) for value in public_factors}) != 1:
+ raise PublisherError(f"{item['kind']} cohort does not control its public factors")
+ if len({_canonical(value[1]) for value in public_factors}) < 2:
+ raise PublisherError(f"{item['kind']} cohort does not vary its declared contrast")
+ if item["kind"] == "routing":
+ if item["publication_tier"] != "comparable-experimental":
+ raise PublisherError("routing cohort must be experimental")
+ has_baseline = sum(
+ member["workload"]["routing"] == "uniform"
+ and not member["workload"]["eplb"]
+ for member in members
+ ) == 1
+ missing_reason = "missing-uniform-baseline" in item["eligibility"]["reasons"]
+ if has_baseline == missing_reason:
+ raise PublisherError("routing baseline and eligibility reason disagree")
+ mismatch = _routing_implementation_mismatch(members)
+ mismatch_reason = "implementation-config-mismatch" in item["eligibility"]["reasons"]
+ if mismatch != mismatch_reason:
+ raise PublisherError("routing implementation control and eligibility disagree")
+ expected_id = _derived_id("cxcohort-v1-", {
+ "kind": item["kind"], "series_ids": item["series_ids"],
+ "controlled_factors": item["controlled_factors"],
+ "varying_factors": item["varying_factors"],
+ })
+ if item["cohort_id"] != expected_id:
+ raise PublisherError("cohort ID differs from its public factors")
+ expected_factors = {
+ "library": (
+ ["system", "workload", "mode", "phase", "measurement", "resource.mode", "source"],
+ ["backend", "resource"],
+ ),
+ "chip": (
+ ["backend", "source", "workload", "mode", "phase", "measurement", "resource.mode"],
+ ["system", "resource"],
+ ),
+ "system": (
+ ["workload", "mode", "phase", "measurement", "source"],
+ ["system", "backend", "resource"],
+ ),
+ "routing": (
+ ["backend", "implementation-static-build", "system", "model-shape", "mode", "phase", "measurement", "resource"],
+ ["workload.routing", "workload.eplb", "implementation-config"],
+ ),
+ }[item["kind"]]
+ member_allocations = {
+ allocation for series_id in item["series_ids"]
+ for allocation in series[series_id]["allocation_ids"]
+ }
+ if (
+ (item["controlled_factors"], item["varying_factors"]) != expected_factors
+ or set(item["eligibility"]["allocation_ids"]) != member_allocations
+ ):
+ raise PublisherError("cohort factors or allocations differ from its members")
+ _eligibility(item["eligibility"], f"cohort {item['cohort_id']}")
+ expected_ranking_keys: set[tuple[str, str, str, int]] = set()
+ for cohort in doc["cohorts"]:
+ if not cohort["eligibility"]["decision_grade"]:
+ continue
+ members = [series[series_id] for series_id in cohort["series_ids"]]
+ tokens = set.intersection(*(
+ {point["tokens_per_rank"] for point in member["points"]}
+ for member in members
+ ))
+ expected_ranking_keys.update(
+ (cohort["cohort_id"], measure, statistic, token)
+ for token in tokens
+ for measure in ("latency_us", "logical_payload_rate_gbps_at_latency_percentile")
+ for statistic in ("p50", "p99")
+ )
+ ranking_top: dict[tuple[str, str, str, int], dict[str, Any]] = {}
+ ranking_ids: set[str] = set()
+ for ranking in doc["rankings"]:
+ cohort = cohorts.get(ranking["cohort_id"])
+ if (
+ cohort is None
+ or not cohort["eligibility"]["decision_grade"]
+ or ranking["eligibility"] != cohort["eligibility"]
+ or ranking["publication_tier"] != cohort["publication_tier"]
+ ):
+ raise PublisherError("ranking references an ineligible cohort")
+ entries = ranking["entries"]
+ _validate_metric(ranking["metric"])
+ if cohort["kind"] == "library" and any(
+ series[series_id]["backend"]["role"] == "reference"
+ for series_id in cohort["series_ids"]
+ ):
+ raise PublisherError("reference evidence cannot drive a library ranking")
+ if {entry["series_id"] for entry in entries} != set(cohort["series_ids"]):
+ raise PublisherError("ranking does not cover its cohort")
+ for entry in entries:
+ point_id, value, unit = _metric_value(series[entry["series_id"]], ranking["metric"])
+ if entry["point_id"] != point_id or entry["unit"] != unit or not math.isclose(entry["value"], value, rel_tol=1e-12):
+ raise PublisherError("ranking entry differs from series data")
+ reverse = ranking["metric"]["objective"] == "max"
+ expected = sorted(entries, key=lambda entry: (entry["value"], entry["series_id"]), reverse=reverse)
+ if entries != expected or [entry["rank"] for entry in entries] != list(range(1, len(entries) + 1)):
+ raise PublisherError("ranking order differs")
+ metric = ranking["metric"]
+ expected_id = _derived_id("cxranking-v1-", {
+ "cohort_id": ranking["cohort_id"], "metric": metric,
+ })
+ if ranking["ranking_id"] != expected_id or expected_id in ranking_ids:
+ raise PublisherError("ranking ID is duplicate or differs")
+ ranking_ids.add(expected_id)
+ ranking_top[(ranking["cohort_id"], metric["measure"], metric["statistic"], metric["tokens_per_rank"])] = entries[0]
+ if set(ranking_top) != expected_ranking_keys:
+ raise PublisherError("rankings do not cover every eligible cohort metric")
+ objective = {
+ "min-p50-latency": ("latency_us", "p50"), "min-p99-latency": ("latency_us", "p99"),
+ "max-payload-rate-at-p50-latency": (
+ "logical_payload_rate_gbps_at_latency_percentile", "p50"
+ ),
+ "max-payload-rate-at-p99-latency": (
+ "logical_payload_rate_gbps_at_latency_percentile", "p99"
+ ),
+ }
+ recommendation_ids: set[str] = set()
+ for item in doc["recommendations"]:
+ measure, statistic = objective[item["objective"]]
+ candidates = [top for key, top in ranking_top.items()
+ if key[:3] == (item["cohort_id"], measure, statistic) and top["point_id"] == item["point_id"]]
+ if len(candidates) != 1 or any(item[field] != candidates[0][field] for field in ("series_id", "point_id", "value", "unit")):
+ raise PublisherError("recommendation is not a ranking winner")
+ matching_ranking = next(
+ ranking for ranking in doc["rankings"]
+ if ranking["cohort_id"] == item["cohort_id"]
+ and ranking["metric"]["measure"] == measure
+ and ranking["metric"]["statistic"] == statistic
+ and ranking["entries"][0]["point_id"] == item["point_id"]
+ )
+ expected_id = _derived_id("cxrecommendation-v1-", {
+ "objective": item["objective"], "ranking_id": matching_ranking["ranking_id"],
+ })
+ cohort = cohorts[item["cohort_id"]]
+ if (item["recommendation_id"] != expected_id or expected_id in recommendation_ids
+ or cohort["publication_tier"] != "official"
+ or item["publication_tier"] != "official"
+ or item["eligibility"] != cohort["eligibility"]):
+ raise PublisherError("recommendation ID/eligibility differs")
+ recommendation_ids.add(expected_id)
+ expected_recommendations = sum(
+ cohorts[ranking["cohort_id"]]["publication_tier"] == "official"
+ for ranking in doc["rankings"]
+ )
+ if len(doc["recommendations"]) != expected_recommendations:
+ raise PublisherError("recommendations do not cover every actionable ranking")
+ sensitivity_ids: set[str] = set()
+ sensitivity_keys: set[tuple[str, str, str, str, str, int]] = set()
+ for item in doc["sensitivities"]:
+ cohort = cohorts.get(item["cohort_id"])
+ if (
+ cohort is None
+ or cohort["kind"] != "routing"
+ or not cohort["eligibility"]["decision_grade"]
+ or item["publication_tier"] != cohort["publication_tier"]
+ or item["eligibility"] != cohort["eligibility"]
+ ):
+ raise PublisherError("sensitivity references a non-routing cohort")
+ if (
+ item["baseline_series_id"] == item["candidate_series_id"]
+ or not {item["baseline_series_id"], item["candidate_series_id"]}.issubset(cohort["series_ids"])
+ ):
+ raise PublisherError("sensitivity series differ from its routing cohort")
+ _validate_metric(item["metric"])
+ baseline_series = series[item["baseline_series_id"]]
+ if (
+ baseline_series["workload"]["routing"] != "uniform"
+ or baseline_series["workload"]["eplb"]
+ ):
+ raise PublisherError("sensitivity baseline is not uniform without EPLB")
+ _, baseline, _ = _metric_value(series[item["baseline_series_id"]], item["metric"])
+ _, candidate, _ = _metric_value(series[item["candidate_series_id"]], item["metric"])
+ if not math.isclose(item["signed_change_ratio"], (candidate - baseline) / baseline, rel_tol=1e-12):
+ raise PublisherError("sensitivity ratio differs")
+ expected_id = _derived_id("cxsensitivity-v1-", {
+ "baseline": item["baseline_series_id"],
+ "candidate": item["candidate_series_id"],
+ "cohort": item["cohort_id"], "metric": item["metric"],
+ })
+ if item["sensitivity_id"] != expected_id or expected_id in sensitivity_ids:
+ raise PublisherError("sensitivity ID is duplicate or differs")
+ sensitivity_ids.add(expected_id)
+ sensitivity_keys.add((
+ item["cohort_id"], item["baseline_series_id"], item["candidate_series_id"],
+ item["metric"]["measure"], item["metric"]["statistic"],
+ item["metric"]["tokens_per_rank"],
+ ))
+ expected_sensitivity_keys: set[tuple[str, str, str, str, str, int]] = set()
+ for cohort in doc["cohorts"]:
+ if cohort["kind"] != "routing" or not cohort["eligibility"]["decision_grade"]:
+ continue
+ members = [series[series_id] for series_id in cohort["series_ids"]]
+ baseline = next((
+ member for member in members
+ if member["workload"]["routing"] == "uniform" and not member["workload"]["eplb"]
+ ), None)
+ if baseline is None:
+ continue
+ tokens = set.intersection(*(
+ {point["tokens_per_rank"] for point in member["points"]}
+ for member in members
+ ))
+ expected_sensitivity_keys.update(
+ (cohort["cohort_id"], baseline["series_id"], candidate["series_id"],
+ measure, statistic, token)
+ for candidate in members if candidate is not baseline
+ for token in tokens
+ for measure in ("latency_us", "logical_payload_rate_gbps_at_latency_percentile")
+ for statistic in ("p50", "p99")
+ )
+ if sensitivity_keys != expected_sensitivity_keys:
+ raise PublisherError("sensitivities do not cover every routing contrast metric")
+ if promotion["status"] == "promoted":
+ run_ids = {item["run_id"] for item in doc["attempts"] if item["selected"]}
+ repeated_cases = all(
+ len({
+ attempts[attempt_id]["run_id"]
+ for attempt_id in coverage["attempt_ids"]
+ if attempts[attempt_id]["selected"]
+ }) == REQUIRED_ALLOCATIONS
+ for coverage in doc["coverage"]
+ )
+ if promotion["matrix_id"] != CANONICAL_FULL_V1_MATRIX_SHA256:
+ raise PublisherError("promotion requires the canonical full-v1 matrix")
+ if (
+ _case_disposition_catalog_sha256(doc["coverage"])
+ != CANONICAL_FULL_V1_CASE_CATALOG_SHA256
+ ):
+ raise PublisherError("promotion requires the canonical case/disposition catalog")
+ if (
+ terminal != len(doc["coverage"])
+ or len(doc["source_bundle_ids"]) != REQUIRED_ALLOCATIONS
+ or len(run_ids) != REQUIRED_ALLOCATIONS
+ or not repeated_cases
+ ):
+ raise PublisherError("promoted dataset lacks complete coverage")
+ expected_outcomes = {
+ item["case_id"]: (
+ "success" if item["disposition"] == "runnable" else "unsupported"
+ )
+ for item in doc["coverage"]
+ }
+ if any(
+ item["selected"]
+ and item["outcome"] != expected_outcomes[item["case_id"]]
+ for item in doc["attempts"]
+ ):
+ raise PublisherError("promoted outcomes differ from requested dispositions")
+ runnable_cases = {
+ item["case_id"] for item in doc["coverage"]
+ if item["disposition"] == "runnable"
+ }
+ if any(
+ item["case_id"] in runnable_cases and item["outcome"] != "success"
+ for item in doc["attempts"]
+ ):
+ raise PublisherError(
+ "promotion rejects runnable cases with failed, invalid, or diagnostic retries"
+ )
+ _require_promotion_series(doc["series"])
+ _require_promotion_cohorts(doc["cohorts"], doc["series"])
+ if not doc["rankings"] or not doc["recommendations"]:
+ raise PublisherError("promoted dataset lacks eligible decisions")
+ if promotion["status"] == "quarantined" and any((
+ doc["source_bundle_ids"], promotion["allocation_ids"], doc["coverage"],
+ doc["attempts"], doc["series"], doc["cohorts"], doc["rankings"],
+ doc["recommendations"], doc["sensitivities"],
+ )):
+ raise PublisherError("quarantined dataset exposes unvalidated evidence")
+ return doc
+
+
+def _file_record(value: Any, path: str) -> dict[str, Any]:
+ item = _exact(value, {"path", "sha256", "bytes"}, path)
+ if not isinstance(item["path"], str) or PurePosixPath(item["path"]).is_absolute() or ".." in PurePosixPath(item["path"]).parts:
+ raise PublisherError(f"{path}.path is unsafe")
+ if not isinstance(item["sha256"], str) or HEX64.fullmatch(item["sha256"]) is None:
+ raise PublisherError(f"{path}.sha256 is invalid")
+ _integer(item["bytes"], f"{path}.bytes", minimum=1)
+ return item
+
+def validate_bundle_manifest(doc: Any) -> dict[str, Any]:
+ _schema("private-bundle-v1.schema.json", doc)
+ attempts = {item["attempt_id"]: item for item in doc["attempts"]}
+ if len(attempts) != len(doc["attempts"]):
+ raise PublisherError("bundle has duplicate attempt IDs")
+ selections = doc["coverage"]["selections"]
+ if len({item["case_id"] for item in selections}) != len(selections):
+ raise PublisherError("bundle has duplicate selected cases")
+ counts = {name: 0 for name in OUTCOMES}
+ for selection in selections:
+ attempt = attempts.get(selection["selected_attempt_id"])
+ if attempt is None or not attempt["selected"] or attempt["case_id"] != selection["case_id"] or attempt["outcome"] != selection["outcome"]:
+ raise PublisherError("bundle selection differs from retained attempt")
+ counts[selection["outcome"]] += 1
+ coverage = doc["coverage"]
+ if coverage["terminal_cases"] != len(selections) or coverage["outcome_counts"] != counts:
+ raise PublisherError("bundle terminal counts differ")
+ if coverage["complete"] != (coverage["expected_cases"] == len(selections)):
+ raise PublisherError("bundle completeness differs from coverage")
+ fingerprints: dict[str, set[str]] = {}
+ for attempt in doc["attempts"]:
+ value = attempt["runtime_fingerprint_sha256"]
+ if value:
+ fingerprints.setdefault(attempt["allocation_id"], set()).add(value)
+ if any(len(values) != 1 for values in fingerprints.values()):
+ raise PublisherError("bundle runtime is heterogeneous within an allocation")
+ return doc
+
+
+def _fsync_dir(path: Path) -> None:
+ descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0))
+ try:
+ os.fsync(descriptor)
+ finally:
+ os.close(descriptor)
+
+
+def _write_bytes(path: Path, data: bytes, *, mode: int) -> None:
+ descriptor = os.open(
+ path,
+ os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0),
+ mode,
+ )
+ try:
+ os.fchmod(descriptor, mode)
+ with os.fdopen(descriptor, "wb", closefd=False) as handle:
+ handle.write(data)
+ handle.flush()
+ os.fsync(handle.fileno())
+ finally:
+ os.close(descriptor)
+
+
+def _write_all(descriptor: int, data: bytes) -> None:
+ view = memoryview(data)
+ while view:
+ view = view[os.write(descriptor, view):]
+
+
+def _write_json(path: Path, value: Any, *, mode: int) -> bytes:
+ data = _canonical(value) + b"\n"
+ _write_bytes(path, data, mode=mode)
+ return data
+
+
+def _file_metadata(path: Path, relative_to: Path) -> dict[str, Any]:
+ return {
+ "path": path.relative_to(relative_to).as_posix(),
+ "sha256": _sha_file(path),
+ "bytes": path.stat().st_size,
+ }
+
+
+def _tree_files(root: Path) -> list[Path]:
+ return sorted(
+ path for path in root.rglob("*")
+ if path.is_file() and not path.is_symlink() and path.name != "COMPLETE"
+ )
+
+
+def _verify_regular_file(path: Path, expected_mode: int) -> None:
+ _reject_symlinked_path(path.parent)
+ try:
+ metadata = os.lstat(path)
+ except FileNotFoundError as exc:
+ raise PublisherError(f"required file is missing: {path.name}") from exc
+ if (
+ not stat.S_ISREG(metadata.st_mode)
+ or metadata.st_uid != os.getuid()
+ or stat.S_IMODE(metadata.st_mode) != expected_mode
+ ):
+ raise PublisherError(
+ f"file is not an owned regular {expected_mode:o} object: {path.name}"
+ )
+
+
+def _verify_frozen_tree(root: Path, *, private: bool) -> None:
+ _reject_symlinked_path(root)
+ directory_mode = 0o500 if private else 0o555
+ file_mode = 0o400 if private else 0o444
+ try:
+ root_metadata = os.lstat(root)
+ except OSError as exc:
+ raise PublisherError(f"cannot inspect immutable object: {root.name}") from exc
+ if not stat.S_ISDIR(root_metadata.st_mode):
+ raise PublisherError(f"immutable object is not a real directory: {root.name}")
+ try:
+ entries = [root, *root.rglob("*")]
+ except OSError as exc:
+ raise PublisherError(f"cannot inspect immutable object: {root.name}") from exc
+ for path in entries:
+ metadata = os.lstat(path)
+ if metadata.st_uid != os.getuid():
+ raise PublisherError(f"immutable object has the wrong owner: {path.name}")
+ if stat.S_ISDIR(metadata.st_mode):
+ expected = directory_mode
+ elif stat.S_ISREG(metadata.st_mode):
+ expected = file_mode
+ else:
+ raise PublisherError(f"immutable object contains a linked or special entry: {path.name}")
+ if stat.S_IMODE(metadata.st_mode) != expected:
+ raise PublisherError(
+ f"immutable object mode differs for {path.name}: expected {expected:o}"
+ )
+
+
+def _freeze_tree(root: Path, *, private: bool) -> None:
+ files: list[Path] = []
+ directories = [root]
+ for path in root.rglob("*"):
+ metadata = os.lstat(path)
+ if stat.S_ISDIR(metadata.st_mode):
+ directories.append(path)
+ elif stat.S_ISREG(metadata.st_mode):
+ files.append(path)
+ else:
+ raise PublisherError(f"immutable object contains a linked or special entry: {path.name}")
+ for path in files:
+ os.chmod(path, 0o400 if private else 0o444)
+ for path in sorted(directories, key=lambda item: len(item.parts), reverse=True):
+ os.chmod(path, 0o500 if private else 0o555)
+ _fsync_dir(path)
+ _verify_frozen_tree(root, private=private)
+
+
+def _reject_symlinked_path(path: Path) -> None:
+ current = Path(path.anchor)
+ for part in path.parts[1:]:
+ current /= part
+ try:
+ metadata = os.lstat(current)
+ except FileNotFoundError:
+ break
+ if stat.S_ISLNK(metadata.st_mode):
+ raise PublisherError("COLLECTIVEX_STORE_ROOT must not traverse a symlinked parent")
+ if not stat.S_ISDIR(metadata.st_mode):
+ raise PublisherError(f"store path component is not a directory: {current}")
+
+
+class Store:
+ """Atomic private/public directory operations on one operator filesystem."""
+
+ def __init__(self, root: str | os.PathLike[str]):
+ candidate = Path(os.path.abspath(os.path.expanduser(root)))
+ _reject_symlinked_path(candidate)
+ candidate.mkdir(parents=True, exist_ok=True, mode=0o750)
+ resolved = candidate.resolve()
+ if candidate != resolved:
+ raise PublisherError(
+ "COLLECTIVEX_STORE_ROOT must not traverse a symlinked parent"
+ )
+ root_metadata = candidate.stat()
+ if root_metadata.st_uid != os.getuid() or stat.S_IMODE(root_metadata.st_mode) & 0o022:
+ raise PublisherError(
+ "COLLECTIVEX_STORE_ROOT must be owned by this user and not group/world writable"
+ )
+ os.chmod(candidate, 0o750)
+ if stat.S_IMODE(candidate.stat().st_mode) != 0o750:
+ raise PublisherError("COLLECTIVEX_STORE_ROOT mode must be 750")
+ self.root = resolved
+ raw = self.root
+ self.private = raw / "private"
+ self.incoming = self.private / "incoming"
+ self.bundles = self.private / "bundles"
+ self.quarantine = self.private / "quarantine"
+ self.public = raw / "public"
+ self.datasets = self.public / "datasets"
+ self.channels = self.public / "channels"
+ self.locks = raw / "locks"
+ for path, mode in (
+ (self.private, 0o700), (self.incoming, 0o700), (self.bundles, 0o700),
+ (self.quarantine, 0o700), (self.public, 0o755), (self.datasets, 0o755),
+ (self.channels, 0o755), (self.locks, 0o700),
+ ):
+ path.mkdir(parents=True, exist_ok=True, mode=mode)
+ if path.is_symlink() or not path.is_dir():
+ raise PublisherError(f"store path is not a real directory: {path}")
+ os.chmod(path, mode)
+
+ @contextlib.contextmanager
+ def locked(self) -> Iterator[None]:
+ lock_path = self.locks / "publisher.lock"
+ descriptor = os.open(
+ lock_path,
+ os.O_RDWR | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0),
+ 0o600,
+ )
+ try:
+ os.fchmod(descriptor, 0o600)
+ metadata = os.fstat(descriptor)
+ if (
+ not stat.S_ISREG(metadata.st_mode)
+ or metadata.st_uid != os.getuid()
+ or stat.S_IMODE(metadata.st_mode) != 0o600
+ ):
+ raise PublisherError("publisher lock is not an owned regular 600 file")
+ fcntl.flock(descriptor, fcntl.LOCK_EX)
+ yield
+ finally:
+ fcntl.flock(descriptor, fcntl.LOCK_UN)
+ os.close(descriptor)
+
+ @contextlib.contextmanager
+ def staging(self, parent: Path, *, private: bool) -> Iterator[Path]:
+ stage = Path(tempfile.mkdtemp(prefix=".staging-", dir=parent))
+ os.chmod(stage, 0o700 if private else 0o755)
+ try:
+ yield stage
+ finally:
+ if stage.exists():
+ for path in stage.rglob("*"):
+ metadata = os.lstat(path)
+ if stat.S_ISDIR(metadata.st_mode):
+ os.chmod(path, 0o700)
+ elif stat.S_ISREG(metadata.st_mode):
+ os.chmod(path, 0o600)
+ os.chmod(stage, 0o700)
+ shutil.rmtree(stage, ignore_errors=True)
+
+ @staticmethod
+ def complete(stage: Path, value: str, *, private: bool) -> None:
+ _write_bytes(stage / "COMPLETE", (value + "\n").encode(), mode=0o600 if private else 0o644)
+ _fsync_dir(stage)
+
+ @staticmethod
+ def install(stage: Path, destination: Path, *, private: bool) -> None:
+ if destination.is_symlink():
+ raise PublisherError(f"immutable destination is a symlink: {destination.name}")
+ if destination.exists():
+ _verify_frozen_tree(destination, private=private)
+ marker = destination / "COMPLETE"
+ if not marker.is_file() or marker.read_text().strip() != destination.name:
+ raise PublisherError(f"immutable destination is incomplete: {destination.name}")
+ return
+ _freeze_tree(stage, private=private)
+ os.rename(stage, destination)
+ _fsync_dir(destination.parent)
+ _verify_frozen_tree(destination, private=private)
+
+ def install_dataset(self, dataset: dict[str, Any]) -> tuple[str, int]:
+ validate_public_dataset(dataset)
+ payload = _canonical(dataset) + b"\n"
+ if len(payload) > MAX_PUBLIC_DATASET_BYTES:
+ raise PublisherError("public dataset exceeds the serving size limit")
+ digest = _sha_bytes(payload)
+ destination = self.datasets / digest
+ with self.staging(self.datasets, private=False) as stage:
+ _write_bytes(stage / "dataset.json", payload, mode=0o644)
+ self.complete(stage, digest, private=False)
+ self.install(stage, destination, private=False)
+ stored = destination / "dataset.json"
+ marker = destination / "COMPLETE"
+ if (not marker.is_file() or marker.read_text().strip() != digest
+ or _sha_file(stored) != digest or stored.stat().st_size != len(payload)):
+ raise PublisherError("stored dataset checksum differs after installation")
+ return digest, len(payload)
+
+ def update_channel(self, channel: str, digest: str, size: int, generated_at: str) -> None:
+ if size > MAX_PUBLIC_DATASET_BYTES:
+ raise PublisherError("channel dataset exceeds the serving size limit")
+ _verify_frozen_tree(self.datasets / digest, private=False)
+ marker = self.datasets / digest / "COMPLETE"
+ if not marker.is_file() or marker.read_text().strip() != digest:
+ raise PublisherError("cannot advance a channel to an incomplete dataset")
+ dataset_path = self.datasets / digest / "dataset.json"
+ dataset = validate_public_dataset(strict_load(dataset_path))
+ if (
+ _sha_file(dataset_path) != digest
+ or dataset_path.stat().st_size != size
+ or dataset["generated_at"] != generated_at
+ ):
+ raise PublisherError("channel metadata differs from its stored dataset")
+ if channel == "dev-latest" and dataset["promotion"]["status"] != "promoted":
+ raise PublisherError("dev-latest may only reference a promoted dataset")
+ pointer = {
+ "format": FORMAT_CHANNEL,
+ "channel": channel,
+ "dataset": {
+ "path": f"datasets/{digest}/dataset.json",
+ "sha256": digest,
+ "bytes": size,
+ },
+ "generated_at": generated_at,
+ }
+ validate_channel(pointer, expected_channel=channel)
+ destination = self.channels / f"{channel}.json"
+ temporary = self.channels / f".{channel}.tmp-{os.getpid()}"
+ try:
+ data = _canonical(pointer) + b"\n"
+ _write_bytes(temporary, data, mode=0o644)
+ os.replace(temporary, destination)
+ _fsync_dir(self.channels)
+ finally:
+ temporary.unlink(missing_ok=True)
+
+ def verify_channel(self, channel: str) -> dict[str, Any]:
+ channel_path = self.channels / f"{channel}.json"
+ _verify_regular_file(channel_path, 0o644)
+ pointer = validate_channel(strict_load(channel_path), expected_channel=channel)
+ target = self.public / pointer["dataset"]["path"]
+ _verify_frozen_tree(target.parent, private=False)
+ if target.stat().st_size != pointer["dataset"]["bytes"] or _sha_file(target) != pointer["dataset"]["sha256"]:
+ raise PublisherError(f"channel {channel} dataset checksum differs")
+ marker = target.parent / "COMPLETE"
+ if not marker.is_file() or marker.read_text().strip() != pointer["dataset"]["sha256"]:
+ raise PublisherError(f"channel {channel} dataset is incomplete")
+ dataset = validate_public_dataset(strict_load(target))
+ if pointer["generated_at"] != dataset["generated_at"]:
+ raise PublisherError(f"channel {channel} metadata differs from its dataset")
+ if channel == "dev-latest" and dataset["promotion"]["status"] != "promoted":
+ raise PublisherError("dev-latest points to a non-promoted dataset")
+ return pointer
+
+
+def _copy_source(source: Path, destination: Path) -> None:
+ if source.is_symlink() or not source.is_file() or not stat.S_ISREG(source.stat().st_mode):
+ raise PublisherError(f"source must be a regular non-symlink file: {source}")
+ descriptor = os.open(source, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
+ try:
+ output = os.open(destination, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
+ try:
+ while True:
+ chunk = os.read(descriptor, 1024 * 1024)
+ if not chunk:
+ break
+ _write_all(output, chunk)
+ os.fsync(output)
+ finally:
+ os.close(output)
+ finally:
+ os.close(descriptor)
+
+
+def _archive_download_directory(source: Path, destination: Path) -> None:
+ if source.is_symlink() or not source.is_dir():
+ raise PublisherError(f"artifact directory is invalid: {source}")
+ files: list[Path] = []
+ for path in source.rglob("*"):
+ if path.is_symlink():
+ raise PublisherError("artifact directory contains a symlink")
+ if path.is_dir():
+ continue
+ if not path.is_file():
+ raise PublisherError("artifact directory contains a non-regular entry")
+ files.append(path)
+ files.sort()
+ if not files or len(files) > MAX_ARCHIVE_MEMBERS:
+ raise PublisherError("artifact directory has an invalid file count")
+ total = 0
+ with zipfile.ZipFile(destination, "x", compression=zipfile.ZIP_STORED) as archive:
+ for path in files:
+ descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
+ with os.fdopen(descriptor, "rb") as handle:
+ metadata = os.fstat(handle.fileno())
+ if not stat.S_ISREG(metadata.st_mode):
+ raise PublisherError("artifact directory member changed type")
+ size = metadata.st_size
+ total += size
+ if size > MAX_ARCHIVE_MEMBER_BYTES or total > MAX_ARCHIVE_TOTAL_BYTES:
+ raise PublisherError("artifact directory exceeds size limits")
+ relative = path.relative_to(source).as_posix()
+ _safe_member(relative)
+ info = zipfile.ZipInfo(relative, date_time=(1980, 1, 1, 0, 0, 0))
+ info.compress_type = zipfile.ZIP_STORED
+ info.external_attr = (stat.S_IFREG | 0o600) << 16
+ with archive.open(info, "w") as output:
+ written = 0
+ for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+ output.write(chunk)
+ written += len(chunk)
+ if written != size:
+ raise PublisherError("artifact directory member changed size")
+ descriptor = os.open(destination, os.O_RDONLY)
+ try:
+ os.fsync(descriptor)
+ finally:
+ os.close(descriptor)
+
+
+def _artifact_name(source: Path) -> str:
+ name = source.name if source.is_dir() else source.name.removesuffix(".zip")
+ if (
+ not source.is_dir() and source.suffix != ".zip"
+ or ARTIFACT_NAME.fullmatch(name) is None
+ ):
+ raise PublisherError(f"artifact source has an invalid GHA name: {source.name}")
+ return name
+
+
+def archive_incoming(
+ store: Store,
+ matrix: Path,
+ artifacts: Sequence[Path],
+ run: dict[str, Any],
+) -> tuple[str, Path, list[dict[str, Any]]]:
+ """Copy exact delivery bytes into immutable incoming before any JSON/ZIP parse."""
+ if not artifacts:
+ raise PublisherError("at least one GitHub artifact archive is required")
+ with store.staging(store.incoming, private=True) as stage:
+ sources = stage / "sources"
+ sources.mkdir(mode=0o700)
+ copied: list[dict[str, Any]] = []
+ named_artifacts = sorted(
+ ((_artifact_name(path), path) for path in artifacts), key=lambda item: item[0]
+ )
+ artifact_names = [name for name, _ in named_artifacts]
+ if len(artifact_names) != len(set(artifact_names)):
+ raise PublisherError("artifact delivery contains duplicate GHA names")
+ inputs = [("matrix.json", matrix, "matrix", None)] + [
+ (f"artifact-{index:04d}.zip", path, "artifact", artifact_name)
+ for index, (artifact_name, path) in enumerate(named_artifacts)
+ ]
+ for name, source, kind, artifact_name in inputs:
+ destination = sources / name
+ if source.is_dir():
+ _archive_download_directory(source, destination)
+ else:
+ if source != matrix and source.stat().st_size > MAX_ARCHIVE_TOTAL_BYTES:
+ raise PublisherError("artifact archive exceeds the size limit")
+ _copy_source(source, destination)
+ copied.append({
+ **_file_metadata(destination, stage),
+ "kind": kind,
+ "artifact_name": artifact_name,
+ })
+ ingest_id = _sha_bytes(_canonical({"run": run, "sources": copied}))
+ incoming_manifest = {
+ "format": "collectivex.incoming.v1",
+ "schema_version": 1,
+ "ingest_id": ingest_id,
+ "run": run,
+ "sources": copied,
+ }
+ _write_json(stage / "incoming.json", incoming_manifest, mode=0o600)
+ store.complete(stage, ingest_id, private=True)
+ destination = store.incoming / ingest_id
+ store.install(stage, destination, private=True)
+ installed = store.incoming / ingest_id
+ if strict_load(installed / "incoming.json") != incoming_manifest:
+ raise PublisherError("existing incoming object differs from archived delivery")
+ for record in copied:
+ _resolve_bundle_file(installed, record)
+ return ingest_id, installed, copied
+
+
+def _safe_member(name: str) -> PurePosixPath:
+ if "\\" in name or "\0" in name:
+ raise PublisherError("archive member has an unsafe separator")
+ path = PurePosixPath(name)
+ if path.is_absolute() or not path.parts or any(part in {"", ".", ".."} for part in path.parts):
+ raise PublisherError("archive member path escapes its artifact")
+ return path
+
+
+def extract_archive(archive: Path, destination: Path) -> list[Path]:
+ """Extract a bounded regular-file ZIP without trusting member paths or links."""
+ try:
+ handle = zipfile.ZipFile(archive)
+ except (OSError, zipfile.BadZipFile) as exc:
+ raise PublisherError("artifact is not a valid ZIP archive") from exc
+ extracted: list[Path] = []
+ seen: set[str] = set()
+ total = 0
+ with handle:
+ members = handle.infolist()
+ if not members or len(members) > MAX_ARCHIVE_MEMBERS:
+ raise PublisherError("artifact has an invalid member count")
+ for member in members:
+ path = _safe_member(member.filename.rstrip("/"))
+ key = path.as_posix()
+ if key in seen:
+ raise PublisherError("artifact contains duplicate member paths")
+ seen.add(key)
+ mode = member.external_attr >> 16
+ if stat.S_ISLNK(mode) or (mode and not (stat.S_ISREG(mode) or stat.S_ISDIR(mode))):
+ raise PublisherError("artifact contains a non-regular member")
+ if member.flag_bits & 0x1:
+ raise PublisherError("encrypted artifact members are not accepted")
+ if member.file_size > MAX_ARCHIVE_MEMBER_BYTES:
+ raise PublisherError("artifact member exceeds the size limit")
+ total += member.file_size
+ if total > MAX_ARCHIVE_TOTAL_BYTES:
+ raise PublisherError("artifact exceeds the expanded size limit")
+ target = destination.joinpath(*path.parts)
+ if member.is_dir():
+ target.mkdir(parents=True, exist_ok=True, mode=0o700)
+ continue
+ target.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
+ output = os.open(target, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
+ try:
+ with handle.open(member, "r") as source:
+ written = 0
+ while True:
+ chunk = source.read(1024 * 1024)
+ if not chunk:
+ break
+ _write_all(output, chunk)
+ written += len(chunk)
+ if written != member.file_size:
+ raise PublisherError("artifact member size changed during extraction")
+ os.fsync(output)
+ finally:
+ os.close(output)
+ extracted.append(target)
+ return extracted
+
+
+def validate_matrix(document: Any) -> list[dict[str, Any]]:
+ try:
+ artifact_safety.assert_publication_safe([document])
+ matrix = sweep_matrix.validate_matrix_document(document)
+ except (SystemExit, ValueError, artifact_safety.ArtifactSafetyError) as exc:
+ raise PublisherError(f"requested matrix is invalid: {exc}") from exc
+ return [
+ {
+ "sku": item["sku"],
+ **item["case"],
+ "_disposition": item["disposition"],
+ "_reason": item["reason"],
+ }
+ for item in matrix["requested_cases"]
+ ]
+
+
+def _expected_deliveries(
+ matrix: dict[str, Any], cases: Sequence[dict[str, Any]], run: dict[str, Any]
+) -> dict[str, tuple[str, str, str]]:
+ shard_by_case: dict[str, str] = {}
+ for shard in matrix["include"]:
+ for case_id in shard["case_ids"]:
+ if case_id in shard_by_case:
+ raise PublisherError("requested case appears in two runnable shards")
+ shard_by_case[case_id] = shard["id"]
+ suffix = f"{run['run_id']}-{run['run_attempt']}"
+ deliveries: dict[str, tuple[str, str, str]] = {}
+ for case in cases:
+ case_id = case["case_id"]
+ if case["_disposition"] == "unsupported":
+ deliveries[case_id] = (
+ f"cxunsupported-{suffix}", "setup",
+ f"{run['run_id']}_{run['run_attempt']}_unsupported",
+ )
+ continue
+ shard_id = shard_by_case.get(case_id)
+ if shard_id is None:
+ raise PublisherError("runnable case has no matrix shard")
+ deliveries[case_id] = (
+ f"cxshard-{shard_id}-{suffix}", "sweep",
+ f"{run['run_id']}_{run['run_attempt']}_{shard_id}",
+ )
+ return deliveries
+
+
+def _document_git_run(document: dict[str, Any]) -> dict[str, Any] | None:
+ provenance = document.get("provenance")
+ if not isinstance(provenance, dict):
+ return None
+ value = provenance.get("git_run", provenance)
+ return value if isinstance(value, dict) else None
+
+
+def _run_matches(document: dict[str, Any], run: dict[str, Any]) -> bool:
+ git_run = _document_git_run(document)
+ if git_run is None:
+ return False
+ return (
+ str(git_run.get("run_id")) == run["run_id"]
+ and str(git_run.get("run_attempt")) == str(run["run_attempt"])
+ and git_run.get("source_sha") == run["source_sha"]
+ and (git_run.get("repo") or git_run.get("repository")) == run["repository"]
+ )
+
+
+def _case_matches(document: dict[str, Any], expected: dict[str, Any]) -> bool:
+ scheduled = {
+ key: value for key, value in expected.items()
+ if key not in {"sku", "case_id"} and not key.startswith("_")
+ }
+ return document.get("identity", {}).get("case_factors") == {
+ "case": scheduled,
+ "profile": identity.V1_CASE_PROFILE,
+ "sku": expected["sku"],
+ }
+
+
+def _outcome(document: dict[str, Any]) -> tuple[str, str | None]:
+ status = document["outcome"]["status"]
+ if status == "success":
+ return status, None
+ native = document["outcome"].get("reason")
+ reason = native if isinstance(native, str) and REASON.fullmatch(native) else {
+ "unsupported": "unsupported-capability", "failed": "execution-failed",
+ "invalid": "validation-failed", "diagnostic": "diagnostic-evidence",
+ }.get(status)
+ if reason is None:
+ raise PublisherError(f"unsupported native outcome {status!r}")
+ return status, reason
+
+
+def _attempt_record(
+ document: dict[str, Any], path: Path, root: Path, *, selected: bool
+) -> dict[str, Any]:
+ normalized = contracts.normalize_attempt(document)
+ runtime = normalized["runtime_fingerprint"]
+ runtime_sha = _sha_bytes(_canonical(runtime)) if runtime is not None else None
+ sample_record = None
+ evidence_ids: list[str] = []
+ series_ids: list[str] = []
+ if document["format"] == contracts.RAW_FORMAT:
+ sample_path = path.with_name(document["sample_artifact"]["path"])
+ sample_record = _file_metadata(sample_path, root)
+ evidence_ids = [row["evidence_id"] for row in document["measurement"]["rows"]]
+ series_ids = [document["identity"]["series_id"]]
+ declared = document["identity"]["series_factors"]["runtime_fingerprint_sha256"]
+ if runtime_sha != declared:
+ raise PublisherError("runtime fingerprint checksum differs from series identity")
+ status, reason = _outcome(document)
+ return {
+ "attempt_id": normalized["attempt_id"],
+ "allocation_id": normalized["allocation_id"],
+ "case_id": normalized["case_id"],
+ "outcome": status,
+ "reason": reason,
+ "selected": selected,
+ "document": _file_metadata(path, root),
+ "samples": sample_record,
+ "runtime_fingerprint_sha256": runtime_sha,
+ "series_ids": series_ids,
+ "evidence_ids": evidence_ids,
+ }
+
+
+def _validate_delivery_binding(
+ document: dict[str, Any], path: Path, raw_root: Path,
+ artifact_by_root: dict[str, str], expected_by_id: dict[str, dict[str, Any]],
+ expected_deliveries: dict[str, tuple[str, str, str]], run: dict[str, Any],
+) -> str:
+ case_id = document["identity"]["case_id"]
+ if case_id not in expected_by_id:
+ raise PublisherError("artifact contains an extra case outcome")
+ expected = expected_by_id[case_id]
+ if not _case_matches(document, expected):
+ raise PublisherError("attempt case coordinates differ from the requested matrix")
+ unsupported = document["outcome"]["status"] == "unsupported"
+ if (expected["_disposition"] == "unsupported") != unsupported:
+ raise PublisherError("terminal outcome differs from requested capability disposition")
+ if unsupported and document["outcome"]["reason"] != expected["_reason"]:
+ raise PublisherError("unsupported outcome reason differs from requested matrix")
+ if not _run_matches(document, run):
+ raise PublisherError("attempt provenance differs from publisher run metadata")
+ relative = path.relative_to(raw_root)
+ if len(relative.parts) < 2:
+ raise PublisherError("attempt document is outside a delivered artifact")
+ delivered_name = artifact_by_root.get(relative.parts[0])
+ expected_name, expected_job, expected_execution = expected_deliveries[case_id]
+ git_run = _document_git_run(document)
+ allocation = document["identity"]["allocation_factors"]
+ if (
+ git_run is None
+ or delivered_name != expected_name
+ or git_run["artifact"] != delivered_name
+ or git_run["job"] != expected_job
+ or allocation["execution_id"] != expected_execution
+ ):
+ raise PublisherError("attempt provenance differs from its delivered GHA shard")
+ return case_id
+
+
+def _parse_extracted(root: Path) -> tuple[list[tuple[Path, dict[str, Any]]], set[Path]]:
+ attempts: list[tuple[Path, dict[str, Any]]] = []
+ consumed_samples: set[Path] = set()
+ json_paths = sorted(path for path in root.rglob("*.json") if path.is_file())
+ for path in json_paths:
+ if path in consumed_samples:
+ continue
+ try:
+ document = contracts.strict_load(path)
+ artifact_safety.assert_publication_safe([document])
+ format_name = document.get("format") if isinstance(document, dict) else None
+ if format_name == contracts.SAMPLES_FORMAT:
+ _schema("samples-v1.schema.json", document)
+ # It must be claimed by a raw document; orphan checking happens after the scan.
+ continue
+ if format_name == contracts.RAW_FORMAT:
+ _schema("raw-case-v1.schema.json", document)
+ sample_path = path.with_name(document["sample_artifact"]["path"])
+ sample_document = contracts.strict_load(sample_path)
+ artifact_safety.assert_publication_safe([sample_document])
+ _schema("samples-v1.schema.json", sample_document)
+ validated = contracts.load_raw_attempt(path)
+ consumed_samples.add(sample_path)
+ elif format_name == contracts.TERMINAL_FORMAT:
+ _schema("terminal-outcome-v1.schema.json", document)
+ validated = contracts.validate_terminal_document(document)
+ else:
+ raise PublisherError(f"artifact contains unknown JSON document {path.name}")
+ except (
+ contracts.ContractError, artifact_safety.ArtifactSafetyError,
+ jsonschema.ValidationError, OSError,
+ ) as exc:
+ raise PublisherError(f"native contract rejected {path.name}: {exc}") from exc
+ attempts.append((path, validated))
+ orphan_samples = [
+ path for path in json_paths
+ if isinstance((doc := contracts.strict_load(path)), dict)
+ and doc.get("format") == contracts.SAMPLES_FORMAT
+ and path not in consumed_samples
+ ]
+ if orphan_samples:
+ raise PublisherError("artifact contains an orphan samples document")
+ if not attempts:
+ raise PublisherError("artifact contains zero native attempt documents")
+ return attempts, consumed_samples
+
+
+def build_bundle(
+ store: Store,
+ incoming_id: str,
+ incoming_path: Path,
+ run: dict[str, Any],
+) -> tuple[str, dict[str, Any], list[dict[str, Any]]]:
+ """Validate one exact workflow delivery and install its immutable private bundle."""
+ incoming_manifest = strict_load(incoming_path / "incoming.json")
+ _exact(
+ incoming_manifest,
+ {"format", "schema_version", "ingest_id", "run", "sources"},
+ "incoming",
+ )
+ artifact_safety.assert_publication_safe([incoming_manifest])
+ if (
+ incoming_manifest["format"] != "collectivex.incoming.v1"
+ or incoming_manifest["schema_version"] != 1
+ or incoming_manifest["ingest_id"] != incoming_id
+ or incoming_manifest["run"] != run
+ or _sha_bytes(_canonical({"run": run, "sources": incoming_manifest["sources"]}))
+ != incoming_id
+ ):
+ raise PublisherError("incoming manifest identity differs from archived delivery")
+ incoming_sources = _array(incoming_manifest["sources"], "incoming.sources", nonempty=True)
+ for index, record in enumerate(incoming_sources):
+ _exact(
+ record,
+ {"path", "sha256", "bytes", "kind", "artifact_name"},
+ f"incoming.sources[{index}]",
+ )
+ _resolve_bundle_file(incoming_path, record)
+ matrix_records = [record for record in incoming_sources if record["kind"] == "matrix"]
+ artifact_records = [record for record in incoming_sources if record["kind"] == "artifact"]
+ if (
+ len(matrix_records) != 1
+ or matrix_records[0]["artifact_name"] is not None
+ or not artifact_records
+ or any(ARTIFACT_NAME.fullmatch(record["artifact_name"] or "") is None
+ for record in artifact_records)
+ or len({record["artifact_name"] for record in artifact_records}) != len(artifact_records)
+ ):
+ raise PublisherError("incoming source catalog is invalid")
+ matrix_source = _resolve_bundle_file(incoming_path, matrix_records[0])
+ matrix_document = strict_load(matrix_source)
+ expected_cases = validate_matrix(matrix_document)
+ expected_by_id = {case["case_id"]: case for case in expected_cases}
+ expected_deliveries = _expected_deliveries(matrix_document, expected_cases, run)
+ if {record["artifact_name"] for record in artifact_records} != {
+ delivery[0] for delivery in expected_deliveries.values()
+ }:
+ raise PublisherError("incoming artifact archive set differs from requested matrix shards")
+ with store.staging(store.bundles, private=True) as stage:
+ source_copy = stage / "source"
+ raw_root = stage / "raw"
+ source_copy.mkdir(mode=0o700)
+ raw_root.mkdir(mode=0o700)
+ matrix_path = stage / "matrix.json"
+ _copy_source(matrix_source, matrix_path)
+ source_records: list[dict[str, Any]] = []
+ artifact_by_root: dict[str, str] = {}
+ for index, source_record in enumerate(artifact_records):
+ archive = _resolve_bundle_file(incoming_path, source_record)
+ copied = source_copy / f"artifact-{index:04d}.zip"
+ _copy_source(archive, copied)
+ source_records.append({
+ **_file_metadata(copied, stage),
+ "artifact_name": source_record["artifact_name"],
+ })
+ artifact_root = raw_root / f"artifact-{index:04d}"
+ artifact_root.mkdir(mode=0o700)
+ artifact_by_root[artifact_root.name] = source_record["artifact_name"]
+ extract_archive(copied, artifact_root)
+ parsed, consumed_samples = _parse_extracted(raw_root)
+ created_at = _latest_timestamp(
+ [document["generated_at"] for _, document in parsed]
+ )
+ consumed_files = {path for path, _ in parsed} | consumed_samples
+ extracted_files = {
+ path for path in raw_root.rglob("*")
+ if path.is_file() and not path.is_symlink()
+ }
+ if consumed_files != extracted_files:
+ raise PublisherError("artifact contains an unconsumed non-native member")
+ by_case: dict[str, list[tuple[Path, dict[str, Any]]]] = {}
+ for path, document in parsed:
+ case_id = _validate_delivery_binding(
+ document, path, raw_root, artifact_by_root, expected_by_id,
+ expected_deliveries, run,
+ )
+ by_case.setdefault(case_id, []).append((path, document))
+ missing = set(expected_by_id) - set(by_case)
+ if missing:
+ raise PublisherError(f"artifact is missing {len(missing)} requested case outcomes")
+ attempt_records: list[dict[str, Any]] = []
+ selections: list[dict[str, Any]] = []
+ selected_documents: list[dict[str, Any]] = []
+ runtime_hashes: set[str] = set()
+ outcome_counts = {name: 0 for name in OUTCOMES}
+ for case_id in sorted(expected_by_id):
+ case_attempts = by_case[case_id]
+ ordinals = [document["identity"]["attempt_ordinal"] for _, document in case_attempts]
+ allocations_for_case = {
+ document["identity"]["allocation_id"] for _, document in case_attempts
+ }
+ if len(allocations_for_case) != 1 or sorted(ordinals) != list(
+ range(1, len(ordinals) + 1)
+ ):
+ raise PublisherError(
+ "case retries must retain contiguous ordinals in one allocation"
+ )
+ _, selected_document = max(
+ case_attempts, key=lambda item: item[1]["identity"]["attempt_ordinal"]
+ )
+ selected_id = selected_document["identity"]["attempt_id"]
+ selected_documents.append(selected_document)
+ selected_status, _ = _outcome(selected_document)
+ selections.append({
+ "case_id": case_id,
+ "selected_attempt_id": selected_id,
+ "outcome": selected_status,
+ })
+ outcome_counts[selected_status] += 1
+ for path, document in sorted(
+ case_attempts, key=lambda item: item[1]["identity"]["attempt_ordinal"]
+ ):
+ normalized = contracts.normalize_attempt(document)
+ if document["format"] == contracts.RAW_FORMAT:
+ sample_path = path.with_name(document["sample_artifact"]["path"])
+ if sample_path not in consumed_samples:
+ raise PublisherError("validated raw attempt lost its samples document")
+ record = _attempt_record(
+ document, path, stage,
+ selected=normalized["attempt_id"] == selected_id,
+ )
+ if record["runtime_fingerprint_sha256"]:
+ runtime_hashes.add(record["runtime_fingerprint_sha256"])
+ attempt_records.append(record)
+ # Every extracted byte is covered; the bundle manifest anchors this checksum catalog.
+ payload_records = [_file_metadata(path, stage) for path in _tree_files(stage)]
+ checksum_document = {
+ "format": "collectivex.checksums.v1",
+ "files": payload_records,
+ }
+ checksum_path = stage / "checksums.json"
+ _write_json(checksum_path, checksum_document, mode=0o600)
+ bundle = {
+ "format": FORMAT_BUNDLE,
+ "schema_version": 1,
+ "created_at": created_at,
+ "ingest_id": incoming_id,
+ "run": run,
+ "matrix": _file_metadata(matrix_path, stage),
+ "sources": source_records,
+ "attempts": attempt_records,
+ "coverage": {
+ "expected_cases": len(expected_cases),
+ "terminal_cases": len(selections),
+ "complete": len(selections) == len(expected_cases),
+ "outcome_counts": outcome_counts,
+ "selections": selections,
+ },
+ "runtime_fingerprints": sorted(runtime_hashes),
+ "checksums": _file_metadata(checksum_path, stage),
+ "validation": {
+ "policy": PUBLISHER_POLICY,
+ "passed": True,
+ "checks": [
+ "archive-safety", "checksums", "exact-coverage", "identity",
+ "native-schema", "privacy", "runtime-homogeneity", "terminal-outcomes",
+ ],
+ },
+ }
+ validate_bundle_manifest(bundle)
+ # Runtime homogeneity is scoped to a realized allocation, not across unlike SKUs.
+ by_allocation: dict[str, set[str]] = {}
+ for attempt in attempt_records:
+ fingerprint = attempt["runtime_fingerprint_sha256"]
+ if fingerprint:
+ by_allocation.setdefault(attempt["allocation_id"], set()).add(fingerprint)
+ if any(len(values) != 1 for values in by_allocation.values()):
+ raise PublisherError("runtime fingerprint is heterogeneous within an allocation")
+ bundle_bytes = _canonical(bundle) + b"\n"
+ bundle_id = _sha_bytes(bundle_bytes)
+ _write_bytes(stage / "bundle.json", bundle_bytes, mode=0o600)
+ store.complete(stage, bundle_id, private=True)
+ store.install(stage, store.bundles / bundle_id, private=True)
+ installed = load_bundle(store, bundle_id)
+ if installed["manifest"] != bundle:
+ raise PublisherError("existing bundle differs from validated manifest")
+ return bundle_id, bundle, selected_documents
+
+
+def _slug(value: Any, fallback: str = "unknown") -> str:
+ text = re.sub(r"[^a-z0-9_.-]+", "-", str(value or "").lower()).strip("-.")
+ return text[:128] if text and SAFE_ID.fullmatch(text[:128]) else fallback
+
+
+def _derived_id(prefix: str, value: Any) -> str:
+ return f"{prefix}{_sha_bytes(_canonical(value))}"
+
+
+def _git_run(document: dict[str, Any]) -> dict[str, Any]:
+ return _document_git_run(document) or {}
+
+
+def _public_attempt(document: dict[str, Any], *, selected: bool = False) -> dict[str, Any]:
+ normalized = contracts.normalize_attempt(document)
+ run = _git_run(document)
+ evidence = (
+ [{"evidence_id": row["evidence_id"], "point_id": row["point_id"]}
+ for row in document["measurement"]["rows"]]
+ if document["format"] == contracts.RAW_FORMAT else []
+ )
+ status, reason = _outcome(document)
+ failure_mode = document["outcome"].get("failure_mode")
+ if not isinstance(failure_mode, str) or REASON.fullmatch(failure_mode) is None:
+ failure_mode = None if status == "success" else reason
+ series_id = normalized["series_id"] if status == "success" and selected else None
+ return {
+ "attempt_id": normalized["attempt_id"],
+ "evidence": evidence,
+ "case_id": normalized["case_id"],
+ "allocation_id": normalized["allocation_id"],
+ "run_id": str(run["run_id"]),
+ "run_attempt": int(run["run_attempt"]),
+ "attempt_index": document["identity"]["attempt_ordinal"],
+ "selected": selected,
+ "outcome": status,
+ "failure_mode": failure_mode,
+ "reason": reason,
+ "series_id": series_id,
+ "completed_at": document["generated_at"],
+ }
+
+
+def _ratio(values: Sequence[float]) -> float | None:
+ return max(values) / min(values) if len(values) >= REQUIRED_ALLOCATIONS and min(values) > 0 else None
+
+
+def _eligibility_record(
+ allocations: Sequence[str],
+ *,
+ complete: bool,
+ correct: bool,
+ measured: bool,
+ stable_ordering: bool,
+ p50_ratio: float | None,
+ p99_ratio: float | None,
+ extra_reasons: Sequence[str] = (),
+) -> dict[str, Any]:
+ ids = sorted(set(allocations))
+ stable_p50 = p50_ratio is not None and p50_ratio <= P50_STABILITY_LIMIT
+ stable_p99 = p99_ratio is not None and p99_ratio <= P99_STABILITY_LIMIT
+ reasons = list(extra_reasons)
+ for condition, reason in (
+ (len(ids) >= REQUIRED_ALLOCATIONS, "insufficient-allocations"),
+ (complete, "incomplete-repeat-coverage"),
+ (correct, "correctness-failed"),
+ (measured, "missing-measured-roundtrip-p99"),
+ (stable_p50, "unstable-p50"),
+ (stable_p99, "unstable-p99"),
+ (stable_ordering, "unstable-ordering"),
+ ):
+ if not condition:
+ reasons.append(reason)
+ reasons = sorted(set(reasons))
+ decision = not reasons
+ return {
+ "decision_grade": decision,
+ "allocation_ids": ids,
+ "complete": complete,
+ "correct": correct,
+ "measured_roundtrip_p99": measured,
+ "stable_p50": stable_p50,
+ "stable_p99": stable_p99,
+ "stable_ordering": stable_ordering,
+ "p50_max_min_ratio": p50_ratio,
+ "p99_max_min_ratio": p99_ratio,
+ "reasons": reasons,
+ }
+
+
+def _aggregate_percentiles(values: Sequence[dict[str, Any]]) -> dict[str, float]:
+ return {
+ name: float(statistics.median(float(value[name]) for value in values))
+ for name in ("p50", "p90", "p95", "p99")
+ }
+
+
+def _aggregate_component(
+ rows: Sequence[dict[str, Any]], name: str
+) -> dict[str, Any] | None:
+ components = [row["components"][name] for row in rows]
+ if all(component["availability"] == "unavailable" for component in components):
+ return None
+ if any(component["availability"] == "unavailable" for component in components):
+ raise PublisherError("component availability differs across repeat allocations")
+ latency = _aggregate_percentiles([component["percentiles_us"] for component in components])
+ if name == "isolated_sum":
+ return {
+ "origin": "derived",
+ "latency_us": latency,
+ "logical_bytes": None,
+ "logical_payload_rate_gbps_at_latency_percentile": None,
+ "sample_count": None,
+ }
+ byte_values = {row["logical_bytes"][name] for row in rows}
+ if len(byte_values) != 1:
+ raise PublisherError("logical byte accounting differs across repeat allocations")
+ logical_bytes = byte_values.pop()
+ rates = {statistic: logical_bytes / (latency[statistic] * 1000.0) for statistic in latency}
+ return {
+ "origin": "measured",
+ "latency_us": latency,
+ "logical_bytes": logical_bytes,
+ "logical_payload_rate_gbps_at_latency_percentile": rates,
+ "sample_count": 512,
+ }
+
+
+def _exact_repeat_value(values: Sequence[Any], label: str) -> Any:
+ if not values or len({_canonical(value) for value in values}) != 1:
+ raise PublisherError(f"{label} differs across repeat allocations")
+ return values[0]
+
+
+def _eplb_descriptor(document: dict[str, Any]) -> dict[str, Any]:
+ value = document["case"]["eplb"]
+ return {
+ "enabled": value["enabled"],
+ "planner": value["planner"],
+ "mapping_sha256": value["mapping_hash"],
+ "logical_experts": value["num_logical_experts"],
+ "physical_experts": value["num_physical_experts"],
+ "redundant_experts": value["num_redundant"],
+ "reference_tokens_per_rank": value["reference_tokens_per_rank"],
+ "replicated_experts": value["replicated_experts"],
+ "max_replicas": value["max_replicas"],
+ "imbalance_before": value["imbalance_before"],
+ "imbalance_after": value["imbalance_after"],
+ }
+
+
+def _routing_facts(row: dict[str, Any]) -> dict[str, Any]:
+ routing = row["routing"]
+ return {
+ "fanout_mean": routing["fanout_mean"],
+ "recv_tokens_max": row["receive"]["max"],
+ "expert_load_cv": routing["expert_load_cv"],
+ "payload_rank_cv": routing["payload_rank_cv"],
+ "hotspot_ratio": routing["hotspot_ratio"],
+ "empty_expert_count": routing["empty_expert_count"],
+ "empty_rank_count": routing["empty_rank_count"],
+ "routed_copies": routing["routed_copies"],
+ }
+
+
+def _series_extra_reasons(documents: Sequence[dict[str, Any]]) -> list[str]:
+ reasons: set[str] = set()
+ for document in documents:
+ validity = document["outcome"]["validity"]
+ rows = document["measurement"]["rows"]
+ if validity.get("provenance_complete") is not True:
+ reasons.add("incomplete-provenance")
+ if validity.get("workload_source") != "canonical-serialized":
+ reasons.add("noncanonical-workload")
+ if validity.get("anomaly_free") is not True or any(row["anomalies"] for row in rows):
+ reasons.add("unresolved-anomaly")
+ if validity.get("semantic_correctness") != "pass":
+ reasons.add("semantic-correctness-failed")
+ if validity.get("measurement_conformance") != "conformant" or validity.get("sampling_conformance") != "conformant":
+ reasons.add("measurement-nonconformant")
+ profile = identity.case_profile(document["case"]["mode"])
+ scopes = {row["correctness"].get("scope") for row in rows}
+ if scopes != {profile["correctness_scope"]}:
+ reasons.add("expert-oracle-incomplete")
+ return sorted(reasons)
+
+
+BACKEND_LABELS = {
+ "deepep": "DeepEP V1",
+ "deepep-v2": "DeepEP V2",
+ "deepep-hybrid": "DeepEP Hybrid",
+ "uccl": "UCCL",
+ "mori": "MoRI",
+ "nccl-ep": "NCCL/RCCL reference",
+}
+
+
+def _build_series(
+ series_id: str,
+ documents: Sequence[dict[str, Any]],
+ expected_repeats: int,
+) -> tuple[dict[str, Any], dict[str, Any]]:
+ if not documents:
+ raise PublisherError("cannot aggregate an empty series")
+ first = documents[0]
+ if any(document["identity"]["series_id"] != series_id for document in documents):
+ raise PublisherError("series aggregation mixed identities")
+ allocations = [document["identity"]["allocation_id"] for document in documents]
+ if len(allocations) != len(set(allocations)):
+ raise PublisherError("series repeats reuse an allocation identity")
+ row_maps = [
+ {row["tokens_per_rank"]: row for row in document["measurement"]["rows"]}
+ for document in documents
+ ]
+ token_sets = {tuple(sorted(rows)) for rows in row_maps}
+ if len(token_sets) != 1:
+ raise PublisherError("series token coverage differs across allocations")
+ tokens = list(next(iter(token_sets)))
+ p50_ratios = [
+ _ratio([rows[token]["components"]["roundtrip"]["percentiles_us"]["p50"] for rows in row_maps])
+ for token in tokens
+ ]
+ p99_ratios = [
+ _ratio([rows[token]["components"]["roundtrip"]["percentiles_us"]["p99"] for rows in row_maps])
+ for token in tokens
+ ]
+ p50_ratio = max((value for value in p50_ratios if value is not None), default=None)
+ p99_ratio = max((value for value in p99_ratios if value is not None), default=None)
+ correct = all(
+ row["correctness"]["passed"]
+ for document in documents for row in document["measurement"]["rows"]
+ )
+ measured = all(
+ row["components"]["roundtrip"]["availability"] == "measured"
+ and row["components"]["roundtrip"]["percentiles_us"].get("p99") is not None
+ for document in documents for row in document["measurement"]["rows"]
+ )
+ eligibility = _eligibility_record(
+ allocations,
+ complete=len(documents) == expected_repeats,
+ correct=correct,
+ measured=measured,
+ # Ordering is defined only across alternatives in a controlled cohort.
+ stable_ordering=True,
+ p50_ratio=p50_ratio,
+ p99_ratio=p99_ratio,
+ extra_reasons=_series_extra_reasons(documents),
+ )
+ case = first["case"]
+ shape = case["shape"]
+ topology = first["topology"]
+ runtime = first["runtime_fingerprint"]
+ workload_id = first["workload"]["workload_id"]
+ if not identity.is_typed_id(workload_id, "workload"):
+ raise PublisherError("raw workload is not canonical")
+ backend_id = case["backend"]
+ resource_raw = first["implementation"]["resource_profile"]
+ public_config = contracts.public_series_config(
+ kernel_generation=first["implementation"]["kernel_generation"],
+ provenance=first["implementation"]["provenance"],
+ resource_profile=resource_raw,
+ resource_mode=case["resource_mode"],
+ device_product=topology["device_product"],
+ )
+ resource_profile = public_config["resource"]["profile"]
+ configured_units = public_config["resource"]["configured_units"]
+ units_kind = public_config["resource"]["comm_units_kind"]
+ resource_label = (
+ f"{configured_units} {str(units_kind).upper()}"
+ if configured_units is not None and units_kind
+ else resource_profile
+ )
+ eplb = _exact_repeat_value(
+ [_eplb_descriptor(document) for document in documents], "EPLB descriptor"
+ )
+ points: list[dict[str, Any]] = []
+ run_metrics: dict[str, dict[int, dict[str, float]]] = {}
+ for document, rows in zip(documents, row_maps, strict=True):
+ run_id = str(_git_run(document)["run_id"])
+ if run_id in run_metrics:
+ raise PublisherError("series has two allocations from one workflow run")
+ run_metrics[run_id] = {}
+ for token in tokens:
+ latency = rows[token]["components"]["roundtrip"]["percentiles_us"]
+ logical_bytes = rows[token]["logical_bytes"]["roundtrip"]
+ run_metrics[run_id][token] = {
+ "latency_us": {statistic: latency[statistic] for statistic in ("p50", "p99")},
+ "logical_payload_rate_gbps_at_latency_percentile": {
+ statistic: logical_bytes / (latency[statistic] * 1000.0)
+ for statistic in ("p50", "p99")
+ },
+ }
+ for token in tokens:
+ rows = [row_map[token] for row_map in row_maps]
+ routing = _exact_repeat_value(
+ [_routing_facts(row) for row in rows], "routing/load facts"
+ )
+ components = {
+ name: _aggregate_component(rows, name)
+ for name in ("dispatch", "combine", "roundtrip")
+ }
+ if components["dispatch"] is None:
+ components["isolated_sum"] = None
+ else:
+ latency = {
+ statistic: components["dispatch"]["latency_us"][statistic]
+ + components["combine"]["latency_us"][statistic]
+ for statistic in ("p50", "p90", "p95", "p99")
+ }
+ components["isolated_sum"] = {
+ "origin": "derived", "latency_us": latency, "logical_bytes": None,
+ "logical_payload_rate_gbps_at_latency_percentile": None, "sample_count": None,
+ }
+ points.append({
+ "point_id": rows[0]["point_id"],
+ "tokens_per_rank": token,
+ "global_tokens": token * case["ep_size"],
+ "correct": all(row["correctness"]["passed"] for row in rows),
+ "routing": routing,
+ "components": components,
+ "roundtrip_token_rate_at_latency_percentile": {
+ statistic: (token * case["ep_size"])
+ / (components["roundtrip"]["latency_us"][statistic] * 1e-6)
+ for statistic in ("p50", "p90", "p95", "p99")
+ },
+ "evidence_ids": [row["evidence_id"] for row in rows],
+ })
+ series = {
+ "series_id": series_id,
+ "label": (
+ f"{case['runner'].upper()} / {BACKEND_LABELS.get(backend_id, backend_id)} / "
+ f"EP{case['ep_size']} / {topology['nodes']} node"
+ f"{'s' if topology['nodes'] != 1 else ''} / {topology['scope']} / "
+ f"{case['mode']} / {case['phase']} / {shape['routing']}"
+ f"{' + EPLB' if case['eplb']['enabled'] else ''} / {resource_label}"
+ ),
+ "status": "decision-grade" if eligibility["decision_grade"] else "diagnostic",
+ "case_ids": sorted({document["identity"]["case_id"] for document in documents}),
+ "allocation_ids": sorted(allocations),
+ "model": _slug(case["workload_name"]),
+ "suite": _slug(case["suite"]),
+ "mode": case["mode"],
+ "phase": case["phase"],
+ "publication_tier": case["required_publication"],
+ "backend": {
+ "id": _slug(backend_id),
+ "label": BACKEND_LABELS.get(backend_id, backend_id),
+ "role": "reference" if backend_id == "nccl-ep" else "library",
+ **public_config["backend"],
+ },
+ "build": {
+ "implementation_contract_sha256": first["identity"]["series_factors"][
+ "implementation_contract_sha256"
+ ],
+ "public_config_sha256": first["identity"]["series_factors"][
+ "public_config_sha256"
+ ],
+ "routing_control_sha256": first["identity"]["series_factors"][
+ "routing_control_sha256"
+ ],
+ "runtime_fingerprint_sha256": first["identity"]["series_factors"][
+ "runtime_fingerprint_sha256"
+ ],
+ "image_digest": first["identity"]["series_factors"]["image_digest"],
+ "source_sha": first["identity"]["series_factors"]["source_sha"],
+ "squash_sha256": first["identity"]["series_factors"]["squash_sha256"],
+ },
+ "system": {
+ "sku": _slug(case["runner"]),
+ "label": public_config["system"]["label"],
+ "vendor": runtime["vendor"],
+ "topology_class": _slug(topology["topology_class"]),
+ "transport": _slug(topology["transport"]),
+ "scale_up_transport": _slug(topology["scale_up_transport"]),
+ "scale_out_transport": (
+ _slug(topology["scale_out_transport"])
+ if topology["scale_out_transport"] is not None
+ else None
+ ),
+ "scope": topology["scope"],
+ "nodes": topology["nodes"],
+ "gpus_per_node": topology["gpus_per_node"],
+ "scale_up_domain": topology["scale_up_domain"],
+ "world_size": topology["world_size"],
+ "ep_size": case["ep_size"],
+ "placement": topology["placement"],
+ },
+ "workload": {
+ "workload_id": workload_id,
+ "hidden": shape["hidden"],
+ "top_k": shape["topk"],
+ "experts": case["eplb"]["num_logical_experts"],
+ "routing": shape["routing"],
+ "eplb": case["eplb"]["enabled"],
+ "dispatch_dtype": shape["dispatch_dtype"],
+ "combine_dtype": shape["quant"]["combine_output_dtype"],
+ "activation_profile": shape["activation_profile"],
+ },
+ "eplb": eplb,
+ "resource": public_config["resource"],
+ "measurement": {
+ "contract": first["measurement"]["contract"],
+ "component_order_contract": first["measurement"]["component_order_contract"],
+ "combine_semantics": identity.case_profile(case["mode"])["combine_semantics"],
+ "payload_unit": identity.case_profile(case["mode"])["payload_unit"],
+ "sampling_contract": first["measurement"]["sampling"]["contract"],
+ "iters": first["measurement"]["sampling"]["iterations_per_trial"],
+ "trials": first["measurement"]["sampling"]["trials"],
+ "warmups": first["measurement"]["sampling"]["warmup_iterations"],
+ "samples_per_component": first["measurement"]["sampling"]["samples_per_component"],
+ "headline_component": "roundtrip",
+ "headline_percentile": "p99",
+ },
+ "points": points,
+ "eligibility": eligibility,
+ }
+ internal = {
+ "documents": list(documents),
+ "run_metrics": run_metrics,
+ "series_factors": first["identity"]["series_factors"],
+ }
+ return series, internal
+
+
+def _resolve_bundle_file(root: Path, record: dict[str, Any]) -> Path:
+ path = root.joinpath(*PurePosixPath(record["path"]).parts)
+ try:
+ path.relative_to(root)
+ except ValueError as exc:
+ raise PublisherError("bundle record escapes its directory") from exc
+ if path.resolve() != path or path.is_symlink() or not path.is_file():
+ raise PublisherError("bundle record points to a missing or linked file")
+ if path.stat().st_size != record["bytes"] or _sha_file(path) != record["sha256"]:
+ raise PublisherError("bundle file checksum differs from its manifest")
+ return path
+
+
+def load_bundle(store: Store, bundle_id: str) -> dict[str, Any]:
+ if HEX64.fullmatch(bundle_id) is None:
+ raise PublisherError("bundle ID must be a SHA-256 digest")
+ root = store.bundles / bundle_id
+ if root.is_symlink() or not (root / "COMPLETE").is_file():
+ raise PublisherError(f"bundle {bundle_id} is missing or incomplete")
+ _verify_frozen_tree(root, private=True)
+ if (root / "COMPLETE").read_text().strip() != bundle_id:
+ raise PublisherError("bundle COMPLETE marker differs")
+ manifest_path = root / "bundle.json"
+ if _sha_file(manifest_path) != bundle_id:
+ raise PublisherError("bundle directory digest differs from bundle.json")
+ manifest = validate_bundle_manifest(strict_load(manifest_path))
+ checksum_path = _resolve_bundle_file(root, manifest["checksums"])
+ checksum_document = strict_load(checksum_path)
+ checksum_document = _exact(checksum_document, {"format", "files"}, "checksums")
+ if checksum_document["format"] != "collectivex.checksums.v1":
+ raise PublisherError("bundle checksum format is invalid")
+ records = [_file_record(value, f"checksums.files[{index}]")
+ for index, value in enumerate(_array(checksum_document["files"], "checksums.files"))]
+ _unique([record["path"] for record in records], "checksums.files[].path")
+ for record in records:
+ _resolve_bundle_file(root, record)
+ expected_paths = {
+ path.relative_to(root).as_posix() for path in _tree_files(root)
+ if path.name not in {"bundle.json", "checksums.json"}
+ }
+ if {record["path"] for record in records} != expected_paths:
+ raise PublisherError("bundle checksum catalog does not cover its payload exactly")
+ artifact_by_root: dict[str, str] = {}
+ for index, source in enumerate(manifest["sources"]):
+ _resolve_bundle_file(root, source)
+ archive_key = f"artifact-{index:04d}"
+ if source["path"] != f"source/{archive_key}.zip":
+ raise PublisherError("bundle source catalog order/path differs")
+ artifact_by_root[archive_key] = source["artifact_name"]
+ if len(set(artifact_by_root.values())) != len(artifact_by_root):
+ raise PublisherError("bundle source catalog repeats an artifact name")
+ matrix_path = _resolve_bundle_file(root, manifest["matrix"])
+ matrix_document = strict_load(matrix_path)
+ cases = validate_matrix(matrix_document)
+ expected_by_id = {case["case_id"]: case for case in cases}
+ expected_deliveries = _expected_deliveries(
+ matrix_document, cases, manifest["run"]
+ )
+ if {item["case_id"] for item in manifest["coverage"]["selections"]} != set(expected_by_id):
+ raise PublisherError("bundle selected coverage differs from requested matrix")
+ documents: dict[str, dict[str, Any]] = {}
+ runtime_fingerprints: set[str] = set()
+ for attempt in manifest["attempts"]:
+ document_path = _resolve_bundle_file(root, attempt["document"])
+ document = contracts.strict_load(document_path)
+ artifact_safety.assert_publication_safe([document])
+ if document.get("format") == contracts.RAW_FORMAT:
+ _schema("raw-case-v1.schema.json", document)
+ sample_path = document_path.with_name(document["sample_artifact"]["path"])
+ if attempt["samples"] is None:
+ raise PublisherError("raw attempt is missing its sample manifest record")
+ manifest_sample_path = _resolve_bundle_file(root, attempt["samples"])
+ if manifest_sample_path != sample_path:
+ raise PublisherError("sample manifest record points to the wrong raw evidence")
+ sample_document = contracts.strict_load(sample_path)
+ artifact_safety.assert_publication_safe([sample_document])
+ _schema("samples-v1.schema.json", sample_document)
+ document = contracts.load_raw_attempt(document_path)
+ else:
+ if attempt["samples"] is not None:
+ raise PublisherError("terminal attempt unexpectedly names a sample artifact")
+ _schema("terminal-outcome-v1.schema.json", document)
+ document = contracts.validate_terminal_document(document)
+ _validate_delivery_binding(
+ document, document_path, root / "raw", artifact_by_root,
+ expected_by_id, expected_deliveries, manifest["run"],
+ )
+ expected_record = _attempt_record(
+ document, document_path, root, selected=attempt["selected"]
+ )
+ if expected_record != attempt:
+ raise PublisherError("bundle attempt record differs from native document")
+ if attempt["runtime_fingerprint_sha256"]:
+ runtime_fingerprints.add(attempt["runtime_fingerprint_sha256"])
+ documents[attempt["attempt_id"]] = document
+ if sorted(runtime_fingerprints) != manifest["runtime_fingerprints"]:
+ raise PublisherError("bundle runtime fingerprint catalog differs from attempts")
+ selected = {
+ selection["case_id"]: documents[selection["selected_attempt_id"]]
+ for selection in manifest["coverage"]["selections"]
+ }
+ return {
+ "id": bundle_id,
+ "root": root,
+ "manifest": manifest,
+ "cases": cases,
+ "documents": documents,
+ "selected": selected,
+ }
+
+
+def _cohort_control(
+ kind: str, series: dict[str, Any], internal: dict[str, Any]
+) -> tuple[dict[str, Any], list[str], list[str], Any]:
+ binary_build = series["build"]
+ source = binary_build["source_sha"]
+ workload = series["workload"]
+ shape = {
+ key: workload[key]
+ for key in ("hidden", "top_k", "experts", "dispatch_dtype", "combine_dtype", "activation_profile")
+ }
+ common = {
+ "model": series["model"], "mode": series["mode"],
+ "phase": series["phase"], "shape": shape,
+ "measurement": series["measurement"], "ep_size": series["system"]["ep_size"],
+ }
+ if kind == "library":
+ control = {**common, "system": series["system"], "workload": workload,
+ "resource_mode": series["resource"]["mode"], "source": source}
+ return control, ["system", "workload", "mode", "phase", "measurement", "resource.mode", "source"], ["backend", "resource"], series["backend"]["id"]
+ if kind == "chip":
+ control = {**common, "backend": series["backend"], "source": source,
+ "workload": workload, "resource_mode": series["resource"]["mode"]}
+ return control, ["backend", "source", "workload", "mode", "phase", "measurement", "resource.mode"], ["system", "resource"], series["system"]
+ if kind == "system":
+ control = {**common, "workload": workload, "source": source}
+ varying = [series["system"]["sku"], series["backend"]["id"], series["resource"]["profile"]]
+ return control, ["workload", "mode", "phase", "measurement", "source"], ["system", "backend", "resource"], varying
+ if kind == "routing":
+ control = {
+ **common,
+ "backend": series["backend"],
+ "system": series["system"],
+ "resource": series["resource"],
+ "build": _routing_build_control(binary_build),
+ }
+ varying = [
+ workload["routing"], workload["eplb"],
+ binary_build["implementation_contract_sha256"],
+ ]
+ return (
+ control,
+ ["backend", "implementation-static-build", "system", "model-shape", "mode", "phase", "measurement", "resource"],
+ ["workload.routing", "workload.eplb", "implementation-config"],
+ varying,
+ )
+ raise PublisherError(f"unknown cohort kind {kind}")
+
+
+def _cohort_ordering(
+ members: Sequence[dict[str, Any]], internals: dict[str, dict[str, Any]], tokens: Sequence[int]
+) -> tuple[bool, int]:
+ run_ids = set.intersection(*(
+ set(internals[member["series_id"]]["run_metrics"]) for member in members
+ ))
+ if len(run_ids) < REQUIRED_ALLOCATIONS:
+ return False, len(run_ids)
+ orders: list[tuple[str, str, int, str, tuple[str, ...]]] = []
+ for run_id in sorted(run_ids):
+ for token in tokens:
+ for measure in ("latency_us", "logical_payload_rate_gbps_at_latency_percentile"):
+ for statistic in ("p50", "p99"):
+ ordered = tuple(
+ member["series_id"]
+ for member in sorted(
+ members,
+ key=lambda item: (
+ internals[item["series_id"]]["run_metrics"][run_id][token][measure][statistic],
+ item["series_id"],
+ ),
+ reverse=measure == "logical_payload_rate_gbps_at_latency_percentile",
+ )
+ )
+ orders.append((measure, statistic, token, run_id, ordered))
+ for token in tokens:
+ for measure in ("latency_us", "logical_payload_rate_gbps_at_latency_percentile"):
+ for statistic in ("p50", "p99"):
+ observed = {
+ entry[4]
+ for entry in orders
+ if entry[0] == measure and entry[1] == statistic and entry[2] == token
+ }
+ if len(observed) != 1:
+ return False, len(run_ids)
+ return True, len(run_ids)
+
+
+def build_decisions(
+ series: Sequence[dict[str, Any]], internals: dict[str, dict[str, Any]]
+) -> tuple[list[dict[str, Any]], list[dict[str, Any]], list[dict[str, Any]], list[dict[str, Any]]]:
+ cohorts: list[dict[str, Any]] = []
+ for kind in ("library", "chip", "system", "routing"):
+ groups: dict[bytes, list[tuple[dict[str, Any], Any, list[str], list[str]]]] = {}
+ for item in series:
+ if kind == "library" and item["backend"]["role"] != "library":
+ continue
+ if kind == "system" and item["backend"]["role"] != "reference":
+ continue
+ control, controlled, varying, variant = _cohort_control(kind, item, internals[item["series_id"]])
+ groups.setdefault(_canonical(control), []).append((item, variant, controlled, varying))
+ for entries in groups.values():
+ variants = {_canonical(entry[1]) for entry in entries}
+ if len(entries) < 2 or len(variants) < 2:
+ continue
+ members = sorted((entry[0] for entry in entries), key=lambda item: item["series_id"])
+ token_sets = [set(point["tokens_per_rank"] for point in member["points"]) for member in members]
+ tokens = sorted(set.intersection(*token_sets))
+ same_points = len({tuple(sorted(values)) for values in token_sets}) == 1
+ ordering, aligned_runs = _cohort_ordering(members, internals, tokens) if tokens else (False, 0)
+ allocations = sorted({value for member in members for value in member["allocation_ids"]})
+ p50_ratio = max(
+ (member["eligibility"]["p50_max_min_ratio"] for member in members
+ if member["eligibility"]["p50_max_min_ratio"] is not None), default=None
+ )
+ p99_ratio = max(
+ (member["eligibility"]["p99_max_min_ratio"] for member in members
+ if member["eligibility"]["p99_max_min_ratio"] is not None), default=None
+ )
+ extra = {
+ reason for member in members for reason in member["eligibility"]["reasons"]
+ if reason not in {"unstable-ordering"}
+ }
+ if aligned_runs < REQUIRED_ALLOCATIONS:
+ extra.add("incomplete-aligned-repeats")
+ if kind == "routing" and sum(
+ member["workload"]["routing"] == "uniform"
+ and not member["workload"]["eplb"]
+ for member in members
+ ) != 1:
+ extra.add("missing-uniform-baseline")
+ if kind == "routing" and {
+ (member["workload"]["routing"], member["workload"]["eplb"])
+ for member in members
+ } != {("uniform", False), ("zipf", False), ("zipf", True)}:
+ extra.add("incomplete-routing-anchors")
+ if kind == "routing" and _routing_implementation_mismatch(members):
+ extra.add("implementation-config-mismatch")
+ if not tokens or (kind != "routing" and not same_points):
+ extra.add("unmatched-token-coverage")
+ eligibility = _eligibility_record(
+ allocations,
+ complete=all(member["eligibility"]["complete"] for member in members)
+ and bool(tokens) and (kind == "routing" or same_points),
+ correct=all(member["eligibility"]["correct"] for member in members),
+ measured=all(member["eligibility"]["measured_roundtrip_p99"] for member in members),
+ stable_ordering=ordering,
+ p50_ratio=p50_ratio,
+ p99_ratio=p99_ratio,
+ extra_reasons=sorted(extra),
+ )
+ member_ids = [member["series_id"] for member in members]
+ publication_tier = (
+ "comparable-experimental"
+ if any(member["publication_tier"] == "comparable-experimental" for member in members)
+ else "official"
+ )
+ controlled, varying = entries[0][2], entries[0][3]
+ cohort_id = _derived_id("cxcohort-v1-", {
+ "kind": kind, "series_ids": member_ids,
+ "controlled_factors": controlled, "varying_factors": varying,
+ })
+ kind_label = "Platform" if kind == "chip" else kind.title()
+ first = members[0]
+ routing_label = first["workload"]["routing"] + (
+ "+EPLB" if first["workload"]["eplb"] else ""
+ )
+ context = {
+ "library": (
+ f"{first['system']['sku'].upper()} EP{first['system']['ep_size']} / "
+ f"{first['mode']} / {first['phase']} / {routing_label}"
+ ),
+ "chip": (
+ f"{first['backend']['label']} EP{first['system']['ep_size']} / "
+ f"{first['mode']} / {first['phase']} / {routing_label}"
+ ),
+ "system": (
+ f"Reference EP{first['system']['ep_size']} / {first['mode']} / "
+ f"{first['phase']} / {routing_label}"
+ ),
+ "routing": (
+ f"{first['system']['sku'].upper()} / {first['backend']['label']} / "
+ f"EP{first['system']['ep_size']} / {first['mode']} / {first['phase']}"
+ ),
+ }[kind]
+ cohorts.append({
+ "cohort_id": cohort_id,
+ "kind": kind,
+ "label": f"{context} / {kind_label} contrast ({len(members)} series)",
+ "description": (
+ "Publisher-controlled NCCL/RCCL system comparison"
+ if kind == "system"
+ else f"Publisher-controlled {kind_label.lower()} comparison"
+ ),
+ "series_ids": member_ids,
+ "controlled_factors": controlled,
+ "varying_factors": varying,
+ "publication_tier": publication_tier,
+ "eligibility": eligibility,
+ })
+ cohorts.sort(key=lambda item: item["cohort_id"])
+ series_by_id = {item["series_id"]: item for item in series}
+ rankings: list[dict[str, Any]] = []
+ recommendations: list[dict[str, Any]] = []
+ sensitivities: list[dict[str, Any]] = []
+ for cohort in cohorts:
+ if not cohort["eligibility"]["decision_grade"]:
+ continue
+ members = [series_by_id[series_id] for series_id in cohort["series_ids"]]
+ tokens = sorted(set.intersection(*(
+ {point["tokens_per_rank"] for point in member["points"]} for member in members
+ )))
+ for token in tokens:
+ for measure, objective, unit in (
+ ("latency_us", "min", "us"), ("logical_payload_rate_gbps_at_latency_percentile", "max", "GB/s")
+ ):
+ for statistic in ("p50", "p99"):
+ metric = {
+ "operation": "roundtrip", "statistic": statistic,
+ "measure": measure, "objective": objective,
+ "tokens_per_rank": token, "phase": members[0]["phase"],
+ }
+ entries = []
+ for member in members:
+ point_id, value, observed_unit = _metric_value(member, metric)
+ if observed_unit != unit:
+ raise PublisherError("publisher metric unit differs")
+ entries.append({
+ "rank": 0, "series_id": member["series_id"], "point_id": point_id,
+ "value": value, "unit": unit,
+ })
+ entries.sort(key=lambda item: (item["value"], item["series_id"]), reverse=objective == "max")
+ for rank, entry in enumerate(entries, 1):
+ entry["rank"] = rank
+ ranking_id = _derived_id("cxranking-v1-", {
+ "cohort_id": cohort["cohort_id"], "metric": metric,
+ })
+ metric_label = _metric_label(measure, statistic)
+ rankings.append({
+ "ranking_id": ranking_id, "cohort_id": cohort["cohort_id"],
+ "label": f"{cohort['kind'].title()} {metric_label} T={token}",
+ "metric": metric, "entries": entries,
+ "publication_tier": cohort["publication_tier"],
+ "eligibility": cohort["eligibility"],
+ })
+ if cohort["publication_tier"] != "official":
+ continue
+ objective_name = (
+ f"min-{statistic}-latency"
+ if measure == "latency_us"
+ else f"max-payload-rate-at-{statistic}-latency"
+ )
+ top = entries[0]
+ recommendation_id = _derived_id("cxrecommendation-v1-", {
+ "objective": objective_name, "ranking_id": ranking_id,
+ })
+ recommendations.append({
+ "recommendation_id": recommendation_id,
+ "cohort_id": cohort["cohort_id"],
+ "label": f"Best {metric_label} at T={token}",
+ "objective": objective_name,
+ "series_id": top["series_id"], "point_id": top["point_id"],
+ "value": top["value"], "unit": top["unit"],
+ "rationale": "Top stable measured roundtrip result in a controlled cohort",
+ "publication_tier": cohort["publication_tier"],
+ "eligibility": cohort["eligibility"],
+ })
+ if cohort["kind"] == "routing":
+ baseline = next(
+ (member for member in members
+ if member["workload"]["routing"] == "uniform" and not member["workload"]["eplb"]),
+ None,
+ )
+ if baseline:
+ for candidate in members:
+ if candidate is baseline:
+ continue
+ for token in tokens:
+ for measure, objective in (("latency_us", "min"), ("logical_payload_rate_gbps_at_latency_percentile", "max")):
+ for statistic in ("p50", "p99"):
+ metric = {
+ "operation": "roundtrip", "statistic": statistic,
+ "measure": measure, "objective": objective,
+ "tokens_per_rank": token, "phase": baseline["phase"],
+ }
+ _, base_value, _ = _metric_value(baseline, metric)
+ _, candidate_value, _ = _metric_value(candidate, metric)
+ sensitivity_id = _derived_id("cxsensitivity-v1-", {
+ "baseline": baseline["series_id"], "candidate": candidate["series_id"],
+ "cohort": cohort["cohort_id"], "metric": metric,
+ })
+ sensitivities.append({
+ "sensitivity_id": sensitivity_id,
+ "cohort_id": cohort["cohort_id"],
+ "label": (
+ f"Routing sensitivity: "
+ f"{_metric_label(measure, statistic)} T={token}"
+ ),
+ "baseline_series_id": baseline["series_id"],
+ "candidate_series_id": candidate["series_id"],
+ "metric": metric,
+ "signed_change_ratio": (candidate_value - base_value) / base_value,
+ "publication_tier": cohort["publication_tier"],
+ "eligibility": cohort["eligibility"],
+ })
+ rankings.sort(key=lambda item: item["ranking_id"])
+ recommendations.sort(key=lambda item: item["recommendation_id"])
+ sensitivities.sort(key=lambda item: item["sensitivity_id"])
+ return cohorts, rankings, recommendations, sensitivities
+
+
+def _require_runnable_promotion_success(
+ bundles: Sequence[dict[str, Any]], cases: dict[str, dict[str, Any]]
+) -> None:
+ for bundle in bundles:
+ for case_id, case in cases.items():
+ if case["_disposition"] != "runnable":
+ continue
+ status, _ = _outcome(bundle["selected"][case_id])
+ if status != "success":
+ raise PublisherError(
+ "promotion requires every runnable matrix case to succeed "
+ "in every selected bundle"
+ )
+ prior_statuses = {
+ _outcome(document)[0]
+ for document in bundle["documents"].values()
+ if document["identity"]["case_id"] == case_id
+ }
+ if prior_statuses != {"success"}:
+ raise PublisherError(
+ "promotion rejects runnable cases with failed, invalid, or diagnostic retries"
+ )
+
+
+def _expected_chip_cohort_count(series: Sequence[dict[str, Any]]) -> int:
+ groups: dict[bytes, set[bytes]] = {}
+ for item in series:
+ control, variant = _public_cohort_factors("chip", item)
+ groups.setdefault(_canonical(control), set()).add(_canonical(variant))
+ return sum(len(variants) >= 2 for variants in groups.values())
+
+
+def _require_promotion_cohorts(
+ cohorts: Sequence[dict[str, Any]], series: Sequence[dict[str, Any]]
+) -> None:
+ eligible_kinds = {
+ cohort["kind"]
+ for cohort in cohorts
+ if cohort["eligibility"]["decision_grade"]
+ }
+ missing = [kind for kind in REQUIRED_COHORT_KINDS if kind not in eligible_kinds]
+ if missing:
+ raise PublisherError(
+ "promotion lacks decision-grade cohort kinds: " + ", ".join(missing)
+ )
+ for kind, expected in REQUIRED_PROMOTION_COHORT_COUNTS.items():
+ members = [cohort for cohort in cohorts if cohort["kind"] == kind]
+ if len(members) != expected or any(
+ not cohort["eligibility"]["decision_grade"] for cohort in members
+ ):
+ raise PublisherError(
+ f"promotion requires exactly {expected} decision-grade {kind} cohorts"
+ )
+
+ chip_cohorts = [cohort for cohort in cohorts if cohort["kind"] == "chip"]
+ expected_chips = _expected_chip_cohort_count(series)
+ if len(chip_cohorts) != expected_chips or any(
+ not cohort["eligibility"]["decision_grade"] for cohort in chip_cohorts
+ ):
+ raise PublisherError(
+ f"promotion requires all {expected_chips} derived chip cohorts to be decision-grade"
+ )
+
+ by_id = {item["series_id"]: item for item in series}
+ anchors = {("uniform", False), ("zipf", False), ("zipf", True)}
+ for cohort in (
+ item for item in cohorts
+ if item["kind"] == "routing" and item["eligibility"]["decision_grade"]
+ ):
+ observed = {
+ (by_id[series_id]["workload"]["routing"], by_id[series_id]["workload"]["eplb"]):
+ by_id[series_id]
+ for series_id in cohort["series_ids"]
+ }
+ if len(cohort["series_ids"]) != len(anchors) or set(observed) != anchors:
+ raise PublisherError(
+ "promotion routing cohorts require exact uniform, zipf, and zipf+EPLB anchors"
+ )
+ if (
+ observed[("uniform", False)]["build"]["implementation_contract_sha256"]
+ != observed[("zipf", False)]["build"]["implementation_contract_sha256"]
+ ):
+ raise PublisherError(
+ "promotion routing cohorts require identical off-EPLB generated implementation"
+ )
+
+
+def _require_promotion_series(series: Sequence[dict[str, Any]]) -> None:
+ if not series or any(item["status"] != "decision-grade" for item in series):
+ raise PublisherError("promotion has unstable or incomplete required series")
+
+
+def build_dataset(
+ store: Store,
+ bundle_ids: Sequence[str],
+ *,
+ promote: bool,
+) -> dict[str, Any]:
+ if not bundle_ids or len(bundle_ids) != len(set(bundle_ids)):
+ raise PublisherError("dataset requires unique explicit bundle IDs")
+ loaded = [load_bundle(store, bundle_id) for bundle_id in bundle_ids]
+ loaded.sort(key=lambda bundle: (
+ int(bundle["manifest"]["run"]["run_id"]),
+ bundle["manifest"]["run"]["run_attempt"],
+ bundle["id"],
+ ))
+ matrix_ids = {bundle["manifest"]["matrix"]["sha256"] for bundle in loaded}
+ case_sets = [{case["case_id"] for case in bundle["cases"]} for bundle in loaded]
+ if len(matrix_ids) != 1 or len({tuple(sorted(values)) for values in case_sets}) != 1:
+ raise PublisherError("dataset bundles do not share one exact requested matrix")
+ run_ids = [bundle["manifest"]["run"]["run_id"] for bundle in loaded]
+ if promote and (
+ len(loaded) != REQUIRED_ALLOCATIONS
+ or len(run_ids) != len(set(run_ids))
+ ):
+ raise PublisherError("promotion requires three independent complete workflow runs")
+ if promote and matrix_ids != {CANONICAL_FULL_V1_MATRIX_SHA256}:
+ raise PublisherError("promotion requires the canonical full-v1 matrix")
+ cases = {case["case_id"]: case for case in loaded[0]["cases"]}
+ if promote:
+ _require_runnable_promotion_success(loaded, cases)
+ all_documents = [
+ document for bundle in loaded for document in bundle["documents"].values()
+ ]
+ selected_ids = {
+ selection["selected_attempt_id"]
+ for bundle in loaded for selection in bundle["manifest"]["coverage"]["selections"]
+ }
+ public_attempts = [
+ _public_attempt(
+ document, selected=document["identity"]["attempt_id"] in selected_ids
+ )
+ for document in all_documents
+ ]
+ _unique([attempt["attempt_id"] for attempt in public_attempts], "dataset attempts")
+ selected_by_case: dict[str, list[dict[str, Any]]] = {
+ case_id: [bundle["selected"][case_id] for bundle in loaded]
+ for case_id in sorted(cases)
+ }
+ coverage: list[dict[str, Any]] = []
+ for case_id, case in sorted(cases.items()):
+ attempts = sorted(
+ (attempt for attempt in public_attempts if attempt["case_id"] == case_id),
+ key=lambda attempt: (
+ int(attempt["run_id"]), attempt["run_attempt"],
+ attempt["attempt_index"], attempt["attempt_id"],
+ ),
+ )
+ selected = _public_attempt(selected_by_case[case_id][-1], selected=True)
+ coverage.append({
+ "case_id": case_id,
+ "label": (
+ f"{case['sku'].upper()} / {case['backend']} / EP{case['ep']} / "
+ f"{case['mode']} / {case['phase']} / {case['routing']}"
+ ),
+ "required": True,
+ "sku": _slug(case["sku"]),
+ "backend": _slug(case["backend"]),
+ "mode": case["mode"],
+ "phase": case["phase"],
+ "topology": _coverage_topology(case),
+ "disposition": case["_disposition"],
+ "selected_attempt_id": selected["attempt_id"],
+ "outcome": selected["outcome"],
+ "failure_mode": selected["failure_mode"],
+ "reason": case["_reason"] if case["_disposition"] == "unsupported" else selected["reason"],
+ "attempt_ids": [attempt["attempt_id"] for attempt in attempts],
+ })
+ by_series: dict[str, list[dict[str, Any]]] = {}
+ for case_documents in selected_by_case.values():
+ for document in case_documents:
+ if (
+ document["format"] == contracts.RAW_FORMAT
+ and document["outcome"]["status"] == "success"
+ ):
+ by_series.setdefault(document["identity"]["series_id"], []).append(document)
+ series: list[dict[str, Any]] = []
+ internals: dict[str, dict[str, Any]] = {}
+ for series_id, documents in sorted(by_series.items()):
+ item, internal = _build_series(series_id, documents, len(loaded))
+ series.append(item)
+ internals[series_id] = internal
+ cohorts, rankings, recommendations, sensitivities = build_decisions(series, internals)
+ allocation_ids = sorted({attempt["allocation_id"] for attempt in public_attempts})
+ status = "promoted" if promote else "diagnostic"
+ dataset = {
+ "format": FORMAT_PUBLIC,
+ "schema_version": 1,
+ "generated_at": _latest_timestamp(
+ [bundle["manifest"]["created_at"] for bundle in loaded]
+ ),
+ "source_bundle_ids": sorted(bundle_ids),
+ "promotion": {
+ "status": status,
+ "reason": None,
+ "matrix_id": next(iter(matrix_ids)),
+ "allocation_ids": allocation_ids,
+ "required_allocations": REQUIRED_ALLOCATIONS,
+ "requested_cases": len(coverage),
+ "terminal_cases": len(coverage),
+ "policy": POLICY,
+ },
+ "coverage": coverage,
+ "attempts": sorted(public_attempts, key=lambda attempt: attempt["attempt_id"]),
+ "series": series,
+ "cohorts": cohorts,
+ "rankings": rankings,
+ "recommendations": recommendations,
+ "sensitivities": sensitivities,
+ }
+ if promote:
+ _require_promotion_series(series)
+ _require_promotion_cohorts(cohorts, series)
+ validate_public_dataset(dataset)
+ return dataset
+
+
+def _quarantine_dataset(reason: str, generated_at: str) -> dict[str, Any]:
+ dataset = {
+ "format": FORMAT_PUBLIC,
+ "schema_version": 1,
+ "generated_at": generated_at,
+ "source_bundle_ids": [],
+ "promotion": {
+ "status": "quarantined",
+ "reason": reason,
+ "matrix_id": None,
+ "allocation_ids": [],
+ "required_allocations": REQUIRED_ALLOCATIONS,
+ "requested_cases": 0,
+ "terminal_cases": 0,
+ "policy": POLICY,
+ },
+ "coverage": [],
+ "attempts": [],
+ "series": [],
+ "cohorts": [],
+ "rankings": [],
+ "recommendations": [],
+ "sensitivities": [],
+ }
+ validate_public_dataset(dataset)
+ return dataset
+
+
+def quarantine_incoming(
+ store: Store, ingest_id: str, reason: str, generated_at: str
+) -> str:
+ if REASON.fullmatch(reason) is None:
+ raise PublisherError("quarantine reason must be a machine code")
+ public_reason = f"{reason}-{ingest_id}"
+ if REASON.fullmatch(public_reason) is None:
+ raise PublisherError("quarantine reason and incoming ID exceed the public reason contract")
+ manifest = {
+ "format": "collectivex.quarantine.v1",
+ "schema_version": 1,
+ "created_at": generated_at,
+ "incoming_id": ingest_id,
+ "reason": reason,
+ }
+ digest = _sha_bytes(_canonical(manifest))
+ with store.staging(store.quarantine, private=True) as stage:
+ _write_json(stage / "quarantine.json", manifest, mode=0o600)
+ store.complete(stage, digest, private=True)
+ store.install(stage, store.quarantine / digest, private=True)
+ if _sha_bytes(_canonical(strict_load(store.quarantine / digest / "quarantine.json"))) != digest:
+ raise PublisherError("existing quarantine object differs")
+ # The incoming digest distinguishes separate rejected deliveries while preserving
+ # byte-identical output when the operator retries the same immutable input.
+ dataset = _quarantine_dataset(public_reason, generated_at)
+ dataset_digest, size = store.install_dataset(dataset)
+ store.update_channel("latest-attempt", dataset_digest, size, generated_at)
+ return digest
+
+
+def _store_from_args(args: argparse.Namespace) -> Store:
+ root = args.store_root or os.environ.get("COLLECTIVEX_STORE_ROOT")
+ if not root:
+ raise PublisherError("COLLECTIVEX_STORE_ROOT or --store-root is required")
+ if not Path(root).is_absolute():
+ raise PublisherError("COLLECTIVEX_STORE_ROOT must be an absolute path")
+ return Store(root)
+
+
+def _run_metadata(args: argparse.Namespace) -> dict[str, Any]:
+ """Validate offline operator assertions about a completed successful GHA run.
+
+ The publisher deliberately performs no network access. The caller must preflight workflow
+ identity and conclusion against GitHub before supplying these values; artifact-internal
+ provenance is then required to match them exactly.
+ """
+ run = {
+ "repository": args.repository,
+ "run_id": args.run_id,
+ "run_attempt": args.run_attempt,
+ "source_sha": args.source_sha,
+ }
+ # Reuse the authoritative private schema constraints before any filesystem mutation.
+ if not re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", run["repository"] or ""):
+ raise PublisherError("--repository must be owner/name")
+ if not re.fullmatch(r"[1-9][0-9]*", run["run_id"] or ""):
+ raise PublisherError("--run-id must be a positive decimal string")
+ if type(run["run_attempt"]) is not int or run["run_attempt"] < 1:
+ raise PublisherError("--run-attempt must be positive")
+ if not re.fullmatch(r"[0-9a-f]{40}", run["source_sha"] or ""):
+ raise PublisherError("--source-sha must be a 40-character lowercase Git SHA")
+ return run
+
+
+def _ingest_inputs(
+ args: argparse.Namespace,
+) -> tuple[dict[str, Any], Path, list[Path]]:
+ run = _run_metadata(args)
+ matrix = Path(args.matrix).absolute()
+ if matrix.is_symlink() or not matrix.is_file():
+ raise PublisherError("--matrix must be a regular non-symlink file")
+ artifacts = [Path(value).absolute() for value in args.artifact]
+ if not artifacts:
+ raise PublisherError("at least one --artifact is required")
+ names = [_artifact_name(path) for path in artifacts]
+ if len(names) != len(set(names)):
+ raise PublisherError("--artifact contains duplicate GHA names")
+ for path in artifacts:
+ if path.is_symlink() or not (path.is_dir() or path.is_file()):
+ raise PublisherError("--artifact must be a regular ZIP or real directory")
+ return run, matrix, artifacts
+
+
+def _bundle_ids(values: Sequence[str], *, promote: bool) -> list[str]:
+ bundle_ids = list(values)
+ if (
+ not bundle_ids
+ or len(bundle_ids) != len(set(bundle_ids))
+ or any(HEX64.fullmatch(value) is None for value in bundle_ids)
+ ):
+ raise PublisherError("bundle IDs must be unique SHA-256 digests")
+ if promote and len(bundle_ids) != REQUIRED_ALLOCATIONS:
+ raise PublisherError("promotion requires exactly three explicit bundle IDs")
+ return bundle_ids
+
+
+def ingest_command(args: argparse.Namespace) -> dict[str, Any]:
+ run, matrix, artifacts = _ingest_inputs(args)
+ store = _store_from_args(args)
+ with store.locked():
+ ingest_id, incoming, _ = archive_incoming(
+ store, matrix, artifacts, run
+ )
+ try:
+ bundle_id, _, _ = build_bundle(store, ingest_id, incoming, run)
+ dataset = build_dataset(store, [bundle_id], promote=False)
+ dataset_id, size = store.install_dataset(dataset)
+ store.update_channel(
+ "latest-attempt", dataset_id, size, dataset["generated_at"]
+ )
+ store.verify_channel("latest-attempt")
+ return {
+ "status": "accepted", "incoming_id": ingest_id,
+ "bundle_id": bundle_id, "dataset_sha256": dataset_id,
+ "channel": "latest-attempt",
+ }
+ except (
+ PublisherError, contracts.ContractError, artifact_safety.ArtifactSafetyError,
+ jsonschema.ValidationError,
+ ) as exc:
+ # Invalid delivery bytes provide no trusted timestamp. A fixed sentinel keeps
+ # repeated quarantine of the same immutable incoming object content-idempotent.
+ generated_at = "1970-01-01T00:00:00Z"
+ quarantine_id = quarantine_incoming(
+ store, ingest_id, "artifact-validation-failed", generated_at
+ )
+ raise PublisherError(
+ f"incoming {ingest_id} quarantined as {quarantine_id}: {exc}"
+ ) from exc
+
+
+def promote_command(args: argparse.Namespace) -> dict[str, Any]:
+ bundle_ids = _bundle_ids(args.bundle, promote=True)
+ store = _store_from_args(args)
+ with store.locked():
+ dataset = build_dataset(store, bundle_ids, promote=True)
+ digest, size = store.install_dataset(dataset)
+ store.update_channel("dev-latest", digest, size, dataset["generated_at"])
+ store.verify_channel("dev-latest")
+ return {
+ "status": "promoted", "bundle_ids": bundle_ids,
+ "dataset_sha256": digest, "channel": "dev-latest",
+ }
+
+
+def verify_command(args: argparse.Namespace) -> dict[str, Any]:
+ bundle_ids = _bundle_ids(args.bundle, promote=False) if args.bundle else []
+ channels = args.channel or ["latest-attempt"]
+ if any(channel not in {"latest-attempt", "dev-latest"} for channel in channels):
+ raise PublisherError("unknown channel")
+ store = _store_from_args(args)
+ if args.channel is None and (store.channels / "dev-latest.json").is_file():
+ channels.append("dev-latest")
+ with store.locked():
+ pointers = {channel: store.verify_channel(channel) for channel in channels}
+ bundles = [load_bundle(store, bundle_id)["id"] for bundle_id in bundle_ids]
+ return {"status": "verified", "channels": pointers, "bundle_ids": bundles}
+
+
+def _parser() -> argparse.ArgumentParser:
+ parser = argparse.ArgumentParser(description="CollectiveX isolated filesystem publisher")
+ parser.add_argument("--store-root", help="defaults to COLLECTIVEX_STORE_ROOT")
+ subparsers = parser.add_subparsers(dest="command", required=True)
+ ingest = subparsers.add_parser("ingest", help="archive and validate one complete GHA run")
+ ingest.add_argument("--matrix", required=True)
+ ingest.add_argument("--artifact", action="append", required=True)
+ ingest.add_argument("--repository", required=True)
+ ingest.add_argument("--run-id", required=True)
+ ingest.add_argument("--run-attempt", required=True, type=int)
+ ingest.add_argument("--source-sha", required=True)
+ promote = subparsers.add_parser("promote", help="publish explicit independent bundles")
+ promote.add_argument("--bundle", action="append", required=True)
+ verify = subparsers.add_parser("verify", help="verify immutable targets and pointers")
+ verify.add_argument("--channel", action="append", choices=["latest-attempt", "dev-latest"])
+ verify.add_argument("--bundle", action="append", default=[])
+ return parser
+
+
+def main() -> int:
+ args = _parser().parse_args()
+ try:
+ if args.command == "ingest":
+ result = ingest_command(args)
+ elif args.command == "promote":
+ result = promote_command(args)
+ elif args.command == "verify":
+ result = verify_command(args)
+ else:
+ raise PublisherError(f"unknown command {args.command!r}")
+ except (
+ PublisherError, contracts.ContractError, artifact_safety.ArtifactSafetyError,
+ jsonschema.ValidationError, OSError,
+ ) as exc:
+ print(json.dumps({"status": "error", "error": str(exc)}), file=sys.stderr)
+ return 2
+ print(json.dumps(result, sort_keys=True))
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/experimental/CollectiveX/requirements.txt b/experimental/CollectiveX/requirements.txt
new file mode 100644
index 0000000000..f68f97d83d
--- /dev/null
+++ b/experimental/CollectiveX/requirements.txt
@@ -0,0 +1,8 @@
+# Host-side matrix generation. GPU libraries are supplied by benchmark images.
+PyYAML==6.0.2
+
+# Canonical workload serialization.
+numpy>=1.26,<3
+
+# Host-only strict artifact publisher schemas (never imported by GPU execution).
+jsonschema==4.25.1
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
new file mode 100644
index 0000000000..1f6a32c2da
--- /dev/null
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -0,0 +1,2339 @@
+# shellcheck shell=bash
+# CollectiveX — shared launcher helpers (sourced, not executed).
+#
+# Cluster-generic scaffolding only (Slurm/container/build/staging); no
+# model-serving. Logging goes to stderr so functions can `echo` a single
+# result on stdout.
+
+_CX_COMMON_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+CX_SQUASH_FORMAT_VERSION="repro-v1"
+CX_SQUASH_SOURCE_DATE_EPOCH=1
+CX_DEEPEP_V2_COMMIT="fa8a9b16898204afd347c663b89e65ef87dc6ce6" # pragma: allowlist secret
+CX_DEEPEP_V2_TREE="29809e75c5874e6609dac4804e7b651d5226959f" # pragma: allowlist secret
+CX_DEEPEP_V2_FMT_COMMIT="a4c7e17133ee9cb6a2f45545f6e974dd3c393efa" # pragma: allowlist secret
+CX_DEEPEP_HYBRID_COMMIT="e0a5b1d9848ab3e7b4a67842bf06f067bfac67f8" # pragma: allowlist secret
+CX_DEEPEP_HYBRID_TREE="d77aeab7f1bb52b615666fe178d26ced41fae08e" # pragma: allowlist secret
+CX_DEEPEP_HYBRID_NCCL_COMMIT="1e0c869c39bb33f1034cb9920bd2a8a8406f04a3" # pragma: allowlist secret
+unset COLLECTIVEX_OPERATOR_CONFIG_LOADED COLLECTIVEX_EPHEMERAL_CONFIG_PATH
+
+cx_log() { printf '[collectivex] %s\n' "$*" >&2; }
+cx_die() { printf '[collectivex] FATAL: %s\n' "$*" >&2; exit 1; }
+
+# Public failure telemetry is a closed vocabulary. Raw scheduler, container,
+# host, and filesystem diagnostics stay in the mode-0600 private logs.
+cx_set_failure_stage() {
+ local stage="$1"
+ case "$stage" in
+ setup|repository-stage|registry-verification|scheduler-allocation|container-import) ;;
+ container-hash|container-launch|backend-setup|execution|artifact-collection) ;;
+ *) cx_die "invalid launcher failure stage" ;;
+ esac
+ export CX_FAILSAFE_MODE="$stage"
+}
+
+cx_fail_stage() {
+ local stage="$1" log_path="${2:-}" diagnostic="unknown"
+ cx_set_failure_stage "$stage"
+ if [ -n "$log_path" ] && [ -f "$log_path" ]; then
+ if grep -aEqi 'no space left|disk quota|quota exceeded' "$log_path"; then
+ diagnostic="storage-capacity"
+ elif grep -aEqi 'permission denied|operation not permitted|read-only file system|source mount (creation|ownership validation|permission inspection|permission normalization|permission validation) failed' "$log_path"; then
+ diagnostic="storage-permission"
+ elif grep -aEqi 'outside one realized LSA domain|lsa(Size| team| domain).*(mismatch|invalid|expected)|ranks.*not in (one|the same) nvlink.domain' "$log_path" \
+ || { [ "${CX_BENCH:-}" = deepep-v2 ] \
+ && grep -aEqi 'nccl[.]cu:(111|112)([^0-9]|$)' "$log_path"; }; then
+ diagnostic="accelerator-topology"
+ elif grep -aEqi 'cuda driver version is insufficient|call requires newer driver|cudaErrorCallRequiresNewerDriver|CUDA_ERROR_SYSTEM_DRIVER_MISMATCH|unsupported toolchain' "$log_path"; then
+ diagnostic="accelerator-driver"
+ elif grep -aEqi 'ncclDevCommCreate|ncclCommWindowRegister|ncclGetLsa(Device)?Pointer|Communicator does not support symmetric memory|Symmetric memory is not supported' "$log_path" \
+ || { [ "${CX_BENCH:-}" = deepep-v2 ] \
+ && grep -aEqi 'nccl[.]cu:(106|127|128|129|135)([^0-9]|$)' "$log_path"; }; then
+ diagnostic="nccl-device-api"
+ elif grep -aEqi 'NVCC (PTX )?compilation failed|cuobjdump failed|invalid device (kernel )?image|no kernel image is available' "$log_path"; then
+ diagnostic="jit-toolchain"
+ elif grep -aEqi 'cuda out of memory|CUDA_ERROR_OUT_OF_MEMORY|out of memory.*cuda' "$log_path"; then
+ diagnostic="accelerator-memory"
+ elif grep -aEqi 'does not match its pinned image contract|requires the exact pinned|version mismatch' "$log_path"; then
+ diagnostic="backend-version"
+ elif grep -aEqi 'nvshmem is unavailable|build-tool installation failed' "$log_path"; then
+ diagnostic="backend-dependency"
+ elif grep -aEqi 'revision fetch failed|submodule fetch failed|package installation failed|staged source is invalid|source (pin resolution|seed validation|seed copy|checkout creation|publication validation|existing source validation) failed' "$log_path"; then
+ diagnostic="backend-source"
+ elif grep -aEqi 'failed to mount|squashfs|enroot|pyxis|mount.*invalid argument|invalid argument.*mount' "$log_path"; then
+ diagnostic="container-runtime"
+ elif grep -aEqi 'backend preparation failed|build (failed|is incomplete)|cache (mount identity )?validation failed|import failed' "$log_path"; then
+ diagnostic="backend-build"
+ elif grep -aEqi 'command not found|not found on this runner|git lookup failed' "$log_path"; then
+ diagnostic="missing-runtime"
+ elif grep -aEqi 'too many requests|rate.?limit' "$log_path"; then
+ diagnostic="registry-rate-limit"
+ elif grep -aEqi 'timed out|operation timeout|wait timeout after|watchdog.*timeout|timeout: sending signal|connection reset|could not resolve|TLS|certificate' "$log_path"; then
+ diagnostic="network-or-timeout"
+ elif grep -aEqi 'salloc:|srun:.*(unable to create step|step creation|invalid partition|invalid account)|unable to create step|job allocation' "$log_path"; then
+ diagnostic="scheduler"
+ elif grep -aEqi 'SHARD done: [0-9]+/[0-9]+ case\(s\) failed|WARN: .* run failed rc=|completed with invalid semantic evidence' "$log_path"; then
+ diagnostic="benchmark-case-failure"
+ elif [ -s "$log_path" ]; then
+ diagnostic="unclassified"
+ else
+ diagnostic="empty-log"
+ fi
+ fi
+ cx_log "ERROR: failure-class=$stage diagnostic=$diagnostic"
+ return 1
+}
+
+# Runner-local deployment settings are strict JSON kept outside the checkout.
+# Only the selected runner's allowlisted values are exported; the document is
+# never sourced or evaluated as shell.
+cx_load_operator_config() {
+ [ -n "${COLLECTIVEX_OPERATOR_CONFIG_LOADED:-}" ] \
+ && [ "$COLLECTIVEX_OPERATOR_CONFIG_LOADED" = "$$" ] && return 0
+ local config_path generated=0 parsed_path config_log key value
+ unset CX_PARTITION CX_ACCOUNT CX_SQUASH_DIR CX_STAGE_DIR CX_ENROOT_CACHE_PATH
+ unset ENROOT_CACHE_PATH
+ unset CX_EXCLUDE_NODES CX_NODELIST CX_LOCK_DIR CX_MASTER_PORT
+ unset CX_SOCKET_IFNAME CX_RDMA_DEVICES CX_IB_GID_INDEX CX_RDMA_SERVICE_LEVEL
+ unset MASTER_ADDR MASTER_PORT RANK WORLD_SIZE LOCAL_RANK LOCAL_WORLD_SIZE
+ config_path="${COLLECTIVEX_OPERATOR_CONFIG:-${XDG_CONFIG_HOME:-${HOME}/.config}/inferencex/collectivex.json}"
+ if [ -n "${COLLECTIVEX_OPERATOR_CONFIG_CONTENT:-}" ]; then
+ umask 077
+ if [[ "${CX_JOB_ROOT:-}" =~ ^/tmp/inferencex-collectivex-[0-9]+-[0-9]+-[A-Za-z0-9._-]+$ ]] \
+ && [ -d "$CX_JOB_ROOT" ] && [ ! -L "$CX_JOB_ROOT" ] \
+ && [ "$(stat -c '%u:%a' "$CX_JOB_ROOT" 2>/dev/null)" = "$(id -u):700" ]; then
+ config_path="$CX_JOB_ROOT/operator-config.json"
+ (set -C; : > "$config_path") 2>/dev/null \
+ || cx_die "cannot create ephemeral runner configuration"
+ else
+ config_path="$(mktemp /tmp/inferencex-collectivex-config.XXXXXX)" \
+ || cx_die "cannot create ephemeral runner configuration"
+ fi
+ COLLECTIVEX_EPHEMERAL_CONFIG_PATH="$config_path"
+ generated=1
+ if ! printf '%s' "$COLLECTIVEX_OPERATOR_CONFIG_CONTENT" > "$config_path"; then
+ unset COLLECTIVEX_OPERATOR_CONFIG_CONTENT
+ rm -f -- "$config_path"
+ unset COLLECTIVEX_EPHEMERAL_CONFIG_PATH
+ cx_die "cannot materialize runner configuration"
+ fi
+ elif [ "${COLLECTIVEX_OPERATOR_CONFIG_REQUIRED:-0}" = 1 ]; then
+ unset COLLECTIVEX_OPERATOR_CONFIG_CONTENT
+ cx_die "runner configuration is unavailable"
+ fi
+ unset COLLECTIVEX_OPERATOR_CONFIG_CONTENT COLLECTIVEX_OPERATOR_CONFIG_REQUIRED
+ if [ ! -e "$config_path" ]; then
+ COLLECTIVEX_OPERATOR_CONFIG_LOADED="$$"
+ return 0
+ fi
+ umask 077
+ parsed_path="$(mktemp /tmp/inferencex-collectivex-parsed.XXXXXX)" || {
+ [ "$generated" = 0 ] || rm -f -- "$config_path"
+ cx_die "cannot parse runner configuration"
+ }
+ config_log="$(cx_private_log_path operator-config)"
+ if ! python3 - "$config_path" "${CX_RUNNER:-${CX_SHARD_SKU:-${CX_PUBLIC_RUNNER:-}}}" \
+ > "$parsed_path" 2> "$config_log" <<'PY'
+import json
+import os
+import posixpath
+import re
+import stat
+import sys
+
+RUNNERS = {
+ "h100-dgxc", "h200-dgxc", "b200-dgxc", "b300",
+ "gb200", "gb300", "mi325x", "mi355x",
+}
+FIELDS = {
+ "partition": "CX_PARTITION",
+ "account": "CX_ACCOUNT",
+ "squash_dir": "CX_SQUASH_DIR",
+ "stage_dir": "CX_STAGE_DIR",
+ "enroot_cache_path": "CX_ENROOT_CACHE_PATH",
+ "exclude_nodes": "CX_EXCLUDE_NODES",
+ "nodelist": "CX_NODELIST",
+ "lock_dir": "CX_LOCK_DIR",
+ "socket_ifname": "CX_SOCKET_IFNAME",
+ "rdma_devices": "CX_RDMA_DEVICES",
+ "ib_gid_index": "CX_IB_GID_INDEX",
+ "rdma_service_level": "CX_RDMA_SERVICE_LEVEL",
+}
+NETWORK_FIELDS = {
+ "socket_ifname", "rdma_devices", "ib_gid_index", "rdma_service_level",
+}
+REQUIRED = {
+ "h100-dgxc": {"partition", "account", "squash_dir", "stage_dir"},
+ "h200-dgxc": {"partition", "squash_dir", "stage_dir"},
+ "b200-dgxc": {"partition", "account", "squash_dir", "stage_dir"},
+ "b300": {
+ "partition", "account", "squash_dir", "stage_dir",
+ },
+ "gb200": {"partition", "account", "storage_roots"},
+ "gb300": {"partition", "account", "squash_dir", "stage_dir", "enroot_cache_path"},
+ "mi325x": {"partition", "squash_dir", "stage_dir"},
+ "mi355x": {"partition", "squash_dir", "stage_dir"},
+}
+ALLOWED = {
+ "h100-dgxc": REQUIRED["h100-dgxc"] | {"exclude_nodes", "stage_dir"} | NETWORK_FIELDS,
+ "h200-dgxc": REQUIRED["h200-dgxc"] | {"account", "exclude_nodes", "stage_dir"} | NETWORK_FIELDS,
+ "b200-dgxc": REQUIRED["b200-dgxc"] | {"exclude_nodes", "stage_dir"} | NETWORK_FIELDS,
+ "b300": REQUIRED["b300"] | {"exclude_nodes"} | NETWORK_FIELDS,
+ "gb200": REQUIRED["gb200"] | NETWORK_FIELDS,
+ "gb300": REQUIRED["gb300"] | NETWORK_FIELDS,
+ "mi325x": REQUIRED["mi325x"] | {"exclude_nodes", "nodelist", "stage_dir", "lock_dir"} | NETWORK_FIELDS,
+ "mi355x": REQUIRED["mi355x"] | {"exclude_nodes", "nodelist", "stage_dir", "lock_dir"} | NETWORK_FIELDS,
+}
+TOKEN = re.compile(r"^[A-Za-z0-9_.\[\],-]+$")
+PATH = re.compile(r"^/[A-Za-z0-9._/+\-]+$")
+IPV4 = re.compile(r"(? 65536
+ ):
+ raise ValueError
+ flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
+ descriptor = os.open(path, flags)
+ try:
+ opened = os.fstat(descriptor)
+ if (opened.st_dev, opened.st_ino) != (metadata.st_dev, metadata.st_ino):
+ raise ValueError
+ payload = b""
+ while len(payload) <= 65536:
+ chunk = os.read(descriptor, 65537 - len(payload))
+ if not chunk:
+ break
+ payload += chunk
+ document = json.loads(
+ payload.decode("utf-8"),
+ object_pairs_hook=pairs,
+ parse_constant=lambda _: (_ for _ in ()).throw(ValueError()),
+ )
+ finally:
+ os.close(descriptor)
+ if (
+ set(document) != {"schema_version", "runners"}
+ or type(document["schema_version"]) is not int
+ or document["schema_version"] != 1
+ ):
+ raise ValueError
+ runners = document["runners"]
+ if (
+ not isinstance(runners, dict) or not runners or set(runners) - RUNNERS
+ or runner not in runners
+ ):
+ raise ValueError
+ selected = None
+ for name, config in runners.items():
+ if (
+ not isinstance(config, dict)
+ or (name == runner and not REQUIRED[name].issubset(config))
+ ):
+ raise ValueError
+ if set(config) - ALLOWED[name]:
+ raise ValueError
+ for field, value in config.items():
+ if field == "storage_roots":
+ if (
+ not isinstance(value, list) or not 1 <= len(value) <= 16
+ or len(value) != len(set(value)) or not all(valid_path(item) for item in value)
+ ):
+ raise ValueError
+ elif field == "socket_ifname":
+ if not isinstance(value, str) or not INTERFACES.fullmatch(value):
+ raise ValueError
+ elif field == "rdma_devices":
+ if not isinstance(value, str) or not RDMA_DEVICES.fullmatch(value):
+ raise ValueError
+ elif field == "ib_gid_index":
+ if type(value) is not int or not 0 <= value <= 255:
+ raise ValueError
+ elif field == "rdma_service_level":
+ if type(value) is not int or not 0 <= value <= 15:
+ raise ValueError
+ elif field.endswith(("_dir", "_path")):
+ if not valid_path(value):
+ raise ValueError
+ elif (
+ not isinstance(value, str) or not value or len(value) > 512
+ or not TOKEN.fullmatch(value) or IPV4.search(value)
+ ):
+ raise ValueError
+ if name == runner:
+ selected = dict(config)
+ if selected is None:
+ raise ValueError
+ roots = selected.pop("storage_roots", None)
+ if roots is not None:
+ for root in roots:
+ squash = posixpath.join(root, "collectivex", "containers")
+ stage = posixpath.join(root, "collectivex", "stage")
+ probes = []
+ try:
+ for directory in (squash, stage):
+ os.makedirs(directory, mode=0o700, exist_ok=True)
+ probe = posixpath.join(directory, f".write-probe-{os.getpid()}")
+ fd = os.open(probe, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
+ os.close(fd)
+ probes.append(probe)
+ selected.update(squash_dir=squash, stage_dir=stage)
+ break
+ except OSError:
+ pass
+ finally:
+ for probe in probes:
+ try:
+ os.unlink(probe)
+ except OSError:
+ pass
+ else:
+ raise ValueError
+ for field, value in selected.items():
+ key = FIELDS[field]
+ sys.stdout.buffer.write(
+ key.encode() + b"\0" + str(value).encode() + b"\0"
+ )
+except (KeyError, OSError, TypeError, UnicodeError, ValueError):
+ raise SystemExit(1)
+PY
+ then
+ rm -f -- "$parsed_path"
+ [ "$generated" = 0 ] || rm -f -- "$config_path"
+ unset COLLECTIVEX_EPHEMERAL_CONFIG_PATH
+ unset COLLECTIVEX_OPERATOR_CONFIG COLLECTIVEX_OPERATOR_CONFIG_EPHEMERAL
+ cx_die "runner-local configuration failed"
+ fi
+ while IFS= read -r -d '' key && IFS= read -r -d '' value; do
+ printf -v "$key" '%s' "$value"
+ export "${key?}"
+ done < "$parsed_path"
+ rm -f -- "$parsed_path"
+ if [ "$generated" = 1 ] || [ "${COLLECTIVEX_OPERATOR_CONFIG_EPHEMERAL:-0}" = 1 ]; then
+ rm -f -- "$config_path" || cx_die "cannot remove ephemeral runner configuration"
+ fi
+ unset COLLECTIVEX_EPHEMERAL_CONFIG_PATH
+ unset COLLECTIVEX_OPERATOR_CONFIG COLLECTIVEX_OPERATOR_CONFIG_EPHEMERAL
+ COLLECTIVEX_OPERATOR_CONFIG_LOADED="$$"
+}
+
+cx_private_log_path() {
+ local label="$1" tag="${COLLECTIVEX_EXECUTION_ID:-manual_$$}" path
+ path="$(python3 - "$tag" "$label" <<'PY' 2>/dev/null
+import os
+import re
+import shutil
+import stat
+import sys
+import time
+
+tag, label = sys.argv[1:]
+if not all(re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]*", value) for value in (tag, label)):
+ raise SystemExit(1)
+root = f"/tmp/inferencex-collectivex-{os.getuid()}"
+old_umask = os.umask(0o077)
+flags = os.O_RDONLY | os.O_DIRECTORY | getattr(os, "O_NOFOLLOW", 0)
+try:
+ try:
+ os.mkdir(root, 0o700)
+ except FileExistsError:
+ pass
+ root_fd = os.open(root, flags)
+ try:
+ metadata = os.fstat(root_fd)
+ if metadata.st_uid != os.getuid() or stat.S_IMODE(metadata.st_mode) != 0o700:
+ raise OSError("unsafe root")
+ cutoff = time.time() - 86400
+ for entry in os.scandir(root):
+ try:
+ if (
+ entry.name != tag and entry.is_dir(follow_symlinks=False)
+ and entry.stat(follow_symlinks=False).st_mtime < cutoff
+ ):
+ shutil.rmtree(entry.path)
+ except OSError:
+ pass
+ try:
+ os.mkdir(tag, 0o700, dir_fd=root_fd)
+ except FileExistsError:
+ pass
+ directory_fd = os.open(tag, flags, dir_fd=root_fd)
+ try:
+ metadata = os.fstat(directory_fd)
+ if metadata.st_uid != os.getuid() or stat.S_IMODE(metadata.st_mode) != 0o700:
+ raise OSError("unsafe directory")
+ log_flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0)
+ log_fd = os.open(f"{label}.log", log_flags, 0o600, dir_fd=directory_fd)
+ os.close(log_fd)
+ finally:
+ os.close(directory_fd)
+ finally:
+ os.close(root_fd)
+finally:
+ os.umask(old_umask)
+print(f"{root}/{tag}/{label}.log", end="")
+PY
+)" || cx_die "cannot create private runtime log"
+ printf '%s' "$path"
+}
+
+# Manual successes delete diagnostics immediately. Canonical workflow logs survive
+# until artifact upload succeeds; failed logs remain private for debugging, and a
+# later run prunes abandoned directories older than 24 hours.
+cx_cleanup_private_logs() {
+ local rc="$1" tag="${COLLECTIVEX_EXECUTION_ID:-manual_$$}"
+ [ "$rc" = 0 ] || return 0
+ python3 - "$tag" <<'PY' >/dev/null 2>&1 || true
+import os
+import re
+import shutil
+import stat
+import sys
+
+tag = sys.argv[1]
+if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]*", tag):
+ raise SystemExit(1)
+root = f"/tmp/inferencex-collectivex-{os.getuid()}"
+flags = os.O_RDONLY | os.O_DIRECTORY | getattr(os, "O_NOFOLLOW", 0)
+root_fd = os.open(root, flags)
+try:
+ metadata = os.fstat(root_fd)
+ if metadata.st_uid != os.getuid() or stat.S_IMODE(metadata.st_mode) != 0o700:
+ raise SystemExit(1)
+finally:
+ os.close(root_fd)
+path = os.path.join(root, tag)
+if os.path.isdir(path) and not os.path.islink(path):
+ shutil.rmtree(path)
+PY
+}
+
+# Explicit Slurm export boundary. Operator config, runner credentials, HOME,
+# workspace paths, and unrelated service secrets never enter the container.
+cx_container_exports() {
+ printf '%s' 'COLLECTIVEX_SOURCE_SHA,COLLECTIVEX_ARTIFACT_NAME,COLLECTIVEX_EXECUTION_ID,COLLECTIVEX_CONTROL_SHA256,COLLECTIVEX_IMAGE,COLLECTIVEX_IMAGE_DIGEST,COLLECTIVEX_IMAGE_DIGEST_VERIFIED,COLLECTIVEX_SQUASH_SHA256,GITHUB_REF_NAME,GITHUB_REF,GITHUB_REPOSITORY,GITHUB_JOB,GITHUB_RUN_ID,GITHUB_RUN_ATTEMPT,GITHUB_SHA,CX_RUNNER,CX_BENCH,CX_NODES,CX_GPUS_PER_NODE,CX_SCALE_UP_DOMAIN,CX_SHARD_FILE,CX_SHARD_SKU,CX_NGPUS,CX_TS,CX_TOPO,CX_SCOPE,CX_TRANSPORT,CX_SCALE_UP_TRANSPORT,CX_SCALE_OUT_TRANSPORT,CX_MODE,CX_PHASE,CX_ROUTING,CX_EPLB,CX_CASE_ID,CX_SUITE,CX_WORKLOAD_NAME,CX_REQUIRED_PUBLICATION,CX_HIDDEN,CX_TOPK,CX_EXPERTS,CX_TOKENS_LADDER,CX_CANONICAL,CX_ITERS,CX_TRIALS,CX_WARMUP,CX_SAMPLES_PER_POINT,CX_WARMUP_SEMANTICS,CX_SEED,CX_RUN_TIMEOUT,CX_NCCL_HOME,CX_ALLOW_MNNVL,CX_ATTEMPT_ID,CX_RUNTIME_MARKER,CX_MORI_KERNEL_TYPE,CX_WORKLOAD_DIR,CX_BACKEND_CACHE_ROOT,CX_BACKEND_CACHE_SENTINEL_SHA256,CX_BACKEND_SOURCE_ROOT,MASTER_ADDR,MASTER_PORT,RANK,WORLD_SIZE,LOCAL_RANK,LOCAL_WORLD_SIZE,NCCL_NET,NCCL_SOCKET_IFNAME,GLOO_SOCKET_IFNAME,NCCL_IB_HCA,NCCL_IB_GID_INDEX,NCCL_IB_SL,NVSHMEM_HCA_LIST,NVSHMEM_IB_GID_INDEX,NVSHMEM_IB_SL,NVSHMEM_IB_ENABLE_IBGDA,NVSHMEM_IBGDA_NIC_HANDLER,EP_NIC_NAME,EP_OVERRIDE_RDMA_SL,UCCL_SOCKET_IFNAME,UCCL_IB_GID_INDEX,UCCL_IB_SL,MORI_RDMA_DEVICES,HYBRID_EP_MULTINODE,USE_NIXL,RDMA_CORE_HOME,DEEPEP_HYBRID_BUILD_MODE,NCCL_CUMEM_ENABLE,NCCL_MNNVL_ENABLE,MC_FORCE_MNNVL,MORI_DISABLE_AUTO_XGMI,MORI_ENABLE_SDMA,MORI_APP_LOG_LEVEL,MORI_SHMEM_LOG_LEVEL,MORI_IO_LOG_LEVEL'
+ printf '%s' ',MORI_COMMIT'
+}
+
+# Host-side utility steps need only the basic login paths. They never receive
+# the complete Actions or runner environment.
+cx_host_exports() {
+ printf '%s' 'HOME,PATH,USER,XDG_CACHE_HOME,ENROOT_CACHE_PATH'
+}
+
+cx_prepare_runtime_marker() {
+ local mount_src="$1" tag="${COLLECTIVEX_EXECUTION_ID:-${CX_TS:-}}" marker
+ [[ "$tag" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] \
+ || cx_die "cannot create runtime stage marker"
+ marker=".shards/runtime-stage-${tag}.txt"
+ mkdir -p "$mount_src/experimental/CollectiveX/.shards" >/dev/null 2>&1 \
+ || cx_die "cannot create runtime stage marker"
+ rm -f -- "$mount_src/experimental/CollectiveX/$marker" >/dev/null 2>&1 \
+ || cx_die "cannot reset runtime stage marker"
+ export CX_RUNTIME_MARKER="$marker"
+}
+
+cx_write_runtime_stage() {
+ local stage="$1" marker="${CX_RUNTIME_MARKER:-}"
+ [ -n "$marker" ] || return 0
+ [[ "$marker" =~ ^\.shards/runtime-stage-[A-Za-z0-9][A-Za-z0-9._-]*\.txt$ ]] \
+ || return 1
+ case "$stage" in backend-setup|execution) ;; *) return 1 ;; esac
+ printf '%s\n' "$stage" > "$marker"
+}
+
+cx_adopt_runtime_stage() {
+ local mount_src="$1" marker="${CX_RUNTIME_MARKER:-}" stage=""
+ [ -n "$marker" ] || return 0
+ if [[ "$marker" =~ ^\.shards/runtime-stage-[A-Za-z0-9][A-Za-z0-9._-]*\.txt$ ]] \
+ && [ -f "$mount_src/experimental/CollectiveX/$marker" ]; then
+ IFS= read -r stage < "$mount_src/experimental/CollectiveX/$marker" || true
+ rm -f -- "$mount_src/experimental/CollectiveX/$marker" >/dev/null 2>&1 || true
+ case "$stage" in
+ backend-setup|execution) cx_set_failure_stage "$stage" ;;
+ esac
+ fi
+}
+
+cx_require_vars() {
+ local name
+ local -a missing=()
+ for name in "$@"; do
+ [ -n "${!name:-}" ] || missing+=("$name")
+ done
+ [ "${#missing[@]}" -eq 0 ] || cx_die \
+ "missing runner-local configuration: ${missing[*]} (set them in COLLECTIVEX_OPERATOR_CONFIG)"
+}
+
+cx_bool_enabled() {
+ local normalized
+ normalized="$(printf '%s' "$1" | tr '[:upper:]' '[:lower:]')"
+ case "$normalized" in
+ 1|true|yes) return 0 ;;
+ *) return 1 ;;
+ esac
+}
+
+cx_require_record_safe() {
+ local value
+ for value in "$@"; do
+ case "$value" in
+ *'|'*|*$'\n'*|*$'\r'*) cx_die "manual case field contains a record delimiter" ;;
+ esac
+ done
+}
+
+cx_require_single_node() {
+ [ "${CX_NODES:-1}" = "1" ] || cx_die "$1 supports one-node EP only"
+}
+
+# Convert private, runner-local network selectors into the public library
+# variables needed inside the container. Values are interface/HCA identifiers,
+# never addresses; the rendezvous hostname is derived from the allocation.
+cx_apply_network_profile() {
+ local nodes="$1" transport="$2" selector rdma_name rdma_names="" ep_nic=""
+ local -a selectors
+ [[ "$nodes" =~ ^[1-9][0-9]*$ ]] || cx_die "invalid network placement"
+ unset NCCL_NET NCCL_SOCKET_IFNAME GLOO_SOCKET_IFNAME NCCL_IB_HCA
+ unset NCCL_IB_GID_INDEX NCCL_IB_SL
+ unset NVSHMEM_HCA_LIST NVSHMEM_IB_GID_INDEX NVSHMEM_IB_SL
+ unset NVSHMEM_IB_ENABLE_IBGDA NVSHMEM_IBGDA_NIC_HANDLER
+ unset EP_NIC_NAME EP_OVERRIDE_RDMA_SL
+ unset UCCL_SOCKET_IFNAME UCCL_IB_GID_INDEX UCCL_IB_SL MORI_RDMA_DEVICES
+ [ "$nodes" -gt 1 ] && [ "$transport" != mnnvl ] || return 0
+ [ -n "${CX_SOCKET_IFNAME:-}" ] && [ -n "${CX_RDMA_DEVICES:-}" ] \
+ || cx_die "multi-node execution requires private socket and RDMA selectors"
+ if [ -n "${CX_SOCKET_IFNAME:-}" ]; then
+ [[ "$CX_SOCKET_IFNAME" =~ ^[A-Za-z][A-Za-z0-9_.-]{0,31}(,[A-Za-z][A-Za-z0-9_.-]{0,31})*$ ]] \
+ || cx_die "invalid private socket interface selector"
+ export NCCL_SOCKET_IFNAME="$CX_SOCKET_IFNAME" GLOO_SOCKET_IFNAME="$CX_SOCKET_IFNAME"
+ export UCCL_SOCKET_IFNAME="$CX_SOCKET_IFNAME"
+ fi
+ if [ -n "${CX_RDMA_DEVICES:-}" ]; then
+ [[ "$CX_RDMA_DEVICES" =~ ^[A-Za-z][A-Za-z0-9_.-]{0,31}(:[1-9][0-9]*)?(,[A-Za-z][A-Za-z0-9_.-]{0,31}(:[1-9][0-9]*)?)*$ ]] \
+ || cx_die "invalid private RDMA device selector"
+ IFS=, read -r -a selectors <<< "$CX_RDMA_DEVICES"
+ for selector in "${selectors[@]}"; do
+ rdma_name="${selector%%:*}"
+ rdma_names="${rdma_names}${rdma_names:+,}${rdma_name}"
+ [ -n "$ep_nic" ] || ep_nic="$rdma_name"
+ done
+ export NCCL_NET=IB NCCL_IB_HCA="=$CX_RDMA_DEVICES"
+ export NVSHMEM_HCA_LIST="$CX_RDMA_DEVICES"
+ export MORI_RDMA_DEVICES="$rdma_names" EP_NIC_NAME="$ep_nic"
+ fi
+ if [ -n "${CX_IB_GID_INDEX:-}" ]; then
+ [[ "$CX_IB_GID_INDEX" =~ ^[0-9]+$ ]] && [ "$CX_IB_GID_INDEX" -le 255 ] \
+ || cx_die "invalid private IB GID index"
+ export NCCL_IB_GID_INDEX="$CX_IB_GID_INDEX" NVSHMEM_IB_GID_INDEX="$CX_IB_GID_INDEX"
+ export UCCL_IB_GID_INDEX="$CX_IB_GID_INDEX"
+ fi
+ if [ -n "${CX_RDMA_SERVICE_LEVEL:-}" ]; then
+ [[ "$CX_RDMA_SERVICE_LEVEL" =~ ^[0-9]+$ ]] && [ "$CX_RDMA_SERVICE_LEVEL" -le 15 ] \
+ || cx_die "invalid private RDMA service level"
+ export NCCL_IB_SL="$CX_RDMA_SERVICE_LEVEL" NVSHMEM_IB_SL="$CX_RDMA_SERVICE_LEVEL"
+ export UCCL_IB_SL="$CX_RDMA_SERVICE_LEVEL"
+ export EP_OVERRIDE_RDMA_SL="$CX_RDMA_SERVICE_LEVEL"
+ fi
+ export NVSHMEM_IB_ENABLE_IBGDA=1 NVSHMEM_IBGDA_NIC_HANDLER=gpu
+}
+
+# Prove that the operator-pinned scale-out fabric exists on every allocated
+# node before image import or backend initialization. Selector values and node
+# diagnostics stay in the runner-private log.
+cx_validate_network_profile_on_job() {
+ local job_id="$1" nodes="$2" transport="$3" log rc=0
+ [ "$nodes" -gt 1 ] && [ "$transport" != mnnvl ] || return 0
+ [[ "$job_id" =~ ^[1-9][0-9]*$ && "$nodes" =~ ^[1-9][0-9]*$ ]] \
+ || return 1
+ [ -n "${CX_SOCKET_IFNAME:-}" ] && [ -n "${CX_RDMA_DEVICES:-}" ] \
+ || return 1
+ log="$(cx_private_log_path network-profile)" || return 1
+ srun --jobid="$job_id" --nodes="$nodes" --ntasks="$nodes" --ntasks-per-node=1 \
+ --chdir=/tmp --input=all \
+ --export="$(cx_host_exports),CX_SOCKET_IFNAME,CX_RDMA_DEVICES,CX_IB_GID_INDEX" \
+ bash -s > "$log" 2>&1 <<'BASH' || rc=$?
+set -euo pipefail
+[[ "$CX_SOCKET_IFNAME" =~ ^[A-Za-z][A-Za-z0-9_.-]{0,31}(,[A-Za-z][A-Za-z0-9_.-]{0,31})*$ ]]
+[[ "$CX_RDMA_DEVICES" =~ ^[A-Za-z][A-Za-z0-9_.-]{0,31}(:[1-9][0-9]*)?(,[A-Za-z][A-Za-z0-9_.-]{0,31}(:[1-9][0-9]*)?)*$ ]]
+if [ -n "${CX_IB_GID_INDEX:-}" ]; then
+ [[ "$CX_IB_GID_INDEX" =~ ^[0-9]+$ ]] && [ "$CX_IB_GID_INDEX" -le 255 ]
+fi
+IFS=, read -r -a interfaces <<< "$CX_SOCKET_IFNAME"
+for interface in "${interfaces[@]}"; do
+ [ -d "/sys/class/net/$interface" ]
+ state="$(cat "/sys/class/net/$interface/operstate")"
+ [ "$state" = up ] || [ "$state" = unknown ]
+done
+check_port() {
+ local port_path="$1" state gid
+ [ -d "$port_path" ] || return 1
+ read -r state _ < "$port_path/state"
+ [ "$state" = 4: ] || return 1
+ if [ -n "${CX_IB_GID_INDEX:-}" ]; then
+ [ -r "$port_path/gids/$CX_IB_GID_INDEX" ] || return 1
+ gid="$(tr -d ':0[:space:]' < "$port_path/gids/$CX_IB_GID_INDEX")"
+ [ -n "$gid" ] || return 1
+ fi
+}
+IFS=, read -r -a devices <<< "$CX_RDMA_DEVICES"
+for selector in "${devices[@]}"; do
+ device="${selector%%:*}"
+ configured_port=""
+ [ "$selector" = "$device" ] || configured_port="${selector#*:}"
+ ports="/sys/class/infiniband/$device/ports"
+ [ -d "$ports" ]
+ if [ -n "$configured_port" ]; then
+ check_port "$ports/$configured_port"
+ else
+ active=0
+ for port_path in "$ports"/*; do
+ if check_port "$port_path"; then
+ active=1
+ break
+ fi
+ done
+ [ "$active" = 1 ]
+ fi
+done
+BASH
+ if [ "$rc" != 0 ]; then
+ cx_fail_stage setup "$log" || true
+ return "$rc"
+ fi
+}
+
+cx_resolve_slurm_rendezvous() {
+ local job_id="$1" nodes master_addr master_port
+ [[ "$job_id" =~ ^[1-9][0-9]*$ ]] || cx_die "invalid rendezvous allocation"
+ nodes="$(squeue -j "$job_id" -h -o %N 2>/dev/null)"
+ master_addr="$(scontrol show hostnames "$nodes" 2>/dev/null | head -n1)"
+ master_port="${CX_MASTER_PORT:-29551}"
+ [[ "$master_addr" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] \
+ || cx_die "could not resolve the allocated primary node"
+ [[ "$master_port" =~ ^[1-9][0-9]*$ ]] && [ "$master_port" -le 65535 ] \
+ || cx_die "invalid distributed rendezvous port"
+ export MASTER_ADDR="$master_addr" MASTER_PORT="$master_port"
+}
+
+# Printed into `bash -c` for one Slurm task per GPU. Every rank derives its
+# identity from Slurm rather than accepting caller-supplied rank values.
+cx_slurm_rank_wrapper() {
+ cat <<'BASH'
+case "${SLURM_PROCID:-}:${SLURM_NTASKS:-}:${SLURM_LOCALID:-}:${SLURM_NODEID:-}" in
+ *[!0-9:]*|:*|*::*|*:) exit 67 ;;
+esac
+[ "$SLURM_NTASKS" = "$CX_NGPUS" ] || exit 67
+[ "$SLURM_LOCALID" -lt "$CX_GPUS_PER_NODE" ] || exit 67
+export RANK="$SLURM_PROCID" WORLD_SIZE="$SLURM_NTASKS"
+export LOCAL_RANK="$SLURM_LOCALID" LOCAL_WORLD_SIZE="$CX_GPUS_PER_NODE"
+exec python3 tests/run_ep.py "$@"
+BASH
+}
+
+# A set shard path is an execution contract, never a hint. Validate it before
+# staging/allocation and again in-container so a missing or stale control file
+# cannot silently fall back to a manual single-case run.
+cx_validate_shard_control() {
+ local cx_root="$1" shard="${CX_SHARD_FILE:-}" path expected_sku control_sha256
+ [ -n "$shard" ] || return 0
+ expected_sku="${CX_SHARD_SKU:-}"
+ [ -n "$expected_sku" ] || cx_die "CX_SHARD_SKU is required with CX_SHARD_FILE"
+ [ -n "${CX_BENCH:-}" ] || cx_die "CX_BENCH is required with CX_SHARD_FILE"
+ [[ "${CX_NODES:-}" =~ ^[1-9][0-9]*$ ]] \
+ || cx_die "positive CX_NODES is required with CX_SHARD_FILE"
+ path="$shard"
+ [ -f "$path" ] || path="${cx_root%/}/$shard"
+ [ -f "$path" ] || cx_die "shard control does not exist"
+ [ -s "$path" ] || cx_die "shard control is empty"
+ python3 "${cx_root%/}/sweep_matrix.py" \
+ --validate-control "$path" --expect-sku "$expected_sku" \
+ --expect-backend "$CX_BENCH" --expect-nodes "$CX_NODES" >/dev/null 2>&1 \
+ || cx_die "invalid shard control"
+ control_sha256="$(sha256sum "$path" | awk '{print $1}')"
+ [[ "$control_sha256" =~ ^[0-9a-f]{64}$ ]] \
+ || cx_die "cannot hash shard control"
+ export COLLECTIVEX_CONTROL_SHA256="$control_sha256"
+}
+
+cx_apply_timing_profile() {
+ [ -n "${CX_TIMING:-}" ] || return 0
+ local iters trials warmup extra
+ IFS=: read -r iters trials warmup extra <<< "$CX_TIMING"
+ [[ "$iters" =~ ^[1-9][0-9]*$ && "$trials" =~ ^[1-9][0-9]*$ \
+ && "$warmup" =~ ^[1-9][0-9]*$ && -z "$extra" ]] \
+ || cx_die "CX_TIMING must be positive iters:trials:warmup"
+ export CX_ITERS="$iters" CX_TRIALS="$trials" CX_WARMUP="$warmup"
+}
+
+# Use an opaque, execution-bound name so a missing grant message can be
+# reconciled without exposing runner or shard details in public logs.
+cx_scheduler_job_name() {
+ local execution_id="${COLLECTIVEX_EXECUTION_ID:-manual-$$}" digest
+ digest="$(printf '%s' "$execution_id" | sha256sum | awk '{print $1}')" \
+ || return 1
+ [[ "$digest" =~ ^[0-9a-f]{64}$ ]] || return 1
+ printf 'cx-%s' "${digest:0:24}"
+}
+
+# Return 0 after recovering one allocation ID, 2 after three successful empty
+# observations, and 1 for every ambiguous or failed lookup. Callers inspect the
+# state variables rather than the status because all missing-ID paths still fail.
+cx_reconcile_salloc_jobid() {
+ local job_name="$1" scheduler_user queue_output line delay attempt
+ local -a ids=()
+ scheduler_user="$(id -un 2>/dev/null)" || return 1
+ [[ "$scheduler_user" =~ ^[A-Za-z0-9_.-]+$ \
+ && "$job_name" =~ ^cx-[0-9a-f]{24}$ ]] || return 1
+ for attempt in 1 2 3; do
+ ids=()
+ if ! queue_output="$(
+ squeue -h --user="$scheduler_user" --name="$job_name" -o %A 2>/dev/null
+ )"; then
+ return 1
+ fi
+ while IFS= read -r line; do
+ [[ "$line" =~ ^[[:space:]]*$ ]] && continue
+ if [[ "$line" =~ ^[[:space:]]*([1-9][0-9]*)[[:space:]]*$ ]]; then
+ ids+=("${BASH_REMATCH[1]}")
+ else
+ return 1
+ fi
+ done <<< "$queue_output"
+ if [ "${#ids[@]}" -eq 1 ]; then
+ JOB_ID="${ids[0]}"
+ CX_ALLOCATION_UNCERTAIN=0
+ return 0
+ fi
+ [ "${#ids[@]}" -eq 0 ] || return 1
+ if [ "$attempt" -eq 3 ]; then
+ CX_ALLOCATION_UNCERTAIN=0
+ return 2
+ fi
+ delay=$((1 << (attempt - 1)))
+ sleep "$delay" || return 1
+ done
+ return 1
+}
+
+# Allocate via salloc's stable grant message and assign JOB_ID in this shell.
+# Raw scheduler output remains in the bounded private execution log.
+cx_salloc_jobid() {
+ local log job_id job_name argument salloc_rc=0
+ log="$(cx_private_log_path scheduler-allocation)"
+ for argument in "$@"; do
+ case "$argument" in
+ --job-name|--job-name=*|-J|-J*)
+ cx_log "ERROR: scheduler job names are managed by CollectiveX"
+ return 1
+ ;;
+ esac
+ done
+ job_name="$(cx_scheduler_job_name)" || return 1
+ CX_ALLOCATION_UNCERTAIN=1
+ # salloc has no portable --parsable option. Parse the stable grant message
+ # used by the production launchers, while also accepting a bare ID from
+ # site wrappers.
+ salloc "$@" --job-name="$job_name" --no-shell > "$log" 2>&1 || salloc_rc=$?
+ job_id="$(sed -nE \
+ -e 's/^([0-9]+)(;[^[:space:]]+)?$/\1/p' \
+ -e 's/.*Granted job allocation ([0-9]+).*/\1/p' \
+ "$log" | head -n1)"
+ if [ -n "$job_id" ]; then
+ [[ "$job_id" =~ ^[0-9]+$ ]] || return 1
+ JOB_ID="$job_id"
+ CX_ALLOCATION_UNCERTAIN=0
+ fi
+ if [ "$salloc_rc" != 0 ]; then
+ if [ "$salloc_rc" -ge 128 ] && [ -z "$JOB_ID" ]; then
+ cx_fail_stage scheduler-allocation "$log"
+ return 1
+ fi
+ [ -n "$JOB_ID" ] || cx_reconcile_salloc_jobid "$job_name" || true
+ cx_fail_stage scheduler-allocation "$log"
+ return 1
+ fi
+ if [ -z "$JOB_ID" ]; then
+ cx_reconcile_salloc_jobid "$job_name" || true
+ cx_fail_stage scheduler-allocation "$log"
+ return 1
+ fi
+}
+
+cx_cancel_job() {
+ local job_id="$1" active delay
+ [[ "$job_id" =~ ^[0-9]+$ ]] || return 1
+ scancel "$job_id" >/dev/null 2>&1 || true
+ for delay in 1 2 4 8 16 32; do
+ if ! active="$(squeue -h -j "$job_id" -o %A 2>/dev/null)"; then
+ sleep "$delay"
+ continue
+ fi
+ [ -n "$active" ] || return 0
+ sleep "$delay"
+ done
+ cx_log "ERROR: scheduled allocation did not terminate during cleanup"
+ return 1
+}
+
+cx_write_cleanup_guard() {
+ local state="$1" root="${CX_JOB_ROOT:-}" safe unsafe
+ [[ "$root" =~ ^/tmp/inferencex-collectivex-[0-9]+-[0-9]+-[A-Za-z0-9._-]+$ ]] \
+ && [ -d "$root" ] && [ ! -L "$root" ] \
+ && [ "$(stat -c '%u:%a' "$root" 2>/dev/null)" = "$(id -u):700" ] || return 0
+ safe="$root/cleanup-safe"
+ unsafe="$root/cleanup-unsafe"
+ umask 077
+ case "$state" in
+ safe) : > "$safe" && rm -f -- "$unsafe" ;;
+ unsafe) rm -f -- "$safe" && : > "$unsafe" ;;
+ *) return 1 ;;
+ esac
+}
+
+# Single multi-arch container for ALL NVIDIA SKUs: tag `v0.5.11-cu130` is an OCI
+# image index covering linux/amd64 (B200) + linux/arm64 (GB200); enroot import
+# pulls the matching arch. (cu130 = CUDA 13, system nccl.h in /usr/include, torch 2.9.x.)
+# Import remains tag-based because Enroot cannot reliably import a digest-qualified
+# Docker Hub reference non-interactively. The registry digest is resolved and checked
+# immediately before import, then recorded as verified provenance.
+CX_IMAGE_MULTIARCH_DIGEST="sha256:061fb71f838e82000a1768c159654d526c2f17ebe751c21e7fc48ca53c8ef975"
+# (v0.5.12-cu130 was rejected: its 62 layers overflow enroot's overlay-based
+# squash creation on these nodes — "failed to mount overlay ... Invalid argument".
+# v0.5.11-cu130 imports cleanly.)
+# Runtime setup verifies the image-bundled DeepEP build for the detected GPU target.
+CX_IMAGE_MULTIARCH="lmsysorg/sglang:v0.5.11-cu130"
+
+# AMD (ROCm/CDNA): separate single-arch images bundle MoRI.
+CX_IMAGE_AMD_MORI="rocm/sgl-dev:sglang-0.5.9-rocm720-mi35x-mori-0227-2"
+CX_IMAGE_AMD_MORI_DIGEST="sha256:24c3b30d64475937abbb6498e3b29528649adcb836dde7a468979f767809b0e8"
+CX_MORI_COMMIT_MI355="99bc0a3a6e7a70aacc6372cd9a4275ccfb4de567" # pragma: allowlist secret
+CX_IMAGE_AMD_MORI_MI325="rocm/sgl-dev:sglang-0.5.14-rocm720-mi35x-mori-0701"
+CX_IMAGE_AMD_MORI_MI325_DIGEST="sha256:ea42375343c2ef8f73b3bdb9e1b7b435556e3ca92aba5e3f74ada29ba217fabc"
+CX_MORI_COMMIT_MI325="bf99bdf18fc69887a346913ca01c315c2aa9bd4c" # pragma: allowlist secret
+cx_default_image() {
+ case "$1" in
+ mi325x*) echo "$CX_IMAGE_AMD_MORI_MI325" ;;
+ mi355x*) echo "$CX_IMAGE_AMD_MORI" ;;
+ b200*|gb200*|b300*|gb300*|h100*|h200*) echo "$CX_IMAGE_MULTIARCH" ;;
+ *) cx_die "no default image for runner prefix: $1" ;;
+ esac
+}
+
+cx_resolve_registry_digest() {
+ local image="$1" repository reference token digest registry
+ if [[ "$image" == *@* ]]; then
+ cx_die "digest-qualified image overrides are unsupported; configure a tag and pinned digest"
+ fi
+ registry="${image%%/*}"
+ if [[ "$image" == */* && ( "$registry" == *.* || "$registry" == *:* || "$registry" = localhost ) ]]; then
+ case "$registry" in
+ docker.io|registry-1.docker.io) image="${image#*/}" ;;
+ *) cx_die "only Docker Hub images are supported by the registry verifier" ;;
+ esac
+ fi
+ repository="${image%:*}"
+ reference="${image##*:}"
+ [ "$repository" != "$image" ] || { repository="$image"; reference=latest; }
+ [ -n "$repository" ] && [ -n "$reference" ] \
+ || cx_die "configured image reference is malformed"
+ [[ "$repository" == */* ]] || repository="library/$repository"
+ token="$(curl -fsSLG --connect-timeout 10 --max-time 30 --retry 2 \
+ --retry-delay 1 --retry-all-errors 'https://auth.docker.io/token' \
+ --data-urlencode 'service=registry.docker.io' \
+ --data-urlencode "scope=repository:${repository}:pull" \
+ | python3 -c 'import json,sys; print(json.load(sys.stdin)["token"])')" \
+ || cx_die "cannot authenticate to the image registry"
+ digest="$(curl -fsSI --connect-timeout 10 --max-time 30 --retry 2 \
+ --retry-delay 1 --retry-all-errors \
+ -H "Authorization: Bearer $token" \
+ -H 'Accept: application/vnd.oci.image.index.v1+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json' \
+ "https://registry-1.docker.io/v2/${repository}/manifests/${reference}" \
+ | tr -d '\r' | awk 'tolower($1)=="docker-content-digest:" {print $2; exit}')" \
+ || cx_die "cannot resolve the configured image digest"
+ [[ "$digest" =~ ^sha256:[0-9a-f]{64}$ ]] \
+ || cx_die "registry returned an invalid image digest"
+ printf '%s' "$digest"
+}
+
+cx_verify_registry_image() {
+ local image="$1" expected actual
+ expected="${CX_IMAGE_DIGEST:-$(cx_default_image_digest "$image")}"
+ [[ "$expected" =~ ^sha256:[0-9a-f]{64}$ ]] \
+ || cx_die "a pinned digest is required for the configured image"
+ actual="$(cx_resolve_registry_digest "$image")"
+ [ "$actual" = "$expected" ] \
+ || cx_die "configured image tag no longer matches its pinned digest"
+ export COLLECTIVEX_IMAGE="$image" COLLECTIVEX_IMAGE_DIGEST="$actual"
+ export COLLECTIVEX_IMAGE_DIGEST_VERIFIED=1
+}
+
+cx_default_image_digest() {
+ case "$1" in
+ "$CX_IMAGE_MULTIARCH") printf '%s' "$CX_IMAGE_MULTIARCH_DIGEST" ;;
+ "$CX_IMAGE_AMD_MORI") printf '%s' "$CX_IMAGE_AMD_MORI_DIGEST" ;;
+ "$CX_IMAGE_AMD_MORI_MI325") printf '%s' "$CX_IMAGE_AMD_MORI_MI325_DIGEST" ;;
+ esac
+}
+
+# Canonical workflow runs must not inherit benchmark controls from a persistent
+# self-hosted runner service. Manual/SSH diagnostics retain their explicit
+# overrides by leaving COLLECTIVEX_CANONICAL_GHA unset.
+cx_gha_workspace_stage_root() {
+ local workspace="${GITHUB_WORKSPACE:-}"
+ python3 - "$workspace" <<'PY'
+import os
+import stat
+import sys
+
+workspace = sys.argv[1]
+try:
+ if (
+ not os.path.isabs(workspace)
+ or os.path.realpath(workspace) != workspace
+ or not os.path.isdir(workspace)
+ ):
+ raise OSError
+ metadata = os.stat(workspace, follow_symlinks=False)
+ # GitHub runner workspaces are runner-owned but commonly writable by the
+ # trusted runner-service group. Keep the child mode 0700 and reject world write.
+ if metadata.st_uid != os.getuid() or stat.S_IMODE(metadata.st_mode) & stat.S_IWOTH:
+ raise OSError
+except OSError:
+ raise SystemExit(1)
+print(workspace, end="")
+PY
+}
+
+# Create a per-UID cache under validated cluster-local storage. Only the fixed
+# /cx-cache mount enters the container; the operator host path does not.
+cx_prepare_backend_cache() {
+ local stage_parent="$1" cache info sentinel_sha256
+ unset CX_PREPARED_BACKEND_CACHE CX_BACKEND_CACHE_SENTINEL_SHA256
+ info="$(python3 - "$stage_parent" <<'PY'
+import hashlib
+import os
+import secrets
+import stat
+import sys
+
+configured_parent = sys.argv[1]
+try:
+ if (
+ not os.path.isabs(configured_parent)
+ or "\n" in configured_parent
+ or "\r" in configured_parent
+ ):
+ raise OSError
+ parent = os.path.realpath(configured_parent)
+ if not os.path.isdir(parent):
+ raise OSError
+ flags = os.O_RDONLY | os.O_DIRECTORY | getattr(os, "O_NOFOLLOW", 0)
+ parent_fd = os.open(parent, flags)
+ try:
+ probe_name = f".collectivex-owner-probe-{os.getpid()}-{secrets.token_hex(8)}"
+ os.mkdir(probe_name, 0o700, dir_fd=parent_fd)
+ try:
+ probe_fd = os.open(probe_name, flags, dir_fd=parent_fd)
+ try:
+ probe = os.fstat(probe_fd)
+ if stat.S_IMODE(probe.st_mode) & 0o777 != 0o700:
+ raise OSError
+ realized_owner = probe.st_uid
+ finally:
+ os.close(probe_fd)
+ finally:
+ os.rmdir(probe_name, dir_fd=parent_fd)
+ for generation in (3, 4):
+ name = f".collectivex-backend-cache-v{generation}-{os.getuid()}"
+ try:
+ os.mkdir(name, 0o700, dir_fd=parent_fd)
+ except FileExistsError:
+ pass
+ try:
+ cache_fd = os.open(name, flags, dir_fd=parent_fd)
+ try:
+ metadata = os.fstat(cache_fd)
+ if (
+ metadata.st_uid != realized_owner
+ or stat.S_IMODE(metadata.st_mode) & 0o777 != 0o700
+ ):
+ raise OSError
+ sentinel_name = ".collectivex-mount-sentinel-v1"
+ temporary_name = (
+ f"{sentinel_name}.tmp.{os.getpid()}.{secrets.token_hex(8)}"
+ )
+ create_flags = (
+ os.O_WRONLY | os.O_CREAT | os.O_EXCL
+ | getattr(os, "O_NOFOLLOW", 0)
+ )
+ payload = secrets.token_bytes(32)
+ temporary_fd = os.open(
+ temporary_name, create_flags, 0o600, dir_fd=cache_fd
+ )
+ try:
+ try:
+ view = memoryview(payload)
+ try:
+ while view:
+ written = os.write(temporary_fd, view)
+ if written <= 0:
+ raise OSError
+ view = view[written:]
+ os.fsync(temporary_fd)
+ finally:
+ view.release()
+ finally:
+ os.close(temporary_fd)
+ try:
+ os.link(
+ temporary_name,
+ sentinel_name,
+ src_dir_fd=cache_fd,
+ dst_dir_fd=cache_fd,
+ follow_symlinks=False,
+ )
+ except FileExistsError:
+ pass
+ finally:
+ try:
+ os.unlink(temporary_name, dir_fd=cache_fd)
+ except FileNotFoundError:
+ pass
+ sentinel_fd = os.open(
+ sentinel_name,
+ os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0),
+ dir_fd=cache_fd,
+ )
+ try:
+ sentinel = os.fstat(sentinel_fd)
+ payload = os.read(sentinel_fd, 33)
+ if (
+ not stat.S_ISREG(sentinel.st_mode)
+ or sentinel.st_uid != realized_owner
+ or stat.S_IMODE(sentinel.st_mode) & 0o777 != 0o600
+ or sentinel.st_size != 32
+ or len(payload) != 32
+ ):
+ raise OSError
+ sentinel_sha256 = hashlib.sha256(payload).hexdigest()
+ finally:
+ os.close(sentinel_fd)
+ finally:
+ os.close(cache_fd)
+ except OSError:
+ if generation == 3:
+ continue
+ raise
+ break
+ finally:
+ os.close(parent_fd)
+except OSError:
+ raise SystemExit(1)
+print(sentinel_sha256, os.path.join(parent, name), end="")
+PY
+)" || return 1
+ sentinel_sha256="${info%% *}"
+ cache="${info#* }"
+ [ "$cache" != "$info" ] && [[ "$sentinel_sha256" =~ ^[0-9a-f]{64}$ ]] \
+ && [[ "$cache" = /* ]] || return 1
+ export CX_PREPARED_BACKEND_CACHE="$cache"
+ export CX_BACKEND_CACHE_SENTINEL_SHA256="$sentinel_sha256"
+}
+
+cx_verify_backend_cache_mount() {
+ python3 - "${CX_BACKEND_CACHE_ROOT:-}" \
+ "${CX_BACKEND_CACHE_SENTINEL_SHA256:-}" <<'PY'
+import hashlib
+import os
+import re
+import stat
+import sys
+
+root, expected = sys.argv[1:]
+try:
+ if (
+ not os.path.isabs(root)
+ or os.path.realpath(root) != root
+ or re.fullmatch(r"[0-9a-f]{64}", expected) is None
+ ):
+ raise OSError
+ flags = os.O_RDONLY | os.O_DIRECTORY | getattr(os, "O_NOFOLLOW", 0)
+ root_fd = os.open(root, flags)
+ try:
+ root_item = os.fstat(root_fd)
+ if (
+ not stat.S_ISDIR(root_item.st_mode)
+ or stat.S_IMODE(root_item.st_mode) & 0o777 != 0o700
+ ):
+ raise OSError
+ sentinel_fd = os.open(
+ ".collectivex-mount-sentinel-v1",
+ os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0),
+ dir_fd=root_fd,
+ )
+ try:
+ sentinel = os.fstat(sentinel_fd)
+ payload = os.read(sentinel_fd, 33)
+ if (
+ not stat.S_ISREG(sentinel.st_mode)
+ or sentinel.st_uid != root_item.st_uid
+ or stat.S_IMODE(sentinel.st_mode) & 0o777 != 0o600
+ or sentinel.st_size != 32
+ or len(payload) != 32
+ or hashlib.sha256(payload).hexdigest() != expected
+ ):
+ raise OSError
+ finally:
+ os.close(sentinel_fd)
+ finally:
+ os.close(root_fd)
+except OSError:
+ raise SystemExit(1)
+PY
+}
+
+cx_git() {
+ GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_TERMINAL_PROMPT=0 \
+ git -c credential.helper= "$@"
+}
+
+cx_git_in_tree() {
+ local directory="$1" canonical
+ shift
+ [[ "$directory" = /* ]] && [ -d "$directory" ] && [ ! -L "$directory" ] \
+ || return 1
+ [[ "$directory" != *'*'* && "$directory" != *$'\n'* && "$directory" != *$'\r'* ]] \
+ || return 1
+ canonical="$(cd -P -- "$directory" && pwd -P)" || return 1
+ cx_git -c "safe.directory=$canonical" -C "$canonical" "$@"
+}
+
+cx_fetch_revision() {
+ local repository="$1" revision="$2" destination="$3" attempt
+ for attempt in 1 2 3; do
+ rm -rf -- "$destination"
+ if cx_git init -q "$destination" \
+ && cx_git_in_tree "$destination" remote add origin "$repository" \
+ && cx_git_in_tree "$destination" fetch -q --no-tags --depth 1 origin "$revision" \
+ && cx_git_in_tree "$destination" -c advice.detachedHead=false \
+ checkout -q --detach FETCH_HEAD \
+ && [ "$(cx_git_in_tree "$destination" rev-parse HEAD)" = "$revision" ]; then
+ return 0
+ fi
+ [ "$attempt" = 3 ] || sleep $((attempt * 5))
+ done
+ return 1
+}
+
+cx_backend_source_pin() {
+ case "$1" in
+ deepep-v2)
+ printf '%s|%s|%s' \
+ "$CX_DEEPEP_V2_COMMIT" "$CX_DEEPEP_V2_TREE" "$CX_DEEPEP_V2_FMT_COMMIT"
+ ;;
+ deepep-hybrid)
+ printf '%s|%s||%s' "$CX_DEEPEP_HYBRID_COMMIT" "$CX_DEEPEP_HYBRID_TREE" \
+ "$CX_DEEPEP_HYBRID_NCCL_COMMIT"
+ ;;
+ *) return 1 ;;
+ esac
+}
+
+cx_backend_source_path() {
+ local root="$1" backend="$2" revision tree fmt nccl pin
+ pin="$(cx_backend_source_pin "$backend")" || return 1
+ IFS='|' read -r revision tree fmt nccl <<< "$pin"
+ printf '%s/%s-%s' "$root" "$backend" "$revision"
+}
+
+cx_backend_source_is_valid() {
+ local backend="$1" source="$2" revision tree fmt nccl pin status ignored
+ pin="$(cx_backend_source_pin "$backend")" || return 1
+ IFS='|' read -r revision tree fmt nccl <<< "$pin"
+ [ -d "$source" ] && [ ! -L "$source" ] \
+ && [ "$(cx_git_in_tree "$source" rev-parse HEAD 2>/dev/null)" = "$revision" ] \
+ && [ "$(cx_git_in_tree "$source" rev-parse 'HEAD^{tree}' 2>/dev/null)" = "$tree" ] \
+ || return 1
+ status="$(cx_git_in_tree "$source" status --porcelain --untracked-files=all \
+ --ignore-submodules=none 2>/dev/null)" || return 1
+ [ -z "$status" ] || return 1
+ ignored="$(cx_git_in_tree "$source" ls-files --others --ignored --exclude-standard \
+ 2>/dev/null)" || return 1
+ [ -z "$ignored" ] || return 1
+ [ -z "$fmt" ] \
+ || [ "$(cx_git_in_tree "$source/third-party/fmt" rev-parse HEAD 2>/dev/null)" = "$fmt" ] \
+ || return 1
+ [ -z "$nccl" ] \
+ || [ "$(cx_git_in_tree "$source/third-party/nccl" rev-parse HEAD 2>/dev/null)" = "$nccl" ]
+}
+
+cx_extension_pair_sha256() {
+ python3 - "$1" "$2" "$3" <<'PY'
+import hashlib
+import os
+from pathlib import Path
+import stat
+import sys
+
+root = Path(sys.argv[1])
+digest = hashlib.sha256()
+try:
+ if root.is_symlink() or not root.is_dir():
+ raise OSError
+ for pattern in sys.argv[2:]:
+ matches = list(root.glob(pattern))
+ if len(matches) != 1 or matches[0].is_symlink():
+ raise OSError
+ path = matches[0]
+ descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
+ try:
+ metadata = os.fstat(descriptor)
+ if not stat.S_ISREG(metadata.st_mode):
+ raise OSError
+ file_digest = hashlib.sha256()
+ with os.fdopen(descriptor, "rb", closefd=False) as stream:
+ for chunk in iter(lambda: stream.read(1024 * 1024), b""):
+ file_digest.update(chunk)
+ digest.update(path.name.encode("utf-8") + b"\0")
+ digest.update(str(metadata.st_size).encode("ascii") + b"\0")
+ digest.update(file_digest.digest())
+ finally:
+ os.close(descriptor)
+except (OSError, UnicodeError):
+ raise SystemExit(1)
+print(digest.hexdigest(), end="")
+PY
+}
+
+# Acquire source before compute allocation, preferring the verified same-run GHA seed.
+_cx_prepare_backend_source() {
+ local mount_src="$1" backend="$2" root source temporary revision tree fmt nccl pin
+ local root_mode stage_mode root_owner stage_owner
+ local seed_root="${CX_BACKEND_SOURCE_SEED_ROOT:-}" seed seed_mode
+ root="$mount_src/experimental/CollectiveX/.cx_sources"
+ CX_BACKEND_SOURCE_STEP="source mount creation"
+ if [ ! -e "$root" ] && [ ! -L "$root" ]; then
+ mkdir -m 700 -- "$root" || return 1
+ fi
+ CX_BACKEND_SOURCE_STEP="source mount ownership validation"
+ [ -d "$mount_src" ] && [ ! -L "$mount_src" ] \
+ && [ -d "$root" ] && [ ! -L "$root" ] || return 1
+ stage_owner="$(stat -c '%u' "$mount_src" 2>/dev/null)" || return 1
+ root_owner="$(stat -c '%u' "$root" 2>/dev/null)" || return 1
+ [ "$root_owner" = "$stage_owner" ] || return 1
+ stage_mode="$(stat -c '%a' "$mount_src" 2>/dev/null)" || return 1
+ case "$stage_mode" in 700|[1-7]700) ;; *) return 1 ;; esac
+ # Shared stage parents may retain harmless special bits despite mkdir -m.
+ CX_BACKEND_SOURCE_STEP="source mount permission inspection"
+ root_mode="$(stat -c '%a' "$root" 2>/dev/null)" || return 1
+ case "$root_mode" in
+ 700|[1-7]700) ;;
+ *)
+ CX_BACKEND_SOURCE_STEP="source mount permission normalization"
+ chmod 700 "$root" || return 1
+ CX_BACKEND_SOURCE_STEP="source mount permission validation"
+ root_mode="$(stat -c '%a' "$root" 2>/dev/null)" || return 1
+ case "$root_mode" in 700|[1-7]700) ;; *) return 1 ;; esac
+ ;;
+ esac
+ CX_BACKEND_SOURCE_STEP="git lookup"
+ command -v git >/dev/null || return 1
+ CX_BACKEND_SOURCE_STEP="source pin resolution"
+ source="$(cx_backend_source_path "$root" "$backend")" || return 1
+ if [ -e "$source" ] || [ -L "$source" ]; then
+ CX_BACKEND_SOURCE_STEP="existing source validation"
+ cx_backend_source_is_valid "$backend" "$source"
+ return
+ fi
+ if [ -n "$seed_root" ]; then
+ CX_BACKEND_SOURCE_STEP="source seed validation"
+ [[ "$seed_root" = /* ]] && [ -d "$seed_root" ] && [ ! -L "$seed_root" ] \
+ || return 1
+ seed_mode="$(stat -c '%a' "$seed_root" 2>/dev/null)" || return 1
+ case "$seed_mode" in 700|[1-7]700) ;; *) return 1 ;; esac
+ seed="$(cx_backend_source_path "$seed_root" "$backend")" || return 1
+ cx_backend_source_is_valid "$backend" "$seed" || return 1
+ CX_BACKEND_SOURCE_STEP="source seed copy"
+ temporary="$(mktemp -d "$root/.${backend}.XXXXXX")" || return 1
+ if ! cp -R -- "$seed/." "$temporary/" \
+ || ! cx_backend_source_is_valid "$backend" "$temporary" \
+ || ! mv -- "$temporary" "$source"; then
+ rm -rf -- "$temporary"
+ return 1
+ fi
+ return
+ fi
+ if [ "${COLLECTIVEX_CANONICAL_GHA:-0}" = 1 ]; then
+ CX_BACKEND_SOURCE_STEP="source seed validation"
+ return 1
+ fi
+ CX_BACKEND_SOURCE_STEP="source checkout creation"
+ temporary="$(mktemp -d "$root/.${backend}.XXXXXX")" || return 1
+ CX_BACKEND_SOURCE_STEP="source pin resolution"
+ pin="$(cx_backend_source_pin "$backend")" || {
+ rm -rf -- "$temporary"
+ return 1
+ }
+ IFS='|' read -r revision tree fmt nccl <<< "$pin"
+ CX_BACKEND_SOURCE_STEP="revision fetch"
+ if ! cx_fetch_revision \
+ https://github.com/deepseek-ai/DeepEP "$revision" "$temporary"; then
+ rm -rf -- "$temporary"
+ return 1
+ fi
+ CX_BACKEND_SOURCE_STEP="submodule fetch"
+ if [ -n "$fmt" ] && ! cx_git_in_tree "$temporary" \
+ -c "safe.directory=$temporary/third-party/fmt" \
+ submodule update -q --init --depth 1 third-party/fmt; then
+ rm -rf -- "$temporary"
+ return 1
+ fi
+ if [ -n "$nccl" ] && ! cx_git_in_tree "$temporary" \
+ -c "safe.directory=$temporary/third-party/nccl" \
+ submodule update -q --init --depth 1 third-party/nccl; then
+ rm -rf -- "$temporary"
+ return 1
+ fi
+ CX_BACKEND_SOURCE_STEP="source publication validation"
+ if ! cx_backend_source_is_valid "$backend" "$temporary" \
+ || ! mv -- "$temporary" "$source"; then
+ rm -rf -- "$temporary"
+ return 1
+ fi
+}
+
+cx_prepare_backend_source() {
+ local log backend="$2" CX_BACKEND_SOURCE_STEP="initialization"
+ log="$(cx_private_log_path "backend-source-$backend")" || return 1
+ if _cx_prepare_backend_source "$@" > "$log" 2>&1; then
+ return 0
+ fi
+ printf '%s failed\n' "$CX_BACKEND_SOURCE_STEP" >> "$log"
+ cx_log "ERROR: backend-source-step=${CX_BACKEND_SOURCE_STEP// /-}"
+ cx_fail_stage backend-setup "$log"
+}
+
+cx_materialize_backend_source() {
+ local backend="$1" destination="$2" source parent temporary
+ [ -n "${CX_BACKEND_SOURCE_ROOT:-}" ] || return 1
+ source="$(cx_backend_source_path "$CX_BACKEND_SOURCE_ROOT" "$backend")" || return 1
+ cx_backend_source_is_valid "$backend" "$source" || return 1
+ parent="${destination%/*}"
+ [ "$parent" != "$destination" ] && [ -d "$parent" ] && [ ! -L "$parent" ] \
+ || return 1
+ temporary="$(mktemp -d "$parent/.collectivex-source.XXXXXX")" || return 1
+ if ! cp -R -- "$source/." "$temporary/" \
+ || ! cx_backend_source_is_valid "$backend" "$temporary"; then
+ rm -rf -- "$temporary"
+ return 1
+ fi
+ if ! rm -rf -- "$destination" || ! mv -- "$temporary" "$destination"; then
+ rm -rf -- "$temporary"
+ return 1
+ fi
+ if ! cx_backend_source_is_valid "$backend" "$destination"; then
+ rm -rf -- "$destination"
+ return 1
+ fi
+ return 0
+}
+
+cx_lock_canonical_gha_env() {
+ local runner="$1" expected_nodes expected_gpn expected_world trusted_lock_dir=""
+ local trusted_stage_dir=""
+ local trusted_socket_ifname="" trusted_rdma_devices=""
+ local trusted_ib_gid_index="" trusted_rdma_service_level=""
+ [ "${COLLECTIVEX_CANONICAL_GHA:-0}" = 1 ] || return 0
+ [ "${GITHUB_ACTIONS:-}" = true ] \
+ || cx_die "canonical CollectiveX execution requires GitHub Actions"
+ [ -n "${CX_SHARD_FILE:-}" ] && [ "${CX_SHARD_SKU:-}" = "$runner" ] \
+ || cx_die "canonical CollectiveX execution requires a matched shard"
+ [[ "${GITHUB_RUN_ID:-}" =~ ^[1-9][0-9]*$ \
+ && "${GITHUB_RUN_ATTEMPT:-}" =~ ^[1-9][0-9]*$ \
+ && "${COLLECTIVEX_SOURCE_SHA:-}" =~ ^[0-9a-f]{40,64}$ ]] \
+ || cx_die "canonical CollectiveX workflow identity is incomplete"
+
+ # cx_load_operator_config clears inherited values before setting this process marker.
+ # Preserve only values parsed from that private strict document.
+ if [ "${COLLECTIVEX_OPERATOR_CONFIG_LOADED:-}" = "$$" ]; then
+ trusted_lock_dir="${CX_LOCK_DIR:-}"
+ trusted_stage_dir="${CX_STAGE_DIR:-}"
+ trusted_socket_ifname="${CX_SOCKET_IFNAME:-}"
+ trusted_rdma_devices="${CX_RDMA_DEVICES:-}"
+ trusted_ib_gid_index="${CX_IB_GID_INDEX:-}"
+ trusted_rdma_service_level="${CX_RDMA_SERVICE_LEVEL:-}"
+ fi
+ unset CX_NCCL_HOME CX_MASTER_PORT CX_MORI_KERNEL_TYPE CX_LOCK_DIR CX_STAGE_DIR
+ unset MASTER_ADDR MASTER_PORT RANK WORLD_SIZE LOCAL_RANK LOCAL_WORLD_SIZE
+ unset CX_SOCKET_IFNAME CX_RDMA_DEVICES CX_IB_GID_INDEX CX_RDMA_SERVICE_LEVEL
+ unset NCCL_NET NCCL_SOCKET_IFNAME GLOO_SOCKET_IFNAME NCCL_IB_HCA
+ unset NCCL_IB_GID_INDEX NCCL_IB_SL
+ unset NVSHMEM_HCA_LIST NVSHMEM_IB_GID_INDEX NVSHMEM_IB_SL
+ unset NVSHMEM_IB_ENABLE_IBGDA NVSHMEM_IBGDA_NIC_HANDLER
+ unset EP_NIC_NAME EP_OVERRIDE_RDMA_SL
+ unset UCCL_SOCKET_IFNAME UCCL_IB_GID_INDEX UCCL_IB_SL MORI_RDMA_DEVICES
+ unset HYBRID_EP_MULTINODE USE_NIXL RDMA_CORE_HOME DEEPEP_HYBRID_BUILD_MODE
+ unset MORI_COMMIT MORI_DISABLE_AUTO_XGMI MORI_ENABLE_SDMA
+ unset MORI_APP_LOG_LEVEL MORI_SHMEM_LOG_LEVEL MORI_IO_LOG_LEVEL
+ unset NCCL_CUMEM_ENABLE NCCL_MNNVL_ENABLE MC_FORCE_MNNVL
+ unset CX_BACKEND_CACHE_ROOT CX_BACKEND_CACHE_SENTINEL_SHA256
+ unset CX_PREPARED_BACKEND_CACHE CX_BACKEND_SOURCE_ROOT
+
+ [ -n "${CX_SQUASH_DIR:-}" ] \
+ || cx_die "canonical CollectiveX execution requires shared container storage"
+ [ -n "$trusted_stage_dir" ] \
+ || cx_die "canonical CollectiveX execution requires a configured shared stage directory"
+
+ case "$runner" in
+ h100-dgxc|h200-dgxc|b200-dgxc|b300)
+ expected_nodes="${CX_NODES:-}"; expected_gpn=8
+ [ "$expected_nodes" = 1 ] || [ "$expected_nodes" = 2 ] \
+ || cx_die "canonical NVIDIA execution requires one or two nodes"
+ CX_IMAGE="$CX_IMAGE_MULTIARCH"
+ CX_IMAGE_DIGEST="$CX_IMAGE_MULTIARCH_DIGEST"
+ CX_NCCL_HOME=/usr
+ ;;
+ gb200|gb300)
+ expected_nodes="${CX_NODES:-}"; expected_gpn=4
+ [ "$expected_nodes" = 2 ] || [ "$expected_nodes" = 4 ] \
+ || cx_die "canonical GB execution requires two or four trays"
+ CX_IMAGE="$CX_IMAGE_MULTIARCH"
+ CX_IMAGE_DIGEST="$CX_IMAGE_MULTIARCH_DIGEST"
+ CX_NCCL_HOME=/usr
+ CX_MASTER_PORT=29551
+ ;;
+ mi325x)
+ expected_nodes="${CX_NODES:-}"; expected_gpn=8
+ [ "$expected_nodes" = 1 ] || [ "$expected_nodes" = 2 ] \
+ || cx_die "canonical AMD execution requires one or two nodes"
+ CX_IMAGE="$CX_IMAGE_AMD_MORI_MI325"
+ CX_IMAGE_DIGEST="$CX_IMAGE_AMD_MORI_MI325_DIGEST"
+ if [ "$expected_nodes" = 2 ]; then
+ CX_MORI_KERNEL_TYPE=internode-v1
+ else
+ CX_MORI_KERNEL_TYPE=asyncll
+ fi
+ MORI_COMMIT="$CX_MORI_COMMIT_MI325"
+ MORI_DISABLE_AUTO_XGMI=0
+ MORI_ENABLE_SDMA=1
+ MORI_APP_LOG_LEVEL=info
+ MORI_SHMEM_LOG_LEVEL=info
+ MORI_IO_LOG_LEVEL=info
+ ;;
+ mi355x)
+ expected_nodes="${CX_NODES:-}"; expected_gpn=8
+ [ "$expected_nodes" = 1 ] || [ "$expected_nodes" = 2 ] \
+ || cx_die "canonical AMD execution requires one or two nodes"
+ CX_IMAGE="$CX_IMAGE_AMD_MORI"
+ CX_IMAGE_DIGEST="$CX_IMAGE_AMD_MORI_DIGEST"
+ if [ "$expected_nodes" = 2 ]; then
+ CX_MORI_KERNEL_TYPE=internode-v1
+ else
+ CX_MORI_KERNEL_TYPE=intranode
+ fi
+ MORI_COMMIT="$CX_MORI_COMMIT_MI355"
+ ;;
+ *) cx_die "canonical CollectiveX runner is not registered" ;;
+ esac
+ case "$runner:$trusted_lock_dir" in
+ mi325x:?*|mi355x:?*) export CX_LOCK_DIR="$trusted_lock_dir" ;;
+ esac
+ CX_STAGE_DIR="$trusted_stage_dir"
+ [ -z "$trusted_socket_ifname" ] \
+ || export CX_SOCKET_IFNAME="$trusted_socket_ifname"
+ [ -z "$trusted_rdma_devices" ] \
+ || export CX_RDMA_DEVICES="$trusted_rdma_devices"
+ [ -z "$trusted_ib_gid_index" ] \
+ || export CX_IB_GID_INDEX="$trusted_ib_gid_index"
+ [ -z "$trusted_rdma_service_level" ] \
+ || export CX_RDMA_SERVICE_LEVEL="$trusted_rdma_service_level"
+ export CX_STAGE_DIR
+ [ "${CX_NODES:-}" = "$expected_nodes" ] \
+ && [ "${CX_GPUS_PER_NODE:-}" = "$expected_gpn" ] \
+ || cx_die "canonical CollectiveX placement differs from the shard"
+ expected_world=$((expected_nodes * expected_gpn))
+ CX_NGPUS="$expected_world"
+ CX_SEED=67
+ case "$runner" in mi325x|mi355x) CX_RUN_TIMEOUT=1800 ;; *) CX_RUN_TIMEOUT=900 ;; esac
+ unset CX_PUBLIC_RUNNER CX_GB_PRODUCT CX_DRYRUN CX_TIMING CX_ALLOW_MNNVL
+ unset CX_ENROOT_LOCAL_IMPORT COLLECTIVEX_IMAGE COLLECTIVEX_IMAGE_DIGEST
+ unset COLLECTIVEX_IMAGE_DIGEST_VERIFIED COLLECTIVEX_SQUASH_SHA256
+ export CX_IMAGE CX_IMAGE_DIGEST CX_NGPUS CX_SEED CX_RUN_TIMEOUT
+ case "$runner" in
+ h100-dgxc|h200-dgxc|b200-dgxc|b300) export CX_NCCL_HOME ;;
+ gb200|gb300) export CX_NCCL_HOME CX_MASTER_PORT ;;
+ mi325x)
+ export CX_MORI_KERNEL_TYPE MORI_COMMIT MORI_DISABLE_AUTO_XGMI MORI_ENABLE_SDMA
+ export MORI_APP_LOG_LEVEL MORI_SHMEM_LOG_LEVEL MORI_IO_LOG_LEVEL
+ ;;
+ mi355x) export CX_MORI_KERNEL_TYPE MORI_COMMIT ;;
+ esac
+}
+
+cx_reverify_registry_image() {
+ local image="$1" actual
+ [[ "${COLLECTIVEX_IMAGE_DIGEST:-}" =~ ^sha256:[0-9a-f]{64}$ ]] \
+ && [ "${COLLECTIVEX_IMAGE_DIGEST_VERIFIED:-0}" = 1 ] || return 1
+ actual="$(cx_resolve_registry_digest "$image")" || return 1
+ [ "$actual" = "$COLLECTIVEX_IMAGE_DIGEST" ] || {
+ cx_log "ERROR: configured image tag changed during container import"
+ return 1
+ }
+}
+
+cx_export_squash_identity() {
+ local image="$1" digest log
+ log="$(cx_private_log_path container-hash)"
+ digest="$(sha256sum "$image" 2>> "$log" | awk '{print $1}')"
+ [[ "$digest" =~ ^[0-9a-f]{64}$ ]] \
+ || { cx_fail_stage container-hash "$log"; return 1; }
+ export COLLECTIVEX_SQUASH_SHA256="$digest"
+}
+
+cx_squash_path() {
+ local squash_dir="$1" image="$2" key platform
+ [[ "${COLLECTIVEX_IMAGE_DIGEST:-}" =~ ^sha256:[0-9a-f]{64}$ ]] \
+ || return 1
+ case "${CX_IMAGE_PLATFORM:-}" in
+ linux/amd64) platform="" ;;
+ linux/arm64) platform="_linux_arm64" ;;
+ *) return 1 ;;
+ esac
+ key="${CX_SQUASH_FORMAT_VERSION}${platform}_${COLLECTIVEX_IMAGE_DIGEST#sha256:}_$(
+ printf '%s' "$image" | sed 's#[/:@#]#_#g'
+ )"
+ printf '%s' "$squash_dir/${key}.sqsh"
+}
+
+# cx_ensure_squash -> echoes the squash file path.
+# Imports via Enroot only if a valid squash is not already present, under a lock.
+cx_ensure_squash() {
+ local squash_dir="$1" image="$2" key sq locks lock_fd log
+ local enroot_local="" import_rc=0 machine
+ log="$(cx_private_log_path container-import)"
+ machine="$(uname -m)"
+ case "${CX_IMAGE_PLATFORM:-}:$machine" in
+ linux/amd64:x86_64|linux/amd64:amd64|linux/arm64:aarch64|linux/arm64:arm64) ;;
+ *) cx_fail_stage container-import "$log"; return 1 ;;
+ esac
+ mkdir -p "$squash_dir" 2>> "$log" \
+ || { cx_fail_stage container-import "$log"; return 1; }
+ sq="$(cx_squash_path "$squash_dir" "$image")" \
+ || { cx_fail_stage container-import "$log"; return 1; }
+ key="${sq##*/}"
+ key="${key%.sqsh}"
+ locks="$squash_dir/.locks"
+ mkdir -p "$locks" 2>> "$log" \
+ || { cx_fail_stage container-import "$log"; return 1; }
+ { exec {lock_fd}>"$locks/${key}.lock"; } 2>> "$log" \
+ || { cx_fail_stage container-import "$log"; return 1; }
+ flock -w 900 "$lock_fd" 2>> "$log" \
+ || { cx_fail_stage container-import "$log"; return 1; }
+ if unsquashfs -l "$sq" >/dev/null 2>&1; then
+ cx_log "container squash ready"
+ else
+ cx_log "importing configured container image"
+ rm -f "$sq" 2>> "$log" \
+ || { cx_fail_stage container-import "$log"; return 1; }
+ # > "$log" 2>&1 || import_rc=$?
+ rm -rf -- "$enroot_local" >/dev/null 2>&1 || true
+ [ "$import_rc" = 0 ] \
+ || { cx_fail_stage container-import "$log"; return 1; }
+ else
+ SOURCE_DATE_EPOCH="$CX_SQUASH_SOURCE_DATE_EPOCH" \
+ enroot import -o "$sq" "docker://$image" > "$log" 2>&1 \
+ || { cx_fail_stage container-import "$log"; return 1; }
+ fi
+ unsquashfs -l "$sq" >> "$log" 2>&1 \
+ || { cx_fail_stage container-import "$log"; return 1; }
+ fi
+ if ! cx_reverify_registry_image "$image" >> "$log" 2>&1; then
+ flock -u "$lock_fd" >/dev/null 2>&1 || true
+ exec {lock_fd}>&-
+ cx_fail_stage container-import "$log"
+ return 1
+ fi
+ flock -u "$lock_fd"
+ exec {lock_fd}>&-
+ echo "$sq"
+}
+
+# Import on an allocated compute node so multiarch tags resolve for the target
+# architecture. The squash directory must be shared with the submit host.
+cx_ensure_squash_on_job() {
+ local job_id="$1" squash_dir="$2" image="$3" lock_dir="${4:-}" sq key lock log
+ [[ "$job_id" =~ ^[0-9]+$ ]] || return 1
+ sq="$(cx_squash_path "$squash_dir" "$image")" || return 1
+ key="${sq##*/}"
+ key="${key%.sqsh}"
+ [ -n "$lock_dir" ] || lock_dir="$squash_dir/.locks"
+ lock="$lock_dir/${key}.lock"
+ log="$(cx_private_log_path container-import)"
+ if ! srun --jobid="$job_id" --nodes=1 --ntasks=1 --chdir=/tmp \
+ --export="$(cx_host_exports)" \
+ bash -s -- "$sq" "$lock" "$image" "$CX_SQUASH_SOURCE_DATE_EPOCH" \
+ "$CX_IMAGE_PLATFORM" \
+ > "$log" 2>&1 <<'BASH'
+set -euo pipefail
+sq="$1"; lock="$2"; image="$3"; source_date_epoch="$4"; platform="$5"
+machine="$(uname -m)"
+case "$platform:$machine" in
+ linux/amd64:x86_64|linux/amd64:amd64|linux/arm64:aarch64|linux/arm64:arm64) ;;
+ *) exit 13 ;;
+esac
+compute_home="$(mktemp -d /tmp/inferencex-collectivex-home.XXXXXX)"
+trap 'rm -rf -- "$compute_home"' EXIT
+export HOME="$compute_home" XDG_CACHE_HOME="$compute_home/.cache"
+export ENROOT_TEMP_PATH="$compute_home/enroot-tmp"
+export ENROOT_CACHE_PATH="$compute_home/enroot-cache"
+export ENROOT_DATA_PATH="$compute_home/enroot-data"
+export ENROOT_RUNTIME_PATH="$compute_home/enroot-run"
+mkdir -p "$(dirname "$sq")" "$(dirname "$lock")" \
+ "$ENROOT_TEMP_PATH" "$ENROOT_CACHE_PATH" "$ENROOT_DATA_PATH" "$ENROOT_RUNTIME_PATH"
+exec 9>"$lock"
+flock -w 900 9
+if unsquashfs -l "$sq" >/dev/null 2>&1; then
+ echo 'container squash ready'
+else
+ rm -f -- "$sq"
+ SOURCE_DATE_EPOCH="$source_date_epoch" \
+ enroot import -o "$sq" "docker://$image" /dev/null 2>&1
+fi
+BASH
+ then
+ cx_fail_stage container-import "$log"
+ return 1
+ fi
+ if ! cx_reverify_registry_image "$image" >> "$log" 2>&1; then
+ cx_fail_stage container-import "$log"
+ return 1
+ fi
+ printf '%s' "$sq"
+}
+
+cx_preflight_allocation() {
+ local job_id="$1" nodes="$2" mount_src="$3" squash="$4" shard="${5:-}"
+ local log rc=0 runtime shard_path="" probe_root probe_token index
+ runtime="$mount_src/experimental/CollectiveX/runtime/run_in_container.sh"
+ [ -z "$shard" ] || shard_path="$mount_src/experimental/CollectiveX/$shard"
+ log="$(cx_private_log_path allocation-preflight)"
+ probe_root="$mount_src/.collectivex-preflight"
+ probe_token="$probe_root/source"
+ if [ -e "$probe_root" ] || [ -L "$probe_root" ] \
+ || ! mkdir -m 700 "$probe_root"; then
+ cx_fail_stage repository-stage "$log"
+ return 1
+ fi
+ if ! printf '%s\n' "${COLLECTIVEX_EXECUTION_ID:-manual-$$}" > "$probe_token" \
+ || ! chmod 600 "$probe_token"; then
+ chmod 700 "$probe_root" >/dev/null 2>&1 || true
+ rm -rf -- "$probe_root" >/dev/null 2>&1 || true
+ cx_fail_stage repository-stage "$log"
+ return 1
+ fi
+ srun --jobid="$job_id" --nodes="$nodes" --ntasks="$nodes" --ntasks-per-node=1 \
+ --chdir=/tmp --input=all \
+ --export="$(cx_host_exports)" bash -s -- "$runtime" "$shard_path" "$squash" \
+ "$CX_IMAGE_PLATFORM" "$probe_root" \
+ > "$log" 2>&1 <<'BASH' || rc=$?
+set -euo pipefail
+machine="$(uname -m)"
+case "$4:$machine" in
+ linux/amd64:x86_64|linux/amd64:amd64|linux/arm64:aarch64|linux/arm64:arm64) ;;
+ *) exit 13 ;;
+esac
+test -r "$1" || exit 10
+[ -z "$2" ] || test -r "$2" || exit 11
+test -r "$3" || exit 12
+unsquashfs -s "$3" >/dev/null 2>&1 || exit 12
+case "${SLURM_NODEID:-}" in ""|*[!0-9]*) exit 10 ;; esac
+[ -d "$5" ] && [ ! -L "$5" ] && [ -r "$5/source" ] || exit 10
+(set -C; cat "$5/source" > "$5/node-$SLURM_NODEID") || exit 10
+cmp -s -- "$5/source" "$5/node-$SLURM_NODEID" || exit 10
+BASH
+ if [ "$rc" = 0 ]; then
+ for ((index = 0; index < nodes; index++)); do
+ if ! cmp -s -- "$probe_token" "$probe_root/node-$index"; then
+ rc=10
+ break
+ fi
+ done
+ fi
+ if [ -d "$probe_root" ] && [ ! -L "$probe_root" ]; then
+ chmod 700 "$probe_root" >/dev/null 2>&1 || rc=10
+ fi
+ rm -rf -- "$probe_root" >/dev/null 2>&1 || rc=10
+ [ "$rc" = 0 ] && return 0
+ case "$rc" in
+ 10|11) cx_fail_stage repository-stage "$log" ;;
+ 12) cx_fail_stage container-hash "$log" ;;
+ *) cx_fail_stage container-launch "$log" ;;
+ esac
+ return 1
+}
+
+# Resolve the exact per-execution child before any copy starts, so the parent
+# EXIT trap can remove an interrupted partial stage. The configured base must
+# already exist on compute-visible storage and must not traverse symlinks.
+cx_stage_path() {
+ local repo_root="$1" stage_base="${2:-}" tag safe_tag stage_path
+ tag="${COLLECTIVEX_EXECUTION_ID:-${GITHUB_RUN_ID:-manual-$$}}"
+ [[ "$tag" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] \
+ || cx_die "invalid staging execution identity"
+ safe_tag="$(printf '%s' "$tag" | tr -c 'A-Za-z0-9._-' '_')"
+ if [ -z "$stage_base" ] || [ "$stage_base" = "$repo_root" ]; then
+ [ "${COLLECTIVEX_CANONICAL_GHA:-0}" != 1 ] \
+ || cx_die "canonical CollectiveX execution requires compute-visible staging"
+ [ -n "${CX_SQUASH_DIR:-}" ] \
+ || cx_die "manual CollectiveX staging requires CX_SQUASH_DIR"
+ stage_base="$CX_SQUASH_DIR"
+ stage_path="${stage_base%/}/.collectivex-stage-$safe_tag"
+ else
+ stage_path="${stage_base%/}/job_$safe_tag"
+ fi
+ python3 - "$repo_root" "$stage_base" "$stage_path" \
+ "${CX_JOB_ROOT:-}" "${GITHUB_WORKSPACE:-}" <<'PY'
+import os
+import stat
+import sys
+
+repo, base, child, job_root, workspace = sys.argv[1:]
+try:
+ if (
+ not os.path.isabs(repo)
+ or os.path.realpath(repo) != repo
+ or not os.path.isabs(base)
+ or os.path.realpath(base) != base
+ or not os.path.isabs(child)
+ or os.path.dirname(child) != base.rstrip("/")
+ or os.path.lexists(child)
+ ):
+ raise OSError
+ metadata = os.stat(base, follow_symlinks=False)
+ excluded = [repo]
+ excluded.extend(path for path in (job_root, workspace) if path)
+ for path in excluded:
+ resolved = os.path.realpath(path)
+ if os.path.commonpath((base, resolved)) == resolved:
+ raise OSError
+ if (
+ not stat.S_ISDIR(metadata.st_mode)
+ or metadata.st_uid != os.getuid()
+ or stat.S_IMODE(metadata.st_mode) & (stat.S_IWGRP | stat.S_IWOTH)
+ or not os.access(base, os.W_OK | os.X_OK)
+ ):
+ raise OSError
+except OSError:
+ raise SystemExit(1)
+print(child, end="")
+PY
+}
+
+# Stage only the public benchmark tree into a pre-resolved, private execution
+# child. A runner-owned marker makes recursive cleanup an explicit capability.
+cx_stage_repo() {
+ local repo_root="$1" stage_dir="$2" expected log tag marker
+ cx_validate_shard_control "$repo_root/experimental/CollectiveX"
+ expected="$(cx_stage_path "$repo_root" "${CX_STAGE_DIR:-}")" \
+ || cx_die "configured stage base is unavailable or unsafe"
+ [ "$stage_dir" = "$expected" ] \
+ || cx_die "execution stage differs from the configured stage base"
+ tag="${COLLECTIVEX_EXECUTION_ID:-${GITHUB_RUN_ID:-manual-$$}}"
+ if [ -e "$stage_dir" ] || [ -L "$stage_dir" ]; then
+ cx_die "refusing to reuse a pre-existing execution stage"
+ fi
+ mkdir -m 700 "$stage_dir" 2>/dev/null \
+ || cx_die "cannot create the configured stage directory"
+ chmod 700 "$stage_dir" 2>/dev/null \
+ || cx_die "cannot protect the configured stage directory"
+ marker="$stage_dir/.collectivex-stage-v1"
+ umask 077
+ (set -C; printf 'collectivex-stage-v1\n%s\n' "$tag" > "$marker") 2>/dev/null \
+ || cx_die "cannot claim the configured stage directory"
+ chmod 600 "$marker" 2>/dev/null \
+ || cx_die "cannot protect the configured stage directory"
+ mkdir -m 700 "$stage_dir/experimental" 2>/dev/null \
+ || cx_die "cannot create the configured stage directory"
+ cx_log "staging CollectiveX on compute-visible storage"
+ log="$(cx_private_log_path repository-stage)"
+ if ! rsync -a --delete --delete-excluded \
+ --exclude='__pycache__/' --exclude='results/' --exclude='.cx_workloads/' \
+ --exclude='.cx_backend/' --exclude='.cx_sources/' \
+ --exclude='configs/platforms.yaml' --exclude='private-infra.md' \
+ --exclude='goal.md' --exclude='notes.md' \
+ "$repo_root/experimental/CollectiveX" "$stage_dir/experimental/" > "$log" 2>&1; then
+ rm -rf -- "$stage_dir" >/dev/null 2>&1 \
+ || cx_log "ERROR: cannot remove the incomplete execution stage"
+ cx_fail_stage repository-stage "$log" || true
+ return 1
+ fi
+}
+
+# cx_collect_results
+# When the run used a staged (compute-visible) mount, copy result JSONs back to
+# the original checkout's results/ so the workflow's upload-artifact (which reads
+# the checkout, not the stage dir) finds them. No-op when no staging was used.
+cx_collect_results() {
+ local mount_src="$1" repo_root="$2" dst log
+ local -a files
+ [ "$mount_src" = "$repo_root" ] && return 0
+ log="$(cx_private_log_path "artifact-collection-$$-${RANDOM}")"
+ dst="$repo_root/experimental/CollectiveX/results"
+ mkdir -p "$dst" 2>> "$log" \
+ || { cx_log "ERROR: cannot create checkout result directory"; return 1; }
+ shopt -s nullglob
+ files=("$mount_src/experimental/CollectiveX/results/"*.json)
+ shopt -u nullglob
+ [ "${#files[@]}" -gt 0 ] || { cx_log "ERROR: staged run produced no result JSON"; return 1; }
+ cp -- "${files[@]}" "$dst/" >> "$log" 2>&1 \
+ || { cx_log "ERROR: staged result collection failed"; return 1; }
+ cx_log "collected staged results for artifact validation"
+}
+
+cx_cleanup_stage() {
+ local mount_src="$1" repo_root="$2" base="${CX_STAGE_DIR:-}" tag safe_tag expected
+ tag="${COLLECTIVEX_EXECUTION_ID:-${GITHUB_RUN_ID:-manual-$$}}"
+ safe_tag="$(printf '%s' "$tag" | tr -c 'A-Za-z0-9._-' '_')"
+ [ "$mount_src" != "$repo_root" ] || return 0
+ if [ -n "$base" ] && [ "$base" != "$repo_root" ]; then
+ expected="${base%/}/job_$safe_tag"
+ else
+ [ -n "${CX_SQUASH_DIR:-}" ] \
+ || { cx_log "ERROR: cannot identify the generated stage directory"; return 1; }
+ expected="${CX_SQUASH_DIR%/}/.collectivex-stage-$safe_tag"
+ fi
+ if [ "$mount_src" != "$expected" ] || [ "$mount_src" = / ] \
+ || { [ -n "$base" ] && [ "$mount_src" = "$base" ]; }; then
+ cx_log "ERROR: refusing to remove an unrecognized stage directory"
+ return 1
+ fi
+ if ! python3 - "$mount_src" "$tag" <<'PY'
+import os
+from pathlib import Path
+import stat
+import sys
+
+root = Path(sys.argv[1])
+expected = f"collectivex-stage-v1\n{sys.argv[2]}\n"
+try:
+ metadata = os.stat(root, follow_symlinks=False)
+ marker = root / ".collectivex-stage-v1"
+ if (
+ not stat.S_ISDIR(metadata.st_mode)
+ or metadata.st_uid != os.getuid()
+ or (stat.S_IMODE(metadata.st_mode) & 0o777) != 0o700
+ ):
+ raise OSError
+ entries = list(root.iterdir())
+ if marker.exists():
+ marker_metadata = os.stat(marker, follow_symlinks=False)
+ if (
+ not stat.S_ISREG(marker_metadata.st_mode)
+ or marker_metadata.st_uid != os.getuid()
+ or stat.S_IMODE(marker_metadata.st_mode) != 0o600
+ ):
+ raise OSError
+ marker_content = marker.read_text()
+ if marker_content != expected and entries != [marker]:
+ raise OSError
+ elif entries:
+ raise OSError
+except (OSError, UnicodeError):
+ raise SystemExit(1)
+PY
+ then
+ cx_log "ERROR: refusing to remove an unowned stage directory"
+ return 1
+ fi
+ rm -rf -- "$mount_src" >/dev/null 2>&1 || {
+ cx_log "ERROR: cannot remove generated stage directory"
+ return 1
+ }
+ cx_log "removed generated per-execution stage directory"
+}
+
+# Return success only when a benchmark output is a complete JSON result object.
+# Callers use this before synthesizing a terminal outcome so an emitted invalid result
+# is not shadowed by a second record for the same attempt.
+cx_has_result_doc() {
+ local path="$1"
+ python3 "$_CX_COMMON_ROOT/contracts.py" probe "$path" >/dev/null 2>&1
+}
+
+cx_result_doc_is() {
+ local path="$1" expected="$2"
+ python3 "$_CX_COMMON_ROOT/contracts.py" probe "$path" --status "$expected" \
+ >/dev/null 2>&1
+}
+
+# A rank-zero result can be written before another rank or backend teardown fails. Preserve its
+# measurements, but make the distributed command's nonzero terminal status authoritative.
+cx_demote_result_doc() {
+ local path="$1" rc="$2"
+ python3 "$_CX_COMMON_ROOT/contracts.py" demote "$path" --return-code "$rc"
+}
+
+cx_quarantine_result_doc() {
+ python3 "$_CX_COMMON_ROOT/contracts.py" quarantine-invalid "$1"
+}
+
+# cx_emit_ep_failed_case
+# Preserve failures from rack launchers that invoke run_ep.py directly and therefore cannot use
+# run_in_container.sh's emitter. Case identity is read from the exported CX_* variables.
+cx_emit_ep_failed_case() {
+ local out="$1" backend="$2" phase="$3" rc="$4"
+ local -a args=(emit-terminal --out "$out" --backend "$backend" --phase "$phase"
+ --return-code "$rc")
+ [ -z "${CX_FAILURE_MODE:-}" ] || args+=(--failure-mode "$CX_FAILURE_MODE")
+ if ! python3 "$_CX_COMMON_ROOT/contracts.py" "${args[@]}"
+ then
+ cx_log "ERROR: could not preserve terminal outcome"
+ return 1
+ fi
+}
+
+cx_case_attempt_exists() {
+ local out_dir="$1" case_id="$2"
+ python3 - "$_CX_COMMON_ROOT" "$out_dir" "$case_id" <<'PY'
+import pathlib, sys
+
+sys.path.insert(0, sys.argv[1])
+import contracts
+
+sample_paths = set()
+referenced_samples = set()
+found = False
+
+def quarantine(path, document):
+ sample = document.get("sample_artifact") if isinstance(document, dict) else None
+ if (
+ isinstance(sample, dict)
+ and isinstance(sample.get("path"), str)
+ and pathlib.Path(sample["path"]).name == sample["path"]
+ ):
+ sample_path = path.with_name(sample["path"])
+ if sample_path.is_file():
+ sample_path.replace(sample_path.with_name(sample_path.name + ".quarantine"))
+ if path.is_file():
+ path.replace(path.with_name(path.name + ".quarantine"))
+
+for path in pathlib.Path(sys.argv[2]).glob("*.json"):
+ document = None
+ try:
+ document = contracts.strict_load(path)
+ if not isinstance(document, dict):
+ continue
+ if document.get("format") == contracts.RAW_FORMAT:
+ document = contracts.load_raw_attempt(path)
+ referenced_samples.add(path.with_name(document["sample_artifact"]["path"]))
+ elif document.get("format") == contracts.TERMINAL_FORMAT:
+ document = contracts.validate_terminal_document(document)
+ elif document.get("format") == contracts.SAMPLES_FORMAT:
+ contracts.validate_samples_document(document)
+ sample_paths.add(path)
+ continue
+ else:
+ continue
+ except (contracts.ContractError, OSError, ValueError):
+ quarantine(path, document)
+ continue
+ if document["identity"]["case_id"] == sys.argv[3]:
+ found = True
+for orphan in sample_paths - referenced_samples:
+ quarantine(orphan, {})
+raise SystemExit(0 if found else 1)
+PY
+}
+
+# Emit one setup-failure record per requested case. Rack launchers call this when
+# backend preparation fails before rank processes can start.
+cx_emit_setup_failures() {
+ local root="$1" out_dir="$2" backend="$3" rc="$4" shard="${CX_SHARD_FILE:-}" path
+ local phase case_id suite workload required routing eplb ep hidden topk experts nodes
+ local gpn domain ladder canonical timing mode scope scale_up_transport scale_out_transport
+ local warmup_semantics
+ local transport topology_class
+ local cases_file expected emitted=0 covered=0
+ mkdir -p "$out_dir" || return 1
+ export CX_FAILURE_MODE="${CX_FAILSAFE_MODE:-setup}" CX_ATTEMPT_ID=1
+ if [ -z "$shard" ]; then
+ local phases="${CX_PHASE:-decode}"
+ [ "$phases" = both ] && phases="decode prefill"
+ for phase in $phases; do
+ if [ -n "${CX_CASE_ID:-}" ] && cx_case_attempt_exists "$out_dir" "$CX_CASE_ID"; then
+ continue
+ fi
+ cx_emit_ep_failed_case "$out_dir/failed_${backend}_${phase}_${CX_TS:-setup}-a01.json" \
+ "$backend" "$phase" "$rc" || return 1
+ done
+ unset CX_FAILURE_MODE
+ return 0
+ fi
+ path="$shard"
+ [ -f "$path" ] || path="${root%/}/$shard"
+ [ -f "$path" ] || {
+ unset CX_FAILURE_MODE
+ cx_log "ERROR: cannot emit setup failures without shard control"
+ return 1
+ }
+ export COLLECTIVEX_CONTROL_SHA256
+ COLLECTIVEX_CONTROL_SHA256="$(sha256sum "$path" | awk '{print $1}')"
+ [[ "$COLLECTIVEX_CONTROL_SHA256" =~ ^[0-9a-f]{64}$ ]] || {
+ unset CX_FAILURE_MODE COLLECTIVEX_CONTROL_SHA256
+ cx_log "ERROR: cannot hash shard for setup-failure records"
+ return 1
+ }
+ cases_file="$(mktemp)" || return 1
+ if ! python3 - "$path" > "$cases_file" <<'PY'
+import json, sys
+
+with open(sys.argv[1]) as handle:
+ cases = json.load(handle)["cases"]
+for case in cases:
+ fields = (
+ case["phase"], case["mode"], case["case_id"], case["suite"], case["workload"],
+ case["required_publication"], case["routing"], "1" if case["eplb"] else "",
+ case["ep"], case["hidden"], case["topk"], case["experts"], case["nodes"],
+ case["gpus_per_node"], case["scale_up_domain"], case["scope"],
+ case["scale_up_transport"], case.get("scale_out_transport") or "",
+ case["transport"], case["topology_class"], case["ladder"],
+ case["warmup_semantics"],
+ "1" if case["canonical"] else "", case["timing"],
+ )
+ print("|".join(map(str, fields)))
+PY
+ then
+ rm -f "$cases_file"
+ unset CX_FAILURE_MODE
+ return 1
+ fi
+ expected="$(wc -l < "$cases_file" | tr -d ' ')"
+ [ "$expected" -gt 0 ] || { rm -f "$cases_file"; unset CX_FAILURE_MODE; return 1; }
+ while IFS='|' read -r phase mode case_id suite workload required routing eplb ep hidden topk \
+ experts nodes gpn domain scope scale_up_transport scale_out_transport transport \
+ topology_class ladder warmup_semantics canonical timing; do
+ export CX_CASE_ID="$case_id" CX_SUITE="$suite" CX_WORKLOAD_NAME="$workload"
+ export CX_REQUIRED_PUBLICATION="$required" CX_ROUTING="$routing" CX_EPLB="$eplb"
+ export CX_EP="$ep" CX_NGPUS="$ep" CX_HIDDEN="$hidden" CX_TOPK="$topk" CX_EXPERTS="$experts"
+ export CX_MODE="$mode" CX_NODES="$nodes" CX_GPUS_PER_NODE="$gpn"
+ export CX_SCALE_UP_DOMAIN="$domain" CX_SCOPE="$scope"
+ export CX_SCALE_UP_TRANSPORT="$scale_up_transport"
+ export CX_SCALE_OUT_TRANSPORT="$scale_out_transport"
+ export CX_TRANSPORT="$transport" CX_TOPO="$topology_class"
+ export CX_TOKENS_LADDER="$ladder" CX_CANONICAL="$canonical"
+ export CX_WARMUP_SEMANTICS="$warmup_semantics"
+ IFS=: read -r CX_ITERS CX_TRIALS CX_WARMUP <<< "$timing"
+ export CX_ITERS CX_TRIALS CX_WARMUP CX_SAMPLES_PER_POINT="$((CX_ITERS * CX_TRIALS))"
+ if cx_case_attempt_exists "$out_dir" "$case_id"; then
+ covered=$((covered + 1))
+ continue
+ fi
+ cx_emit_ep_failed_case "$out_dir/failed_${case_id}-a01.json" "$backend" "$phase" "$rc" || return 1
+ emitted=$((emitted + 1))
+ done < "$cases_file"
+ rm -f "$cases_file"
+ unset CX_FAILURE_MODE
+ [ "$((emitted + covered))" -eq "$expected" ] || {
+ cx_log "ERROR: covered $((emitted + covered))/$expected terminal cases"
+ return 1
+ }
+}
+
+# Run one validated shard with one Slurm task per GPU. Launchers provide only
+# allocation/container policy through globals and CX_DISTRIBUTED_CONTAINER_ARGS.
+# shellcheck disable=SC2153
+cx_run_distributed_shard() {
+ local build_log build_rc cases_file expected_cases ci=0 failed_cases=0
+ local ph mode routing eplb hidden topk experts ladder suite workload required_pub
+ local canonical case_id ep timing case_iters case_trials case_warmup case_stem
+ local scope scale_up_transport scale_out_transport transport topology_class nodes gpn domain
+ local workload_dir workload_ladder workload_log stage_rc attempt_tag out failure_out
+ local runtime_log run_rc expected_out case_ok summary_log
+ local -a container_args workload_args ep_args
+ [ "${NODES:-0}" -gt 1 ] && [ "${NGPUS:-0}" = "$((NODES * GPN))" ] \
+ || cx_die "invalid distributed launcher placement"
+ [ -n "${JOB_ID:-}" ] && [ -n "${SQUASH_FILE:-}" ] \
+ && [ -n "${CONTAINER_MOUNTS:-}" ] || cx_die "distributed launcher is incomplete"
+ [ -n "${SOURCE_BACKEND_ENV:-}" ] && [ -n "${BACKEND_PROBE:-}" ] \
+ && [ -n "${WRAP:-}" ] || cx_die "distributed rank wrapper is incomplete"
+
+ cx_resolve_slurm_rendezvous "$JOB_ID"
+ mkdir -p "$MOUNT_SRC/experimental/CollectiveX/results"
+ container_args=(--container-mounts="$CONTAINER_MOUNTS" --no-container-mount-home
+ --container-workdir=/ix/experimental/CollectiveX --no-container-entrypoint)
+ if declare -p CX_DISTRIBUTED_CONTAINER_ARGS >/dev/null 2>&1; then
+ container_args+=("${CX_DISTRIBUTED_CONTAINER_ARGS[@]}")
+ fi
+ local container_name="cxep_${JOB_ID}"
+
+ cx_log "distributed backend preparation: bench=$CX_BENCH nodes=$NODES"
+ cx_set_failure_stage backend-setup
+ build_log="$(cx_private_log_path backend-prepare)"
+ set +e
+ srun --jobid="$JOB_ID" --nodes="$NODES" --ntasks-per-node=1 --chdir=/tmp \
+ --container-name="$container_name" --container-image="$SQUASH_FILE" \
+ "${container_args[@]}" --export="$(cx_container_exports),CX_BUILD_ONLY=1" \
+ bash /ix/experimental/CollectiveX/runtime/run_in_container.sh \
+ "$build_log" 2>&1
+ build_rc=$?
+ if [ "$build_rc" = 0 ]; then
+ srun --jobid="$JOB_ID" --nodes="$NODES" --ntasks-per-node=1 --chdir=/tmp \
+ --container-name="$container_name" "${container_args[@]}" \
+ --export="$(cx_container_exports)" bash -c "$BACKEND_PROBE" \
+ >"$build_log" 2>&1
+ build_rc=$?
+ fi
+ set -e
+ if [ "$build_rc" != 0 ]; then
+ cx_fail_stage backend-setup "$build_log" || true
+ cx_emit_setup_failures "$CX_DIR" "$MOUNT_SRC/experimental/CollectiveX/results" \
+ "$CX_BENCH" "$build_rc"
+ return "$build_rc"
+ fi
+ cx_set_failure_stage execution
+
+ cases_file="$(mktemp)" || return 1
+ local shard="${CX_SHARD_FILE:-}"
+ [ -z "$shard" ] || [ -f "$shard" ] || shard="$CX_DIR/$shard"
+ if [ -n "$shard" ]; then
+ if [ ! -f "$shard" ] || ! python3 - "$shard" > "$cases_file" <<'PY'
+import json
+import sys
+
+with open(sys.argv[1]) as handle:
+ cases = json.load(handle)["cases"]
+for case in cases:
+ get = lambda key, default="": str(case.get(key) or default)
+ fields = (
+ get("phase", "decode"), get("mode", "normal"), get("routing", "uniform"),
+ "1" if case.get("eplb") else "", get("hidden", "7168"),
+ get("topk", "8"), get("experts", "256"), get("ladder"),
+ get("suite"), get("workload"), get("required_publication"),
+ "1" if case.get("canonical") else "", get("case_id"), get("ep"),
+ get("timing", "8:64:32"), get("nodes"), get("gpus_per_node"),
+ get("scale_up_domain"), get("scope"), get("scale_up_transport"),
+ get("scale_out_transport"), get("transport"), get("topology_class"),
+ )
+ print("|".join(fields))
+PY
+ then
+ rm -f "$cases_file"
+ cx_die "could not enumerate validated shard cases"
+ fi
+ else
+ local phases="${CX_PHASE:-decode}" phase
+ [ "$phases" = both ] && phases="decode prefill"
+ cx_require_record_safe "$phases" "${CX_MODE:-normal}" "${CX_ROUTING:-uniform}" \
+ "${CX_EPLB:-}" "${CX_HIDDEN:-7168}" "${CX_TOPK:-8}" "${CX_EXPERTS:-256}" \
+ "${CX_TOKENS_LADDER:-}" "${CX_SUITE:-}" "${CX_WORKLOAD_NAME:-}" \
+ "${CX_REQUIRED_PUBLICATION:-}" "${CX_CANONICAL:-}" "${CX_CASE_ID:-}" \
+ "${CX_ITERS:-8}" "${CX_TRIALS:-64}" "${CX_WARMUP:-32}" \
+ "${CX_SCOPE:-scale-up}" \
+ "${CX_SCALE_UP_TRANSPORT:-unknown}" "${CX_SCALE_OUT_TRANSPORT:-}" \
+ "${CX_TRANSPORT:-unknown}" "${CX_TOPO:-manual}"
+ for phase in $phases; do
+ (IFS='|'; printf '%s\n' "$phase|${CX_MODE:-normal}|${CX_ROUTING:-uniform}|${CX_EPLB:-}|${CX_HIDDEN:-7168}|${CX_TOPK:-8}|${CX_EXPERTS:-256}|${CX_TOKENS_LADDER:-}|${CX_SUITE:-}|${CX_WORKLOAD_NAME:-}|${CX_REQUIRED_PUBLICATION:-}|${CX_CANONICAL:-}|${CX_CASE_ID:-}|$NGPUS|${CX_ITERS:-8}:${CX_TRIALS:-64}:${CX_WARMUP:-32}|$NODES|$GPN|$SCALE_UP_DOMAIN|${CX_SCOPE:-scale-up}|${CX_SCALE_UP_TRANSPORT:-unknown}|${CX_SCALE_OUT_TRANSPORT:-}|${CX_TRANSPORT:-unknown}|${CX_TOPO:-manual}")
+ done > "$cases_file"
+ fi
+ expected_cases="$(wc -l < "$cases_file" | tr -d ' ')"
+ [ "$expected_cases" -gt 0 ] \
+ || { rm -f "$cases_file"; cx_die "distributed case list is empty"; }
+
+ while IFS='|' read -r ph mode routing eplb hidden topk experts ladder suite workload \
+ required_pub canonical case_id ep timing nodes gpn domain scope scale_up_transport \
+ scale_out_transport transport topology_class; do
+ [ -n "$ph" ] || continue
+ ci=$((ci + 1))
+ case_stem="${RUNNER}_${CX_BENCH}_${ph}_${TS}-c$(printf '%03d' "$ci")"
+ IFS=: read -r case_iters case_trials case_warmup <<< "${timing:-8:64:32}"
+ case_iters="${case_iters:-8}"
+ case_trials="${case_trials:-64}"
+ case_warmup="${case_warmup:-32}"
+ ep="${ep:-$NGPUS}"
+ export CX_MODE="$mode" CX_CASE_ID="$case_id" CX_SUITE="$suite" CX_WORKLOAD_NAME="$workload"
+ export CX_REQUIRED_PUBLICATION="$required_pub" CX_CANONICAL="$canonical" CX_EP="$ep"
+ export CX_ROUTING="$routing" CX_EPLB="$eplb" CX_TOKENS_LADDER="$ladder"
+ export CX_HIDDEN="$hidden" CX_TOPK="$topk" CX_EXPERTS="$experts"
+ export CX_NODES="$nodes" CX_GPUS_PER_NODE="$gpn" CX_SCALE_UP_DOMAIN="$domain"
+ export CX_SCOPE="$scope" CX_SCALE_UP_TRANSPORT="$scale_up_transport"
+ export CX_SCALE_OUT_TRANSPORT="$scale_out_transport"
+ export CX_TRANSPORT="$transport" CX_TOPO="$topology_class"
+ export CX_ITERS="$case_iters" CX_TRIALS="$case_trials" CX_WARMUP="$case_warmup"
+ export CX_SAMPLES_PER_POINT="$((case_iters * case_trials))"
+ export CX_WARMUP_SEMANTICS="full-roundtrip-before-each-component-trial-point-v1"
+ cx_log "EP${NGPUS}[$ci] id=${case_id:-manual} $mode/$ph $CX_BENCH"
+ if [ "$ep" != "$NGPUS" ] || [ "$nodes" != "$NODES" ] || [ "$gpn" != "$GPN" ] \
+ || [ "$domain" != "$SCALE_UP_DOMAIN" ]; then
+ export CX_ATTEMPT_ID=1
+ failure_out="$MOUNT_SRC/experimental/CollectiveX/results/failed_${case_stem}-a01.json"
+ cx_emit_ep_failed_case "$failure_out" "$CX_BENCH" "$ph" 5
+ failed_cases=$((failed_cases + 1))
+ continue
+ fi
+
+ workload_dir=""
+ if cx_bool_enabled "$canonical"; then
+ workload_dir=".cx_workloads/c$(printf '%03d' "$ci")"
+ workload_ladder="$ladder"
+ [ -n "$workload_ladder" ] \
+ || workload_ladder="1 2 4 8 16 32 64 128 256 512 1024 2048 4096"
+ workload_args=(python3 tests/make_workloads.py --out-dir "$workload_dir"
+ --routing "$routing" --ep "$ep" --hidden "$hidden" --topk "$topk"
+ --experts "$experts" --seed "${CX_SEED:-67}" --tokens-ladder "$workload_ladder")
+ workload_log="$(cx_private_log_path "workload-c$(printf '%03d' "$ci")")"
+ set +e
+ srun --jobid="$JOB_ID" --nodes=1 --ntasks=1 --chdir=/tmp \
+ --container-name="$container_name" "${container_args[@]}" \
+ --export="$(cx_container_exports)" "${workload_args[@]}" \
+ "$workload_log" 2>&1
+ stage_rc=$?
+ set -e
+ if [ "$stage_rc" != 0 ]; then
+ export CX_ATTEMPT_ID=1
+ failure_out="$MOUNT_SRC/experimental/CollectiveX/results/failed_${case_stem}-a01.json"
+ cx_emit_ep_failed_case "$failure_out" "$CX_BENCH" "$ph" "$stage_rc"
+ failed_cases=$((failed_cases + 1))
+ continue
+ fi
+ fi
+
+ ep_args=(--backend "$CX_BENCH" --mode "$mode" --phase "$ph" --routing "$routing"
+ --gpus-per-node "$gpn" --scale-up-domain "$domain" --scope "$scope"
+ --scale-up-transport "$scale_up_transport" --scale-out-transport "$scale_out_transport"
+ --tokens-ladder "$ladder" --hidden "$hidden" --topk "$topk" --experts "$experts"
+ --warmup "$case_warmup" --iters "$case_iters" --trials "$case_trials"
+ --seed "${CX_SEED:-67}" --runner "$RUNNER" --topology-class "$topology_class"
+ --transport "$transport" --case-id "$case_id" --suite "$suite"
+ --workload-name "$workload" --required-publication "$required_pub")
+ cx_bool_enabled "$eplb" && ep_args+=(--eplb)
+ [ -z "$workload_dir" ] || ep_args+=(--workload-dir "$workload_dir")
+ export CX_ATTEMPT_ID=1
+ attempt_tag=a01
+ out="results/${case_stem}_${attempt_tag}.json"
+ failure_out="$MOUNT_SRC/experimental/CollectiveX/results/failed_${case_stem}-${attempt_tag}.json"
+ runtime_log="$(cx_private_log_path "runtime-c$(printf '%03d' "$ci")-$attempt_tag")"
+ set +e
+ timeout -k 30 "${CX_RUN_TIMEOUT:-900}" srun --jobid="$JOB_ID" --nodes="$NODES" \
+ --ntasks="$NGPUS" --ntasks-per-node="$GPN" --chdir=/tmp \
+ --container-name="$container_name" "${container_args[@]}" \
+ --export="$(cx_container_exports)" \
+ bash -c "$WRAP" _ "${ep_args[@]}" --out "$out" \
+ "$runtime_log" 2>&1
+ run_rc=$?
+ set -e
+ expected_out="$MOUNT_SRC/experimental/CollectiveX/$out"
+ case_ok=0
+ if [ "$run_rc" = 0 ] && cx_result_doc_is "$expected_out" success; then
+ case_ok=1
+ elif [ "$run_rc" = 0 ] && cx_result_doc_is "$expected_out" invalid; then
+ cx_log "ERROR: EP${NGPUS}[$ci] completed with invalid semantic evidence"
+ else
+ [ "$run_rc" != 0 ] || run_rc=1
+ if cx_has_result_doc "$expected_out"; then
+ cx_demote_result_doc "$expected_out" "$run_rc" \
+ || { cx_quarantine_result_doc "$expected_out"; cx_emit_ep_failed_case "$failure_out" "$CX_BENCH" "$ph" "$run_rc"; }
+ else
+ cx_quarantine_result_doc "$expected_out"
+ cx_emit_ep_failed_case "$failure_out" "$CX_BENCH" "$ph" "$run_rc"
+ fi
+ fi
+ if [ "$case_ok" = 0 ]; then
+ [ "$run_rc" = 0 ] || cx_fail_stage execution "$runtime_log" || true
+ failed_cases=$((failed_cases + 1))
+ fi
+ done < "$cases_file"
+ rm -f "$cases_file"
+ [ "$ci" -eq "$expected_cases" ] \
+ || cx_die "enumerated $expected_cases cases but executed $ci"
+ if [ "$failed_cases" -ne 0 ]; then
+ summary_log="$(cx_private_log_path shard-summary)"
+ printf 'SHARD done: %s/%s case(s) failed\n' "$failed_cases" "$expected_cases" \
+ > "$summary_log"
+ cx_fail_stage execution "$summary_log" || true
+ return 1
+ fi
+ return 0
+}
+
+cx_launcher_cleanup() {
+ local rc="$1" stage_root="${MOUNT_SRC:-}" source_root out_dir allocation_stopped=1
+ source_root="${stage_root:-${REPO_ROOT:-}}"
+ trap - EXIT
+ if [ -n "${COLLECTIVEX_EPHEMERAL_CONFIG_PATH:-}" ]; then
+ rm -f -- "$COLLECTIVEX_EPHEMERAL_CONFIG_PATH" >/dev/null 2>&1 || true
+ unset COLLECTIVEX_EPHEMERAL_CONFIG_PATH
+ fi
+ if [ -n "${JOB_ID:-}" ]; then
+ if ! cx_cancel_job "$JOB_ID"; then
+ allocation_stopped=0
+ [ "$rc" != 0 ] || rc=1
+ fi
+ elif [ "${CX_ALLOCATION_UNCERTAIN:-0}" = 1 ]; then
+ allocation_stopped=0
+ [ "$rc" != 0 ] || rc=1
+ fi
+ if [ "$allocation_stopped" = 1 ]; then
+ cx_write_cleanup_guard safe || true
+ else
+ cx_write_cleanup_guard unsafe || true
+ fi
+ [ "$allocation_stopped" = 1 ] || source_root="${REPO_ROOT:-$source_root}"
+ if [ "$rc" != 0 ] && [ -n "${REPO_ROOT:-}" ] && [ -n "${CX_BENCH:-}" ]; then
+ cx_log "ERROR: terminal-failure-class=${CX_FAILSAFE_MODE:-setup}"
+ [ -d "$source_root/experimental/CollectiveX" ] || source_root="$REPO_ROOT"
+ out_dir="$source_root/experimental/CollectiveX/results"
+ cx_emit_setup_failures \
+ "$source_root/experimental/CollectiveX" "$out_dir" "$CX_BENCH" "$rc" || true
+ [ "$source_root" = "$REPO_ROOT" ] \
+ || cx_collect_results "$source_root" "$REPO_ROOT" || true
+ fi
+ if [ "$allocation_stopped" = 1 ] && [ -n "${REPO_ROOT:-}" ] \
+ && [ -n "$stage_root" ] && [ "$stage_root" != "$REPO_ROOT" ]; then
+ if ! cx_cleanup_stage "$stage_root" "$REPO_ROOT"; then
+ [ "$rc" != 0 ] || rc=1
+ fi
+ fi
+ [ "${COLLECTIVEX_CANONICAL_GHA:-0}" = 1 ] || cx_cleanup_private_logs "$rc"
+ exit "$rc"
+}
+
+cx_install_launcher_fail_safe() {
+ CX_ALLOCATION_UNCERTAIN=0
+ trap 'cx_launcher_cleanup "$?"' EXIT
+}
diff --git a/experimental/CollectiveX/runtime/run_in_container.sh b/experimental/CollectiveX/runtime/run_in_container.sh
new file mode 100644
index 0000000000..63396c48a0
--- /dev/null
+++ b/experimental/CollectiveX/runtime/run_in_container.sh
@@ -0,0 +1,1087 @@
+#!/usr/bin/env bash
+# CollectiveX — generic in-container benchmark dispatcher (single-node).
+#
+# Runs INSIDE the container under `srun` for single-node shards. The GB EP8 launcher invokes
+# run_ep.py directly across nodes. The SKU adapter handles allocation/container/transport-env;
+# this script selects one EP backend from CX_BENCH and writes result JSON under results/.
+#
+# Required env (exported by the adapter): CX_RUNNER CX_NGPUS CX_TS CX_TOPO
+# Selector: CX_BENCH = deepep | deepep-v2 | mori | uccl | nccl-ep | deepep-hybrid
+# EP knobs passed to tests/run_ep.py:
+# CX_PHASE = decode | prefill | both (default decode) <- picks the token sweep
+# CX_TOKENS_LADDER (space/comma sep; blank = phase default)
+# CX_HIDDEN CX_TOPK CX_EXPERTS CX_ROUTING CX_SEED CX_ITERS
+set -euo pipefail
+
+cd /ix/experimental/CollectiveX
+# shellcheck source=../runtime/common.sh
+source runtime/common.sh
+mkdir -p results
+cx_write_runtime_stage backend-setup || cx_die "cannot record runtime stage"
+
+: "${CX_RUNNER:?CX_RUNNER not set}"
+: "${CX_NGPUS:?CX_NGPUS not set}"
+: "${CX_TS:?CX_TS not set}"
+: "${CX_TOPO:?CX_TOPO not set}"
+CX_BENCH="${CX_BENCH:-deepep}"
+CX_TRANSPORT="${CX_TRANSPORT:-}"
+
+cx_apply_timing_profile
+
+cx_log "in-container: runner=$CX_RUNNER ngpus=$CX_NGPUS bench=$CX_BENCH topo=$CX_TOPO"
+
+# Blank ladders use the phase default in tests/run_ep.py.
+cx_ep_ladder() {
+ printf '%s' "${CX_TOKENS_LADDER:-}"
+}
+
+# Canonical workload staging. Every SKU/backend generates identical canonical array bytes and
+# content IDs in-container; the NPZ container bytes themselves are not an identity boundary. When CX_CANONICAL=1
+# (and CX_WORKLOAD_DIR not already provided) we generate routing traces for the run's ladder
+# into a NON-results dir (.cx_workloads/ — so the *.manifest.json never pollute the results glob) and
+# point run_ep at it. Raw attempts remain diagnostic until the publisher validates full coverage.
+cx_stage_canonical() {
+ cx_bool_enabled "${CX_CANONICAL:-0}" || return 0
+ [ -n "${CX_WORKLOAD_DIR:-}" ] && return 0
+ local dir="$PWD/.cx_workloads"
+ local ladder; ladder="$(cx_ep_ladder)"
+ # cover both phase ladders when none is given, so either phase finds its files.
+ [ -z "$ladder" ] && ladder="1 2 4 8 16 32 64 128 256 512 1024 2048 4096"
+ cx_log "staging canonical workloads (routing=${CX_ROUTING:-uniform} ep=$CX_NGPUS ladder='$ladder')"
+ python3 tests/make_workloads.py --out-dir "$dir" --routing "${CX_ROUTING:-uniform}" \
+ --ep "$CX_NGPUS" --hidden "${CX_HIDDEN:-7168}" --topk "${CX_TOPK:-8}" \
+ --experts "${CX_EXPERTS:-256}" --seed "${CX_SEED:-67}" --tokens-ladder "$ladder" \
+ || { cx_log "ERROR: canonical workload staging failed"; return 1; }
+ export CX_WORKLOAD_DIR="$dir"
+ cx_log "canonical workloads staged at $dir"
+}
+
+# run_ep_suite
+# One tests/run_ep.py invocation per phase (decode/prefill/both); dispatch and
+# combine are timed separately inside it. One JSON per (backend, phase).
+# Preserve a failed case with its full scheduled identity instead of letting it vanish.
+emit_failed_case() { # backend phase rc
+ cx_emit_ep_failed_case \
+ "results/failed_${CX_RUNNER}_${1}_${2}_${CX_TS}.json" "$1" "$2" "$3" || true
+}
+
+run_ep_suite() {
+ local backend="$1" phase phases ladder failure_kind rc=0 rc_run
+ ladder="$(cx_ep_ladder)"
+ phases="${CX_PHASE:-decode}"
+ [ "$phases" = "both" ] && phases="decode prefill"
+ if ! cx_stage_canonical; then
+ for phase in $phases; do
+ emit_failed_case "$backend" "$phase" 2
+ done
+ return 1
+ fi
+ for phase in $phases; do
+ cx_log "ep backend=$backend phase=$phase ngpus=$CX_NGPUS ladder='${ladder:-}'"
+ local out="results/${CX_RUNNER}_${backend}_${phase}_${CX_TS}.json"
+ local -a EPARGS=(--backend "$backend" --mode "${CX_MODE:-normal}" --phase "$phase"
+ --tokens-ladder "$ladder"
+ --hidden "${CX_HIDDEN:-7168}" --topk "${CX_TOPK:-8}" --experts "${CX_EXPERTS:-256}"
+ --routing "${CX_ROUTING:-uniform}" --seed "${CX_SEED:-67}" --iters "${CX_ITERS:-8}"
+ --trials "${CX_TRIALS:-64}" --warmup "${CX_WARMUP:-32}"
+ --gpus-per-node "${CX_GPUS_PER_NODE:-0}" --scale-up-domain "${CX_SCALE_UP_DOMAIN:-0}"
+ --scope "${CX_SCOPE:-scale-up}" --scale-up-transport "${CX_SCALE_UP_TRANSPORT:-unknown}"
+ --scale-out-transport "${CX_SCALE_OUT_TRANSPORT:-}"
+ --case-id "${CX_CASE_ID:-}" --suite "${CX_SUITE:-}" --workload-name "${CX_WORKLOAD_NAME:-}"
+ --required-publication "${CX_REQUIRED_PUBLICATION:-}"
+ --runner "$CX_RUNNER" --topology-class "$CX_TOPO" --transport "$CX_TRANSPORT"
+ --out "$out")
+ cx_bool_enabled "${CX_EPLB:-0}" && EPARGS+=(--eplb)
+ [ -n "${CX_WORKLOAD_DIR:-}" ] && EPARGS+=(--workload-dir "$CX_WORKLOAD_DIR")
+ cx_write_runtime_stage execution || cx_die "cannot record runtime stage"
+ if timeout -k 30 "${CX_RUN_TIMEOUT:-900}" \
+ torchrun --nproc_per_node="$CX_NGPUS" tests/run_ep.py "${EPARGS[@]}"; then
+ rc_run=0
+ else
+ rc_run=$?
+ fi
+ if [ "$rc_run" = 0 ] && cx_result_doc_is "$out" invalid; then
+ cx_log "WARN: $backend $phase completed with invalid semantic evidence"
+ rc=1
+ continue
+ fi
+ if [ "$rc_run" = 0 ] && ! cx_result_doc_is "$out" success; then
+ rc_run=1
+ fi
+ if [ "$rc_run" != 0 ]; then
+ failure_kind=failed
+ [ "$rc_run" != 124 ] && [ "$rc_run" != 137 ] || failure_kind="timed out"
+ if [ "$failure_kind" = "timed out" ]; then
+ cx_log "WARN: $backend $phase run timed out rc=$rc_run (limit=${CX_RUN_TIMEOUT:-900}s)"
+ else
+ cx_log "WARN: $backend $phase run failed rc=$rc_run"
+ fi
+ if cx_has_result_doc "$out"; then
+ cx_demote_result_doc "$out" "$rc_run" \
+ || { cx_quarantine_result_doc "$out"; emit_failed_case "$backend" "$phase" "$rc_run"; }
+ cx_log "preserved benchmark output as a failed attempt"
+ else
+ cx_quarantine_result_doc "$out"
+ emit_failed_case "$backend" "$phase" "$rc_run"
+ fi
+ rc=1
+ fi
+ done
+ return "$rc"
+}
+
+# Resolve and verify the actual CUDA target before compiling source kernels.
+cx_cuda_arch() {
+ local expected detected
+ case "$CX_RUNNER" in
+ h100*|h200*) expected="9.0" ;;
+ b200*|gb200*) expected="10.0" ;;
+ b300*|gb300*) expected="10.3" ;;
+ *) cx_log "ERROR: no CUDA target registered for $CX_RUNNER"; return 1 ;;
+ esac
+ detected="$(python3 - <<'PY'
+import torch
+
+major, minor = torch.cuda.get_device_capability()
+print(f"{major}.{minor}")
+PY
+)" || return 1
+ [ "$detected" = "$expected" ] || {
+ cx_log "ERROR: $CX_RUNNER expected CUDA target $expected, detected $detected"
+ return 1
+ }
+ printf '%s' "$detected"
+}
+
+cx_nvidia_package_root() {
+ local package="$1" component="$2"
+ python3 - "$package" "$component" <<'PY'
+from importlib import metadata
+from pathlib import Path, PurePosixPath
+import sys
+
+package, component = sys.argv[1:]
+try:
+ distribution = metadata.distribution(package)
+ prefix = f"nvidia/{component}/"
+ entries = [str(entry).replace("\\", "/") for entry in distribution.files or ()]
+ if not any(entry.startswith(prefix) for entry in entries):
+ raise ValueError
+ root = Path(distribution.locate_file(PurePosixPath("nvidia") / component)).resolve()
+ if not root.is_dir():
+ raise ValueError
+except (metadata.PackageNotFoundError, OSError, TypeError, ValueError):
+ raise SystemExit(1)
+print(root, end="")
+PY
+}
+
+cx_prepare_cuda_cccl() {
+ local cccl="" candidate cuda_home nvcc
+ nvcc="$(command -v nvcc)" \
+ || { cx_log "ERROR: CUDA nvcc is unavailable"; return 1; }
+ nvcc="$(readlink -f -- "$nvcc")" \
+ || { cx_log "ERROR: CUDA nvcc cannot be resolved"; return 1; }
+ case "$nvcc" in
+ */bin/nvcc) cuda_home="${nvcc%/bin/nvcc}" ;;
+ *) cx_log "ERROR: CUDA nvcc has an unexpected path"; return 1 ;;
+ esac
+ [ -x "$cuda_home/bin/nvcc" ] && [ -d "$cuda_home/include" ] \
+ && [ -d "$cuda_home/lib64" ] \
+ || { cx_log "ERROR: CUDA toolkit root is incomplete"; return 1; }
+ for candidate in "$cuda_home"/targets/*/include/cccl; do
+ if [ -d "$candidate" ]; then
+ cccl="$candidate"
+ break
+ fi
+ done
+ [ -n "$cccl" ] || { cx_log "ERROR: CUDA CCCL headers are unavailable"; return 1; }
+ export CUDA_HOME="$cuda_home" CX_CUDA_CCCL="$cccl"
+ export CPATH="$cccl:${CPATH:-}"
+ export NVCC_PREPEND_FLAGS="-I$cccl ${NVCC_PREPEND_FLAGS:-}"
+}
+
+cx_prepare_deepep_toolchain() {
+ local packaged overlay path root temporary
+ packaged="$(cx_nvidia_package_root nvidia-nvshmem-cu12 nvshmem)" \
+ || { cx_log "ERROR: nvidia.nvshmem is unavailable"; return 1; }
+ root="$(cx_deepep_v2_root)" || return 1
+ overlay="$root/nvshmem-overlay"
+ if ! (
+ umask 077
+ exec 8>"$root/nvshmem-overlay.lock" || exit 1
+ flock 8 || exit 1
+ if [ ! -d "$overlay" ]; then
+ temporary="$root/.nvshmem-overlay.$$"
+ rm -rf "$temporary" || exit 1
+ mkdir -p "$temporary/lib" || exit 1
+ ln -s "$packaged/include" "$temporary/include" || exit 1
+ for path in "$packaged"/lib/*; do
+ ln -s "$path" "$temporary/lib/${path##*/}" || exit 1
+ done
+ [ ! -e "$packaged/lib/libnvshmem_host.so.3" ] \
+ || ln -sf "$packaged/lib/libnvshmem_host.so.3" \
+ "$temporary/lib/libnvshmem_host.so" || exit 1
+ mv "$temporary" "$overlay" || exit 1
+ fi
+ [ ! -L "$overlay" ] \
+ && [ "$(readlink -f "$overlay/include")" = "$(readlink -f "$packaged/include")" ] \
+ && [ -e "$overlay/lib/libnvshmem_host.so" ] \
+ && [ -e "$overlay/lib/libnvshmem_device.a" ]
+ ); then
+ cx_log "ERROR: DeepEP V2 NVSHMEM overlay is invalid"
+ return 1
+ fi
+ NVSHMEM_DIR="$overlay"
+ export NVSHMEM_DIR
+ cx_prepare_cuda_cccl || return 1
+ export LD_LIBRARY_PATH="$NVSHMEM_DIR/lib:${LD_LIBRARY_PATH:-}"
+}
+
+cx_probe_deepep() {
+ local expected_record_sha256 expected_version expected_wheel_sha256
+ if [ "${COLLECTIVEX_IMAGE:-}" != "$CX_IMAGE_MULTIARCH" ] \
+ || [ "${COLLECTIVEX_IMAGE_DIGEST:-}" != "$CX_IMAGE_MULTIARCH_DIGEST" ] \
+ || [ "${COLLECTIVEX_IMAGE_DIGEST_VERIFIED:-0}" != 1 ]; then
+ cx_log "ERROR: DeepEP V1 requires the exact pinned multi-architecture image"
+ return 1
+ fi
+ cx_cuda_arch >/dev/null || return 1
+ case "$CX_RUNNER" in
+ gb200|gb300)
+ expected_version="1.1.0+814e508"
+ expected_wheel_sha256="784dabec0877b6cf72619b7e93eda7e2f365648487bd37fc3ff6960e53669313"
+ expected_record_sha256="2671cff7baf8c2c214ff4bac721af875d513130670bec57601998bd1aae82882"
+ DEEPEP_COMMIT="814e508537c6ffc775d59f6f1b9ba43f3a65968c"
+ ;;
+ *)
+ expected_version="1.2.1"
+ expected_wheel_sha256="7c02c29306ea0fe2dd474618e72e0f310f260187a9c0700a656d2f6964e8c307"
+ expected_record_sha256="6548e9c504a12b2471af4b7f4d9546321210a57a456b5dc55bd4a8dad0f932ac"
+ DEEPEP_COMMIT="9af0e0d0e74f3577af1979c9b9e1ac2cad0104ee"
+ ;;
+ esac
+ export DEEPEP_COMMIT
+ python3 - "$expected_version" "$expected_wheel_sha256" "$expected_record_sha256" <<'PY' || {
+import base64
+import csv
+import hashlib
+import importlib.metadata as metadata
+import io
+import json
+from pathlib import Path
+import sys
+
+import deep_ep
+from deep_ep import Buffer
+
+distribution = metadata.distribution("deep_ep")
+assert distribution.version == sys.argv[1]
+assert Buffer.__name__ == "Buffer"
+recorded_files = {
+ Path(distribution.locate_file(entry)).resolve() for entry in distribution.files or ()
+}
+buffer_module = sys.modules.get(Buffer.__module__)
+assert Path(deep_ep.__file__).resolve() in recorded_files
+assert buffer_module is not None and Path(buffer_module.__file__).resolve() in recorded_files
+direct_url = json.loads(distribution.read_text("direct_url.json"))
+assert direct_url["archive_info"]["hashes"]["sha256"] == sys.argv[2]
+record_entry = next(
+ entry for entry in distribution.files or ()
+ if str(entry).endswith(".dist-info/RECORD")
+)
+record = distribution.locate_file(record_entry).read_bytes()
+assert hashlib.sha256(record).hexdigest() == sys.argv[3]
+for path, encoded_digest, size in csv.reader(io.StringIO(record.decode())):
+ if not encoded_digest:
+ continue
+ algorithm, expected = encoded_digest.split("=", 1)
+ assert algorithm == "sha256"
+ payload = distribution.locate_file(path).read_bytes()
+ observed = base64.urlsafe_b64encode(hashlib.sha256(payload).digest()).decode().rstrip("=")
+ assert observed == expected
+ assert not size or len(payload) == int(size)
+PY
+ cx_log "ERROR: container DeepEP build does not match its pinned image contract"
+ return 1
+ }
+ cx_log "DeepEP image build ready ($DEEPEP_COMMIT)"
+}
+
+# DeepEP V2 is PR #605's ElasticBuffer implementation with upstream PR #630's pure scale-up
+# initialization fix. Canonical launchers stage the pinned source and mount a private cluster-local
+# build cache at /cx-cache.
+cx_deepep_v2_root() {
+ local arch cpu base identity key image_digest
+ arch="$(cx_cuda_arch)" || return 1
+ cpu="$(uname -m)"
+ [[ "$cpu" =~ ^[A-Za-z0-9._-]+$ ]] || return 1
+ base="${CX_BACKEND_CACHE_ROOT:-}"
+ [[ "$base" = /* ]] || return 1
+ image_digest="${COLLECTIVEX_IMAGE_DIGEST:-manual-unverified}"
+ [[ "$image_digest" = manual-unverified || "$image_digest" =~ ^sha256:[0-9a-f]{64}$ ]] \
+ || return 1
+ # Bump the recipe generation whenever the build procedure changes. Benchmark-only
+ # source revisions must reuse the same immutable environment instead of leaking GBs.
+ identity="deepep-v2-cache-v2|$cpu|sm${arch/./}|image=$image_digest|recipe=aot-persistent-nvshmem-active-cuda-maxjobs16-v2|$CX_DEEPEP_V2_COMMIT|$CX_DEEPEP_V2_TREE|$CX_DEEPEP_V2_FMT_COMMIT|pip=26.1.2|setuptools=82.0.1|wheel=0.47.0|ninja=1.13.0|numpy=2.2.6|torch=2.10.0+cu130|nccl=2.30.4|nvshmem=3.3.9|max-jobs=16"
+ key="$(printf '%s' "$identity" | sha256sum | awk '{print $1}')"
+ [[ "$key" =~ ^[0-9a-f]{64}$ ]] || return 1
+ printf '%s/deepep-v2-%s' "$base" "$key"
+}
+
+cx_activate_deepep_v2() {
+ local root venv stage_root
+ root="$(cx_deepep_v2_root)" || return 1
+ venv="$root/venv"
+ [ -x "$venv/bin/python" ] \
+ || { cx_log "ERROR: DeepEP V2 venv interpreter is unavailable"; return 1; }
+ export VIRTUAL_ENV="$venv"
+ export PATH="$venv/bin:${PATH#"$venv/bin:"}"
+ EP_NCCL_ROOT_DIR="$(cx_nvidia_package_root nvidia-nccl-cu13 nccl)" \
+ || { cx_log "ERROR: DeepEP V2 NCCL package root is unavailable"; return 1; }
+ EP_NVSHMEM_ROOT_DIR="$(cx_nvidia_package_root nvidia-nvshmem-cu12 nvshmem)" \
+ || { cx_log "ERROR: DeepEP V2 NVSHMEM package root is unavailable"; return 1; }
+ export EP_NCCL_ROOT_DIR EP_NVSHMEM_ROOT_DIR
+ export LD_LIBRARY_PATH="$EP_NCCL_ROOT_DIR/lib:$EP_NVSHMEM_ROOT_DIR/lib:${LD_LIBRARY_PATH:-}"
+ case "${CX_BACKEND_SOURCE_ROOT:-}" in
+ /*/.cx_sources) stage_root="${CX_BACKEND_SOURCE_ROOT%/.cx_sources}" ;;
+ *) cx_log "ERROR: DeepEP V2 job-local source root is unavailable"; return 1 ;;
+ esac
+ [ -d "$stage_root" ] && [ ! -L "$stage_root" ] \
+ || { cx_log "ERROR: DeepEP V2 job-local stage is invalid"; return 1; }
+ # JIT CUBINs are evidence from this shard, not part of the persistent AOT environment.
+ # Keeping them on the isolated staged tree prevents a prior driver/topology attempt
+ # from seeding a later run; all ranks and cases in this shard still share one cold build.
+ export EP_JIT_CACHE_DIR="$stage_root/.cx_backend/deepep-v2-jit"
+ export EP_REUSE_NCCL_COMM=1
+ export DEEPEP_V2_PR=605 DEEPEP_V2_FIX_PR=630
+ DEEPEP_V2_COMMIT="$CX_DEEPEP_V2_COMMIT"
+ DEEPEP_V2_TREE="$CX_DEEPEP_V2_TREE"
+ DEEPEP_V2_FMT_COMMIT="$CX_DEEPEP_V2_FMT_COMMIT"
+ export DEEPEP_V2_COMMIT DEEPEP_V2_TREE DEEPEP_V2_FMT_COMMIT
+ [ ! -L "$stage_root/.cx_backend" ] && [ ! -L "$EP_JIT_CACHE_DIR" ] \
+ || { cx_log "ERROR: DeepEP V2 JIT cache path is unsafe"; return 1; }
+ if ! mkdir -p "$EP_JIT_CACHE_DIR" \
+ || ! chmod 700 "$stage_root/.cx_backend" "$EP_JIT_CACHE_DIR"; then
+ cx_log "ERROR: DeepEP V2 JIT cache is unavailable"
+ return 1
+ fi
+ unset EP_SUPPRESS_NCCL_CHECK
+}
+
+cx_enable_deepep_v2_jit_reproducibility() {
+ local seed="collectivex-deepep-v2-fa8a9b1" cccl
+ [ -n "${CUDA_HOME:-}" ] \
+ || { cx_log "ERROR: active CUDA toolkit is unavailable"; return 1; }
+ cccl="${CX_CUDA_CCCL:-}"
+ case "$cccl" in
+ "$CUDA_HOME"/targets/*/include/cccl) ;;
+ *) cx_log "ERROR: CUDA CCCL headers differ from the active toolkit"; return 1 ;;
+ esac
+ [ -d "$cccl" ] || { cx_log "ERROR: CUDA CCCL headers are unavailable"; return 1; }
+ CPATH="$cccl"
+ NVCC_PREPEND_FLAGS="--frandom-seed=$seed -I$cccl"
+ DEEPEP_V2_JIT_RANDOM_SEED="$seed"
+ EP_JIT_DUMP_SASS=1
+ unset EP_JIT_DEBUG EP_JIT_DUMP_ASM EP_JIT_DUMP_PTX EP_JIT_WITH_LINEINFO
+ unset EP_JIT_PTXAS_VERBOSE EP_JIT_PRINT_COMPILER_COMMAND EP_JIT_NVCC_COMPILER
+ unset EP_JIT_CPP_STANDARD EP_JIT_PTXAS_CHECK EP_GIN_GDAKI_DEBUG EP_NUM_TOPK_IDX_BITS
+ export CPATH DEEPEP_V2_JIT_RANDOM_SEED EP_JIT_DUMP_SASS NVCC_PREPEND_FLAGS
+}
+
+cx_probe_deepep_v2() {
+ python3 - <<'PY'
+import ctypes
+import importlib.metadata as metadata
+import inspect
+import os
+
+import torch
+
+assert torch.__version__ == "2.10.0+cu130", torch.__version__
+assert metadata.version("nvidia-nccl-cu13") == "2.30.4"
+assert metadata.version("nvidia-nvshmem-cu12") == "3.3.9"
+assert metadata.version("numpy") == "2.2.6"
+
+import deep_ep
+assert deep_ep.__version__ == "2.0.0", deep_ep.__version__
+assert metadata.version("deep_ep") == "2.0.0+fa8a9b1"
+assert inspect.isclass(deep_ep.ElasticBuffer)
+assert deep_ep.ElasticBuffer.__name__ == "ElasticBuffer"
+assert os.environ.get("EP_SUPPRESS_NCCL_CHECK") is None
+with open("/proc/self/maps", encoding="utf-8") as handle:
+ loaded_nccl = {
+ os.path.realpath(line.rstrip().split()[-1])
+ for line in handle
+ if "libnccl.so" in line and os.path.isfile(line.rstrip().split()[-1])
+ }
+assert len(loaded_nccl) == 1
+runtime_version = ctypes.c_int()
+assert ctypes.CDLL(loaded_nccl.pop()).ncclGetVersion(ctypes.byref(runtime_version)) == 0
+assert runtime_version.value == 23004, runtime_version.value
+PY
+}
+
+cx_deepep_v2_content_sha256() {
+ python3 - <<'PY'
+import hashlib
+from importlib import metadata
+import os
+from pathlib import Path, PurePosixPath
+import stat
+
+distribution = metadata.distribution("deep_ep")
+entries = sorted(distribution.files or (), key=lambda entry: entry.as_posix())
+if not entries:
+ raise SystemExit(1)
+venv_path = Path(os.environ["VIRTUAL_ENV"]).absolute()
+if venv_path.is_symlink() or not venv_path.is_dir():
+ raise SystemExit(1)
+venv = venv_path.resolve(strict=True)
+digest = hashlib.sha256()
+extension = False
+for entry in entries:
+ relative = PurePosixPath(entry.as_posix())
+ if (
+ relative.is_absolute()
+ or ".." in relative.parts
+ or not relative.parts
+ or not (
+ relative.parts[0] == "deep_ep"
+ or relative.parts[0].startswith("deep_ep-")
+ and relative.parts[0].endswith(".dist-info")
+ )
+ ):
+ raise SystemExit(1)
+ path = Path(distribution.locate_file(entry)).absolute()
+ resolved = path.resolve(strict=True)
+ try:
+ path.relative_to(venv_path)
+ resolved.relative_to(venv)
+ except ValueError:
+ raise SystemExit(1)
+ parent = path.parent
+ while parent != venv_path:
+ if parent.is_symlink():
+ raise SystemExit(1)
+ parent = parent.parent
+ item = os.lstat(path)
+ if not stat.S_ISREG(item.st_mode):
+ raise SystemExit(1)
+ descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
+ try:
+ opened = os.fstat(descriptor)
+ if (opened.st_dev, opened.st_ino) != (item.st_dev, item.st_ino):
+ raise SystemExit(1)
+ file_digest = hashlib.sha256()
+ while chunk := os.read(descriptor, 1024 * 1024):
+ file_digest.update(chunk)
+ finally:
+ os.close(descriptor)
+ name = relative.as_posix()
+ extension |= name.startswith("deep_ep/") and name.endswith(".so")
+ digest.update(name.encode())
+ digest.update(b"\0")
+ digest.update(str(item.st_size).encode())
+ digest.update(b"\0")
+ digest.update(file_digest.digest())
+if not extension:
+ raise SystemExit(1)
+print(digest.hexdigest(), end="")
+PY
+}
+
+cx_deepep_v2_marker_content_sha256() {
+ local root="$1" marker="$2" revision="$3" tree="$4" fmt_revision="$5" cache_key="$6"
+ python3 - "$root" "$marker" "$revision" "$tree" "$fmt_revision" "$cache_key" <<'PY'
+import os
+import re
+import stat
+import sys
+
+root, marker, revision, tree, fmt_revision, cache_key = sys.argv[1:]
+try:
+ root_item = os.lstat(root)
+ marker_item = os.lstat(marker)
+ children = [os.lstat(os.path.join(root, name)) for name in ("source", "venv")]
+ if (
+ not stat.S_ISDIR(root_item.st_mode)
+ or stat.S_IMODE(root_item.st_mode) & 0o777 != 0o700
+ or not stat.S_ISREG(marker_item.st_mode)
+ or marker_item.st_uid != root_item.st_uid
+ or stat.S_IMODE(marker_item.st_mode) & 0o777 != 0o600
+ or marker_item.st_size > 1024
+ or any(
+ not stat.S_ISDIR(child.st_mode)
+ or child.st_uid != root_item.st_uid
+ or stat.S_IMODE(child.st_mode) & 0o022
+ for child in children
+ )
+ ):
+ raise OSError
+ descriptor = os.open(marker, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
+ try:
+ opened = os.fstat(descriptor)
+ if (opened.st_dev, opened.st_ino) != (marker_item.st_dev, marker_item.st_ino):
+ raise OSError
+ payload = os.read(descriptor, 1025)
+ finally:
+ os.close(descriptor)
+ lines = payload.decode("ascii").splitlines()
+ if lines[:4] != [revision, tree, fmt_revision, cache_key] or len(lines) != 5:
+ raise ValueError
+ if not re.fullmatch(r"[0-9a-f]{64}", lines[4]):
+ raise ValueError
+except (OSError, UnicodeError, ValueError):
+ raise SystemExit(1)
+print(lines[4], end="")
+PY
+}
+
+cx_deepep_v2_cache_is_valid() {
+ local root="$1" marker="$2" revision="$3" tree="$4" fmt_revision="$5" cache_key="$6"
+ local expected_content actual_content
+ expected_content="$(
+ cx_deepep_v2_marker_content_sha256 \
+ "$root" "$marker" "$revision" "$tree" "$fmt_revision" "$cache_key"
+ )" || return 1
+ [ -d "$root/source" ] && [ ! -L "$root/source" ] \
+ && [ "$(cx_git_in_tree "$root/source" rev-parse 'HEAD^{tree}' 2>/dev/null)" = "$tree" ] \
+ && [ "$(cx_git_in_tree "$root/source/third-party/fmt" rev-parse HEAD 2>/dev/null)" = "$fmt_revision" ] \
+ || return 1
+ cx_activate_deepep_v2 || return 1
+ actual_content="$(cx_deepep_v2_content_sha256)" || return 1
+ [ "$actual_content" = "$expected_content" ]
+}
+
+cx_build_deepep_v2() {
+ local root venv source marker marker_tmp lock_path arch cache_key cache_ready content_sha256
+ local revision="fa8a9b16898204afd347c663b89e65ef87dc6ce6"
+ local tree="29809e75c5874e6609dac4804e7b651d5226959f"
+ local fmt_revision="a4c7e17133ee9cb6a2f45545f6e974dd3c393efa"
+ cx_verify_backend_cache_mount \
+ || { cx_log "ERROR: DeepEP V2 cache mount identity validation failed"; return 1; }
+ arch="$(cx_cuda_arch)" || return 1
+ root="$(cx_deepep_v2_root)" || return 1
+ cache_key="${root##*/deepep-v2-}"
+ [[ "$cache_key" =~ ^[0-9a-f]{64}$ ]] || return 1
+ venv="$root/venv"; source="$root/source"; marker="$root/.collectivex-complete"
+ lock_path="${root}.lock"
+ command -v flock >/dev/null || { cx_log "ERROR: flock is required for DeepEP V2"; return 1; }
+ mkdir -p "${root%/*}" || return 1
+ cx_log "DeepEP V2: preparing PR #605 implementation with upstream PR #630 fix ($revision)"
+ if ! (
+ [ ! -L "$lock_path" ] \
+ || { cx_log "ERROR: DeepEP V2 cache lock is unsafe"; exit 1; }
+ (umask 077; : >> "$lock_path") && chmod 600 "$lock_path" \
+ || { cx_log "ERROR: DeepEP V2 cache-lock-create failed"; exit 1; }
+ exec 9<>"$lock_path" \
+ || { cx_log "ERROR: DeepEP V2 cache-lock-open failed"; exit 1; }
+ flock 9 \
+ || { cx_log "ERROR: DeepEP V2 cache-lock-acquire failed"; exit 1; }
+ cache_ready=0
+ if [ -e "$marker" ] || [ -L "$marker" ]; then
+ if (
+ cx_deepep_v2_cache_is_valid \
+ "$root" "$marker" "$revision" "$tree" "$fmt_revision" "$cache_key"
+ ); then
+ cache_ready=1
+ else
+ cx_log "ERROR: published DeepEP V2 cache failed integrity validation; refusing reset"
+ exit 1
+ fi
+ fi
+ if [ "$cache_ready" != 1 ]; then
+ if [ -e "$root" ] || [ -L "$root" ]; then
+ rm -rf "$root" \
+ || { cx_log "ERROR: incomplete DeepEP V2 cache-reset failed"; exit 1; }
+ fi
+ mkdir -m 700 "$root" \
+ || { cx_log "ERROR: DeepEP V2 cache-create failed"; exit 1; }
+ python3 -m venv "$venv" \
+ || { cx_log "ERROR: DeepEP V2 venv creation failed"; exit 1; }
+ "$venv/bin/python" -m pip install -q --disable-pip-version-check --no-input \
+ "pip==26.1.2" "setuptools==82.0.1" "wheel==0.47.0" "ninja==1.13.0" \
+ "numpy==2.2.6" "nvidia-nvshmem-cu12==3.3.9" >&2 2>&1 \
+ || { cx_log "ERROR: DeepEP V2 build-tool installation failed"; exit 1; }
+ "$venv/bin/python" -m pip install -q --disable-pip-version-check --no-input \
+ --index-url https://download.pytorch.org/whl/cu130 \
+ --extra-index-url https://pypi.org/simple "torch==2.10.0" >&2 2>&1 \
+ || { cx_log "ERROR: torch 2.10.0+cu130 installation failed"; exit 1; }
+ # Torch pins NCCL 2.28.9; the PR #605 ElasticBuffer implementation requires 2.30.4.
+ "$venv/bin/python" -m pip install -q --disable-pip-version-check --no-input \
+ --force-reinstall --no-deps "nvidia-nccl-cu13==2.30.4" >&2 2>&1 \
+ || { cx_log "ERROR: NCCL 2.30.4 installation failed"; exit 1; }
+ cx_activate_deepep_v2 \
+ || { cx_log "ERROR: DeepEP V2 environment activation failed"; exit 1; }
+ cx_prepare_deepep_toolchain \
+ || { cx_log "ERROR: DeepEP V2 toolchain preparation failed"; exit 1; }
+ EP_NVSHMEM_ROOT_DIR="$NVSHMEM_DIR"
+ export EP_NVSHMEM_ROOT_DIR
+ cx_materialize_backend_source deepep-v2 "$source" \
+ || { cx_log "ERROR: DeepEP V2 staged source is invalid"; exit 1; }
+ (cd "$source" && SOURCE_DATE_EPOCH="$(cx_git_in_tree "$source" show -s --format=%ct HEAD)" \
+ TORCH_CUDA_ARCH_LIST="$arch" MAX_JOBS=16 \
+ python3 -m pip install -q --no-build-isolation --no-deps --force-reinstall .) >&2 2>&1 \
+ || { cx_log "ERROR: DeepEP V2 build failed"; exit 1; }
+ cx_probe_deepep_v2 \
+ || { cx_log "ERROR: DeepEP V2 ElasticBuffer/runtime probe failed"; exit 1; }
+ content_sha256="$(cx_deepep_v2_content_sha256)" \
+ || { cx_log "ERROR: DeepEP V2 installed-content hashing failed"; exit 1; }
+ marker_tmp="$(mktemp "$root/.collectivex-complete.tmp.XXXXXX")" \
+ || { cx_log "ERROR: DeepEP V2 cache-marker-create failed"; exit 1; }
+ chmod 600 "$marker_tmp" \
+ || { cx_log "ERROR: DeepEP V2 cache-marker-permission failed"; exit 1; }
+ printf '%s\n%s\n%s\n%s\n%s\n' \
+ "$revision" "$tree" "$fmt_revision" "$cache_key" "$content_sha256" > "$marker_tmp" \
+ || { cx_log "ERROR: DeepEP V2 cache-marker-write failed"; exit 1; }
+ mv -f -- "$marker_tmp" "$marker" \
+ || { cx_log "ERROR: DeepEP V2 cache-marker-publish failed"; exit 1; }
+ fi
+ cx_deepep_v2_cache_is_valid \
+ "$root" "$marker" "$revision" "$tree" "$fmt_revision" "$cache_key" \
+ || { cx_log "ERROR: DeepEP V2 cache validation failed"; exit 1; }
+ ); then
+ cx_log "ERROR: shared DeepEP V2 environment is incomplete"
+ return 1
+ fi
+ cx_activate_deepep_v2 || return 1
+ cx_prepare_deepep_toolchain || return 1
+ cx_enable_deepep_v2_jit_reproducibility || return 1
+ EP_NVSHMEM_ROOT_DIR="$NVSHMEM_DIR"
+ export EP_NVSHMEM_ROOT_DIR
+ cx_probe_deepep_v2 || { cx_log "ERROR: DeepEP V2 shared runtime probe failed"; return 1; }
+ cx_log "DeepEP V2 ready ($DEEPEP_V2_COMMIT, ElasticBuffer, NCCL Device API; LSA/Gin selected by adapter)"
+}
+
+# Build the pinned DeepEP `hybrid-ep` implementation. MNNVL remains one scale-up
+# domain; true x86 scale-out uses the upstream DOCA/RDMA build explicitly.
+cx_configure_deepep_hybrid_build() {
+ local interface device rdma_name
+ local -a interfaces devices
+ unset HYBRID_EP_MULTINODE USE_NIXL RDMA_CORE_HOME DEEPEP_HYBRID_BUILD_MODE
+ if [ "${CX_NODES:-1}" -le 1 ] || [ "${CX_TRANSPORT:-}" = mnnvl ]; then
+ export DEEPEP_HYBRID_BUILD_MODE=intradomain
+ return 0
+ fi
+ [ "$(uname -m)" = x86_64 ] \
+ || { cx_log "ERROR: hybrid-ep scale-out is registered only on x86_64"; return 1; }
+ [ -n "${GLOO_SOCKET_IFNAME:-}" ] && [ -n "${NCCL_IB_HCA:-}" ] \
+ || { cx_log "ERROR: hybrid-ep scale-out network selectors are unavailable"; return 1; }
+ IFS=, read -r -a interfaces <<< "$GLOO_SOCKET_IFNAME"
+ for interface in "${interfaces[@]}"; do
+ [ -d "/sys/class/net/$interface" ] \
+ || { cx_log "ERROR: configured hybrid-ep socket interface is absent"; return 1; }
+ done
+ IFS=, read -r -a devices <<< "$NCCL_IB_HCA"
+ for device in "${devices[@]}"; do
+ rdma_name="${device%%:*}"
+ [ -d "/sys/class/infiniband/$rdma_name" ] \
+ || { cx_log "ERROR: configured hybrid-ep RDMA device is absent"; return 1; }
+ done
+ command -v make >/dev/null \
+ || { cx_log "ERROR: make is required for hybrid-ep scale-out"; return 1; }
+ [ -r /usr/include/infiniband/verbs.h ] && [ -r /usr/include/infiniband/mlx5dv.h ] \
+ || { cx_log "ERROR: pinned hybrid-ep RDMA headers are unavailable"; return 1; }
+ python3 - <<'PY' >/dev/null 2>&1 || {
+import ctypes.util
+import sys
+sys.exit(0 if all(ctypes.util.find_library(name) for name in ("ibverbs", "mlx5")) else 1)
+PY
+ cx_log "ERROR: pinned hybrid-ep RDMA libraries are unavailable"
+ return 1
+ }
+ export HYBRID_EP_MULTINODE=1 USE_NIXL=0 RDMA_CORE_HOME=/usr
+ export DEEPEP_HYBRID_BUILD_MODE=multinode-doca
+}
+
+cx_deepep_hybrid_marker_content_sha256() {
+ python3 - "$1" "$2" "$3" "$4" "${5:-}" <<'PY'
+import os
+import re
+import stat
+import sys
+
+root, marker, revision, tree, build_mode = sys.argv[1:]
+try:
+ root_item = os.lstat(root)
+ marker_item = os.lstat(marker)
+ if (
+ not stat.S_ISDIR(root_item.st_mode)
+ or stat.S_IMODE(root_item.st_mode) & 0o777 != 0o700
+ or not stat.S_ISREG(marker_item.st_mode)
+ or marker_item.st_uid != root_item.st_uid
+ or stat.S_IMODE(marker_item.st_mode) & 0o777 != 0o600
+ or marker_item.st_size > 512
+ ):
+ raise OSError
+ descriptor = os.open(marker, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
+ try:
+ opened = os.fstat(descriptor)
+ if (opened.st_dev, opened.st_ino) != (marker_item.st_dev, marker_item.st_ino):
+ raise OSError
+ payload = os.read(descriptor, 513)
+ finally:
+ os.close(descriptor)
+ lines = payload.decode("ascii").splitlines()
+ expected = [revision, tree, build_mode] if build_mode else [revision, tree]
+ if len(lines) != len(expected) + 1 or lines[:-1] != expected:
+ raise ValueError
+ if not re.fullmatch(r"[0-9a-f]{64}", lines[-1]):
+ raise ValueError
+except (OSError, UnicodeError, ValueError):
+ raise SystemExit(1)
+print(lines[-1], end="")
+PY
+}
+
+cx_deepep_hybrid_cache_is_valid() {
+ local root="$1" marker="$2" revision="$3" tree="$4" build_mode="${5:-}"
+ local expected actual status extra
+ expected="$(cx_deepep_hybrid_marker_content_sha256 \
+ "$root" "$marker" "$revision" "$tree" "$build_mode")" || return 1
+ [ "$(cx_git_in_tree "$root" rev-parse HEAD 2>/dev/null)" = "$revision" ] \
+ && [ "$(cx_git_in_tree "$root" rev-parse 'HEAD^{tree}' 2>/dev/null)" = "$tree" ] \
+ || return 1
+ status="$(cx_git_in_tree "$root" status --porcelain --untracked-files=no \
+ --ignore-submodules=none 2>/dev/null)" || return 1
+ [ -z "$status" ] || return 1
+ extra="$(cx_git_in_tree "$root" ls-files --others --exclude-standard -- \
+ 'deep_ep/*.py' 'deep_ep/*.so' 2>/dev/null)" || return 1
+ [ -z "$extra" ] || return 1
+ extra="$(cx_git_in_tree "$root" ls-files --others --ignored --exclude-standard -- \
+ 'deep_ep/*.py' 'deep_ep/*.so' 2>/dev/null)" || return 1
+ [ -z "$extra" ] || return 1
+ actual="$(cx_extension_pair_sha256 "$root" 'deep_ep_cpp*.so' 'hybrid_ep_cpp*.so')" \
+ || return 1
+ [ "$actual" = "$expected" ]
+}
+
+cx_build_deepep_hybrid() {
+ local arch revision="$CX_DEEPEP_HYBRID_COMMIT" tree="$CX_DEEPEP_HYBRID_TREE"
+ local build_root marker marker_tmp lock_path content_sha256 cache_ready build_mode
+ export DEEPEP_COMMIT="$revision" DEEPEP_TREE="$tree"
+ arch="$(cx_cuda_arch)" || return 1
+ cx_configure_deepep_hybrid_build || return 1
+ build_mode="$DEEPEP_HYBRID_BUILD_MODE"
+ build_root="$PWD/.cx_backend/deepep-hybrid-${arch/./}-${build_mode}"
+ marker="$build_root/.collectivex-complete"
+ lock_path="${build_root}.lock"
+ cx_log "DeepEP hybrid-ep: building $revision for CUDA target $arch"
+ unset NVSHMEM_DIR
+ cx_prepare_cuda_cccl || return 1
+ command -v flock >/dev/null || { cx_log "ERROR: flock is required for hybrid-ep"; return 1; }
+ mkdir -p "$PWD/.cx_backend" || return 1
+ if ! (
+ [ ! -L "$lock_path" ] || exit 1
+ (umask 077; : >> "$lock_path") && chmod 600 "$lock_path" || exit 1
+ exec 9<>"$lock_path" || exit 1
+ flock 9 || exit 1
+ cache_ready=0
+ if [ -e "$marker" ] || [ -L "$marker" ]; then
+ cx_deepep_hybrid_cache_is_valid \
+ "$build_root" "$marker" "$revision" "$tree" "$build_mode" \
+ || exit 1
+ cache_ready=1
+ fi
+ if [ "$cache_ready" != 1 ]; then
+ cx_materialize_backend_source deepep-hybrid "$build_root" \
+ || { cx_log "ERROR: hybrid-ep staged source is invalid"; exit 1; }
+ if [ "$build_mode" = multinode-doca ]; then
+ [ "$(cx_git_in_tree "$build_root/third-party/nccl" rev-parse HEAD 2>/dev/null)" \
+ = "$CX_DEEPEP_HYBRID_NCCL_COMMIT" ] \
+ || { cx_log "ERROR: pinned hybrid-ep NCCL transport source is absent"; exit 1; }
+ fi
+ (cd "$build_root" && \
+ SOURCE_DATE_EPOCH="$(cx_git_in_tree "$build_root" show -s --format=%ct HEAD)" \
+ TORCH_CUDA_ARCH_LIST="$arch" MAX_JOBS=16 \
+ python3 setup.py build_ext --inplace) >&2 2>&1 \
+ || { cx_log "ERROR: hybrid-ep build failed"; exit 1; }
+ content_sha256="$(cx_extension_pair_sha256 \
+ "$build_root" 'deep_ep_cpp*.so' 'hybrid_ep_cpp*.so')" || exit 1
+ marker_tmp="$(mktemp "$build_root/.collectivex-complete.tmp.XXXXXX")" || exit 1
+ chmod 600 "$marker_tmp" || exit 1
+ printf '%s\n%s\n%s\n%s\n' \
+ "$revision" "$tree" "$build_mode" "$content_sha256" > "$marker_tmp" \
+ || exit 1
+ mv -f -- "$marker_tmp" "$marker" || exit 1
+ fi
+ cx_deepep_hybrid_cache_is_valid \
+ "$build_root" "$marker" "$revision" "$tree" "$build_mode"
+ ); then
+ cx_log "ERROR: shared hybrid-ep build is incomplete"
+ return 1
+ fi
+ export PYTHONPATH="$build_root:${PYTHONPATH:-}"
+ python3 -c "import deep_ep; assert hasattr(deep_ep,'HybridEPBuffer'); print('built hybrid-ep deep_ep', getattr(deep_ep,'__version__','?'))" >&2 \
+ || { cx_log "ERROR: hybrid-ep import / HybridEPBuffer missing after build"; return 1; }
+ cx_log "DeepEP hybrid-ep ready ($DEEPEP_COMMIT, mode=$build_mode)"
+}
+
+# UCCL EP (uccl.ep.Buffer is a DeepEP-API clone). The prebuilt wheel is cu12; on a cu13
+# image its kernels need a cu12 CUDA runtime on LD_LIBRARY_PATH (probe-confirmed). PEP-668
+# images need PIP_BREAK_SYSTEM_PACKAGES. Best-effort; failure to import fails loudly.
+cx_build_uccl() {
+ if [ -f /tmp/.cx_built_uccl ]; then
+ cx_log "UCCL EP already prepared this allocation — skip rebuild"
+ python3 -c "import torch; from uccl_deepep import Buffer" 2>/dev/null || return 1
+ return 0
+ fi
+ local version="0.1.1" tag="v0.1.1"
+ local wheel_sha256="390c1320918972206546e44d79b132988f2818ec07e23afcd0595f7183916cec"
+ cx_log "UCCL EP: installing uccl==$version + cu12 runtime shim"
+ export PIP_BREAK_SYSTEM_PACKAGES=1
+ pip install -q --no-deps "sortedcontainers==2.4.0" "intervaltree==3.1.0" >&2 2>&1 \
+ || { cx_log "ERROR: UCCL support dependency installation failed"; return 1; }
+ printf 'uccl==%s --hash=sha256:%s\n' "$version" "$wheel_sha256" \
+ | pip install -q --no-deps --only-binary=:all: --require-hashes -r /dev/stdin >&2 2>&1 \
+ || { cx_log "ERROR: pip install uccl==$version failed"; return 1; }
+ pip install -q --no-deps "nvidia-cuda-runtime-cu12==12.9.79" >&2 2>&1 \
+ || { cx_log "ERROR: CUDA 12 runtime shim install failed"; return 1; }
+ local cu12lib
+ cu12lib="$(python3 -c "import nvidia.cuda_runtime as m, os; print(os.path.join(os.path.dirname(m.__file__),'lib'))" 2>/dev/null)"
+ [ -n "$cu12lib" ] && export LD_LIBRARY_PATH="$cu12lib:${LD_LIBRARY_PATH:-}"
+ local installed
+ installed="$(python3 -c 'import importlib.metadata as m; print(m.version("uccl"))')" \
+ || { cx_log "ERROR: cannot read installed UCCL version"; return 1; }
+ [ "$installed" = "$version" ] \
+ || { cx_log "ERROR: expected UCCL $version, installed $installed"; return 1; }
+ UCCL_COMMIT="pkg-$installed"
+ export UCCL_COMMIT
+ # import torch FIRST: uccl.ep's C extension links libc10.so (torch), which is only on the loader
+ # path once torch is imported (rpath). The adapter (ep_uccl.py) imports torch before uccl.ep too.
+ python3 -c "import torch; from uccl.ep import Buffer; print('uccl.ep ready')" >&2 \
+ || { cx_log "ERROR: uccl.ep import failed (cu12 runtime on LD_LIBRARY_PATH?)"; return 1; }
+ # Vendor UCCL's DeepEP-API wrapper (ep/deep_ep_wrapper/deep_ep) under a NON-conflicting name
+ # (uccl_deepep) so it doesn't shadow the container's real deep_ep. Its Buffer(group, num_nvl_bytes,
+ # ...) takes a torch ProcessGroup (matching DeepEP + ep_uccl.py's calls) and runs the full
+ # proxy/IPC-handle/runtime.sync bootstrap that the low-level uccl.ep.Buffer(rank,num_ranks) lacks.
+ rm -rf /tmp/uccl_src /tmp/uccl_deepep_pkg
+ # Pin the wrapper to the SAME tag as the installed wheel (pkg-0.1.1 -> v0.1.1): the wrapper's
+ # dispatch calls into uccl.ep (get_rdma_buffer etc.), so a main-branch wrapper vs a 0.1.1 wheel
+ # mismatches signatures. Match them.
+ if git clone --depth 1 --branch "$tag" https://github.com/uccl-project/uccl /tmp/uccl_src >&2 2>&1 \
+ && [ "$(git -C /tmp/uccl_src rev-parse HEAD)" = "73ee4f12ba71717d6de34ba06806e1baaabe3f42" ] \
+ && [ -d /tmp/uccl_src/ep/deep_ep_wrapper/deep_ep ]; then
+ mkdir -p /tmp/uccl_deepep_pkg/uccl_deepep
+ cp /tmp/uccl_src/ep/deep_ep_wrapper/deep_ep/*.py /tmp/uccl_deepep_pkg/uccl_deepep/ 2>/dev/null
+ export PYTHONPATH="/tmp/uccl_deepep_pkg:${PYTHONPATH:-}"
+ python3 -c "import torch; from uccl_deepep import Buffer; print('uccl_deepep wrapper ready')" >&2 \
+ || { cx_log "ERROR: uccl_deepep wrapper import failed"; return 1; }
+ export CX_UCCL_WRAPPER=1
+ export UCCL_WRAPPER_COMMIT="73ee4f12ba71717d6de34ba06806e1baaabe3f42"
+ else
+ cx_log "ERROR: uccl deep_ep_wrapper not available"
+ return 1
+ fi
+ : > /tmp/.cx_built_uccl
+ cx_log "UCCL EP ready ($UCCL_COMMIT, wrapper=${CX_UCCL_WRAPPER:-0})"
+}
+
+# Rack build and rank steps may enter different container instances. Persist each node's
+# loader/import path and build identity on the shared staged mount, then require it from every rank.
+cx_persist_backend_env() {
+ local root="$PWD/.cx_backend/env" node_id="${SLURM_NODEID:-0}" path temporary name
+ local -a names=(PATH VIRTUAL_ENV LD_LIBRARY_PATH PYTHONPATH CUDA_HOME CPATH NVCC_PREPEND_FLAGS
+ NVSHMEM_DIR DEEPEP_COMMIT DEEPEP_TREE
+ EP_NCCL_ROOT_DIR EP_NVSHMEM_ROOT_DIR EP_JIT_CACHE_DIR EP_REUSE_NCCL_COMM
+ EP_JIT_DUMP_SASS
+ DEEPEP_V2_PR DEEPEP_V2_FIX_PR DEEPEP_V2_COMMIT DEEPEP_V2_TREE DEEPEP_V2_FMT_COMMIT
+ DEEPEP_V2_JIT_RANDOM_SEED
+ HYBRID_EP_MULTINODE USE_NIXL RDMA_CORE_HOME DEEPEP_HYBRID_BUILD_MODE
+ UCCL_COMMIT UCCL_WRAPPER_COMMIT CX_UCCL_WRAPPER)
+ [[ "$node_id" =~ ^[0-9]+$ ]] || return 1
+ mkdir -p "$root" || return 1
+ chmod 700 "$root" || return 1
+ temporary="$(mktemp "$root/.node-${node_id}.XXXXXX")" || return 1
+ chmod 600 "$temporary" || { rm -f "$temporary"; return 1; }
+ for name in "${names[@]}"; do
+ if declare -p "$name" >/dev/null 2>&1; then
+ printf 'export %s=%q\n' "$name" "${!name}" >> "$temporary" \
+ || { rm -f "$temporary"; return 1; }
+ fi
+ done
+ path="$root/node-${node_id}.sh"
+ mv -f -- "$temporary" "$path" || { rm -f "$temporary"; return 1; }
+}
+
+# Validate private scale-out selectors on every allocated compute node before a
+# backend can initialize or build transport code.
+cx_probe_scaleout_network() {
+ local interface device rdma_name
+ local -a interfaces devices
+ if [ "${CX_NODES:-1}" -le 1 ] || [ "${CX_TRANSPORT:-}" = mnnvl ]; then
+ return 0
+ fi
+ [ -n "${GLOO_SOCKET_IFNAME:-}" ] && [ -n "${NCCL_IB_HCA:-}" ] \
+ || { cx_log "ERROR: scale-out network selectors are unavailable"; return 1; }
+ IFS=, read -r -a interfaces <<< "$GLOO_SOCKET_IFNAME"
+ for interface in "${interfaces[@]}"; do
+ [ -d "/sys/class/net/$interface" ] \
+ || { cx_log "ERROR: configured scale-out socket interface is absent"; return 1; }
+ done
+ IFS=, read -r -a devices <<< "$NCCL_IB_HCA"
+ for device in "${devices[@]}"; do
+ rdma_name="${device%%:*}"
+ [ -d "/sys/class/infiniband/$rdma_name" ] \
+ || { cx_log "ERROR: configured scale-out RDMA device is absent"; return 1; }
+ done
+}
+
+# Prepare and probe one backend without running a benchmark. The same hook is used
+# by normal in-container runs and by rack launchers' persistent build-only step.
+cx_prepare_backend() {
+ local backend="${1:-}"
+ case "$backend" in
+ deepep)
+ cx_probe_deepep || return 1
+ ;;
+ deepep-v2)
+ cx_build_deepep_v2 || return 1
+ ;;
+ deepep-hybrid)
+ cx_build_deepep_hybrid || return 1
+ ;;
+ uccl)
+ cx_build_uccl || return 1
+ ;;
+ mori)
+ python3 -c "import mori" 2>/dev/null || return 1
+ ;;
+ nccl-ep)
+ ;;
+ *)
+ cx_log "ERROR: unknown backend preparation request"
+ return 1
+ ;;
+ esac
+}
+
+prepare_backend_or_record() {
+ local backend="$1" phases="${CX_PHASE:-decode}" phase
+ cx_write_runtime_stage backend-setup || return 1
+ if cx_prepare_backend "$backend"; then
+ return 0
+ fi
+ cx_log "WARN: $backend preparation failed"
+ [ "$phases" = "both" ] && phases="decode prefill"
+ for phase in $phases; do
+ CX_FAILURE_MODE=backend-setup emit_failed_case "$backend" "$phase" 6
+ done
+ return 1
+}
+
+# dispatch_bench runs the CURRENT CX_BENCH (+ CX_* config env) once. The sweep workflow runs many
+# of these per allocation (SHARD mode below), reusing this single container + its built backend.
+dispatch_bench() {
+ case "$CX_BENCH" in
+ nccl-ep)
+ run_ep_suite "$CX_BENCH"
+ ;;
+ deepep|deepep-v2|deepep-hybrid|mori|uccl)
+ prepare_backend_or_record "$CX_BENCH" && run_ep_suite "$CX_BENCH"
+ ;;
+ *)
+ cx_die "unknown CX_BENCH=$CX_BENCH (want deepep|deepep-v2|mori|uccl|nccl-ep|deepep-hybrid)"
+ ;;
+ esac
+}
+
+rc=0
+cx_validate_shard_control "$PWD"
+# Build-only mode: rack launchers run the shared backend preparation hook once per
+# node inside a persistent named container, then direct rank processes reuse it.
+if [ -n "${CX_BUILD_ONLY:-}" ]; then
+ if cx_probe_scaleout_network && cx_prepare_backend "${CX_BENCH:-}"; then
+ cx_persist_backend_env || rc=1
+ else
+ rc=1
+ fi
+ cx_log "backend preparation: bench=${CX_BENCH:-unknown} rc=$rc"
+ exit "$rc"
+fi
+if [ -n "${CX_SHARD_FILE:-}" ]; then
+ # SHARD/SWEEP mode (collectivex-sweep.yml): run EVERY case of this shard in THIS one allocation.
+ # All cases share (sku, backend, nodes), so backend preparation is paid once and cached.
+ ncases="$(python3 -c "import json;print(len(json.load(open('$CX_SHARD_FILE'))['cases']))")"
+ cx_log "SHARD mode: $ncases case(s) in one allocation (shard=$CX_SHARD_FILE)"
+ _cx_ts_base="$CX_TS" # per-case CX_TS suffix below keeps each case's result file UNIQUE (else
+ # cases sharing backend+phase overwrite each other at the same timestamp).
+ ci=0
+ failed_cases=0
+ while [ "$ci" -lt "$ncases" ]; do
+ CX_TS="${_cx_ts_base}-c$(printf '%03d' "$ci")"
+ export CX_TS
+ # Map varying case fields plus the frozen v1 defaults into CX_* env.
+ _exports="$(python3 - "$CX_SHARD_FILE" "$ci" <<'PY'
+import json, sys, shlex
+c = json.load(open(sys.argv[1]))["cases"][int(sys.argv[2])]
+def g(k, d=""):
+ v = c.get(k, d); return "" if v is None else str(v)
+env = {
+ "CX_BENCH": g("backend"),
+ "CX_MODE": g("mode", "normal"),
+ "CX_ROUTING": g("routing", "uniform"), "CX_PHASE": g("phase", "decode"),
+ "CX_EP": g("ep", "1"),
+ "CX_EPLB": "1" if c.get("eplb") else "",
+ "CX_CASE_ID": g("case_id"), "CX_SUITE": g("suite"), "CX_WORKLOAD_NAME": g("workload"),
+ "CX_REQUIRED_PUBLICATION": g("required_publication"),
+ "CX_HIDDEN": g("hidden"), "CX_TOPK": g("topk"), "CX_EXPERTS": g("experts"),
+ "CX_TOKENS_LADDER": g("ladder"), "CX_CANONICAL": ("1" if c.get("canonical") else ""),
+ "CX_NODES": g("nodes"), "CX_GPUS_PER_NODE": g("gpus_per_node"),
+ "CX_SCALE_UP_DOMAIN": g("scale_up_domain"), "CX_SCOPE": g("scope"),
+ "CX_SCALE_UP_TRANSPORT": g("scale_up_transport"),
+ "CX_SCALE_OUT_TRANSPORT": g("scale_out_transport"),
+ "CX_TRANSPORT": g("transport"), "CX_TOPO": g("topology_class"),
+ "CX_SAMPLES_PER_POINT": g("samples_per_point"),
+ "CX_WARMUP_SEMANTICS": g("warmup_semantics"),
+}
+lines = [f"export {k}={shlex.quote(v)}" for k, v in env.items()]
+# Per-case timing "iters:trials:warmup" (fixed-512-v1 requires 8:64:32 everywhere);
+# cases without one must fall back to the harness defaults, so UNSET rather than export-empty
+# (an empty CX_ITERS would defeat the 8-iter default and break the run_ep argparse; NOTE no
+# apostrophes in this heredoc — bash command-substitution scanning chokes on unbalanced quotes).
+timing = g("timing")
+if timing:
+ parts = (timing.split(":") + ["", "", ""])[:3]
+ for k, v in zip(("CX_ITERS", "CX_TRIALS", "CX_WARMUP"), parts):
+ if v:
+ lines.append(f"export {k}={shlex.quote(v)}")
+else:
+ lines.append("unset CX_ITERS CX_TRIALS CX_WARMUP 2>/dev/null || true")
+print("\n".join(lines))
+PY
+)"
+ eval "$_exports"
+ # Each case has its OWN routing/dims -> its own canonical workload manifest. cx_stage_canonical
+ # short-circuits when CX_WORKLOAD_DIR is already set, so without this unset the first case's
+ # staged dir is reused for the rest and run_ep.py can't find the later cases' manifests
+ # (FileNotFoundError .cx_workloads/.manifest.json). Unset so every case re-stages its own.
+ unset CX_WORKLOAD_DIR 2>/dev/null || true
+ cx_log " [$((ci+1))/$ncases] $CX_BENCH $CX_MODE/$CX_PHASE routing=$CX_ROUTING eplb=${CX_EPLB:-0}"
+ _cx_case_ts="$CX_TS"
+ CX_TS="${_cx_case_ts}-a01"
+ export CX_ATTEMPT_ID=1 CX_TS
+ dispatch_bench || {
+ failed_cases=$((failed_cases+1))
+ cx_log " [$((ci+1))/$ncases] $CX_BENCH case FAILED; failed-case record preserved"
+ }
+ export CX_TS="$_cx_case_ts"
+ ci=$((ci + 1))
+ done
+ if [ "${failed_cases:-0}" -gt 0 ]; then
+ cx_log "SHARD done: $failed_cases/$ncases case(s) failed"
+ rc=1
+ fi
+ # The base timestamp matches every per-case file, so the final summary covers the whole shard.
+ export CX_TS="$_cx_ts_base"
+else
+ _cx_single_ts="$CX_TS"
+ CX_TS="${_cx_single_ts}-a01"
+ export CX_ATTEMPT_ID=1 CX_TS
+ dispatch_bench || rc=1
+fi
+
+# Summary table for the log; also fails the job if no valid results were produced.
+python3 summarize.py --results-dir results --runner "$CX_RUNNER" --ts "$CX_TS" || rc=1
+exit "$rc"
diff --git a/experimental/CollectiveX/schemas/channel-v1.schema.json b/experimental/CollectiveX/schemas/channel-v1.schema.json
new file mode 100644
index 0000000000..663e22914b
--- /dev/null
+++ b/experimental/CollectiveX/schemas/channel-v1.schema.json
@@ -0,0 +1,23 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "https://inferencex.com/schemas/collectivex/channel-v1.schema.json",
+ "title": "CollectiveX public channel v1",
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["format","channel","dataset","generated_at"],
+ "properties": {
+ "format": {"const": "collectivex.channel.v1"},
+ "channel": {"enum": ["latest-attempt","dev-latest"]},
+ "dataset": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["path","sha256","bytes"],
+ "properties": {
+ "path": {"type": "string","pattern": "^datasets/[0-9a-f]{64}/dataset\\.json$"},
+ "sha256": {"type": "string","pattern": "^[0-9a-f]{64}$"},
+ "bytes": {"type": "integer","minimum": 1,"maximum": 33554432}
+ }
+ },
+ "generated_at": {"type": "string","format": "date-time"}
+ }
+}
diff --git a/experimental/CollectiveX/schemas/private-bundle-v1.schema.json b/experimental/CollectiveX/schemas/private-bundle-v1.schema.json
new file mode 100644
index 0000000000..166c808930
--- /dev/null
+++ b/experimental/CollectiveX/schemas/private-bundle-v1.schema.json
@@ -0,0 +1,162 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "https://inferencex.com/schemas/collectivex/private-bundle-v1.schema.json",
+ "title": "CollectiveX private attempt bundle v1",
+ "type": "object",
+ "additionalProperties": false,
+ "required": [
+ "format",
+ "schema_version",
+ "created_at",
+ "ingest_id",
+ "run",
+ "matrix",
+ "sources",
+ "attempts",
+ "coverage",
+ "runtime_fingerprints",
+ "checksums",
+ "validation"
+ ],
+ "properties": {
+ "format": {"const": "collectivex.private.bundle.v1"},
+ "schema_version": {"const": 1},
+ "created_at": {"type": "string","format": "date-time"},
+ "ingest_id": {"$ref": "#/$defs/sha256"},
+ "run": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["repository","run_id","run_attempt","source_sha"],
+ "properties": {
+ "repository": {"type": "string","pattern": "^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$"},
+ "run_id": {"type": "string","pattern": "^[1-9][0-9]*$"},
+ "run_attempt": {"type": "integer","minimum": 1},
+ "source_sha": {"type": "string","pattern": "^[0-9a-f]{40}$"}
+ }
+ },
+ "matrix": {"$ref": "#/$defs/file"},
+ "sources": {"type": "array","minItems": 1,"uniqueItems": true,"items": {"$ref": "#/$defs/source"}},
+ "attempts": {
+ "type": "array",
+ "minItems": 1,
+ "items": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": [
+ "attempt_id",
+ "allocation_id",
+ "case_id",
+ "outcome",
+ "reason",
+ "selected",
+ "document",
+ "samples",
+ "runtime_fingerprint_sha256",
+ "series_ids",
+ "evidence_ids"
+ ],
+ "properties": {
+ "attempt_id": {"$ref": "#/$defs/attemptId"},
+ "allocation_id": {"$ref": "#/$defs/allocationId"},
+ "case_id": {"$ref": "#/$defs/caseId"},
+ "outcome": {"$ref": "#/$defs/outcome"},
+ "reason": {"$ref": "#/$defs/reason"},
+ "selected": {"type": "boolean"},
+ "document": {"$ref": "#/$defs/file"},
+ "samples": {"oneOf": [{"type": "null"},{"$ref": "#/$defs/file"}]},
+ "runtime_fingerprint_sha256": {"oneOf": [{"type": "null"},{"$ref": "#/$defs/sha256"}]},
+ "series_ids": {"type": "array","uniqueItems": true,"items": {"$ref": "#/$defs/seriesId"}},
+ "evidence_ids": {"type": "array","uniqueItems": true,"items": {"$ref": "#/$defs/evidenceId"}}
+ }
+ }
+ },
+ "coverage": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["expected_cases","terminal_cases","complete","outcome_counts","selections"],
+ "properties": {
+ "expected_cases": {"type": "integer","minimum": 1},
+ "terminal_cases": {"type": "integer","minimum": 0},
+ "complete": {"type": "boolean"},
+ "outcome_counts": {"$ref": "#/$defs/outcomeCounts"},
+ "selections": {
+ "type": "array",
+ "minItems": 1,
+ "items": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["case_id","selected_attempt_id","outcome"],
+ "properties": {
+ "case_id": {"$ref": "#/$defs/caseId"},
+ "selected_attempt_id": {"$ref": "#/$defs/attemptId"},
+ "outcome": {"$ref": "#/$defs/outcome"}
+ }
+ }
+ }
+ }
+ },
+ "runtime_fingerprints": {"type": "array","uniqueItems": true,"items": {"$ref": "#/$defs/sha256"}},
+ "checksums": {"$ref": "#/$defs/file"},
+ "validation": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["policy","passed","checks"],
+ "properties": {
+ "policy": {"const": "collectivex-publisher-v1"},
+ "passed": {"const": true},
+ "checks": {
+ "type": "array",
+ "minItems": 1,
+ "uniqueItems": true,
+ "items": {"type": "string","pattern": "^[a-z0-9][a-z0-9.-]*$"}
+ }
+ }
+ }
+ },
+ "$defs": {
+ "sha256": {"type": "string","pattern": "^[0-9a-f]{64}$"},
+ "caseId": {"type": "string","pattern": "^cxcase-v1-[0-9a-f]{64}$"},
+ "seriesId": {"type": "string","pattern": "^cxseries-v1-[0-9a-f]{64}$"},
+ "evidenceId": {"type": "string","pattern": "^cxevidence-v1-[0-9a-f]{64}$"},
+ "allocationId": {"type": "string","pattern": "^cxallocation-v1-[0-9a-f]{64}$"},
+ "attemptId": {"type": "string","pattern": "^cxattempt-v1-[0-9a-f]{64}$"},
+ "reason": {"oneOf": [{"type": "null"},{"type": "string","pattern": "^[a-z0-9][a-z0-9.-]*$","maxLength": 96}]},
+ "outcome": {"enum": ["success","unsupported","failed","invalid","diagnostic"]},
+ "outcomeCounts": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["success","unsupported","failed","invalid","diagnostic"],
+ "properties": {
+ "success": {"type": "integer","minimum": 0},
+ "unsupported": {"type": "integer","minimum": 0},
+ "failed": {"type": "integer","minimum": 0},
+ "invalid": {"type": "integer","minimum": 0},
+ "diagnostic": {"type": "integer","minimum": 0}
+ }
+ },
+ "file": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["path","sha256","bytes"],
+ "properties": {
+ "path": {"type": "string","pattern": "^[A-Za-z0-9_.-]+(?:/[A-Za-z0-9_.-]+)*$"},
+ "sha256": {"$ref": "#/$defs/sha256"},
+ "bytes": {"type": "integer","minimum": 1}
+ }
+ },
+ "source": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["path","sha256","bytes","artifact_name"],
+ "properties": {
+ "path": {"type": "string","pattern": "^[A-Za-z0-9_.-]+(?:/[A-Za-z0-9_.-]+)*$"},
+ "sha256": {"$ref": "#/$defs/sha256"},
+ "bytes": {"type": "integer","minimum": 1},
+ "artifact_name": {
+ "type": "string",
+ "pattern": "^cx(?:unsupported|shard-[a-z0-9][a-z0-9_.-]{0,127})-[1-9][0-9]*-[1-9][0-9]*$"
+ }
+ }
+ }
+ }
+}
diff --git a/experimental/CollectiveX/schemas/public-dataset-v1.schema.json b/experimental/CollectiveX/schemas/public-dataset-v1.schema.json
new file mode 100644
index 0000000000..1b51a8c579
--- /dev/null
+++ b/experimental/CollectiveX/schemas/public-dataset-v1.schema.json
@@ -0,0 +1,606 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "https://inferencex.com/schemas/collectivex/public-dataset-v1.schema.json",
+ "title": "CollectiveX sanitized public dataset v1",
+ "type": "object",
+ "additionalProperties": false,
+ "required": [
+ "format",
+ "schema_version",
+ "generated_at",
+ "source_bundle_ids",
+ "promotion",
+ "coverage",
+ "attempts",
+ "series",
+ "cohorts",
+ "rankings",
+ "recommendations",
+ "sensitivities"
+ ],
+ "properties": {
+ "format": {"const": "collectivex.public.v1"},
+ "schema_version": {"const": 1},
+ "generated_at": {"type": "string","format": "date-time"},
+ "source_bundle_ids": {"type": "array","uniqueItems": true,"items": {"$ref": "#/$defs/sha256"}},
+ "promotion": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": [
+ "status",
+ "reason",
+ "matrix_id",
+ "allocation_ids",
+ "required_allocations",
+ "requested_cases",
+ "terminal_cases",
+ "policy"
+ ],
+ "properties": {
+ "status": {"enum": ["promoted","diagnostic","quarantined"]},
+ "reason": {"$ref": "#/$defs/reason"},
+ "matrix_id": {"oneOf": [{"type": "null"},{"$ref": "#/$defs/sha256"}]},
+ "allocation_ids": {"type": "array","uniqueItems": true,"items": {"$ref": "#/$defs/allocationId"}},
+ "required_allocations": {"const": 3},
+ "requested_cases": {"type": "integer","minimum": 0},
+ "terminal_cases": {"type": "integer","minimum": 0},
+ "policy": {"const": "collectivex-decision-grade-v1"}
+ }
+ },
+ "coverage": {"type": "array","items": {"$ref": "#/$defs/coverage"}},
+ "attempts": {"type": "array","items": {"$ref": "#/$defs/attempt"}},
+ "series": {"type": "array","items": {"$ref": "#/$defs/series"}},
+ "cohorts": {"type": "array","items": {"$ref": "#/$defs/cohort"}},
+ "rankings": {"type": "array","items": {"$ref": "#/$defs/ranking"}},
+ "recommendations": {"type": "array","items": {"$ref": "#/$defs/recommendation"}},
+ "sensitivities": {"type": "array","items": {"$ref": "#/$defs/sensitivity"}}
+ },
+ "$defs": {
+ "sha256": {"type": "string","pattern": "^[0-9a-f]{64}$"},
+ "caseId": {"type": "string","pattern": "^cxcase-v1-[0-9a-f]{64}$"},
+ "workloadId": {"type": "string","pattern": "^cxwork-v1-[0-9a-f]{64}$"},
+ "seriesId": {"type": "string","pattern": "^cxseries-v1-[0-9a-f]{64}$"},
+ "pointId": {"type": "string","pattern": "^cxpoint-v1-[0-9a-f]{64}$"},
+ "evidenceId": {"type": "string","pattern": "^cxevidence-v1-[0-9a-f]{64}$"},
+ "allocationId": {"type": "string","pattern": "^cxallocation-v1-[0-9a-f]{64}$"},
+ "attemptId": {"type": "string","pattern": "^cxattempt-v1-[0-9a-f]{64}$"},
+ "safeId": {"type": "string","pattern": "^[a-z0-9][a-z0-9_.-]*$","maxLength": 128},
+ "publicationTier": {"enum": ["official","comparable-experimental"]},
+ "label": {"type": "string","minLength": 1,"maxLength": 160},
+ "nullableLabel": {"oneOf": [{"type": "null"},{"$ref": "#/$defs/label"}]},
+ "reason": {"oneOf": [{"type": "null"},{"type": "string","pattern": "^[a-z0-9][a-z0-9.-]*$","maxLength": 96}]},
+ "outcome": {"enum": ["success","unsupported","failed","invalid","diagnostic"]},
+ "coverageTopology": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": [
+ "ep_size",
+ "nodes",
+ "gpus_per_node",
+ "scale_up_domain",
+ "scope",
+ "scale_up_transport",
+ "scale_out_transport",
+ "transport",
+ "topology_class"
+ ],
+ "properties": {
+ "ep_size": {"type": "integer","minimum": 1},
+ "nodes": {"type": "integer","minimum": 1},
+ "gpus_per_node": {"type": "integer","minimum": 1},
+ "scale_up_domain": {"type": "integer","minimum": 1},
+ "scope": {"enum": ["scale-up","scale-out"]},
+ "scale_up_transport": {"$ref": "#/$defs/safeId"},
+ "scale_out_transport": {"oneOf": [{"type": "null"},{"$ref": "#/$defs/safeId"}]},
+ "transport": {"$ref": "#/$defs/safeId"},
+ "topology_class": {"$ref": "#/$defs/safeId"}
+ }
+ },
+ "coverage": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": [
+ "case_id",
+ "label",
+ "required",
+ "sku",
+ "backend",
+ "mode",
+ "phase",
+ "topology",
+ "disposition",
+ "selected_attempt_id",
+ "outcome",
+ "failure_mode",
+ "reason",
+ "attempt_ids"
+ ],
+ "properties": {
+ "case_id": {"$ref": "#/$defs/caseId"},
+ "label": {"$ref": "#/$defs/label"},
+ "required": {"type": "boolean"},
+ "sku": {"$ref": "#/$defs/safeId"},
+ "backend": {"$ref": "#/$defs/safeId"},
+ "mode": {"enum": ["normal","low-latency"]},
+ "phase": {"enum": ["decode","prefill"]},
+ "topology": {"$ref": "#/$defs/coverageTopology"},
+ "disposition": {"enum": ["runnable","unsupported"]},
+ "selected_attempt_id": {"oneOf": [{"type": "null"},{"$ref": "#/$defs/attemptId"}]},
+ "outcome": {"$ref": "#/$defs/outcome"},
+ "failure_mode": {"$ref": "#/$defs/reason"},
+ "reason": {"$ref": "#/$defs/reason"},
+ "attempt_ids": {"type": "array","uniqueItems": true,"items": {"$ref": "#/$defs/attemptId"}}
+ }
+ },
+ "attempt": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": [
+ "attempt_id",
+ "evidence",
+ "case_id",
+ "allocation_id",
+ "run_id",
+ "run_attempt",
+ "attempt_index",
+ "selected",
+ "outcome",
+ "failure_mode",
+ "reason",
+ "series_id",
+ "completed_at"
+ ],
+ "properties": {
+ "attempt_id": {"$ref": "#/$defs/attemptId"},
+ "evidence": {
+ "type": "array",
+ "uniqueItems": true,
+ "items": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["evidence_id","point_id"],
+ "properties": {"evidence_id": {"$ref": "#/$defs/evidenceId"},"point_id": {"$ref": "#/$defs/pointId"}}
+ }
+ },
+ "case_id": {"$ref": "#/$defs/caseId"},
+ "allocation_id": {"$ref": "#/$defs/allocationId"},
+ "run_id": {"type": "string","pattern": "^[1-9][0-9]*$"},
+ "run_attempt": {"type": "integer","minimum": 1},
+ "attempt_index": {"type": "integer","minimum": 1},
+ "selected": {"type": "boolean"},
+ "outcome": {"$ref": "#/$defs/outcome"},
+ "failure_mode": {"$ref": "#/$defs/reason"},
+ "reason": {"$ref": "#/$defs/reason"},
+ "series_id": {"oneOf": [{"type": "null"},{"$ref": "#/$defs/seriesId"}]},
+ "completed_at": {"oneOf": [{"type": "null"},{"type": "string","format": "date-time"}]}
+ }
+ },
+ "eligibility": {
+ "type": "object",
+ "additionalProperties": false,
+ "allOf": [{
+ "if": {"properties": {"decision_grade": {"const": true}},"required": ["decision_grade"]},
+ "then": {"properties": {"reasons": {"maxItems": 0}}},
+ "else": {"properties": {"reasons": {"minItems": 1}}}
+ }],
+ "required": [
+ "decision_grade",
+ "allocation_ids",
+ "complete",
+ "correct",
+ "measured_roundtrip_p99",
+ "stable_p50",
+ "stable_p99",
+ "stable_ordering",
+ "p50_max_min_ratio",
+ "p99_max_min_ratio",
+ "reasons"
+ ],
+ "properties": {
+ "decision_grade": {"type": "boolean"},
+ "allocation_ids": {"type": "array","uniqueItems": true,"items": {"$ref": "#/$defs/allocationId"}},
+ "complete": {"type": "boolean"},
+ "correct": {"type": "boolean"},
+ "measured_roundtrip_p99": {"type": "boolean"},
+ "stable_p50": {"type": "boolean"},
+ "stable_p99": {"type": "boolean"},
+ "stable_ordering": {"type": "boolean"},
+ "p50_max_min_ratio": {"oneOf": [{"type": "null"},{"type": "number","minimum": 1}]},
+ "p99_max_min_ratio": {"oneOf": [{"type": "null"},{"type": "number","minimum": 1}]},
+ "reasons": {
+ "type": "array",
+ "uniqueItems": true,
+ "items": {"type": "string","pattern": "^[a-z0-9][a-z0-9.-]*$","maxLength": 96}
+ }
+ }
+ },
+ "percentiles": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["p50","p90","p95","p99"],
+ "properties": {
+ "p50": {"type": "number","exclusiveMinimum": 0},
+ "p90": {"type": "number","exclusiveMinimum": 0},
+ "p95": {"type": "number","exclusiveMinimum": 0},
+ "p99": {"type": "number","exclusiveMinimum": 0}
+ }
+ },
+ "component": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["origin","latency_us","logical_bytes","logical_payload_rate_gbps_at_latency_percentile","sample_count"],
+ "properties": {
+ "origin": {"enum": ["measured","derived"]},
+ "latency_us": {"$ref": "#/$defs/percentiles"},
+ "logical_bytes": {"oneOf": [{"type": "null"},{"type": "integer","minimum": 1}]},
+ "logical_payload_rate_gbps_at_latency_percentile": {"oneOf": [{"type": "null"},{"$ref": "#/$defs/percentiles"}]},
+ "sample_count": {"oneOf": [{"type": "null"},{"type": "integer","minimum": 1}]}
+ }
+ },
+ "nullableComponent": {"oneOf": [{"type": "null"},{"$ref": "#/$defs/component"}]},
+ "point": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": [
+ "point_id",
+ "tokens_per_rank",
+ "global_tokens",
+ "correct",
+ "routing",
+ "components",
+ "roundtrip_token_rate_at_latency_percentile",
+ "evidence_ids"
+ ],
+ "properties": {
+ "point_id": {"$ref": "#/$defs/pointId"},
+ "tokens_per_rank": {"type": "integer","minimum": 1},
+ "global_tokens": {"type": "integer","minimum": 1},
+ "correct": {"type": "boolean"},
+ "routing": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": [
+ "fanout_mean",
+ "recv_tokens_max",
+ "expert_load_cv",
+ "payload_rank_cv",
+ "hotspot_ratio",
+ "empty_expert_count",
+ "empty_rank_count",
+ "routed_copies"
+ ],
+ "properties": {
+ "fanout_mean": {"type": "number","minimum": 0},
+ "recv_tokens_max": {"type": "integer","minimum": 0},
+ "expert_load_cv": {"type": "number","minimum": 0},
+ "payload_rank_cv": {"type": "number","minimum": 0},
+ "hotspot_ratio": {"type": "number","minimum": 0},
+ "empty_expert_count": {"type": "integer","minimum": 0},
+ "empty_rank_count": {"type": "integer","minimum": 0},
+ "routed_copies": {"type": "integer","minimum": 1}
+ }
+ },
+ "components": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["dispatch","combine","roundtrip","isolated_sum"],
+ "properties": {
+ "dispatch": {"$ref": "#/$defs/nullableComponent"},
+ "combine": {"$ref": "#/$defs/nullableComponent"},
+ "roundtrip": {"$ref": "#/$defs/nullableComponent"},
+ "isolated_sum": {"$ref": "#/$defs/nullableComponent"}
+ }
+ },
+ "roundtrip_token_rate_at_latency_percentile": {"$ref": "#/$defs/percentiles"},
+ "evidence_ids": {"type": "array","uniqueItems": true,"items": {"$ref": "#/$defs/evidenceId"}}
+ }
+ },
+ "series": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": [
+ "series_id",
+ "label",
+ "status",
+ "case_ids",
+ "allocation_ids",
+ "model",
+ "suite",
+ "mode",
+ "phase",
+ "publication_tier",
+ "backend",
+ "build",
+ "system",
+ "workload",
+ "eplb",
+ "resource",
+ "measurement",
+ "points",
+ "eligibility"
+ ],
+ "properties": {
+ "series_id": {"$ref": "#/$defs/seriesId"},
+ "label": {"$ref": "#/$defs/label"},
+ "status": {"enum": ["decision-grade","diagnostic"]},
+ "case_ids": {"type": "array","minItems": 1,"uniqueItems": true,"items": {"$ref": "#/$defs/caseId"}},
+ "allocation_ids": {"type": "array","minItems": 1,"uniqueItems": true,"items": {"$ref": "#/$defs/allocationId"}},
+ "model": {"$ref": "#/$defs/safeId"},
+ "suite": {"$ref": "#/$defs/safeId"},
+ "mode": {"enum": ["normal","low-latency"]},
+ "phase": {"enum": ["decode","prefill"]},
+ "publication_tier": {"$ref": "#/$defs/publicationTier"},
+ "backend": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["id","label","role","generation","version"],
+ "properties": {
+ "id": {"$ref": "#/$defs/safeId"},
+ "label": {"$ref": "#/$defs/label"},
+ "role": {"enum": ["library","reference"]},
+ "generation": {"$ref": "#/$defs/nullableLabel"},
+ "version": {"$ref": "#/$defs/nullableLabel"}
+ }
+ },
+ "build": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["implementation_contract_sha256","public_config_sha256","routing_control_sha256","runtime_fingerprint_sha256","image_digest","source_sha","squash_sha256"],
+ "properties": {
+ "implementation_contract_sha256": {"$ref": "#/$defs/sha256"},
+ "public_config_sha256": {"$ref": "#/$defs/sha256"},
+ "routing_control_sha256": {"$ref": "#/$defs/sha256"},
+ "runtime_fingerprint_sha256": {"$ref": "#/$defs/sha256"},
+ "image_digest": {"type": "string","pattern": "^sha256:[0-9a-f]{64}$"},
+ "source_sha": {"type": "string","pattern": "^[0-9a-f]{40,64}$"},
+ "squash_sha256": {"$ref": "#/$defs/sha256"}
+ }
+ },
+ "system": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["sku","label","vendor","topology_class","transport","scale_up_transport","scale_out_transport","scope","nodes","gpus_per_node","scale_up_domain","world_size","ep_size","placement"],
+ "properties": {
+ "sku": {"$ref": "#/$defs/safeId"},
+ "label": {"$ref": "#/$defs/label"},
+ "vendor": {"enum": ["nvidia","amd"]},
+ "topology_class": {"$ref": "#/$defs/safeId"},
+ "transport": {"$ref": "#/$defs/safeId"},
+ "scale_up_transport": {"$ref": "#/$defs/safeId"},
+ "scale_out_transport": {"oneOf": [{"type": "null"},{"$ref": "#/$defs/safeId"}]},
+ "scope": {"enum": ["scale-up","scale-out"]},
+ "nodes": {"type": "integer","minimum": 1},
+ "gpus_per_node": {"type": "integer","minimum": 1},
+ "scale_up_domain": {"type": "integer","minimum": 1},
+ "world_size": {"type": "integer","minimum": 1},
+ "ep_size": {"type": "integer","minimum": 1},
+ "placement": {"enum": ["packed"]}
+ }
+ },
+ "workload": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": [
+ "workload_id",
+ "hidden",
+ "top_k",
+ "experts",
+ "routing",
+ "eplb",
+ "dispatch_dtype",
+ "combine_dtype",
+ "activation_profile"
+ ],
+ "properties": {
+ "workload_id": {"$ref": "#/$defs/workloadId"},
+ "hidden": {"type": "integer","minimum": 1},
+ "top_k": {"type": "integer","minimum": 1},
+ "experts": {"type": "integer","minimum": 1},
+ "routing": {"enum": ["uniform","zipf"]},
+ "eplb": {"type": "boolean"},
+ "dispatch_dtype": {"const": "bf16"},
+ "combine_dtype": {"const": "bf16"},
+ "activation_profile": {"const": "canonical-counter-source-v3"}
+ }
+ },
+ "eplb": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": [
+ "enabled",
+ "planner",
+ "mapping_sha256",
+ "logical_experts",
+ "physical_experts",
+ "redundant_experts",
+ "reference_tokens_per_rank",
+ "replicated_experts",
+ "max_replicas",
+ "imbalance_before",
+ "imbalance_after"
+ ],
+ "properties": {
+ "enabled": {"type": "boolean"},
+ "planner": {"$ref": "#/$defs/nullableLabel"},
+ "mapping_sha256": {"oneOf": [{"type": "null"},{"$ref": "#/$defs/sha256"}]},
+ "logical_experts": {"type": "integer","minimum": 1},
+ "physical_experts": {"type": "integer","minimum": 1},
+ "redundant_experts": {"type": "integer","minimum": 0},
+ "reference_tokens_per_rank": {"oneOf": [{"type": "null"},{"type": "integer","minimum": 1}]},
+ "replicated_experts": {"type": "integer","minimum": 0},
+ "max_replicas": {"oneOf": [{"type": "null"},{"type": "integer","minimum": 0}]},
+ "imbalance_before": {"oneOf": [{"type": "null"},{"type": "number","minimum": 0}]},
+ "imbalance_after": {"oneOf": [{"type": "null"},{"type": "number","minimum": 0}]}
+ }
+ },
+ "resource": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["mode","profile","comm_units_kind","configured_units"],
+ "properties": {
+ "mode": {"const": "tuned"},
+ "profile": {"$ref": "#/$defs/safeId"},
+ "comm_units_kind": {"$ref": "#/$defs/nullableLabel"},
+ "configured_units": {"oneOf": [{"type": "null"},{"type": "integer","minimum": 1}]}
+ }
+ },
+ "measurement": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": [
+ "contract",
+ "component_order_contract",
+ "combine_semantics",
+ "payload_unit",
+ "sampling_contract",
+ "iters",
+ "trials",
+ "warmups",
+ "samples_per_component",
+ "headline_component",
+ "headline_percentile"
+ ],
+ "properties": {
+ "contract": {"enum": ["layout-and-dispatch-v1","expert-packed-weighted-combine-v1"]},
+ "component_order_contract": {"enum": ["roundtrip-dispatch-activation-only-combine-v2","roundtrip-dispatch-gate-weighted-combine-v1"]},
+ "combine_semantics": {"enum": ["activation-only","gate-weighted"]},
+ "payload_unit": {"enum": ["token-rank","token-expert"]},
+ "sampling_contract": {"const": "fixed-512-v1"},
+ "iters": {"const": 8},
+ "trials": {"const": 64},
+ "warmups": {"const": 32},
+ "samples_per_component": {"const": 512},
+ "headline_component": {"const": "roundtrip"},
+ "headline_percentile": {"const": "p99"}
+ }
+ },
+ "points": {"type": "array","minItems": 1,"items": {"$ref": "#/$defs/point"}},
+ "eligibility": {"$ref": "#/$defs/eligibility"}
+ }
+ },
+ "cohort": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": [
+ "cohort_id",
+ "kind",
+ "label",
+ "description",
+ "series_ids",
+ "controlled_factors",
+ "varying_factors",
+ "publication_tier",
+ "eligibility"
+ ],
+ "properties": {
+ "cohort_id": {"type": "string","pattern": "^cxcohort-v1-[0-9a-f]{64}$"},
+ "kind": {"enum": ["library","chip","system","routing"]},
+ "label": {"$ref": "#/$defs/label"},
+ "description": {"$ref": "#/$defs/label"},
+ "series_ids": {"type": "array","minItems": 2,"uniqueItems": true,"items": {"$ref": "#/$defs/seriesId"}},
+ "controlled_factors": {"type": "array","minItems": 1,"uniqueItems": true,"items": {"$ref": "#/$defs/safeId"}},
+ "varying_factors": {"type": "array","minItems": 1,"uniqueItems": true,"items": {"$ref": "#/$defs/safeId"}},
+ "publication_tier": {"$ref": "#/$defs/publicationTier"},
+ "eligibility": {"$ref": "#/$defs/eligibility"}
+ }
+ },
+ "metric": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["operation","statistic","measure","objective","tokens_per_rank","phase"],
+ "properties": {
+ "operation": {"const": "roundtrip"},
+ "statistic": {"enum": ["p50","p99"]},
+ "measure": {"enum": ["latency_us","logical_payload_rate_gbps_at_latency_percentile"]},
+ "objective": {"enum": ["min","max"]},
+ "tokens_per_rank": {"type": "integer","minimum": 1},
+ "phase": {"enum": ["decode","prefill"]}
+ }
+ },
+ "ranking": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["ranking_id","cohort_id","label","metric","entries","publication_tier","eligibility"],
+ "properties": {
+ "ranking_id": {"type": "string","pattern": "^cxranking-v1-[0-9a-f]{64}$"},
+ "cohort_id": {"type": "string","pattern": "^cxcohort-v1-[0-9a-f]{64}$"},
+ "label": {"$ref": "#/$defs/label"},
+ "metric": {"$ref": "#/$defs/metric"},
+ "entries": {
+ "type": "array",
+ "minItems": 2,
+ "items": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["rank","series_id","point_id","value","unit"],
+ "properties": {
+ "rank": {"type": "integer","minimum": 1},
+ "series_id": {"$ref": "#/$defs/seriesId"},
+ "point_id": {"$ref": "#/$defs/pointId"},
+ "value": {"type": "number","exclusiveMinimum": 0},
+ "unit": {"enum": ["us","GB/s"]}
+ }
+ }
+ },
+ "publication_tier": {"$ref": "#/$defs/publicationTier"},
+ "eligibility": {"$ref": "#/$defs/eligibility"}
+ }
+ },
+ "recommendation": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": [
+ "recommendation_id",
+ "cohort_id",
+ "label",
+ "objective",
+ "series_id",
+ "point_id",
+ "value",
+ "unit",
+ "rationale",
+ "publication_tier",
+ "eligibility"
+ ],
+ "properties": {
+ "recommendation_id": {"type": "string","pattern": "^cxrecommendation-v1-[0-9a-f]{64}$"},
+ "cohort_id": {"type": "string","pattern": "^cxcohort-v1-[0-9a-f]{64}$"},
+ "label": {"$ref": "#/$defs/label"},
+ "objective": {"enum": ["min-p50-latency","min-p99-latency","max-payload-rate-at-p50-latency","max-payload-rate-at-p99-latency"]},
+ "series_id": {"$ref": "#/$defs/seriesId"},
+ "point_id": {"$ref": "#/$defs/pointId"},
+ "value": {"type": "number","exclusiveMinimum": 0},
+ "unit": {"enum": ["us","GB/s"]},
+ "rationale": {"$ref": "#/$defs/label"},
+ "publication_tier": {"const": "official"},
+ "eligibility": {"$ref": "#/$defs/eligibility"}
+ }
+ },
+ "sensitivity": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": [
+ "sensitivity_id",
+ "cohort_id",
+ "label",
+ "baseline_series_id",
+ "candidate_series_id",
+ "metric",
+ "signed_change_ratio",
+ "publication_tier",
+ "eligibility"
+ ],
+ "properties": {
+ "sensitivity_id": {"type": "string","pattern": "^cxsensitivity-v1-[0-9a-f]{64}$"},
+ "cohort_id": {"type": "string","pattern": "^cxcohort-v1-[0-9a-f]{64}$"},
+ "label": {"$ref": "#/$defs/label"},
+ "baseline_series_id": {"$ref": "#/$defs/seriesId"},
+ "candidate_series_id": {"$ref": "#/$defs/seriesId"},
+ "metric": {"$ref": "#/$defs/metric"},
+ "signed_change_ratio": {"type": "number"},
+ "publication_tier": {"$ref": "#/$defs/publicationTier"},
+ "eligibility": {"$ref": "#/$defs/eligibility"}
+ }
+ }
+ }
+}
diff --git a/experimental/CollectiveX/schemas/raw-case-v1.schema.json b/experimental/CollectiveX/schemas/raw-case-v1.schema.json
new file mode 100644
index 0000000000..be0d98d76c
--- /dev/null
+++ b/experimental/CollectiveX/schemas/raw-case-v1.schema.json
@@ -0,0 +1,1199 @@
+{
+ "$id": "https://inferencex.com/schemas/collectivex/raw-case-v1.schema.json",
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$defs": {
+ "deepep_v2_jit_cubin": {
+ "additionalProperties": false,
+ "properties": {
+ "cache_key": {
+ "pattern":"^kernel\\.[A-Za-z0-9_+-]+\\.[0-9a-f]{32}$",
+ "type":"string"
+ },
+ "cubin_sha256": {"pattern":"^[0-9a-f]{64}$","type":"string"},
+ "sass_sha256": {"pattern":"^[0-9a-f]{64}$","type":"string"},
+ "source_sha256": {"pattern":"^[0-9a-f]{64}$","type":"string"}
+ },
+ "required": ["cache_key","cubin_sha256","sass_sha256","source_sha256"],
+ "type": "object"
+ },
+ "hybrid_jit_rank_artifact": {
+ "additionalProperties": false,
+ "properties": {
+ "bytes": {"minimum":1,"type":"integer"},
+ "rank": {"minimum":0,"type":"integer"},
+ "sha256": {"pattern":"^[0-9a-f]{64}$","type":"string"}
+ },
+ "required": ["bytes","rank","sha256"],
+ "type": "object"
+ },
+ "hybrid_realized_config": {
+ "additionalProperties": false,
+ "properties": {
+ "backward_combine_api": {"type":"boolean"},
+ "device_side_sync_combine_api": {"type":"boolean"},
+ "device_side_sync_dispatch_api": {"type":"boolean"},
+ "forward_dispatch_api": {"type":"boolean"},
+ "hidden_dim": {"minimum":1,"type":"integer"},
+ "max_num_of_tokens_per_rank": {"minimum":1,"type":"integer"},
+ "num_of_additional_in_flight_s2g_combine_api": {"minimum":0,"type":"integer"},
+ "num_of_additional_in_flight_s2g_dispatch_api": {"minimum":0,"type":"integer"},
+ "num_of_blocks_combine_api": {"minimum":0,"type":"integer"},
+ "num_of_blocks_dispatch_api": {"minimum":0,"type":"integer"},
+ "num_of_blocks_permute": {"minimum":0,"type":"integer"},
+ "num_of_blocks_preprocessing_api": {"minimum":0,"type":"integer"},
+ "num_of_blocks_unpermute": {"minimum":0,"type":"integer"},
+ "num_of_experts_per_rank": {"minimum":1,"type":"integer"},
+ "num_of_in_flight_s2g_dispatch_api": {"minimum":0,"type":"integer"},
+ "num_of_in_flight_s2g_permute_block_dispatch_api": {"minimum":0,"type":"integer"},
+ "num_of_nodes": {"minimum":1,"type":"integer"},
+ "num_of_ranks_per_node": {"minimum":1,"type":"integer"},
+ "num_of_stages_dispatch_api": {"minimum":0,"type":"integer"},
+ "num_of_stages_g2s_combine_api": {"minimum":0,"type":"integer"},
+ "num_of_stages_permute_block_dispatch_api": {"minimum":0,"type":"integer"},
+ "num_of_stages_s2g_combine_api": {"minimum":0,"type":"integer"},
+ "num_of_threads_per_block_preprocessing_api": {"minimum":0,"type":"integer"},
+ "num_of_tokens_per_chunk_combine_api": {"minimum":0,"type":"integer"},
+ "num_of_tokens_per_chunk_dispatch_api": {"minimum":0,"type":"integer"},
+ "num_of_tokens_per_chunk_preprocessing_api": {"minimum":0,"type":"integer"},
+ "num_of_tokens_per_group_combine_api": {"minimum":0,"type":"integer"},
+ "pad_multiple": {"minimum":0,"type":"integer"},
+ "token_data_type": {"enum":["UINT8","UINT16"]}
+ },
+ "required": [
+ "backward_combine_api","device_side_sync_combine_api","device_side_sync_dispatch_api",
+ "forward_dispatch_api","hidden_dim","max_num_of_tokens_per_rank",
+ "num_of_additional_in_flight_s2g_combine_api",
+ "num_of_additional_in_flight_s2g_dispatch_api","num_of_blocks_combine_api",
+ "num_of_blocks_dispatch_api","num_of_blocks_permute","num_of_blocks_preprocessing_api",
+ "num_of_blocks_unpermute","num_of_experts_per_rank",
+ "num_of_in_flight_s2g_dispatch_api","num_of_in_flight_s2g_permute_block_dispatch_api",
+ "num_of_nodes","num_of_ranks_per_node","num_of_stages_dispatch_api",
+ "num_of_stages_g2s_combine_api","num_of_stages_permute_block_dispatch_api",
+ "num_of_stages_s2g_combine_api","num_of_threads_per_block_preprocessing_api",
+ "num_of_tokens_per_chunk_combine_api","num_of_tokens_per_chunk_dispatch_api",
+ "num_of_tokens_per_chunk_preprocessing_api","num_of_tokens_per_group_combine_api",
+ "pad_multiple","token_data_type"
+ ],
+ "type": "object"
+ },
+ "nullable_sha256": {"oneOf":[{"type":"null"},{"pattern":"^[0-9a-f]{64}$","type":"string"}]},
+ "oracle": {
+ "additionalProperties": false,
+ "properties": {
+ "checks": {
+ "additionalProperties": false,
+ "properties": {
+ "combine_values": {"type":"boolean"},
+ "counts": {"type":"boolean"},
+ "metadata": {"type":"boolean"},
+ "multiplicity": {"type":"boolean"},
+ "payload": {"type":"boolean"},
+ "source_set": {"type":"boolean"},
+ "weights": {"type":"boolean"}
+ },
+ "required": ["combine_values","counts","metadata","multiplicity","payload","source_set","weights"],
+ "type": "object"
+ },
+ "atol": {"const":0.02},
+ "combine_weight_semantics": {"enum":["unweighted-rank-sum","gate-weighted-sum"]},
+ "contract": {"enum":["expert-specific-transform-v1","expert-assignment-transform-v1"]},
+ "dispatch_sha256": {"$ref":"#/$defs/nullable_sha256"},
+ "max_absolute_error": {"oneOf":[{"type":"null"},{"minimum":0,"type":"number"}]},
+ "max_elementwise_relative_error": {"oneOf":[{"type":"null"},{"minimum":0,"type":"number"}]},
+ "max_relative_error": {"oneOf":[{"type":"null"},{"minimum":0,"type":"number"}]},
+ "max_weight_error": {"oneOf":[{"type":"null"},{"minimum":0,"type":"number"}]},
+ "order_sha256": {"$ref":"#/$defs/nullable_sha256"},
+ "ordering_contract": {"minLength":1,"type":"string"},
+ "passed": {"type":"boolean"},
+ "receive_count": {"minimum":0,"type":"integer"},
+ "rtol": {"const":0.05}
+ },
+ "required": [
+ "atol",
+ "checks",
+ "combine_weight_semantics",
+ "contract",
+ "dispatch_sha256",
+ "max_absolute_error",
+ "max_elementwise_relative_error",
+ "max_relative_error",
+ "max_weight_error",
+ "order_sha256",
+ "ordering_contract",
+ "passed",
+ "receive_count",
+ "rtol"
+ ],
+ "type": "object"
+ },
+ "percentiles": {
+ "additionalProperties": false,
+ "properties": {
+ "p50": {"minimum":0,"type":"number"},
+ "p90": {"minimum":0,"type":"number"},
+ "p95": {"minimum":0,"type":"number"},
+ "p99": {"minimum":0,"type":"number"}
+ },
+ "required": ["p50","p90","p95","p99"],
+ "type": "object"
+ },
+ "component": {
+ "additionalProperties": false,
+ "properties": {
+ "availability": {"enum":["measured","derived","unavailable"]},
+ "origin": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]},
+ "percentiles_us": {"oneOf":[{"type":"null"},{"$ref":"#/$defs/percentiles"}]},
+ "sample_count": {"minimum":0,"type":"integer"}
+ },
+ "required": ["availability","origin","percentiles_us","sample_count"],
+ "type": "object"
+ },
+ "histogram": {
+ "additionalProperties": false,
+ "properties": {
+ "bins": {"minimum":1,"type":"integer"},
+ "counts": {"items":{"minimum":0,"type":"integer"},"minItems":1,"type":"array"},
+ "max": {"minimum":0,"type":"number"},
+ "min": {"minimum":0,"type":"number"},
+ "n": {"minimum":1,"type":"integer"}
+ },
+ "required": ["n","min","max","bins","counts"],
+ "type": "object"
+ },
+ "scheduled_case": {
+ "additionalProperties": false,
+ "properties": {
+ "backend": {"maxLength":128,"pattern":"^[a-z0-9][a-z0-9_.-]*$","type":"string"},
+ "canonical": {"const":true},
+ "ep": {"minimum":1,"type":"integer"},
+ "eplb": {"type":"boolean"},
+ "experts": {"minimum":1,"type":"integer"},
+ "gpus_per_node": {"minimum":1,"type":"integer"},
+ "hidden": {"minimum":1,"type":"integer"},
+ "ladder": {"pattern":"^[1-9][0-9]*( [1-9][0-9]*)*$","type":"string"},
+ "mode": {"enum":["normal","low-latency"]},
+ "nodes": {"minimum":1,"type":"integer"},
+ "phase": {"enum":["decode","prefill"]},
+ "required_publication": {"enum":["official","comparable-experimental"]},
+ "routing": {"enum":["uniform","zipf"]},
+ "samples_per_point": {"const":512},
+ "scale_out_transport": {"oneOf":[{"type":"null"},{"maxLength":128,"pattern":"^[a-z0-9][a-z0-9_.-]*$","type":"string"}]},
+ "scale_up_domain": {"minimum":1,"type":"integer"},
+ "scale_up_transport": {"maxLength":128,"pattern":"^[a-z0-9][a-z0-9_.-]*$","type":"string"},
+ "scope": {"enum":["scale-up","scale-out"]},
+ "suite": {"maxLength":128,"pattern":"^[a-z0-9][a-z0-9_.-]*$","type":"string"},
+ "timing": {"const":"8:64:32"},
+ "topk": {"minimum":1,"type":"integer"},
+ "topology_class": {"maxLength":128,"pattern":"^[a-z0-9][a-z0-9_.-]*$","type":"string"},
+ "transport": {"maxLength":128,"pattern":"^[a-z0-9][a-z0-9_.-]*$","type":"string"},
+ "warmup_semantics": {"const":"full-roundtrip-before-each-component-trial-point-v1"},
+ "workload": {"maxLength":128,"pattern":"^[a-z0-9][a-z0-9_.-]*$","type":"string"}
+ },
+ "required": [
+ "backend",
+ "canonical",
+ "eplb",
+ "ep",
+ "experts",
+ "gpus_per_node",
+ "hidden",
+ "ladder",
+ "mode",
+ "nodes",
+ "phase",
+ "required_publication",
+ "routing",
+ "samples_per_point",
+ "scale_out_transport",
+ "scale_up_domain",
+ "scale_up_transport",
+ "scope",
+ "suite",
+ "timing",
+ "topk",
+ "topology_class",
+ "transport",
+ "warmup_semantics",
+ "workload"
+ ],
+ "type": "object"
+ },
+ "git_run": {
+ "additionalProperties": false,
+ "properties": {
+ "artifact": {"minLength":1,"type":"string"},
+ "job": {"minLength":1,"type":"string"},
+ "ref": {"minLength":1,"type":"string"},
+ "repo": {"pattern":"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$","type":"string"},
+ "run_attempt": {"pattern":"^[1-9][0-9]*$","type":"string"},
+ "run_id": {"pattern":"^[1-9][0-9]*$","type":"string"},
+ "source_sha": {"pattern":"^[0-9a-f]{40}$","type":"string"}
+ },
+ "required": ["artifact","job","ref","repo","run_attempt","run_id","source_sha"],
+ "type": "object"
+ }
+ },
+ "additionalProperties": false,
+ "properties": {
+ "case": {
+ "additionalProperties": false,
+ "properties": {
+ "attempt_ordinal": {"minimum":1,"type":"integer"},
+ "backend": {"maxLength":128,"pattern":"^[a-z0-9][a-z0-9_.-]*$","type":"string"},
+ "ep_size": {"minimum":1,"type":"integer"},
+ "eplb": {
+ "additionalProperties": false,
+ "properties": {
+ "enabled": {"type":"boolean"},
+ "imbalance_after": {"oneOf":[{"type":"null"},{"minimum":0,"type":"number"}]},
+ "imbalance_before": {"oneOf":[{"type":"null"},{"minimum":0,"type":"number"}]},
+ "mapping_hash": {"oneOf":[{"type":"null"},{"pattern":"^[0-9a-f]{64}$","type":"string"}]},
+ "max_replicas": {"oneOf":[{"type":"null"},{"minimum":0,"type":"integer"}]},
+ "num_logical_experts": {"minimum":1,"type":"integer"},
+ "num_physical_experts": {"minimum":1,"type":"integer"},
+ "num_redundant": {"minimum":0,"type":"integer"},
+ "planner": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]},
+ "reference_tokens_per_rank": {"oneOf":[{"type":"null"},{"minimum":1,"type":"integer"}]},
+ "replicated_experts": {"minimum":0,"type":"integer"}
+ },
+ "required": [
+ "enabled",
+ "imbalance_after",
+ "imbalance_before",
+ "mapping_hash",
+ "max_replicas",
+ "num_logical_experts",
+ "num_physical_experts",
+ "num_redundant",
+ "planner",
+ "reference_tokens_per_rank",
+ "replicated_experts"
+ ],
+ "type": "object"
+ },
+ "mode": {"enum":["normal","low-latency"]},
+ "phase": {"enum":["decode","prefill"]},
+ "required_publication": {"enum":["official","comparable-experimental"]},
+ "resource_mode": {"const":"tuned"},
+ "runner": {"maxLength":128,"pattern":"^[a-z0-9][a-z0-9_.-]*$","type":"string"},
+ "shape": {
+ "additionalProperties": false,
+ "properties": {
+ "activation_profile": {"const":"canonical-counter-source-v3"},
+ "dispatch_dtype": {"const":"bf16"},
+ "eplb": {"type":"boolean"},
+ "experts": {"minimum":1,"type":"integer"},
+ "experts_per_rank": {"minimum":1,"type":"integer"},
+ "hidden": {"minimum":1,"type":"integer"},
+ "kernel_gen": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]},
+ "num_logical_experts": {"minimum":1,"type":"integer"},
+ "quant": {
+ "additionalProperties": false,
+ "properties": {
+ "combine_accum_dtype": {"minLength":1,"type":"string"},
+ "combine_input_dtype": {"const":"bf16"},
+ "combine_output_dtype": {"const":"bf16"},
+ "combine_quant_mode": {"const":"none"},
+ "scale_layout": {"type":"null"}
+ },
+ "required": [
+ "combine_accum_dtype",
+ "combine_input_dtype",
+ "combine_output_dtype",
+ "combine_quant_mode",
+ "scale_layout"
+ ],
+ "type": "object"
+ },
+ "routing": {"enum":["uniform","zipf"]},
+ "topk": {"minimum":1,"type":"integer"}
+ },
+ "required": [
+ "activation_profile",
+ "dispatch_dtype",
+ "eplb",
+ "experts",
+ "experts_per_rank",
+ "hidden",
+ "kernel_gen",
+ "num_logical_experts",
+ "quant",
+ "routing",
+ "topk"
+ ],
+ "type": "object"
+ },
+ "suite": {"maxLength":128,"pattern":"^[a-z0-9][a-z0-9_.-]*$","type":"string"},
+ "workload_name": {"maxLength":128,"pattern":"^[a-z0-9][a-z0-9_.-]*$","type":"string"}
+ },
+ "required": [
+ "attempt_ordinal",
+ "backend",
+ "eplb",
+ "ep_size",
+ "mode",
+ "phase",
+ "required_publication",
+ "resource_mode",
+ "runner",
+ "shape",
+ "suite",
+ "workload_name"
+ ],
+ "type": "object"
+ },
+ "format": {"const":"collectivex.ep.v1"},
+ "generated_at": {"format":"date-time","type":"string"},
+ "identity": {
+ "additionalProperties": false,
+ "properties": {
+ "allocation_factors": {
+ "additionalProperties": false,
+ "properties": {
+ "artifact": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]},
+ "execution_id": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]},
+ "job": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]},
+ "repo": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]},
+ "run_attempt": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]},
+ "run_id": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]},
+ "runner": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]},
+ "source_sha": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]}
+ },
+ "required": ["artifact","execution_id","job","repo","run_attempt","run_id","runner","source_sha"],
+ "type": "object"
+ },
+ "allocation_id": {"pattern":"^cxallocation-v1-[0-9a-f]{64}$","type":"string"},
+ "attempt_id": {"pattern":"^cxattempt-v1-[0-9a-f]{64}$","type":"string"},
+ "attempt_ordinal": {"minimum":1,"type":"integer"},
+ "case_factors": {
+ "additionalProperties": false,
+ "properties": {
+ "case": {"$ref":"#/$defs/scheduled_case"},
+ "profile": {
+ "oneOf": [
+ {"const": {
+ "activation_generator": "collectivex-activation-counter-v3",
+ "activation_profile": "canonical-counter-source-v3",
+ "combine_dtype": "bf16",
+ "combine_quant_mode": "none",
+ "combine_semantics": "activation-only",
+ "component_order_contract": "roundtrip-dispatch-activation-only-combine-v2",
+ "conditioning_contract": "fixed-phase-ramp-8-roundtrips-v1",
+ "contract": "layout-and-dispatch-v1",
+ "correctness_scope": "dispatch-metadata-and-transformed-combine",
+ "dtype": "bf16",
+ "eplb_planner": "greedy-rank-major-v1",
+ "eplb_redundant_experts": 32,
+ "eplb_reference_tokens_per_rank": 2048,
+ "mode": "normal",
+ "oracle_contract": "expert-specific-transform-v1",
+ "oracle_tolerances": "rtol=0.05,atol=0.02",
+ "payload_unit": "token-rank",
+ "placement": "packed",
+ "percentile_method": "nearest-rank",
+ "rank_reduction": "cross-rank-max-per-iteration",
+ "resource_mode": "tuned",
+ "routing_generator": "collectivex-routing-counter-v3",
+ "sampling_contract": "fixed-512-v1",
+ "seed": 67
+ }},
+ {"const": {
+ "activation_generator": "collectivex-activation-counter-v3",
+ "activation_profile": "canonical-counter-source-v3",
+ "combine_dtype": "bf16",
+ "combine_quant_mode": "none",
+ "combine_semantics": "gate-weighted",
+ "component_order_contract": "roundtrip-dispatch-gate-weighted-combine-v1",
+ "conditioning_contract": "fixed-phase-ramp-8-roundtrips-v1",
+ "contract": "expert-packed-weighted-combine-v1",
+ "correctness_scope": "expert-assignment-and-weighted-combine",
+ "dtype": "bf16",
+ "eplb_planner": "greedy-rank-major-v1",
+ "eplb_redundant_experts": 32,
+ "eplb_reference_tokens_per_rank": 2048,
+ "mode": "low-latency",
+ "oracle_contract": "expert-assignment-transform-v1",
+ "oracle_tolerances": "rtol=0.05,atol=0.02",
+ "payload_unit": "token-expert",
+ "placement": "packed",
+ "percentile_method": "nearest-rank",
+ "rank_reduction": "cross-rank-max-per-iteration",
+ "resource_mode": "tuned",
+ "routing_generator": "collectivex-routing-counter-v3",
+ "sampling_contract": "fixed-512-v1",
+ "seed": 67
+ }}]
+ },
+ "sku": {"maxLength":128,"pattern":"^[a-z0-9][a-z0-9_.-]*$","type":"string"}
+ },
+ "required": ["case","profile","sku"],
+ "type": "object"
+ },
+ "case_id": {"pattern":"^cxcase-v1-[0-9a-f]{64}$","type":"string"},
+ "series_factors": {
+ "additionalProperties": false,
+ "properties": {
+ "backend": {"maxLength":128,"pattern":"^[a-z0-9][a-z0-9_.-]*$","type":"string"},
+ "implementation_contract_sha256": {"pattern":"^[0-9a-f]{64}$","type":"string"},
+ "public_config_sha256": {"pattern":"^[0-9a-f]{64}$","type":"string"},
+ "routing_control_sha256": {"pattern":"^[0-9a-f]{64}$","type":"string"},
+ "case_id": {"pattern":"^cxcase-v1-[0-9a-f]{64}$","type":"string"},
+ "image_digest": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]},
+ "runtime_fingerprint_sha256": {"pattern":"^[0-9a-f]{64}$","type":"string"},
+ "source_sha": {"oneOf":[{"type":"null"},{"pattern":"^[0-9a-f]{40}$","type":"string"}]},
+ "squash_sha256": {"oneOf":[{"type":"null"},{"pattern":"^[0-9a-f]{64}$","type":"string"}]},
+ "workload_id": {"pattern":"^cxwork-v1-[0-9a-f]{64}$","type":"string"}
+ },
+ "required": [
+ "backend",
+ "implementation_contract_sha256",
+ "public_config_sha256",
+ "routing_control_sha256",
+ "case_id",
+ "image_digest",
+ "runtime_fingerprint_sha256",
+ "source_sha",
+ "squash_sha256",
+ "workload_id"
+ ],
+ "type": "object"
+ },
+ "series_id": {"pattern":"^cxseries-v1-[0-9a-f]{64}$","type":"string"}
+ },
+ "required": [
+ "allocation_factors",
+ "allocation_id",
+ "attempt_id",
+ "attempt_ordinal",
+ "case_factors",
+ "case_id",
+ "series_factors",
+ "series_id"
+ ],
+ "type": "object"
+ },
+ "implementation": {
+ "additionalProperties": false,
+ "properties": {
+ "kernel_generation": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]},
+ "name": {"maxLength":128,"pattern":"^[a-z0-9][a-z0-9_.-]*$","type":"string"},
+ "provenance": {
+ "properties": {
+ "allow_hybrid_mode": {"type":"boolean"},
+ "communication_backend": {"enum":["nccl-device-lsa","nccl-gin"]},
+ "deepep_fix_pr": {"const":630},
+ "deepep_pr": {"const":605},
+ "deterministic": {"type": "boolean"},
+ "gin_enabled": {"type":"boolean"},
+ "jit_cubins": {
+ "items": {"$ref":"#/$defs/deepep_v2_jit_cubin"},
+ "maxItems": 5,
+ "minItems": 5,
+ "type": "array",
+ "uniqueItems": true
+ },
+ "jit_kernel_keys": {
+ "items": {"maxLength":512,"pattern":"^[A-Za-z0-9][A-Za-z0-9_.+-]*$","type":"string"},
+ "maxItems": 3,
+ "minItems": 3,
+ "type": "array",
+ "uniqueItems": true
+ },
+ "jit_random_seed": {"const":"collectivex-deepep-v2-fa8a9b1"},
+ "jit_shared_objects": {
+ "items": {
+ "additionalProperties": false,
+ "properties": {
+ "kernel_key": {"maxLength":512,"pattern":"^[A-Za-z0-9][A-Za-z0-9_.+-]*$","type":"string"},
+ "rank_artifacts": {
+ "items": {"$ref":"#/$defs/hybrid_jit_rank_artifact"},
+ "minItems": 1,
+ "type": "array"
+ }
+ },
+ "required": ["kernel_key","rank_artifacts"],
+ "type": "object"
+ },
+ "maxItems": 3,
+ "minItems": 3,
+ "type": "array"
+ },
+ "num_experts": {"minimum": 1, "type": "integer"},
+ "num_nvl_bytes": {"minimum": 0, "type": "integer"},
+ "num_qps_per_rank": {"minimum": 1, "type": "integer"},
+ "num_rdma_bytes": {"minimum": 0, "type": "integer"},
+ "rdma_block_num": {"minimum": 0, "type": "integer"},
+ "realized_config": {"$ref":"#/$defs/hybrid_realized_config"},
+ "tuning_num_experts": {"minimum": 1, "type": "integer"},
+ "uccl_dependency_versions": {
+ "additionalProperties": false,
+ "properties": {
+ "intervaltree": {"const":"3.1.0"},
+ "nvidia-cuda-runtime-cu12": {"const":"12.9.79"},
+ "sortedcontainers": {"const":"2.4.0"}
+ },
+ "required": ["intervaltree","nvidia-cuda-runtime-cu12","sortedcontainers"],
+ "type": "object"
+ },
+ "use_external_inp_buf": {"type": "boolean"}
+ },
+ "type": "object",
+ "propertyNames": {
+ "enum": [
+ "allocated_qps",
+ "allow_hybrid_mode",
+ "allow_mnnvl",
+ "allow_multiple_reduction",
+ "api",
+ "api_signature_sha256",
+ "backend",
+ "backend_lineage",
+ "block_num",
+ "block_num_floored",
+ "block_num_target",
+ "branch",
+ "collective_library",
+ "combine_dtype",
+ "combine_warps",
+ "communication_backend",
+ "cuda_version",
+ "deepep_commit",
+ "deepep_distribution_version",
+ "deepep_fix_pr",
+ "deepep_pr",
+ "deepep_tree",
+ "deepep_version",
+ "deterministic",
+ "device_cus",
+ "device_sms",
+ "dispatch_dtype",
+ "dispatch_warps",
+ "enable_sdma",
+ "fmt_commit",
+ "gpus_per_node",
+ "gin_enabled",
+ "heap_size",
+ "impl",
+ "jit_cache_key",
+ "jit_cubins",
+ "jit_kernel_keys",
+ "jit_random_seed",
+ "jit_shared_objects",
+ "kernel_type",
+ "loaded_libraries",
+ "local_experts",
+ "logical_scaleout_ranks",
+ "logical_scaleup_ranks",
+ "mapping_variant",
+ "max_num_inp_token_per_rank",
+ "max_num_tokens",
+ "max_total_recv_tokens",
+ "mnnvl_comm",
+ "mode",
+ "mori_commit",
+ "nccl_communicator",
+ "nccl_package_version",
+ "nccl_version",
+ "num_experts",
+ "num_max_tokens_per_rank",
+ "num_nvl_bytes",
+ "num_qps",
+ "num_qps_per_rank",
+ "num_rdma_bytes",
+ "num_sms",
+ "nvshmem_package_version",
+ "path",
+ "physical_nvlink_ranks",
+ "physical_rdma_ranks",
+ "prefer_overlap_with_compute",
+ "rdma_block_num",
+ "reference_semantics",
+ "realized_config",
+ "requested_num_sms",
+ "resource_mode",
+ "routing_factor",
+ "routing_metadata",
+ "sm_fraction",
+ "top_k",
+ "torch_git_version",
+ "torch_version",
+ "transport",
+ "trtllm",
+ "tuned_source",
+ "tuning_num_experts",
+ "uccl_commit",
+ "uccl_dependency_versions",
+ "uccl_version",
+ "uccl_wrapper_commit",
+ "use_external_inp_buf",
+ "workspace"
+ ]
+ }
+ },
+ "resource_profile": {
+ "additionalProperties": false,
+ "properties": {
+ "achieved_fraction": {},
+ "comm_units_kind": {},
+ "configured_units": {},
+ "conformance_class": {},
+ "device_units": {},
+ "fixed_kernel": {},
+ "nonconforming": {},
+ "pareto_eligible": {},
+ "persistent_bytes": {},
+ "qps_per_rank": {},
+ "requested_fraction": {},
+ "tuned_source": {},
+ "target_achieved_within_tol": {},
+ "tolerance": {},
+ "resource_class": {},
+ "warps_combine": {},
+ "warps_dispatch": {}
+ },
+ "required": [
+ "comm_units_kind",
+ "requested_fraction",
+ "configured_units",
+ "device_units",
+ "achieved_fraction",
+ "warps_dispatch",
+ "warps_combine",
+ "qps_per_rank",
+ "persistent_bytes",
+ "tuned_source",
+ "resource_class",
+ "conformance_class",
+ "tolerance",
+ "target_achieved_within_tol",
+ "nonconforming",
+ "fixed_kernel",
+ "pareto_eligible"
+ ],
+ "type": "object"
+ }
+ },
+ "required": ["kernel_generation","name","provenance","resource_profile"],
+ "type": "object"
+ },
+ "measurement": {
+ "additionalProperties": false,
+ "properties": {
+ "component_order_contract": {"enum":["roundtrip-dispatch-activation-only-combine-v2","roundtrip-dispatch-gate-weighted-combine-v1"]},
+ "conditioning": {
+ "additionalProperties": false,
+ "properties": {
+ "contract": {"const":"fixed-phase-ramp-8-roundtrips-v1"},
+ "ladder": {"items":{"minimum":1,"type":"integer"},"minItems":1,"type":"array"},
+ "roundtrips_per_shape": {"const":8}
+ },
+ "required": ["contract","ladder","roundtrips_per_shape"],
+ "type": "object"
+ },
+ "contract": {"enum":["layout-and-dispatch-v1","expert-packed-weighted-combine-v1"]},
+ "rows": {
+ "items": {
+ "additionalProperties": false,
+ "properties": {
+ "anomalies": {
+ "items": {
+ "additionalProperties": false,
+ "properties": {
+ "T": {"minimum":1,"type":"integer"},
+ "component_floor_p50": {"minimum":0,"type":"number"},
+ "isolated_sum_p99": {"minimum":0,"type":"number"},
+ "ratio": {"minimum":0,"type":"number"},
+ "roundtrip_p50": {"minimum":0,"type":"number"},
+ "roundtrip_p99": {"minimum":0,"type":"number"},
+ "threshold": {"minimum":0,"type":"number"},
+ "type": {"enum":["roundtrip_gt_isolated_sum","roundtrip_lt_component_floor"]}
+ },
+ "required": ["type","T"],
+ "type": "object"
+ },
+ "type": "array"
+ },
+ "components": {
+ "additionalProperties": false,
+ "properties": {
+ "combine": {"$ref":"#/$defs/component"},
+ "dispatch": {"$ref":"#/$defs/component"},
+ "isolated_sum": {"$ref":"#/$defs/component"},
+ "roundtrip": {"$ref":"#/$defs/component"}
+ },
+ "required": ["combine","dispatch","isolated_sum","roundtrip"],
+ "type": "object"
+ },
+ "correctness": {
+ "additionalProperties": false,
+ "properties": {
+ "contract": {"enum":["expert-specific-transform-v1","expert-assignment-transform-v1"]},
+ "max_relative_error": {"minimum":0,"type":"number"},
+ "passed": {"type":"boolean"},
+ "rank_evidence": {
+ "items": {
+ "additionalProperties": false,
+ "properties": {
+ "input_unchanged": {"type":"boolean"},
+ "order_stable": {"type":"boolean"},
+ "post_timing": {"$ref":"#/$defs/oracle"},
+ "pre_timing": {"$ref":"#/$defs/oracle"},
+ "rank": {"minimum":0,"type":"integer"}
+ },
+ "required": ["input_unchanged","order_stable","post_timing","pre_timing","rank"],
+ "type": "object"
+ },
+ "minItems": 1,
+ "type": "array"
+ },
+ "scope": {"enum":["dispatch-metadata-and-transformed-combine","expert-assignment-and-weighted-combine"]}
+ },
+ "required": ["contract","max_relative_error","passed","rank_evidence","scope"],
+ "type": "object"
+ },
+ "evidence_id": {"pattern":"^cxevidence-v1-[0-9a-f]{64}$","type":"string"},
+ "global_tokens": {"minimum":1,"type":"integer"},
+ "logical_bytes": {
+ "additionalProperties": false,
+ "properties": {
+ "combine": {"minimum":1,"type":"integer"},
+ "dispatch": {"minimum":1,"type":"integer"},
+ "roundtrip": {"minimum":1,"type":"integer"}
+ },
+ "required": ["combine","dispatch","roundtrip"],
+ "type": "object"
+ },
+ "point_id": {"pattern":"^cxpoint-v1-[0-9a-f]{64}$","type":"string"},
+ "receive": {
+ "additionalProperties": false,
+ "properties": {
+ "max": {"minimum":0,"type":"integer"},
+ "mean": {"minimum":0,"type":"number"},
+ "min": {"minimum":0,"type":"integer"},
+ "total": {"minimum":0,"type":"integer"}
+ },
+ "required": ["max","mean","min","total"],
+ "type": "object"
+ },
+ "routing": {
+ "additionalProperties": false,
+ "properties": {
+ "empty_expert_count": {"minimum":0,"type":"integer"},
+ "empty_rank_count": {"minimum":0,"type":"integer"},
+ "expert_assignment_rank_cv": {"minimum":0,"type":"number"},
+ "expert_assignments_per_rank": {"items":{"minimum":0,"type":"integer"},"type":"array"},
+ "expert_load_cv": {"minimum":0,"type":"number"},
+ "expert_load_max": {"minimum":0,"type":"integer"},
+ "expert_load_mean": {"minimum":0,"type":"number"},
+ "expert_load_min": {"minimum":0,"type":"integer"},
+ "fanout_histogram": {"items":{"minimum":0,"type":"integer"},"type":"array"},
+ "fanout_max": {"minimum":1,"type":"integer"},
+ "fanout_mean": {"minimum":0,"type":"number"},
+ "fanout_min": {"minimum":1,"type":"integer"},
+ "hash": {"pattern":"^[0-9a-f]{64}$","type":"string"},
+ "hotspot_ratio": {"minimum":0,"type":"number"},
+ "locality": {
+ "oneOf": [
+ {"type":"null"},
+ {
+ "additionalProperties": false,
+ "properties": {
+ "copies": {"minimum":0,"type":"integer"},
+ "cross_domain_fraction": {"minimum":0,"type":"number"},
+ "cross_node_fraction": {"minimum":0,"type":"number"},
+ "gpus_per_node": {"minimum":1,"type":"integer"},
+ "local_rank_fraction": {"minimum":0,"type":"number"},
+ "placement": {"const":"packed"},
+ "same_node_fraction": {"minimum":0,"type":"number"},
+ "same_scaleup_domain_fraction": {"minimum":0,"type":"number"},
+ "scale_up_domain": {"minimum":1,"type":"integer"}
+ },
+ "required": [
+ "placement",
+ "local_rank_fraction",
+ "same_node_fraction",
+ "same_scaleup_domain_fraction",
+ "cross_node_fraction",
+ "cross_domain_fraction",
+ "gpus_per_node",
+ "scale_up_domain",
+ "copies"
+ ],
+ "type": "object"
+ }
+ ]
+ },
+ "payload_copies_per_rank": {"items":{"minimum":0,"type":"integer"},"type":"array"},
+ "payload_rank_cv": {"minimum":0,"type":"number"},
+ "routed_copies": {"minimum":1,"type":"integer"},
+ "source_token_stats": {
+ "oneOf": [
+ {"type":"null"},
+ {
+ "additionalProperties": false,
+ "properties": {
+ "cv": {"minimum":0,"type":"number"},
+ "empty_ranks": {"minimum":0,"type":"integer"},
+ "max": {"minimum":0,"type":"integer"},
+ "mean": {"minimum":0,"type":"number"},
+ "min": {"minimum":0,"type":"integer"},
+ "ranks": {"minimum":1,"type":"integer"},
+ "total": {"minimum":0,"type":"integer"}
+ },
+ "required": ["min","mean","max","cv","empty_ranks","total","ranks"],
+ "type": "object"
+ }
+ ]
+ }
+ },
+ "required": [
+ "empty_expert_count",
+ "empty_rank_count",
+ "expert_assignment_rank_cv",
+ "expert_assignments_per_rank",
+ "expert_load_cv",
+ "expert_load_max",
+ "expert_load_mean",
+ "expert_load_min",
+ "fanout_histogram",
+ "fanout_max",
+ "fanout_mean",
+ "fanout_min",
+ "hash",
+ "hotspot_ratio",
+ "locality",
+ "payload_copies_per_rank",
+ "payload_rank_cv",
+ "routed_copies",
+ "source_token_stats"
+ ],
+ "type": "object"
+ },
+ "sample_histograms": {
+ "additionalProperties": false,
+ "properties": {
+ "combine": {"oneOf":[{"type":"null"},{"$ref":"#/$defs/histogram"}]},
+ "dispatch": {"oneOf":[{"type":"null"},{"$ref":"#/$defs/histogram"}]},
+ "roundtrip": {"$ref":"#/$defs/histogram"}
+ },
+ "required": ["dispatch","combine","roundtrip"],
+ "type": "object"
+ },
+ "sample_sha256": {"pattern":"^[0-9a-f]{64}$","type":"string"},
+ "token_rate_at_latency_percentile": {"$ref":"#/$defs/percentiles"},
+ "tokens_per_rank": {"minimum":1,"type":"integer"}
+ },
+ "required": [
+ "anomalies",
+ "components",
+ "correctness",
+ "evidence_id",
+ "global_tokens",
+ "logical_bytes",
+ "point_id",
+ "receive",
+ "routing",
+ "sample_histograms",
+ "sample_sha256",
+ "token_rate_at_latency_percentile",
+ "tokens_per_rank"
+ ],
+ "type": "object"
+ },
+ "minItems": 1,
+ "type": "array"
+ },
+ "sampling": {
+ "additionalProperties": false,
+ "properties": {
+ "contract": {"const":"fixed-512-v1"},
+ "iterations_per_trial": {"const":8},
+ "percentile_method": {"const":"nearest-rank"},
+ "reduction": {"const":"cross-rank-max-per-iteration"},
+ "samples_per_component": {"const":512},
+ "trials": {"const":64},
+ "warmup_iterations": {"const":32},
+ "warmup_semantics": {"const":"full-roundtrip-before-each-component-trial-point-v1"}
+ },
+ "required": [
+ "contract",
+ "iterations_per_trial",
+ "percentile_method",
+ "reduction",
+ "samples_per_component",
+ "trials",
+ "warmup_iterations",
+ "warmup_semantics"
+ ],
+ "type": "object"
+ },
+ "source_allocation": {"const":"even"}
+ },
+ "required": [
+ "component_order_contract",
+ "conditioning",
+ "contract",
+ "rows",
+ "sampling",
+ "source_allocation"
+ ],
+ "type": "object"
+ },
+ "outcome": {
+ "additionalProperties": false,
+ "properties": {
+ "publication_status": {"enum":["diagnostic","invalid"]},
+ "reasons": {"items":{"type":"string"},"type":"array"},
+ "status": {"enum":["success","invalid"]},
+ "validity": {
+ "additionalProperties": false,
+ "properties": {
+ "anomaly_free": {"type":"boolean"},
+ "execution_status": {"enum":["complete","failed"]},
+ "measurement_conformance": {"enum":["conformant","nonconformant"]},
+ "provenance_complete": {"type":"boolean"},
+ "resource_conformance": {"minLength":1,"type":"string"},
+ "sampling_conformance": {"enum":["conformant","nonconformant"]},
+ "semantic_correctness": {"enum":["pass","fail"]},
+ "workload_identity": {"enum":["consistent-across-ranks","inconsistent"]},
+ "workload_source": {"enum":["canonical-serialized","seeded-runtime"]}
+ },
+ "required": [
+ "execution_status",
+ "semantic_correctness",
+ "workload_identity",
+ "workload_source",
+ "measurement_conformance",
+ "sampling_conformance",
+ "resource_conformance",
+ "provenance_complete",
+ "anomaly_free"
+ ],
+ "type": "object"
+ }
+ },
+ "required": ["publication_status","reasons","status","validity"],
+ "type": "object"
+ },
+ "provenance": {
+ "additionalProperties": false,
+ "properties": {
+ "command": {"minLength":1,"type":"string"},
+ "distributed_launcher": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]},
+ "git_run": {"oneOf":[{"type":"null"},{"$ref":"#/$defs/git_run"}]},
+ "image": {
+ "additionalProperties": false,
+ "properties": {
+ "arch": {"maxLength":128,"pattern":"^[a-z0-9][a-z0-9_.-]*$","type":"string"},
+ "digest": {
+ "oneOf": [{"type":"null"},{"pattern":"^sha256:[0-9a-f]{64}$","type":"string"}]
+ },
+ "digest_verified": {"type":"boolean"},
+ "reference": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]},
+ "squash_sha256": {"oneOf":[{"type":"null"},{"pattern":"^[0-9a-f]{64}$","type":"string"}]}
+ },
+ "required": ["arch","digest","digest_verified","reference","squash_sha256"],
+ "type": "object"
+ },
+ "redaction": {"const":"sanitized-v1"}
+ },
+ "required": ["command","distributed_launcher","git_run","image","redaction"],
+ "type": "object"
+ },
+ "record_type": {"const":"case-attempt"},
+ "runtime_fingerprint": {
+ "additionalProperties": false,
+ "properties": {
+ "accelerator_runtime": {
+ "additionalProperties": false,
+ "properties": {
+ "kind": {"enum":["cuda","hip"]},
+ "version": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]}
+ },
+ "required": ["kind","version"],
+ "type": "object"
+ },
+ "collective_library": {
+ "additionalProperties": false,
+ "properties": {
+ "kind": {"enum":["nccl","rccl"]},
+ "version": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]}
+ },
+ "required": ["kind","version"],
+ "type": "object"
+ },
+ "device": {
+ "additionalProperties": false,
+ "properties": {
+ "arch": {"minLength":1,"type":"string"},
+ "compute_units": {"minimum":1,"type":"integer"},
+ "memory_bytes": {"minimum":1,"type":"integer"},
+ "product": {"minLength":1,"type":"string"},
+ "warp_size": {"minimum":1,"type":"integer"}
+ },
+ "required": ["arch","compute_units","memory_bytes","product","warp_size"],
+ "type": "object"
+ },
+ "driver_version": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]},
+ "framework": {
+ "additionalProperties": false,
+ "properties": {"kind":{"const":"torch"},"version":{"minLength":1,"type":"string"}},
+ "required": ["kind","version"],
+ "type": "object"
+ },
+ "machine": {"maxLength":128,"pattern":"^[a-z0-9][a-z0-9_.-]*$","type":"string"},
+ "python_version": {"minLength":1,"type":"string"},
+ "vendor": {"enum":["nvidia","amd"]}
+ },
+ "required": [
+ "accelerator_runtime",
+ "collective_library",
+ "device",
+ "driver_version",
+ "framework",
+ "machine",
+ "python_version",
+ "vendor"
+ ],
+ "type": "object"
+ },
+ "sample_artifact": {
+ "additionalProperties": false,
+ "properties": {
+ "bytes": {"minimum":1,"type":"integer"},
+ "format": {"const":"collectivex.samples.v1"},
+ "path": {"pattern":"^[A-Za-z0-9_.-]+$","type":"string"},
+ "sha256": {"pattern":"^[0-9a-f]{64}$","type":"string"}
+ },
+ "required": ["bytes","format","path","sha256"],
+ "type": "object"
+ },
+ "schema_version": {"const":1},
+ "topology": {
+ "additionalProperties": false,
+ "properties": {
+ "device_count": {"minimum":1,"type":"integer"},
+ "device_product": {"minLength":1,"type":"string"},
+ "gpus_per_node": {"minimum":1,"type":"integer"},
+ "nodes": {"minimum":1,"type":"integer"},
+ "placement": {"const":"packed"},
+ "realized_placement": {
+ "additionalProperties": false,
+ "properties": {
+ "gpus_per_node": {"minimum":1,"type":"integer"},
+ "nodes": {"minimum":1,"type":"integer"},
+ "ranks_per_node": {"minimum":1,"type":"integer"},
+ "unique_local_ranks": {"const":true},
+ "valid": {"const":true}
+ },
+ "required": ["gpus_per_node","nodes","ranks_per_node","unique_local_ranks","valid"],
+ "type": "object"
+ },
+ "scale_out_transport": {"oneOf":[{"type":"null"},{"maxLength":128,"pattern":"^[a-z0-9][a-z0-9_.-]*$","type":"string"}]},
+ "scale_up_domain": {"minimum":1,"type":"integer"},
+ "scale_up_transport": {"maxLength":128,"pattern":"^[a-z0-9][a-z0-9_.-]*$","type":"string"},
+ "scope": {"enum":["scale-up","scale-out"]},
+ "topology_class": {"maxLength":128,"pattern":"^[a-z0-9][a-z0-9_.-]*$","type":"string"},
+ "transport": {"maxLength":128,"pattern":"^[a-z0-9][a-z0-9_.-]*$","type":"string"},
+ "world_size": {"minimum":1,"type":"integer"}
+ },
+ "required": [
+ "device_count",
+ "device_product",
+ "gpus_per_node",
+ "nodes",
+ "placement",
+ "realized_placement",
+ "scale_out_transport",
+ "scale_up_domain",
+ "scale_up_transport",
+ "scope",
+ "topology_class",
+ "transport",
+ "world_size"
+ ],
+ "type": "object"
+ },
+ "workload": {
+ "additionalProperties": false,
+ "properties": {
+ "activation_generator": {"const":"collectivex-activation-counter-v3"},
+ "activation_identity": {"pattern":"^[0-9a-f]{64}$","type":"string"},
+ "activation_profile": {"const":"canonical-counter-source-v3"},
+ "cross_rank_consistent": {"const":true},
+ "manifest_checksums": {
+ "oneOf": [
+ {"type":"null"},
+ {
+ "additionalProperties": {
+ "additionalProperties": false,
+ "properties": {
+ "topk_idx": {"pattern":"^[0-9a-f]{64}$","type":"string"},
+ "topk_weights": {"pattern":"^[0-9a-f]{64}$","type":"string"},
+ "trace": {"pattern":"^[0-9a-f]{64}$","type":"string"}
+ },
+ "required": ["topk_idx", "topk_weights", "trace"],
+ "type": "object"
+ },
+ "type": "object"
+ }
+ ]
+ },
+ "members": {
+ "oneOf": [
+ {"type":"null"},
+ {
+ "items": {"pattern":"^cxwork-v1-[0-9a-f]{64}$","type":"string"},
+ "minItems": 1,
+ "uniqueItems": true,
+ "type": "array"
+ }
+ ]
+ },
+ "routing_generator": {"const":"collectivex-routing-counter-v3"},
+ "source": {"enum":["canonical-serialized","seeded-runtime"]},
+ "trace_hashes": {
+ "items": {"pattern":"^[0-9a-f]{64}$","type":"string"},
+ "minItems": 1,
+ "type": "array"
+ },
+ "trace_signature": {"pattern":"^[0-9a-f]{64}$","type":"string"},
+ "workload_id": {
+ "oneOf": [{"type":"null"},{"pattern":"^cxwork-v1-[0-9a-f]{64}$","type":"string"}]
+ }
+ },
+ "required": [
+ "activation_generator",
+ "activation_identity",
+ "activation_profile",
+ "cross_rank_consistent",
+ "manifest_checksums",
+ "members",
+ "routing_generator",
+ "source",
+ "trace_hashes",
+ "trace_signature",
+ "workload_id"
+ ],
+ "type": "object"
+ }
+ },
+ "required": [
+ "case",
+ "format",
+ "generated_at",
+ "identity",
+ "implementation",
+ "measurement",
+ "outcome",
+ "provenance",
+ "record_type",
+ "runtime_fingerprint",
+ "sample_artifact",
+ "schema_version",
+ "topology",
+ "workload"
+ ],
+ "title": "CollectiveX raw case attempt v1",
+ "type": "object"
+}
diff --git a/experimental/CollectiveX/schemas/samples-v1.schema.json b/experimental/CollectiveX/schemas/samples-v1.schema.json
new file mode 100644
index 0000000000..b9a1df0541
--- /dev/null
+++ b/experimental/CollectiveX/schemas/samples-v1.schema.json
@@ -0,0 +1,80 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "https://inferencex.com/schemas/collectivex/samples-v1.schema.json",
+ "title": "CollectiveX exact private samples v1",
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["allocation_id","attempt_id","case_id","format","points","sampling","schema_version","series_id"],
+ "properties": {
+ "allocation_id": {"$ref": "#/$defs/allocationId"},
+ "attempt_id": {"$ref": "#/$defs/attemptId"},
+ "case_id": {"$ref": "#/$defs/caseId"},
+ "format": {"const": "collectivex.samples.v1"},
+ "points": {
+ "type": "array",
+ "minItems": 1,
+ "items": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["components","evidence_id","point_id","sample_sha256","tokens_per_rank"],
+ "properties": {
+ "components": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["combine","dispatch","roundtrip"],
+ "properties": {
+ "combine": {"$ref": "#/$defs/component"},
+ "dispatch": {"$ref": "#/$defs/component"},
+ "roundtrip": {"$ref": "#/$defs/component"}
+ }
+ },
+ "evidence_id": {"$ref": "#/$defs/evidenceId"},
+ "point_id": {"$ref": "#/$defs/pointId"},
+ "sample_sha256": {"$ref": "#/$defs/sha256"},
+ "tokens_per_rank": {"type": "integer","minimum": 1}
+ }
+ }
+ },
+ "sampling": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["iterations_per_trial","reduction","trials"],
+ "properties": {
+ "iterations_per_trial": {"const": 8},
+ "reduction": {"const": "cross-rank-max-per-iteration"},
+ "trials": {"const": 64}
+ }
+ },
+ "schema_version": {"const": 1},
+ "series_id": {"$ref": "#/$defs/seriesId"}
+ },
+ "$defs": {
+ "sha256": {"type": "string","pattern": "^[0-9a-f]{64}$"},
+ "caseId": {"type": "string","pattern": "^cxcase-v1-[0-9a-f]{64}$"},
+ "seriesId": {"type": "string","pattern": "^cxseries-v1-[0-9a-f]{64}$"},
+ "pointId": {"type": "string","pattern": "^cxpoint-v1-[0-9a-f]{64}$"},
+ "evidenceId": {"type": "string","pattern": "^cxevidence-v1-[0-9a-f]{64}$"},
+ "allocationId": {"type": "string","pattern": "^cxallocation-v1-[0-9a-f]{64}$"},
+ "attemptId": {"type": "string","pattern": "^cxattempt-v1-[0-9a-f]{64}$"},
+ "component": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["availability","sample_count","trials"],
+ "properties": {
+ "availability": {"enum": ["measured","unavailable"]},
+ "sample_count": {"type": "integer","minimum": 0,"maximum": 512},
+ "trials": {
+ "oneOf": [
+ {"type": "null"},
+ {
+ "type": "array",
+ "minItems": 64,
+ "maxItems": 64,
+ "items": {"type": "array","minItems": 8,"maxItems": 8,"items": {"type": "number","minimum": 0}}
+ }
+ ]
+ }
+ }
+ }
+ }
+}
diff --git a/experimental/CollectiveX/schemas/terminal-outcome-v1.schema.json b/experimental/CollectiveX/schemas/terminal-outcome-v1.schema.json
new file mode 100644
index 0000000000..dede78a43c
--- /dev/null
+++ b/experimental/CollectiveX/schemas/terminal-outcome-v1.schema.json
@@ -0,0 +1,289 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "https://inferencex.com/schemas/collectivex/terminal-outcome-v1.schema.json",
+ "title": "CollectiveX terminal outcome v1",
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["case","format","generated_at","identity","outcome","provenance","record_type","schema_version"],
+ "properties": {
+ "case": {"$ref": "#/$defs/case"},
+ "format": {"const": "collectivex.terminal.v1"},
+ "generated_at": {"type": "string","format": "date-time"},
+ "identity": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["allocation_factors","allocation_id","attempt_id","attempt_ordinal","case_factors","case_id"],
+ "properties": {
+ "allocation_factors": {"$ref": "#/$defs/allocationFactors"},
+ "allocation_id": {"$ref": "#/$defs/allocationId"},
+ "attempt_id": {"$ref": "#/$defs/attemptId"},
+ "attempt_ordinal": {"type": "integer","minimum": 1},
+ "case_factors": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["case","profile","sku"],
+ "properties": {
+ "case": {"$ref": "#/$defs/case"},
+ "profile": {
+ "oneOf": [
+ {"const": {
+ "activation_generator": "collectivex-activation-counter-v3",
+ "activation_profile": "canonical-counter-source-v3",
+ "combine_dtype": "bf16",
+ "combine_quant_mode": "none",
+ "combine_semantics": "activation-only",
+ "component_order_contract": "roundtrip-dispatch-activation-only-combine-v2",
+ "conditioning_contract": "fixed-phase-ramp-8-roundtrips-v1",
+ "contract": "layout-and-dispatch-v1",
+ "correctness_scope": "dispatch-metadata-and-transformed-combine",
+ "dtype": "bf16",
+ "eplb_planner": "greedy-rank-major-v1",
+ "eplb_redundant_experts": 32,
+ "eplb_reference_tokens_per_rank": 2048,
+ "mode": "normal",
+ "oracle_contract": "expert-specific-transform-v1",
+ "oracle_tolerances": "rtol=0.05,atol=0.02",
+ "payload_unit": "token-rank",
+ "placement": "packed",
+ "percentile_method": "nearest-rank",
+ "rank_reduction": "cross-rank-max-per-iteration",
+ "resource_mode": "tuned",
+ "routing_generator": "collectivex-routing-counter-v3",
+ "sampling_contract": "fixed-512-v1",
+ "seed": 67
+ }},
+ {"const": {
+ "activation_generator": "collectivex-activation-counter-v3",
+ "activation_profile": "canonical-counter-source-v3",
+ "combine_dtype": "bf16",
+ "combine_quant_mode": "none",
+ "combine_semantics": "gate-weighted",
+ "component_order_contract": "roundtrip-dispatch-gate-weighted-combine-v1",
+ "conditioning_contract": "fixed-phase-ramp-8-roundtrips-v1",
+ "contract": "expert-packed-weighted-combine-v1",
+ "correctness_scope": "expert-assignment-and-weighted-combine",
+ "dtype": "bf16",
+ "eplb_planner": "greedy-rank-major-v1",
+ "eplb_redundant_experts": 32,
+ "eplb_reference_tokens_per_rank": 2048,
+ "mode": "low-latency",
+ "oracle_contract": "expert-assignment-transform-v1",
+ "oracle_tolerances": "rtol=0.05,atol=0.02",
+ "payload_unit": "token-expert",
+ "placement": "packed",
+ "percentile_method": "nearest-rank",
+ "rank_reduction": "cross-rank-max-per-iteration",
+ "resource_mode": "tuned",
+ "routing_generator": "collectivex-routing-counter-v3",
+ "sampling_contract": "fixed-512-v1",
+ "seed": 67
+ }}
+ ]
+ },
+ "sku": {"$ref": "#/$defs/safeId"}
+ }
+ },
+ "case_id": {"$ref": "#/$defs/caseId"}
+ }
+ },
+ "outcome": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["failure_mode","reason","return_code","status"],
+ "properties": {
+ "failure_mode": {"$ref": "#/$defs/safeId"},
+ "reason": {"type": "string","minLength": 1,"maxLength": 240},
+ "return_code": {"type": "integer","minimum": 0},
+ "status": {"enum": ["failed","invalid","unsupported"]}
+ }
+ },
+ "provenance": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["git_run","control_sha256","redaction","source"],
+ "properties": {
+ "git_run": {"oneOf": [{"type": "null"},{"$ref": "#/$defs/gitRun"}]},
+ "control_sha256": {"oneOf": [{"type": "null"},{"$ref": "#/$defs/sha256"}]},
+ "redaction": {"const": "sanitized-v1"},
+ "source": {
+ "enum": [
+ "runtime-emitter",
+ "post-emit-command",
+ "matrix-capability-resolver"
+ ]
+ }
+ }
+ },
+ "record_type": {"const": "terminal-outcome"},
+ "schema_version": {"const": 1}
+ },
+ "allOf": [
+ {
+ "oneOf": [
+ {
+ "properties": {
+ "provenance": {
+ "properties": {"source": {"const": "runtime-emitter"}}
+ },
+ "outcome": {"$ref": "#/$defs/runtimeOutcome"}
+ }
+ },
+ {
+ "properties": {
+ "provenance": {
+ "properties": {"source": {"const": "post-emit-command"}}
+ },
+ "outcome": {"$ref": "#/$defs/postEmitOutcome"}
+ }
+ },
+ {
+ "properties": {
+ "provenance": {
+ "properties": {"source": {"const": "matrix-capability-resolver"}}
+ },
+ "outcome": {"$ref": "#/$defs/capabilityOutcome"}
+ }
+ }
+ ]
+ }
+ ],
+ "$defs": {
+ "runtimeOutcome": {
+ "type": "object",
+ "properties": {"status": {"const": "failed"}},
+ "allOf": [
+ {
+ "oneOf": [
+ {"properties": {"failure_mode": {"const": "setup"}, "reason": {"const": "launcher-setup-failed"}}},
+ {"properties": {"failure_mode": {"const": "repository-stage"}, "reason": {"const": "repository-staging-failed"}}},
+ {"properties": {"failure_mode": {"const": "registry-verification"}, "reason": {"const": "container-registry-verification-failed"}}},
+ {"properties": {"failure_mode": {"const": "scheduler-allocation"}, "reason": {"const": "scheduler-allocation-failed"}}},
+ {"properties": {"failure_mode": {"const": "container-import"}, "reason": {"const": "container-image-preparation-failed"}}},
+ {"properties": {"failure_mode": {"const": "container-hash"}, "reason": {"const": "container-image-identity-failed"}}},
+ {"properties": {"failure_mode": {"const": "container-launch"}, "reason": {"const": "container-runtime-launch-failed"}}},
+ {"properties": {"failure_mode": {"const": "backend-setup"}, "reason": {"const": "backend-setup-failed"}}},
+ {"properties": {"failure_mode": {"const": "artifact-collection"}, "reason": {"const": "artifact-collection-failed"}}},
+ {"properties": {"failure_mode": {"const": "runtime-identity"}, "reason": {"const": "runtime-identity-mismatch"}}},
+ {"properties": {"failure_mode": {"const": "timeout"}, "reason": {"const": "execution-timeout"}}},
+ {"properties": {"failure_mode": {"const": "deadlock"}, "reason": {"const": "execution-deadlock"}}},
+ {"properties": {"failure_mode": {"const": "execution"}, "reason": {"const": "distributed-command-failed"}}}
+ ]
+ }
+ ]
+ },
+ "postEmitOutcome": {
+ "type": "object",
+ "properties": {
+ "status": {"const": "failed"},
+ "failure_mode": {"enum": ["runtime-identity", "timeout", "deadlock", "execution"]},
+ "reason": {"const": "post-emit-distributed-command-failed"}
+ }
+ },
+ "capabilityOutcome": {
+ "type": "object",
+ "properties": {
+ "status": {"const": "unsupported"},
+ "failure_mode": {"const": "capability"},
+ "reason": {
+ "enum": [
+ "backend-platform-unsupported",
+ "backend-token-capacity"
+ ]
+ }
+ }
+ },
+ "sha256": {"type": "string","pattern": "^[0-9a-f]{64}$"},
+ "safeId": {"type": "string","pattern": "^[a-z0-9][a-z0-9_.-]*$","maxLength": 128},
+ "caseId": {"type": "string","pattern": "^cxcase-v1-[0-9a-f]{64}$"},
+ "allocationId": {"type": "string","pattern": "^cxallocation-v1-[0-9a-f]{64}$"},
+ "attemptId": {"type": "string","pattern": "^cxattempt-v1-[0-9a-f]{64}$"},
+ "nullableText": {"oneOf": [{"type": "null"},{"type": "string","minLength": 1}]},
+ "allocationFactors": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["artifact","execution_id","job","repo","run_attempt","run_id","runner","source_sha"],
+ "properties": {
+ "artifact": {"$ref": "#/$defs/nullableText"},
+ "execution_id": {"$ref": "#/$defs/nullableText"},
+ "job": {"$ref": "#/$defs/nullableText"},
+ "repo": {"$ref": "#/$defs/nullableText"},
+ "run_attempt": {"$ref": "#/$defs/nullableText"},
+ "run_id": {"$ref": "#/$defs/nullableText"},
+ "runner": {"$ref": "#/$defs/nullableText"},
+ "source_sha": {"$ref": "#/$defs/nullableText"}
+ }
+ },
+ "gitRun": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["artifact","job","ref","repo","run_attempt","run_id","source_sha"],
+ "properties": {
+ "artifact": {"type": "string","minLength": 1},
+ "job": {"type": "string","minLength": 1},
+ "ref": {"type": "string","minLength": 1},
+ "repo": {"type": "string","pattern": "^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$"},
+ "run_attempt": {"type": "string","pattern": "^[1-9][0-9]*$"},
+ "run_id": {"type": "string","pattern": "^[1-9][0-9]*$"},
+ "source_sha": {"type": "string","pattern": "^[0-9a-f]{40}$"}
+ }
+ },
+ "case": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": [
+ "backend",
+ "canonical",
+ "eplb",
+ "ep",
+ "experts",
+ "gpus_per_node",
+ "hidden",
+ "ladder",
+ "mode",
+ "nodes",
+ "phase",
+ "required_publication",
+ "routing",
+ "samples_per_point",
+ "scale_out_transport",
+ "scale_up_domain",
+ "scale_up_transport",
+ "scope",
+ "suite",
+ "timing",
+ "topk",
+ "topology_class",
+ "transport",
+ "warmup_semantics",
+ "workload"
+ ],
+ "properties": {
+ "backend": {"$ref": "#/$defs/safeId"},
+ "canonical": {"type": "boolean"},
+ "eplb": {"type": "boolean"},
+ "ep": {"type": "integer","minimum": 1},
+ "experts": {"type": "integer","minimum": 1},
+ "gpus_per_node": {"type": "integer","minimum": 1},
+ "hidden": {"type": "integer","minimum": 1},
+ "ladder": {"type": "string","pattern": "^[1-9][0-9]*( [1-9][0-9]*)*$"},
+ "mode": {"enum": ["normal","low-latency"]},
+ "nodes": {"type": "integer","minimum": 1},
+ "phase": {"enum": ["decode","prefill"]},
+ "required_publication": {"enum": ["official","comparable-experimental","diagnostic"]},
+ "routing": {"enum": ["uniform","zipf"]},
+ "samples_per_point": {"const": 512},
+ "scale_out_transport": {"oneOf": [{"type": "null"},{"$ref": "#/$defs/safeId"}]},
+ "scale_up_domain": {"type": "integer","minimum": 1},
+ "scale_up_transport": {"$ref": "#/$defs/safeId"},
+ "scope": {"enum": ["scale-up","scale-out"]},
+ "suite": {"$ref": "#/$defs/safeId"},
+ "timing": {"const": "8:64:32"},
+ "topk": {"type": "integer","minimum": 1},
+ "topology_class": {"$ref": "#/$defs/safeId"},
+ "transport": {"$ref": "#/$defs/safeId"},
+ "warmup_semantics": {"const": "full-roundtrip-before-each-component-trial-point-v1"},
+ "workload": {"$ref": "#/$defs/safeId"}
+ }
+ }
+ }
+}
diff --git a/experimental/CollectiveX/source_archive.py b/experimental/CollectiveX/source_archive.py
new file mode 100644
index 0000000000..c027490a65
--- /dev/null
+++ b/experimental/CollectiveX/source_archive.py
@@ -0,0 +1,349 @@
+#!/usr/bin/env python3
+"""Validate and extract one pinned backend from a shared source tar."""
+from __future__ import annotations
+
+import argparse
+import os
+from pathlib import Path, PurePosixPath
+import stat
+import tarfile
+from typing import Optional, Sequence
+
+
+PathParts = tuple[str, ...]
+_DIRECTORY_FLAGS = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC
+_FILE_FLAGS = os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC
+MAX_ARCHIVE_MEMBERS = 20_000
+MAX_MEMBER_BYTES = 512 * 1024 * 1024
+MAX_EXPANDED_BYTES = 2 * 1024 * 1024 * 1024
+MAX_ARCHIVE_BYTES = 4 * 1024 * 1024 * 1024
+MAX_ARCHIVE_HEADERS = 40_000
+MAX_EXTENSION_BYTES = 64 * 1024 * 1024
+MAX_EXTENSION_MEMBER_BYTES = 1024 * 1024
+MAX_EXTENSION_CHAIN = 8
+_TAR_BLOCK = 512
+_EXTENSION_TYPES = {b"L", b"K", b"x", b"g", b"X"}
+
+
+class SourceArchiveError(ValueError):
+ """The backend source archive cannot be extracted safely."""
+
+
+def _tar_size(field: bytes) -> int:
+ if field[0] in (0o200, 0o377):
+ value = int.from_bytes(field[1:], "big")
+ if field[0] == 0o377:
+ value -= 256 ** (len(field) - 1)
+ return value
+ try:
+ text = field.split(b"\0", 1)[0].decode("ascii").strip()
+ return int(text or "0", 8)
+ except (UnicodeDecodeError, ValueError) as exc:
+ raise SourceArchiveError("archive contains an invalid size field") from exc
+
+
+def _preflight_archive(descriptor: int, archive_size: int) -> None:
+ if archive_size <= 0 or archive_size > MAX_ARCHIVE_BYTES:
+ raise SourceArchiveError("backend source archive exceeds the raw size limit")
+ offset = headers = extension_bytes = extension_chain = 0
+ while offset < archive_size:
+ header = os.pread(descriptor, _TAR_BLOCK, offset)
+ if len(header) != _TAR_BLOCK:
+ raise SourceArchiveError("archive header is truncated")
+ if not any(header):
+ return
+ headers += 1
+ if headers > MAX_ARCHIVE_HEADERS:
+ raise SourceArchiveError("archive has too many physical headers")
+ size = _tar_size(header[124:136])
+ if size < 0:
+ raise SourceArchiveError("archive contains a negative payload size")
+ type_flag = header[156:157]
+ if type_flag in _EXTENSION_TYPES:
+ extension_chain += 1
+ extension_bytes += size
+ if (
+ extension_chain > MAX_EXTENSION_CHAIN
+ or size > MAX_EXTENSION_MEMBER_BYTES
+ or extension_bytes > MAX_EXTENSION_BYTES
+ ):
+ raise SourceArchiveError("archive extension metadata exceeds its limit")
+ if type_flag in {b"x", b"g", b"X"}:
+ payload = os.pread(descriptor, size, offset + _TAR_BLOCK)
+ if len(payload) != size:
+ raise SourceArchiveError("archive extension metadata is truncated")
+ if b"GNU.sparse." in payload:
+ raise SourceArchiveError("archive contains sparse extension metadata")
+ else:
+ extension_chain = 0
+ if type_flag == b"S":
+ raise SourceArchiveError("archive contains a sparse member")
+ blocks = (size + _TAR_BLOCK - 1) // _TAR_BLOCK
+ offset += _TAR_BLOCK + blocks * _TAR_BLOCK
+ if offset > archive_size:
+ raise SourceArchiveError("archive payload is truncated")
+
+
+def _member_parts(name: str) -> PathParts:
+ if not name or "\\" in name or "\0" in name:
+ raise SourceArchiveError("archive contains a noncanonical member path")
+ path = PurePosixPath(name)
+ if (
+ path.is_absolute()
+ or path.as_posix() != name
+ or not path.parts
+ or path.parts[0] != ".cx_sources"
+ or any(part in {"", ".", ".."} for part in path.parts)
+ ):
+ raise SourceArchiveError("archive contains a noncanonical member path")
+ return path.parts
+
+
+def _root_parts(root_basename: str) -> PathParts:
+ path = PurePosixPath(root_basename)
+ if (
+ not root_basename
+ or "\\" in root_basename
+ or "\0" in root_basename
+ or path.is_absolute()
+ or path.as_posix() != root_basename
+ or len(path.parts) != 1
+ or path.parts[0] in {"", ".", ".."}
+ ):
+ raise SourceArchiveError("invalid backend source root")
+ return (".cx_sources", root_basename)
+
+
+def _read_members(archive: tarfile.TarFile) -> list[tarfile.TarInfo]:
+ members: list[tarfile.TarInfo] = []
+ for member in archive:
+ if len(members) >= MAX_ARCHIVE_MEMBERS:
+ raise SourceArchiveError("archive has an invalid member count")
+ members.append(member)
+ return members
+
+
+def _validate_members(
+ members: list[tarfile.TarInfo], selected_root: PathParts
+) -> dict[PathParts, tarfile.TarInfo]:
+ if not members or len(members) > MAX_ARCHIVE_MEMBERS:
+ raise SourceArchiveError("archive has an invalid member count")
+ entries: dict[PathParts, tarfile.TarInfo] = {}
+ expanded_bytes = 0
+ for member in members:
+ parts = _member_parts(member.name)
+ if parts in entries:
+ raise SourceArchiveError("archive contains duplicate member paths")
+ if member.sparse is not None:
+ raise SourceArchiveError("archive contains a sparse member")
+ if member.isdir():
+ if member.size != 0:
+ raise SourceArchiveError("archive contains an invalid directory")
+ elif member.isfile():
+ if member.size < 0 or member.size > MAX_MEMBER_BYTES:
+ raise SourceArchiveError("archive member exceeds the size limit")
+ expanded_bytes += member.size
+ if expanded_bytes > MAX_EXPANDED_BYTES:
+ raise SourceArchiveError("archive exceeds the expanded size limit")
+ elif member.issym():
+ if member.size != 0:
+ raise SourceArchiveError("archive contains an invalid symbolic link")
+ else:
+ raise SourceArchiveError("archive contains a non-file member")
+ entries[parts] = member
+
+ source_parent = entries.get((".cx_sources",))
+ selected = entries.get(selected_root)
+ if source_parent is None or not source_parent.isdir():
+ raise SourceArchiveError("archive is missing its source directory")
+ if selected is None or not selected.isdir():
+ raise SourceArchiveError("archive is missing the selected backend source")
+
+ for parts in entries:
+ for depth in range(1, len(parts)):
+ parent = entries.get(parts[:depth])
+ if parent is None or not parent.isdir():
+ raise SourceArchiveError("archive member has an unsafe parent")
+
+ for parts, member in entries.items():
+ if not member.issym():
+ continue
+ target_name = member.linkname
+ target_path = PurePosixPath(target_name)
+ if (
+ not target_name
+ or "\\" in target_name
+ or "\0" in target_name
+ or target_path.is_absolute()
+ or target_path.as_posix() != target_name
+ ):
+ raise SourceArchiveError("archive contains an unsafe symbolic link")
+ target = list(parts[:-1])
+ for component in target_path.parts:
+ if component == "..":
+ if len(target) <= 2:
+ raise SourceArchiveError("symbolic link escapes its backend source")
+ target.pop()
+ else:
+ target.append(component)
+ resolved = tuple(target)
+ if resolved[:2] != parts[:2]:
+ raise SourceArchiveError("symbolic link crosses backend sources")
+ target_member = entries.get(resolved)
+ if target_member is None or not target_member.isfile():
+ raise SourceArchiveError("symbolic link target is not a regular archive file")
+ return entries
+
+
+def _open_directory(root_fd: int, parts: PathParts) -> int:
+ descriptor = os.dup(root_fd)
+ try:
+ for part in parts:
+ child = os.open(part, _DIRECTORY_FLAGS, dir_fd=descriptor)
+ os.close(descriptor)
+ descriptor = child
+ return descriptor
+ except BaseException:
+ os.close(descriptor)
+ raise
+
+
+def _create_directory(root_fd: int, parts: PathParts) -> None:
+ parent_fd = _open_directory(root_fd, parts[:-1])
+ try:
+ os.mkdir(parts[-1], mode=0o700, dir_fd=parent_fd)
+ finally:
+ os.close(parent_fd)
+
+
+def _extract_file(
+ archive: tarfile.TarFile, root_fd: int, parts: PathParts, member: tarfile.TarInfo
+) -> None:
+ parent_fd = _open_directory(root_fd, parts[:-1])
+ descriptor = -1
+ source = None
+ try:
+ mode = 0o700 if member.mode & 0o111 else 0o600
+ descriptor = os.open(parts[-1], _FILE_FLAGS, mode, dir_fd=parent_fd)
+ source = archive.extractfile(member)
+ if source is None:
+ raise SourceArchiveError("archive file has no readable payload")
+ remaining = member.size
+ while remaining:
+ chunk = source.read(min(1024 * 1024, remaining))
+ if not chunk:
+ raise SourceArchiveError("archive file payload is truncated")
+ view = memoryview(chunk)
+ while view:
+ written = os.write(descriptor, view)
+ view = view[written:]
+ remaining -= len(chunk)
+ os.fchmod(descriptor, mode)
+ finally:
+ if source is not None:
+ source.close()
+ if descriptor >= 0:
+ os.close(descriptor)
+ os.close(parent_fd)
+
+
+def _extract_symlink(root_fd: int, parts: PathParts, member: tarfile.TarInfo) -> None:
+ parent_fd = _open_directory(root_fd, parts[:-1])
+ try:
+ os.symlink(member.linkname, parts[-1], dir_fd=parent_fd)
+ finally:
+ os.close(parent_fd)
+
+
+def _extract_selected(
+ archive: tarfile.TarFile,
+ destination_fd: int,
+ entries: dict[PathParts, tarfile.TarInfo],
+ selected_root: PathParts,
+) -> None:
+ try:
+ os.stat(".cx_sources", dir_fd=destination_fd, follow_symlinks=False)
+ except FileNotFoundError:
+ pass
+ else:
+ raise SourceArchiveError("backend source output already exists")
+
+ selected = {
+ parts: member
+ for parts, member in entries.items()
+ if parts[: len(selected_root)] == selected_root
+ }
+ _create_directory(destination_fd, (".cx_sources",))
+ directories = sorted(
+ (parts for parts, member in selected.items() if member.isdir()),
+ key=lambda parts: (len(parts), parts),
+ )
+ for parts in directories:
+ _create_directory(destination_fd, parts)
+ for parts, member in sorted(selected.items()):
+ if member.isfile():
+ _extract_file(archive, destination_fd, parts, member)
+ for parts, member in sorted(selected.items()):
+ if member.issym():
+ _extract_symlink(destination_fd, parts, member)
+
+
+def extract_source_archive(
+ archive_path: Path, destination: Path, root_basename: str
+) -> None:
+ """Validate the complete tar, then safely extract one backend source root."""
+ selected_root = _root_parts(root_basename)
+ archive_fd = os.open(archive_path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC)
+ try:
+ metadata = os.fstat(archive_fd)
+ if (
+ not stat.S_ISREG(metadata.st_mode)
+ or metadata.st_uid != os.getuid()
+ or stat.S_IMODE(metadata.st_mode) & 0o022
+ ):
+ raise SourceArchiveError("backend source archive has unsafe metadata")
+ _preflight_archive(archive_fd, metadata.st_size)
+ with os.fdopen(os.dup(archive_fd), "rb") as stream:
+ try:
+ with tarfile.open(fileobj=stream, mode="r:") as archive:
+ entries = _validate_members(_read_members(archive), selected_root)
+ destination_fd = os.open(destination, _DIRECTORY_FLAGS)
+ try:
+ destination_metadata = os.fstat(destination_fd)
+ if (
+ destination_metadata.st_uid != os.getuid()
+ or stat.S_IMODE(destination_metadata.st_mode) != 0o700
+ ):
+ raise SourceArchiveError("backend source destination is unsafe")
+ previous_umask = os.umask(0o077)
+ try:
+ _extract_selected(
+ archive, destination_fd, entries, selected_root
+ )
+ finally:
+ os.umask(previous_umask)
+ finally:
+ os.close(destination_fd)
+ except RecursionError as exc:
+ raise SourceArchiveError("archive extension metadata is recursive") from exc
+ finally:
+ os.close(archive_fd)
+
+
+def main(argv: Optional[Sequence[str]] = None) -> int:
+ parser = argparse.ArgumentParser(
+ description="Safely install one pinned backend source archive"
+ )
+ parser.add_argument("archive", type=Path)
+ parser.add_argument("destination", type=Path)
+ parser.add_argument("root_basename")
+ args = parser.parse_args(argv)
+ try:
+ extract_source_archive(args.archive, args.destination, args.root_basename)
+ except (OSError, SourceArchiveError, tarfile.TarError) as exc:
+ parser.error(f"backend source archive rejected: {exc}")
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/experimental/CollectiveX/summarize.py b/experimental/CollectiveX/summarize.py
new file mode 100644
index 0000000000..3752db6b9d
--- /dev/null
+++ b/experimental/CollectiveX/summarize.py
@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+"""Render a small native-v1 shard summary and gate on a successful case."""
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+
+import contracts
+
+
+def load_results(directory: str, runner: str | None, timestamp: str | None) -> list[dict]:
+ documents: list[dict] = []
+ for path in sorted(Path(directory).glob("*.json")):
+ if runner and not path.name.startswith(f"{runner}_"):
+ continue
+ if timestamp and timestamp not in path.name:
+ continue
+ try:
+ document = contracts.strict_load(path)
+ if document.get("format") == contracts.RAW_FORMAT:
+ documents.append(contracts.load_raw_attempt(path))
+ elif document.get("format") == contracts.TERMINAL_FORMAT:
+ documents.append(contracts.validate_terminal_document(document))
+ except (contracts.ContractError, OSError):
+ continue
+ return documents
+
+
+def _identity(document: dict) -> tuple[str, str, str, str, bool, str, int]:
+ case = document["case"]
+ if document["format"] == contracts.RAW_FORMAT:
+ routing = case["shape"]["routing"]
+ eplb = case["eplb"]["enabled"]
+ else:
+ routing = case["routing"]
+ eplb = case["eplb"]
+ sku = document["identity"]["case_factors"]["sku"]
+ return (
+ sku, case["suite"], routing, case["phase"], eplb,
+ case["required_publication"], case.get("ep_size", case.get("ep", 0)),
+ )
+
+
+def _headline(document: dict) -> tuple[int | str, float | str, float | str]:
+ if document["format"] != contracts.RAW_FORMAT:
+ return "-", "-", "-"
+ rows = document["measurement"]["rows"]
+ row = next((item for item in rows if item["tokens_per_rank"] == 64), rows[len(rows) // 2])
+ latency = row["components"]["roundtrip"]["percentiles_us"]
+ return row["tokens_per_rank"], latency["p50"], latency["p99"]
+
+
+def render(documents: list[dict], markdown: bool) -> str:
+ documents = sorted(documents, key=_identity)
+ if markdown:
+ lines = [
+ "## CollectiveX EP results", "",
+ "| sku | backend | suite | phase | routing | tier | ep | outcome | T* | p50 us | p99 us |",
+ "|---|---|---|---|---|---|--:|---|--:|--:|--:|",
+ ]
+ for document in documents:
+ sku, suite, routing, phase, eplb, tier, ep = _identity(document)
+ backend = document["case"]["backend"]
+ token, p50, p99 = _headline(document)
+ lines.append(
+ f"| {sku} | `{backend}` | {suite} | {phase} | "
+ f"{routing}{'+eplb' if eplb else ''} | {tier} | {ep} | "
+ f"{document['outcome']['status']} | {token} | {p50} | {p99} |"
+ )
+ if not documents:
+ lines.append("\n> No valid native v1 outcome documents found.")
+ return "\n".join(lines)
+ lines = ["CollectiveX EP results", "======================"]
+ for document in documents:
+ sku, suite, routing, phase, eplb, tier, ep = _identity(document)
+ backend = document["case"]["backend"]
+ token, _, p99 = _headline(document)
+ lines.append(
+ f" {sku:<10} {backend:<16} {suite:<13} {phase:<7} "
+ f"{routing}{'+eplb' if eplb else ''} {tier} ep{ep} "
+ f"{document['outcome']['status']} T={token} roundtrip_p99_us={p99}"
+ )
+ return "\n".join(lines)
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description="Summarize CollectiveX native v1 outcomes")
+ parser.add_argument("--results-dir", default="results")
+ parser.add_argument("--runner")
+ parser.add_argument("--ts")
+ parser.add_argument("--markdown", action="store_true")
+ args = parser.parse_args()
+ documents = load_results(args.results_dir, args.runner, args.ts)
+ print(render(documents, args.markdown))
+ if args.markdown:
+ return 0
+ return 0 if any(
+ document["format"] == contracts.RAW_FORMAT
+ and document["outcome"]["status"] == "success"
+ for document in documents
+ ) else 1
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/experimental/CollectiveX/sweep_matrix.py b/experimental/CollectiveX/sweep_matrix.py
new file mode 100644
index 0000000000..fcf1ffc233
--- /dev/null
+++ b/experimental/CollectiveX/sweep_matrix.py
@@ -0,0 +1,1031 @@
+#!/usr/bin/env python3
+"""Resolve CollectiveX v1 suites and extract validated execution shards.
+
+The promoted v1 profile contains normal and explicit low-latency BF16 contracts.
+Mode changes measurement semantics and therefore participates in case identity;
+resource mode and quantization remain fixed rather than becoming matrix axes.
+"""
+from __future__ import annotations
+
+import argparse
+import datetime as dt
+import hashlib
+import itertools
+import json
+import os
+from pathlib import Path
+import re
+import sys
+from typing import Any
+
+HERE = Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE))
+sys.path.insert(0, str(HERE / "tests"))
+
+try: # Shard extraction on GPU runners is intentionally stdlib-only.
+ import yaml # type: ignore
+except ModuleNotFoundError: # pragma: no cover - exercised by the workflow environment
+ yaml = None
+
+import capability as cap # noqa: E402
+import contracts # noqa: E402
+import ep_harness # noqa: E402
+import identity # noqa: E402
+
+
+EP_TIMING_PROFILE = (
+ f"{ep_harness.TIMED_ITERS_PER_TRIAL}:{ep_harness.TRIALS_PER_POINT}:"
+ f"{ep_harness.WARMUP_ITERS_PER_TRIAL}"
+)
+V1_WORKLOAD = ("deepseek-v3-v1", 7168, 8, 256)
+V1_SUITE_CONTRACTS = {
+ "ep-core-v1": {
+ "mode": "normal",
+ "publication": "official",
+ "coordinates": {
+ ("normal", "decode", "uniform", False),
+ ("normal", "prefill", "uniform", False),
+ },
+ "ladders": {
+ "decode": tuple(ep_harness.DECODE_LADDER),
+ "prefill": (256, 512),
+ },
+ },
+ "ep-routing-v1": {
+ "mode": "normal",
+ "publication": "comparable-experimental",
+ "coordinates": {
+ ("normal", "decode", "zipf", False),
+ ("normal", "decode", "zipf", True),
+ ("normal", "prefill", "zipf", False),
+ ("normal", "prefill", "zipf", True),
+ },
+ "ladders": {"decode": (128,), "prefill": (512,)},
+ },
+ "ep-low-latency-v1": {
+ "mode": "low-latency",
+ "publication": "official",
+ "backends": {"deepep", "uccl"},
+ "coordinates": {("low-latency", "decode", "uniform", False)},
+ "ladders": {"decode": tuple(ep_harness.DECODE_LADDER)},
+ },
+}
+IDENTIFIER = re.compile(r"[a-z0-9][a-z0-9.-]*")
+SUITE_FIELDS = {
+ "backends", "ep_degrees", "eplb", "mode", "phases", "platforms",
+ "required_publication", "routings", "token_points", "token_points_decode",
+ "token_points_prefill", "workloads",
+}
+SUITE_REQUIRED = {
+ "ep_degrees", "mode", "phases", "platforms", "required_publication", "routings",
+ "workloads",
+}
+TOPOLOGY_FIELDS = (
+ "nodes", "gpus_per_node", "scale_up_domain", "scope", "scale_up_transport",
+ "scale_out_transport", "transport", "topology_class",
+)
+
+
+class MatrixError(ValueError):
+ """A matrix or shard-control document violates the execution contract."""
+
+
+if yaml is not None:
+ class _UniqueKeyLoader(yaml.SafeLoader):
+ pass
+
+ def _unique_mapping(loader: Any, node: Any, deep: bool = False) -> dict[Any, Any]:
+ result: dict[Any, Any] = {}
+ for key_node, value_node in node.value:
+ key = loader.construct_object(key_node, deep=deep)
+ if key in result:
+ raise SystemExit(f"duplicate YAML key {key!r} at line {key_node.start_mark.line + 1}")
+ result[key] = loader.construct_object(value_node, deep=deep)
+ return result
+
+ _UniqueKeyLoader.add_constructor(
+ yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, _unique_mapping
+ )
+
+
+def _load(name: str) -> dict[str, Any]:
+ if yaml is None:
+ raise SystemExit("matrix generation requires PyYAML; shard extraction does not")
+ try:
+ with (HERE / "configs" / name).open() as fh:
+ document = yaml.load(fh, Loader=_UniqueKeyLoader)
+ except yaml.YAMLError as exc:
+ raise SystemExit(f"configs/{name} is not valid YAML: {exc}") from exc
+ if not isinstance(document, dict):
+ raise SystemExit(f"configs/{name} must contain a YAML object")
+ return document
+
+
+def _workload_registry(workloads: dict[str, Any]) -> dict[str, dict[str, Any]]:
+ return {
+ name: cfg
+ for section in ("synthetic", "model_derived")
+ for name, cfg in (workloads.get(section) or {}).items()
+ }
+
+
+def _fields(value: Any, path: str, allowed: set[str], required: set[str]) -> dict[str, Any]:
+ if not isinstance(value, dict):
+ raise SystemExit(f"{path} must be an object")
+ if any(not isinstance(key, str) for key in value):
+ raise SystemExit(f"{path} field names must be strings")
+ unknown, missing = set(value) - allowed, required - set(value)
+ if unknown or missing:
+ raise SystemExit(f"{path} fields: unknown={sorted(unknown)}, missing={sorted(missing)}")
+ return value
+
+
+def _list(value: Any, path: str, item_type: type, allowed: set[Any] | None = None) -> list[Any]:
+ if (not isinstance(value, list) or not value
+ or any(type(item) is not item_type for item in value)
+ or len(value) != len(set(value))
+ or (allowed is not None and any(item not in allowed for item in value))):
+ raise SystemExit(f"{path} must be a non-empty unique list of valid {item_type.__name__}s")
+ return value
+
+
+def validate_config_documents(
+ suites_document: dict[str, Any], workloads: dict[str, Any]
+) -> None:
+ """Reject configuration that is ambiguous, unused, or outside the v1 grid."""
+ _fields(
+ suites_document, "configs/suites.yaml",
+ {"schema_version", "suites"}, {"schema_version", "suites"},
+ )
+ _fields(
+ workloads, "configs/workloads.yaml",
+ {"schema_version", "synthetic", "model_derived"}, {"schema_version"},
+ )
+ if type(suites_document["schema_version"]) is not int or suites_document["schema_version"] != 1:
+ raise SystemExit("configs/suites.yaml schema_version must be integer 1")
+ if type(workloads["schema_version"]) is not int or workloads["schema_version"] != 1:
+ raise SystemExit("configs/workloads.yaml schema_version must be integer 1")
+ registry: dict[str, dict[str, Any]] = {}
+ for section, expert_field in (
+ ("synthetic", "experts"),
+ ("model_derived", "routed_experts"),
+ ):
+ entries = workloads.get(section, {})
+ if not isinstance(entries, dict):
+ raise SystemExit(f"workloads.{section} must be an object")
+ for name, value in entries.items():
+ if not isinstance(name, str) or not IDENTIFIER.fullmatch(name) or name in registry:
+ raise SystemExit(f"workloads.{section} has invalid or duplicate name {name!r}")
+ fields = {"hidden", "topk", expert_field, "verified_against"}
+ config = _fields(value, f"workload {name}", fields, fields - {"verified_against"})
+ dimensions = [config[key] for key in ("hidden", "topk", expert_field)]
+ if any(type(item) is not int or item <= 0 for item in dimensions):
+ raise SystemExit(f"workload {name} dimensions must be positive integers")
+ if dimensions[1] > dimensions[2]:
+ raise SystemExit(f"workload {name}.topk exceeds its expert count")
+ source = config.get("verified_against")
+ if source is not None and (not isinstance(source, str) or not source.strip()):
+ raise SystemExit(f"workload {name}.verified_against must be a non-empty string")
+ registry[name] = config
+ if not registry:
+ raise SystemExit("configs/workloads.yaml must define at least one workload")
+
+ suites = suites_document["suites"]
+ if not isinstance(suites, dict) or not suites:
+ raise SystemExit("configs/suites.yaml suites must be a non-empty object")
+ referenced: set[str] = set()
+ for name, value in suites.items():
+ if not isinstance(name, str) or not IDENTIFIER.fullmatch(name):
+ raise SystemExit(f"invalid suite name {name!r}")
+ suite = _fields(value, f"suite {name}", SUITE_FIELDS, SUITE_REQUIRED)
+ contract = V1_SUITE_CONTRACTS.get(name)
+ if contract is None:
+ raise SystemExit(f"suite {name} is outside the frozen v1 catalog")
+ mode = suite["mode"]
+ if mode not in identity.V1_CASE_PROFILES or mode != contract["mode"]:
+ raise SystemExit(f"suite {name}.mode differs from the frozen v1 catalog")
+ suite_backends = _list(
+ suite.get("backends", list(cap.SWEEP_BACKENDS)),
+ f"suite {name}.backends",
+ str,
+ set(cap.SWEEP_BACKENDS),
+ )
+ expected_backends = contract.get("backends")
+ if expected_backends is not None and set(suite_backends) != expected_backends:
+ raise SystemExit(f"suite {name}.backends differs from the frozen v1 catalog")
+ if expected_backends is None and "backends" in suite:
+ raise SystemExit(f"suite {name}.backends must be omitted")
+ suite_workloads = _list(suite["workloads"], f"suite {name}.workloads", str)
+ unknown = sorted(set(suite_workloads) - set(registry))
+ if unknown:
+ raise SystemExit(f"suite {name}: unknown workloads {unknown}")
+ referenced.update(suite_workloads)
+ platforms = _list(
+ suite["platforms"], f"suite {name}.platforms", str, set(cap.PLATFORMS)
+ )
+ phases = _list(suite["phases"], f"suite {name}.phases", str, {"decode", "prefill"})
+ routings = _list(suite["routings"], f"suite {name}.routings", str, {"uniform", "zipf"})
+ eplb = _list(suite.get("eplb", [False]), f"suite {name}.eplb", bool)
+ if True in eplb and routings != ["zipf"]:
+ raise SystemExit(f"suite {name}: EPLB is only valid for Zipf routing")
+ if suite["required_publication"] not in {"official", "comparable-experimental"}:
+ raise SystemExit(f"suite {name}.required_publication is invalid")
+ if suite["required_publication"] == "official":
+ unverified = [item for item in suite_workloads if not registry[item].get("verified_against")]
+ if unverified:
+ raise SystemExit(f"suite {name}: official workloads need verified_against: {unverified}")
+ degrees = _list(suite["ep_degrees"], f"suite {name}.ep_degrees", int)
+ if degrees != [8, 16]:
+ raise SystemExit(f"suite {name}.ep_degrees must be exactly [8, 16]")
+ for platform in platforms:
+ if not set(degrees).issubset(cap.PLATFORMS[platform]["ep_degrees"]):
+ raise SystemExit(f"suite {name}: invalid EP degree for {platform}")
+ for phase in {"decode", "prefill"} - set(phases):
+ if f"token_points_{phase}" in suite:
+ raise SystemExit(f"suite {name}.token_points_{phase} is unreachable")
+ if "token_points" in suite and all(
+ f"token_points_{phase}" in suite for phase in phases
+ ):
+ raise SystemExit(f"suite {name}.token_points is unreachable")
+ for phase in phases:
+ _ladder(suite, phase)
+ coordinates = {
+ (mode, phase, routing, enabled)
+ for phase, routing, enabled in itertools.product(phases, routings, eplb)
+ }
+ if coordinates != contract["coordinates"] or any(
+ tuple(map(int, _ladder(suite, phase).split())) != contract["ladders"][phase]
+ for phase in phases
+ ):
+ raise SystemExit(f"suite {name} coordinates differ from the frozen v1 catalog")
+ unused = sorted(set(registry) - referenced)
+ if unused:
+ raise SystemExit(f"unreferenced workloads: {unused}")
+
+
+def _dims(workloads: dict[str, Any], name: str) -> tuple[int, int, int]:
+ config = _workload_registry(workloads)[name]
+ values = (
+ config.get("hidden"),
+ config.get("topk"),
+ config.get("experts", config.get("routed_experts")),
+ )
+ return values # type: ignore[return-value]
+
+
+def _ladder(suite: dict[str, Any], phase: str) -> str:
+ points = suite.get(f"token_points_{phase}", suite.get("token_points"))
+ if points is None:
+ points = ep_harness.DECODE_LADDER if phase == "decode" else ep_harness.PREFILL_LADDER
+ if (not isinstance(points, list) or not points
+ or any(isinstance(point, bool) or not isinstance(point, int) or point <= 0
+ for point in points)
+ or points != sorted(set(points))):
+ raise SystemExit(f"invalid {phase} token ladder: {points!r}")
+ return " ".join(map(str, points))
+
+
+def _v1_requested_ladder(case: dict[str, Any]) -> str:
+ """Bind extracted controls to the frozen v1 suite and workload catalog."""
+ suite = V1_SUITE_CONTRACTS.get(case.get("suite"))
+ coordinate = (
+ case.get("mode"), case.get("phase"), case.get("routing"), case.get("eplb")
+ )
+ if (
+ suite is None
+ or coordinate not in suite["coordinates"]
+ or case.get("required_publication") != suite["publication"]
+ or (
+ case.get("workload"), case.get("hidden"), case.get("topk"), case.get("experts")
+ ) != V1_WORKLOAD
+ ):
+ raise MatrixError("case differs from the frozen v1 suite/workload catalog")
+ return " ".join(map(str, suite["ladders"][case["phase"]]))
+
+
+def _expected_disposition(
+ sku: str, case: dict[str, Any]
+) -> tuple[str, str | None, str | None]:
+ requested_ladder = _v1_requested_ladder(case)
+ ok, detail = cap.resolve(
+ sku, case["backend"], ep=case["ep"], nodes=case["nodes"],
+ routing=case["routing"], eplb=case["eplb"], mode=case["mode"],
+ )
+ if ok:
+ if case["ladder"] != requested_ladder:
+ raise MatrixError("case ladder differs from the frozen v1 suite catalog")
+ return "runnable", None, None
+ if case["ladder"] != requested_ladder:
+ raise MatrixError("unsupported case ladder differs from the frozen v1 suite catalog")
+ return "unsupported", "backend-platform-unsupported", detail
+
+
+def _case_id(sku: str, case: dict[str, Any]) -> str:
+ return identity.case_id(
+ sku=sku, profile=identity.profile_for_case(case), case=case
+ )
+
+
+def _semantic_points(sku: str, case: dict[str, Any]) -> list[str]:
+ execution = {
+ key: value for key, value in case.items()
+ if key not in {"canonical", "case_id", "ladder", "required_publication", "suite", "workload"}
+ }
+ return [
+ json.dumps(
+ {"sku": sku, "tokens_per_rank": int(point), **execution},
+ sort_keys=True,
+ separators=(",", ":"),
+ )
+ for point in case["ladder"].split()
+ ]
+
+
+def _select_backends(backend: str, backends: str) -> list[str]:
+ available = list(cap.SWEEP_BACKENDS)
+ if backend and backends:
+ raise SystemExit("--backend and --backends are mutually exclusive")
+ if backends:
+ names = available if backends == "all" else [
+ value.strip() for value in backends.split(",") if value.strip()
+ ]
+ else:
+ names = [backend or "deepep"]
+ unknown = sorted(set(names) - set(available))
+ if unknown:
+ raise SystemExit(f"unknown backend values {unknown}; have {available}")
+ if len(names) != len(set(names)):
+ raise SystemExit("backend selection contains duplicates")
+ return names
+
+
+def resolve_matrix(
+ suites: str = "all",
+ backend: str = "",
+ backends: str = "",
+ only_sku: str = "",
+ min_nodes: int = 0,
+ max_nodes: int = 0,
+ max_cases: int = 128,
+) -> dict[str, Any]:
+ """Resolve suite configuration into allocation-sized workflow shards."""
+ if max_cases <= 0:
+ raise SystemExit("--max-cases must be positive")
+ if min_nodes < 0 or max_nodes < 0 or (min_nodes and max_nodes and min_nodes > max_nodes):
+ raise SystemExit("invalid node bounds")
+ if only_sku and only_sku not in cap.PLATFORMS:
+ raise SystemExit(f"unknown --only-sku {only_sku!r}; have {sorted(cap.PLATFORMS)}")
+
+ workloads = _load("workloads.yaml")
+ suites_document = _load("suites.yaml")
+ validate_config_documents(suites_document, workloads)
+ registry = suites_document["suites"]
+ names = list(registry) if suites == "all" else [
+ value.strip() for value in suites.split(",") if value.strip()
+ ]
+ if not names or len(names) != len(set(names)):
+ raise SystemExit("suite selection must be non-empty and unique")
+ unknown = sorted(set(names) - set(registry))
+ if unknown:
+ raise SystemExit(f"unknown suites {unknown}; have {sorted(registry)}")
+ targets = _select_backends(backend, backends)
+
+ shards: dict[tuple[str, str, int], list[dict[str, Any]]] = {}
+ requested_cases: list[dict[str, Any]] = []
+ scheduled: set[str] = set()
+ for suite_name in names:
+ suite = registry[suite_name]
+ mode = suite["mode"]
+ phases = suite["phases"]
+ routings = suite["routings"]
+ eplb_values = suite.get("eplb", [False])
+ suite_backends = set(suite.get("backends", cap.SWEEP_BACKENDS))
+ suite_targets = [target for target in targets if target in suite_backends]
+ if not suite_targets:
+ continue
+ for platform_name in suite["platforms"]:
+ if only_sku and platform_name != only_sku:
+ continue
+ ep_degrees = suite["ep_degrees"]
+ for workload, ep, phase, routing, eplb, target in itertools.product(
+ suite["workloads"], ep_degrees, phases, routings, eplb_values, suite_targets
+ ):
+ topology = cap.topology_for(platform_name, ep)
+ if topology is None:
+ raise SystemExit(
+ f"suite {suite_name}: {platform_name} EP{ep} is not registered"
+ )
+ nodes = int(topology["nodes"])
+ if min_nodes and nodes < min_nodes:
+ continue
+ if max_nodes and nodes > max_nodes:
+ continue
+ ok, capability_detail = cap.resolve(
+ platform_name,
+ target,
+ ep=ep,
+ nodes=nodes,
+ routing=routing,
+ eplb=bool(eplb),
+ mode=mode,
+ )
+ hidden, topk, experts = _dims(workloads, workload)
+
+ def add_case(
+ case_ladder: str,
+ disposition: str,
+ reason: str | None,
+ detail: str | None,
+ ) -> None:
+ case: dict[str, Any] = {
+ "suite": suite_name,
+ "workload": workload,
+ "required_publication": suite["required_publication"],
+ "backend": target,
+ "routing": routing,
+ "phase": phase,
+ "ep": ep,
+ "eplb": eplb,
+ "hidden": hidden,
+ "topk": topk,
+ "experts": experts,
+ "samples_per_point": ep_harness.TIMED_SAMPLES_PER_POINT,
+ "warmup_semantics": ep_harness.WARMUP_SEMANTICS,
+ "ladder": case_ladder,
+ "mode": mode,
+ "timing": EP_TIMING_PROFILE,
+ "canonical": True,
+ **{field: topology[field] for field in TOPOLOGY_FIELDS},
+ }
+ for signature in _semantic_points(platform_name, case):
+ if signature in scheduled:
+ raise SystemExit(
+ f"suite {suite_name}: duplicate semantic point for {platform_name}"
+ )
+ scheduled.add(signature)
+ case["case_id"] = _case_id(platform_name, case)
+ requested_cases.append(
+ {
+ "sku": platform_name,
+ "case": case,
+ "disposition": disposition,
+ "reason": reason,
+ "detail": detail,
+ }
+ )
+ if disposition == "runnable":
+ shards.setdefault((platform_name, target, nodes), []).append(case)
+
+ requested_ladder = _ladder(suite, phase)
+ if not ok:
+ add_case(
+ requested_ladder,
+ "unsupported",
+ "backend-platform-unsupported",
+ capability_detail,
+ )
+ continue
+ add_case(requested_ladder, "runnable", None, None)
+
+ shards_by_sku: dict[str, list[dict[str, Any]]] = {}
+ for (sku, target, nodes), cases in sorted(shards.items()):
+ chunk_size = max_cases
+ for offset in range(0, len(cases), chunk_size):
+ chunk = cases[offset:offset + chunk_size]
+ part = offset // chunk_size
+ shard_id = f"{sku}-{target}-n{nodes}"
+ if len(cases) > chunk_size:
+ shard_id += f"-p{part}"
+ shards_by_sku.setdefault(sku, []).append({
+ "id": shard_id,
+ "sku": sku,
+ "backend": target,
+ "launcher": cap.PLATFORMS[sku]["launcher"],
+ **{field: chunk[0][field] for field in TOPOLOGY_FIELDS},
+ "n": len(chunk),
+ "case_ids": [case["case_id"] for case in chunk],
+ })
+ include = [
+ shards_by_sku[sku][round_index]
+ for round_index in range(max(map(len, shards_by_sku.values()), default=0))
+ for sku in sorted(shards_by_sku)
+ if round_index < len(shards_by_sku[sku])
+ ]
+ return {
+ "format": "collectivex.matrix.v1",
+ "schema_version": 1,
+ "requested_cases": requested_cases,
+ "include": include,
+ }
+
+
+def _strict_json_load(path: Path) -> Any:
+ def reject_constant(value: str) -> None:
+ raise MatrixError(f"non-finite JSON number {value}")
+
+ def reject_duplicates(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
+ result: dict[str, Any] = {}
+ for key, value in pairs:
+ if key in result:
+ raise MatrixError(f"duplicate JSON key {key!r}")
+ result[key] = value
+ return result
+
+ if not path.is_file():
+ raise MatrixError(f"matrix does not exist: {path}")
+ if path.stat().st_size == 0:
+ raise MatrixError(f"matrix is empty: {path}")
+ try:
+ with path.open() as fh:
+ return json.load(
+ fh, parse_constant=reject_constant, object_pairs_hook=reject_duplicates
+ )
+ except (OSError, json.JSONDecodeError) as exc:
+ raise MatrixError(f"matrix is not valid JSON: {exc}") from exc
+
+
+def _positive_int(value: Any, field: str) -> int:
+ if type(value) is not int:
+ raise MatrixError(f"{field} must be a positive integer")
+ if value <= 0:
+ raise MatrixError(f"{field} must be a positive integer")
+ return value
+
+
+def validate_shard_control(
+ shard: dict[str, Any],
+ *,
+ sku: str,
+ backend: str,
+ nodes: int,
+ require_runnable: bool = True,
+) -> None:
+ """Validate one shard against the workflow cell that requested it."""
+ if not isinstance(shard, dict):
+ raise MatrixError("shard must be a JSON object")
+ if sku not in cap.PLATFORMS or backend not in cap.SWEEP_BACKENDS:
+ raise MatrixError("shard platform/backend is not registered")
+ top_fields = {"schema_version", "id", "sku", "backend", "nodes", "n", "cases"}
+ if (
+ set(shard) != top_fields
+ or type(shard.get("schema_version")) is not int
+ or shard["schema_version"] != 1
+ ):
+ raise MatrixError("shard fields or schema version differ from v1 contract")
+ if not isinstance(shard.get("id"), str) or not IDENTIFIER.fullmatch(shard["id"]):
+ raise MatrixError("shard has invalid id")
+ for field, expected in (("sku", sku), ("backend", backend)):
+ if shard.get(field) != expected:
+ raise MatrixError(
+ f"shard {field} mismatch: expected {expected!r}, got {shard.get(field)!r}"
+ )
+ if _positive_int(shard.get("nodes"), "shard.nodes") != nodes:
+ raise MatrixError(
+ f"shard nodes mismatch: expected {nodes}, got {shard.get('nodes')!r}"
+ )
+ cases = shard.get("cases")
+ if not isinstance(cases, list) or not cases:
+ raise MatrixError("shard must contain at least one case")
+ if _positive_int(shard.get("n"), "shard.n") != len(cases):
+ raise MatrixError("shard.n does not match the number of cases")
+ seen: set[str] = set()
+ required = {
+ "case_id", "suite", "workload", "required_publication", "backend", "routing",
+ "mode", "phase", "ep", "eplb", "hidden", "topk", "experts",
+ "samples_per_point",
+ "warmup_semantics", "ladder", "timing", "canonical",
+ } | set(TOPOLOGY_FIELDS)
+ for index, case in enumerate(cases):
+ if not isinstance(case, dict):
+ raise MatrixError(f"case {index} must be a JSON object")
+ fields = set(case)
+ if fields != required:
+ raise MatrixError(
+ f"case {index} fields differ from v1 contract: "
+ f"missing={sorted(required - fields)}, extra={sorted(fields - required)}"
+ )
+ case_id = case["case_id"]
+ if not identity.is_typed_id(case_id, "case"):
+ raise MatrixError(f"case {index} has invalid case_id")
+ if case_id in seen:
+ raise MatrixError(f"duplicate case_id {case_id}")
+ seen.add(case_id)
+ for field in (
+ "suite", "workload", "required_publication", "backend", "mode", "routing",
+ "phase", "warmup_semantics", "ladder", "timing",
+ ):
+ if not isinstance(case[field], str) or not case[field]:
+ raise MatrixError(f"case {index}.{field} must be a non-empty string")
+ for field in ("suite", "workload", "required_publication", "backend", "routing", "phase"):
+ if not IDENTIFIER.fullmatch(case[field]):
+ raise MatrixError(f"case {index}.{field} is not a safe identifier")
+ if case["required_publication"] not in {"official", "comparable-experimental"}:
+ raise MatrixError(f"case {index} has invalid publication requirement")
+ case_identity = {key: value for key, value in case.items() if key != "case_id"}
+ if case_id != _case_id(sku, case_identity):
+ raise MatrixError(f"case {index} case_id does not match its contents")
+ if case["backend"] != backend:
+ raise MatrixError(f"case {index} backend does not match shard")
+ if case["mode"] not in identity.V1_CASE_PROFILES:
+ raise MatrixError(f"case {index} mode is invalid")
+ if _positive_int(case["nodes"], f"case {index}.nodes") != nodes:
+ raise MatrixError(f"case {index} nodes does not match shard")
+ ep = _positive_int(case["ep"], f"case {index}.ep")
+ gpus_per_node = _positive_int(
+ case["gpus_per_node"], f"case {index}.gpus_per_node"
+ )
+ topology = cap.topology_for(sku, ep)
+ if topology is None or any(case[field] != topology[field] for field in TOPOLOGY_FIELDS):
+ raise MatrixError(f"case {index} differs from the platform registry")
+ if ep != nodes * gpus_per_node:
+ raise MatrixError(f"case {index} ep does not equal nodes * gpus_per_node")
+ if case["samples_per_point"] != ep_harness.TIMED_SAMPLES_PER_POINT:
+ raise MatrixError(f"case {index} violates fixed-512-v1")
+ if case["timing"] != EP_TIMING_PROFILE:
+ raise MatrixError(f"case {index} has invalid timing profile")
+ if case["warmup_semantics"] != ep_harness.WARMUP_SEMANTICS:
+ raise MatrixError(f"case {index} has invalid warmup semantics")
+ if case["phase"] not in {"decode", "prefill"}:
+ raise MatrixError(f"case {index} has invalid phase")
+ if case["routing"] not in {"uniform", "zipf"}:
+ raise MatrixError(f"case {index} has invalid routing")
+ if not isinstance(case["eplb"], bool) or (case["eplb"] and case["routing"] != "zipf"):
+ raise MatrixError(f"case {index} has invalid EPLB setting")
+ if not isinstance(case["canonical"], bool) or not case["canonical"]:
+ raise MatrixError(f"case {index} must use a canonical workload")
+ for field in ("ep", "nodes", "gpus_per_node", "hidden", "topk", "experts",
+ "samples_per_point", "scale_up_domain"):
+ if isinstance(case[field], bool) or not isinstance(case[field], int):
+ raise MatrixError(f"case {index}.{field} must be an integer")
+ _positive_int(case[field], f"case {index}.{field}")
+ scale_up_domain = _positive_int(
+ case["scale_up_domain"], f"case {index}.scale_up_domain"
+ )
+ expected_scope = "scale-up" if ep <= scale_up_domain else "scale-out"
+ if case["scope"] != expected_scope or (
+ expected_scope == "scale-out" and ep % scale_up_domain
+ ):
+ raise MatrixError(f"case {index} has invalid scale-up/scale-out geometry")
+ try:
+ ladder = [int(value) for value in case["ladder"].split()]
+ except (AttributeError, ValueError) as exc:
+ raise MatrixError(f"case {index} has invalid token ladder") from exc
+ if (not ladder or any(value <= 0 for value in ladder)
+ or ladder != sorted(set(ladder))
+ or case["ladder"] != " ".join(map(str, ladder))):
+ raise MatrixError(f"case {index} has invalid token ladder")
+ if require_runnable:
+ disposition, reason, _ = _expected_disposition(sku, case)
+ if disposition != "runnable":
+ raise MatrixError(f"case {index} violates capability registry: {reason}")
+ else:
+ _v1_requested_ladder(case)
+
+
+def validate_matrix_document(document: Any) -> dict[str, Any]:
+ """Validate the complete requested grid and its runnable shard partition."""
+ if not isinstance(document, dict) or set(document) != {
+ "format", "schema_version", "requested_cases", "include"
+ }:
+ raise MatrixError("matrix fields differ from the v1 contract")
+ if (
+ document["format"] != "collectivex.matrix.v1"
+ or type(document["schema_version"]) is not int
+ or document["schema_version"] != 1
+ ):
+ raise MatrixError("matrix format/schema differs from v1")
+ requested = document["requested_cases"]
+ include = document["include"]
+ if not isinstance(requested, list) or not requested:
+ raise MatrixError("matrix.requested_cases must be non-empty")
+ if not isinstance(include, list):
+ raise MatrixError("matrix.include must be an array")
+
+ cases_by_id: dict[str, dict[str, Any]] = {}
+ runnable_ids: set[str] = set()
+ semantic_points: set[str] = set()
+ for index, value in enumerate(requested):
+ path = f"matrix.requested_cases[{index}]"
+ if not isinstance(value, dict) or set(value) != {
+ "sku", "case", "disposition", "reason", "detail"
+ }:
+ raise MatrixError(f"{path} fields differ from the v1 contract")
+ sku = value["sku"]
+ case = value["case"]
+ disposition = value["disposition"]
+ if sku not in cap.PLATFORMS:
+ raise MatrixError(f"{path}.sku is unknown")
+ if disposition not in {"runnable", "unsupported"}:
+ raise MatrixError(f"{path}.disposition is invalid")
+ if disposition == "runnable":
+ if value["reason"] is not None or value["detail"] is not None:
+ raise MatrixError(f"{path} runnable cases cannot have a reason")
+ else:
+ if (
+ not isinstance(value["reason"], str)
+ or not IDENTIFIER.fullmatch(value["reason"])
+ or not isinstance(value["detail"], str)
+ or not value["detail"]
+ ):
+ raise MatrixError(f"{path} unsupported cases need a public reason and detail")
+ if not isinstance(case, dict):
+ raise MatrixError(f"{path}.case must be an object")
+ backend = case.get("backend")
+ nodes = case.get("nodes")
+ if not isinstance(backend, str) or type(nodes) is not int:
+ raise MatrixError(f"{path}.case backend/nodes are invalid")
+ validate_shard_control(
+ {
+ "schema_version": 1,
+ "id": "requested-case",
+ "sku": sku,
+ "backend": backend,
+ "nodes": nodes,
+ "n": 1,
+ "cases": [case],
+ },
+ sku=sku,
+ backend=backend,
+ nodes=nodes,
+ require_runnable=disposition == "runnable",
+ )
+ case_id = case["case_id"]
+ if case_id in cases_by_id:
+ raise MatrixError(f"duplicate requested case_id {case_id}")
+ for signature in _semantic_points(sku, case):
+ if signature in semantic_points:
+ raise MatrixError(f"{path} duplicates a semantic token point")
+ semantic_points.add(signature)
+ cases_by_id[case_id] = value
+ expected = _expected_disposition(sku, case)
+ if (disposition, value["reason"], value["detail"]) != expected:
+ raise MatrixError(f"{path} disposition differs from the frozen v1 catalog")
+ if disposition == "runnable":
+ runnable_ids.add(case_id)
+
+ shard_ids: set[str] = set()
+ assigned: list[str] = []
+ for index, shard in enumerate(include):
+ path = f"matrix.include[{index}]"
+ expected = {
+ "id", "sku", "backend", "launcher", "n", "case_ids",
+ } | set(TOPOLOGY_FIELDS)
+ if not isinstance(shard, dict) or set(shard) != expected:
+ raise MatrixError(f"{path} fields differ from the v1 contract")
+ shard_id = shard["id"]
+ if not isinstance(shard_id, str) or not IDENTIFIER.fullmatch(shard_id):
+ raise MatrixError(f"{path}.id is invalid")
+ if shard_id in shard_ids:
+ raise MatrixError(f"duplicate shard id {shard_id}")
+ shard_ids.add(shard_id)
+ sku = shard["sku"]
+ if sku not in cap.PLATFORMS:
+ raise MatrixError(f"{path}.sku is unknown")
+ platform = cap.PLATFORMS[sku]
+ if shard["launcher"] != platform["launcher"]:
+ raise MatrixError(f"{path}.launcher differs from the platform registry")
+ case_ids = shard["case_ids"]
+ if not isinstance(case_ids, list) or not case_ids or len(case_ids) != len(set(case_ids)):
+ raise MatrixError(f"{path}.case_ids must be a non-empty unique array")
+ if _positive_int(shard["n"], f"{path}.n") != len(case_ids):
+ raise MatrixError(f"{path}.n differs from case_ids")
+ nodes = _positive_int(shard["nodes"], f"{path}.nodes")
+ for case_id in case_ids:
+ wrapper = cases_by_id.get(case_id)
+ if wrapper is None or wrapper["disposition"] != "runnable":
+ raise MatrixError(f"{path} references a missing or unsupported case")
+ case = wrapper["case"]
+ if (
+ wrapper["sku"] != sku
+ or case["backend"] != shard["backend"]
+ or case["nodes"] != nodes
+ or any(shard[field] != case[field] for field in TOPOLOGY_FIELDS)
+ ):
+ raise MatrixError(f"{path} case does not match shard coordinates")
+ assigned.append(case_id)
+ if len(assigned) != len(set(assigned)):
+ raise MatrixError("a runnable case is assigned to more than one shard")
+ if set(assigned) != runnable_ids:
+ raise MatrixError("runnable requested cases and shard assignments differ")
+ return document
+
+
+def extract_shard(
+ matrix_path: str | os.PathLike[str],
+ shard_id: str,
+ output_path: str | os.PathLike[str],
+ *,
+ sku: str,
+ backend: str,
+ nodes: int,
+) -> dict[str, Any]:
+ """Extract one strictly matched shard control file, writing it atomically."""
+ document = validate_matrix_document(_strict_json_load(Path(matrix_path)))
+ include = document["include"]
+ matches = [item for item in include if isinstance(item, dict) and item.get("id") == shard_id]
+ if len(matches) != 1:
+ raise MatrixError(f"expected exactly one shard {shard_id!r}, found {len(matches)}")
+ source = matches[0]
+ requested = {
+ item["case"]["case_id"]: item
+ for item in document["requested_cases"]
+ }
+ cases = [requested[case_id]["case"] for case_id in source["case_ids"]]
+ control = {
+ "schema_version": 1,
+ "id": source.get("id"),
+ "sku": source.get("sku"),
+ "backend": source.get("backend"),
+ "nodes": source.get("nodes"),
+ "n": source.get("n"),
+ "cases": cases,
+ }
+ validate_shard_control(control, sku=sku, backend=backend, nodes=nodes)
+ output = Path(output_path)
+ output.parent.mkdir(parents=True, exist_ok=True)
+ temporary = output.with_name(f".{output.name}.tmp-{os.getpid()}")
+ try:
+ with temporary.open("w") as fh:
+ json.dump(control, fh, sort_keys=True, separators=(",", ":"))
+ fh.write("\n")
+ os.replace(temporary, output)
+ finally:
+ temporary.unlink(missing_ok=True)
+ return control
+
+
+def emit_unsupported(
+ matrix_path: str | os.PathLike[str], output_dir: str | os.PathLike[str]
+) -> list[Path]:
+ """Materialize one strict terminal outcome for each unsupported requested case."""
+ source = Path(matrix_path)
+ document = validate_matrix_document(_strict_json_load(source))
+ control_sha256 = hashlib.sha256(source.read_bytes()).hexdigest()
+ generated_at = dt.datetime.now(dt.timezone.utc).isoformat()
+ git_run = {
+ "run_id": os.environ.get("GITHUB_RUN_ID"),
+ "run_attempt": os.environ.get("GITHUB_RUN_ATTEMPT"),
+ "ref": os.environ.get("GITHUB_REF_NAME") or os.environ.get("GITHUB_REF"),
+ "source_sha": os.environ.get("COLLECTIVEX_SOURCE_SHA") or os.environ.get("GITHUB_SHA"),
+ "repo": os.environ.get("GITHUB_REPOSITORY"),
+ "job": os.environ.get("GITHUB_JOB"),
+ "artifact": os.environ.get("COLLECTIVEX_ARTIFACT_NAME"),
+ }
+ allocation_factors = {
+ "artifact": git_run["artifact"],
+ "execution_id": os.environ.get("COLLECTIVEX_EXECUTION_ID"),
+ "job": git_run["job"],
+ "repo": git_run["repo"],
+ "run_attempt": git_run["run_attempt"],
+ "run_id": git_run["run_id"],
+ "runner": "capability-resolver",
+ "source_sha": git_run["source_sha"],
+ }
+ destination = Path(output_dir)
+ destination.mkdir(parents=True, exist_ok=True)
+ written: list[Path] = []
+ for wrapper in document["requested_cases"]:
+ if wrapper["disposition"] != "unsupported":
+ continue
+ scheduled = wrapper["case"]
+ case = {key: value for key, value in scheduled.items() if key != "case_id"}
+ case_factors = {
+ "case": case,
+ "profile": identity.profile_for_case(case),
+ "sku": wrapper["sku"],
+ }
+ case_id = identity.digest("case", case_factors)
+ if case_id != scheduled["case_id"]:
+ raise MatrixError(f"unsupported case identity differs for {scheduled['case_id']}")
+ attempt_ordinal = 1
+ record = contracts.make_terminal_document(
+ allocation_factors=allocation_factors,
+ attempt_ordinal=attempt_ordinal,
+ case=case,
+ case_factors=case_factors,
+ control_sha256=control_sha256,
+ failure_mode="capability",
+ generated_at=generated_at,
+ git_run=git_run,
+ reason=wrapper["reason"],
+ return_code=5,
+ source="matrix-capability-resolver",
+ status="unsupported",
+ expected_case_id=case_id,
+ )
+ path = destination / f"unsupported_{case_id}.json"
+ temporary = path.with_name(f".{path.name}.tmp-{os.getpid()}")
+ try:
+ with temporary.open("x") as handle:
+ json.dump(record, handle, allow_nan=False, sort_keys=True, separators=(",", ":"))
+ handle.write("\n")
+ handle.flush()
+ os.fsync(handle.fileno())
+ os.replace(temporary, path)
+ finally:
+ temporary.unlink(missing_ok=True)
+ written.append(path)
+ return written
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description="CollectiveX v1 matrix resolver")
+ parser.add_argument("--suites", default="all", help="'all' or comma-list of suites")
+ parser.add_argument("--backend", default="", help="select one EP backend")
+ parser.add_argument("--backends", default="", help="'all' or comma-list of EP backends")
+ parser.add_argument("--only-sku", default="")
+ parser.add_argument("--min-nodes", type=int, default=0)
+ parser.add_argument("--max-nodes", type=int, default=0)
+ parser.add_argument("--max-cases", type=int, default=128)
+ parser.add_argument("--extract-from", default="", metavar="MATRIX")
+ parser.add_argument("--validate-control", default="", metavar="SHARD")
+ parser.add_argument("--emit-unsupported-from", default="", metavar="MATRIX")
+ parser.add_argument("--out-dir", default="")
+ parser.add_argument("--shard-id", default="")
+ parser.add_argument("--expect-sku", default="")
+ parser.add_argument("--expect-backend", default="")
+ parser.add_argument("--expect-nodes", type=int, default=0)
+ parser.add_argument("--out", default="")
+ args = parser.parse_args()
+
+ if args.emit_unsupported_from:
+ if not args.out_dir:
+ parser.error("unsupported outcome emission requires --out-dir")
+ try:
+ written = emit_unsupported(args.emit_unsupported_from, args.out_dir)
+ except MatrixError as exc:
+ parser.error(str(exc))
+ print(f"emitted {len(written)} unsupported terminal outcomes", file=sys.stderr)
+ return 0
+
+ if args.validate_control:
+ if not all((args.expect_sku, args.expect_backend, args.expect_nodes)):
+ parser.error(
+ "control validation requires --expect-sku, --expect-backend, and --expect-nodes"
+ )
+ try:
+ control = _strict_json_load(Path(args.validate_control))
+ validate_shard_control(
+ control,
+ sku=args.expect_sku,
+ backend=args.expect_backend,
+ nodes=args.expect_nodes,
+ )
+ except MatrixError as exc:
+ parser.error(str(exc))
+ print(f"validated {control.get('id')}: {control['n']} cases", file=sys.stderr)
+ return 0
+
+ if args.extract_from:
+ if not all((args.shard_id, args.expect_sku, args.expect_backend, args.expect_nodes, args.out)):
+ parser.error(
+ "shard extraction requires --shard-id, --expect-sku, --expect-backend, "
+ "--expect-nodes, and --out"
+ )
+ try:
+ control = extract_shard(
+ args.extract_from,
+ args.shard_id,
+ args.out,
+ sku=args.expect_sku,
+ backend=args.expect_backend,
+ nodes=args.expect_nodes,
+ )
+ except MatrixError as exc:
+ parser.error(str(exc))
+ print(f"extracted {control['id']}: {control['n']} cases", file=sys.stderr)
+ print(json.dumps(control, separators=(",", ":")))
+ return 0
+
+ matrix = resolve_matrix(
+ suites=args.suites,
+ backend=args.backend,
+ backends=args.backends,
+ only_sku=args.only_sku,
+ min_nodes=args.min_nodes,
+ max_nodes=args.max_nodes,
+ max_cases=args.max_cases,
+ )
+ try:
+ validate_matrix_document(matrix)
+ except MatrixError as exc:
+ parser.error(str(exc))
+ if args.out:
+ with open(args.out, "w") as fh:
+ json.dump(matrix, fh, sort_keys=True, separators=(",", ":"))
+ fh.write("\n")
+ runnable = sum(
+ item["disposition"] == "runnable" for item in matrix["requested_cases"]
+ )
+ unsupported = len(matrix["requested_cases"]) - runnable
+ print(
+ f"resolved {len(matrix['include'])} shard-cells, "
+ f"{runnable} runnable and {unsupported} unsupported cases",
+ file=sys.stderr,
+ )
+ print(json.dumps(matrix))
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/experimental/CollectiveX/tests/ep_deepep.py b/experimental/CollectiveX/tests/ep_deepep.py
new file mode 100644
index 0000000000..26cc889b5d
--- /dev/null
+++ b/experimental/CollectiveX/tests/ep_deepep.py
@@ -0,0 +1,294 @@
+#!/usr/bin/env python3
+"""CollectiveX DeepEP adapter for the v1 BF16 normal-mode workload."""
+from __future__ import annotations
+
+import inspect
+import os
+import sys
+import types
+
+import torch
+import torch.distributed as dist
+import contracts
+
+try:
+ import deep_ep
+ from deep_ep import Buffer # type: ignore
+except Exception as exc: # pragma: no cover - requires the benchmark image
+ print(f"ERROR: deep_ep import failed: {exc!r}", file=sys.stderr)
+ raise
+
+
+def _deepep_version() -> str:
+ try:
+ import importlib.metadata as metadata
+
+ return metadata.version("deep_ep")
+ except Exception:
+ return getattr(deep_ep, "__version__", "unknown")
+
+
+def _mnnvl_buffer_configuration() -> tuple[dict[str, bool], str]:
+ """Resolve the explicit DeepEP MNNVL API contract."""
+ requested_value = os.environ.get("CX_ALLOW_MNNVL")
+ if requested_value not in {None, "", "0", "1"}:
+ raise RuntimeError("CX_ALLOW_MNNVL must be unset, 0, or 1")
+ requested = requested_value == "1"
+ if not requested:
+ return contracts.resolve_deepep_mnnvl(
+ requested=False, signature_parameters=(),
+ deepep_commit=os.environ.get("DEEPEP_COMMIT"),
+ )
+ try:
+ parameters = inspect.signature(Buffer.__init__).parameters
+ except (TypeError, ValueError) as exc:
+ raise RuntimeError("cannot inspect DeepEP Buffer MNNVL API") from exc
+ try:
+ return contracts.resolve_deepep_mnnvl(
+ requested=True, signature_parameters=parameters,
+ deepep_commit=os.environ.get("DEEPEP_COMMIT"),
+ )
+ except contracts.ContractError as exc:
+ raise RuntimeError(str(exc)) from exc
+
+
+def _normal_buffer_sizes(hidden: int, world_size: int) -> tuple[int, int]:
+ """Apply DeepEP's dispatch/combine buffer sizing contract for this EP world."""
+ hidden_bytes = hidden * torch.tensor([], dtype=torch.bfloat16).element_size()
+ configs = (Buffer.get_dispatch_config(world_size), Buffer.get_combine_config(world_size))
+ num_nvl_bytes = max(
+ int(config.get_nvl_buffer_size_hint(hidden_bytes, world_size)) for config in configs
+ )
+ num_rdma_bytes = max(
+ int(config.get_rdma_buffer_size_hint(hidden_bytes, world_size)) for config in configs
+ )
+ if num_nvl_bytes <= 0 or num_rdma_bytes < 0:
+ raise RuntimeError("DeepEP returned invalid normal-mode buffer size hints")
+ return num_nvl_bytes, num_rdma_bytes
+
+
+class DeepEPBackend:
+ name = "deepep"
+ combine_needs_redispatch = False
+ # DeepEP reduces activations and top-k weights independently. The activation
+ # tensor must therefore carry the complete local weighted expert sum.
+ combine_weight_semantics = "unweighted-rank-sum"
+ oracle_layout = "token-rank"
+ payload_unit = "token-rank"
+
+ def __init__(self, args, rank, world_size, local_rank, device):
+ self.args = args
+ self.rank = rank
+ self.world_size = world_size
+ self.device = device
+ self.mode = getattr(args, "mode", "normal")
+ if self.mode not in {"normal", "low-latency"}:
+ raise ValueError(f"unsupported DeepEP mode {self.mode!r}")
+
+ self.group = dist.group.WORLD
+ device_sms = torch.cuda.get_device_properties(device).multi_processor_count
+ mnnvl_kwargs, mnnvl_comm = _mnnvl_buffer_configuration()
+ if self.mode == "low-latency":
+ if args.phase != "decode":
+ raise ValueError("DeepEP low-latency mode only supports the decode ladder")
+ if args.experts % world_size:
+ raise ValueError("DeepEP low-latency experts must divide the EP group")
+ self.combine_needs_redispatch = True
+ self.combine_weight_semantics = "gate-weighted-sum"
+ self.oracle_layout = "expert-packed"
+ self.payload_unit = "token-expert"
+ self.max_tokens_per_rank = 128
+ num_qps_per_rank = args.experts // world_size
+ num_rdma_bytes = Buffer.get_low_latency_rdma_size_hint(
+ self.max_tokens_per_rank, args.hidden, world_size, args.experts
+ )
+ self.buffer = Buffer(
+ self.group,
+ num_nvl_bytes=0,
+ num_rdma_bytes=num_rdma_bytes,
+ low_latency_mode=True,
+ num_qps_per_rank=num_qps_per_rank,
+ allow_nvlink_for_low_latency_mode=True,
+ explicitly_destroy=True,
+ **mnnvl_kwargs,
+ )
+ self.buffer.clean_low_latency_buffer(
+ self.max_tokens_per_rank, args.hidden, args.experts
+ )
+ resource_provenance = {
+ "requested_num_sms": None,
+ "num_sms": None,
+ "sm_fraction": None,
+ "tuned_source": "deepep-low-latency-fixed-kernel",
+ "num_max_tokens_per_rank": self.max_tokens_per_rank,
+ "num_nvl_bytes": 0,
+ "num_rdma_bytes": num_rdma_bytes,
+ "num_qps_per_rank": num_qps_per_rank,
+ }
+ else:
+ num_nvl_bytes, num_rdma_bytes = _normal_buffer_sizes(args.hidden, world_size)
+ if world_size > args.scale_up_domain and num_rdma_bytes == 0:
+ raise RuntimeError("DeepEP scale-out configuration returned no RDMA buffer")
+ self.buffer = Buffer(
+ self.group, num_nvl_bytes, num_rdma_bytes, **mnnvl_kwargs
+ )
+ num_sms = int(getattr(Buffer, "num_sms", args.num_sms))
+ try:
+ Buffer.set_num_sms(num_sms)
+ except Exception as exc: # pragma: no cover - version dependent
+ raise RuntimeError(
+ f"DeepEP did not apply requested num_sms={num_sms}: {exc!r}"
+ ) from exc
+ applied_num_sms = int(getattr(Buffer, "num_sms", num_sms))
+ if applied_num_sms != num_sms:
+ raise RuntimeError(
+ f"DeepEP num_sms mismatch: requested={num_sms} applied={applied_num_sms}"
+ )
+ resource_provenance = {
+ "requested_num_sms": num_sms,
+ "num_sms": applied_num_sms,
+ "sm_fraction": applied_num_sms / device_sms,
+ "tuned_source": "deepep-default-num_sms",
+ "num_nvl_bytes": num_nvl_bytes,
+ "num_rdma_bytes": num_rdma_bytes,
+ }
+ version = _deepep_version()
+ self.backend_provenance = {
+ "deepep_version": version,
+ "deepep_commit": os.environ.get("DEEPEP_COMMIT") or f"pkg-{version}",
+ "backend_lineage": "deepep-v1",
+ "mode": self.mode,
+ "dispatch_dtype": "bf16",
+ "combine_dtype": "bf16",
+ "resource_mode": "tuned",
+ "device_sms": device_sms,
+ "allow_mnnvl": bool(mnnvl_kwargs),
+ "mnnvl_comm": mnnvl_comm,
+ **resource_provenance,
+ }
+
+ def buffer_cap(self, args):
+ return self.max_tokens_per_rank if self.mode == "low-latency" else None
+
+ def make_problem(self, T, idx, weights, x):
+ return types.SimpleNamespace(
+ T=T,
+ x=x,
+ topk_idx=idx.to(torch.int64),
+ topk_weights=weights.to(torch.float32),
+ )
+
+ def dispatch(self, p):
+ if self.mode == "low-latency":
+ recv_x, recv_counts, handle, _, _ = self.buffer.low_latency_dispatch(
+ p.x,
+ p.topk_idx,
+ self.max_tokens_per_rank,
+ self.args.experts,
+ use_fp8=False,
+ async_finish=False,
+ return_recv_hook=False,
+ )
+ return types.SimpleNamespace(
+ recv_x=recv_x,
+ recv_counts=recv_counts,
+ handle=handle,
+ )
+ (
+ num_tokens_per_rank,
+ num_tokens_per_rdma_rank,
+ num_tokens_per_expert,
+ is_token_in_rank,
+ _,
+ ) = self.buffer.get_dispatch_layout(p.topk_idx, self.args.experts)
+ recv_x, recv_topk_idx, recv_topk_weights, recv_counts, handle, _ = self.buffer.dispatch(
+ p.x,
+ topk_idx=p.topk_idx,
+ topk_weights=p.topk_weights,
+ num_tokens_per_rank=num_tokens_per_rank,
+ num_tokens_per_rdma_rank=num_tokens_per_rdma_rank,
+ is_token_in_rank=is_token_in_rank,
+ num_tokens_per_expert=num_tokens_per_expert,
+ )
+ return types.SimpleNamespace(
+ recv_x=recv_x,
+ recv_topk_idx=recv_topk_idx,
+ recv_topk_weights=recv_topk_weights,
+ recv_counts=recv_counts,
+ handle=handle,
+ )
+
+ def stage(self, p, h):
+ h.combine_input = h.recv_x
+
+ def combine(self, p, h):
+ if self.mode == "low-latency":
+ combined_x, _, _ = self.buffer.low_latency_combine(
+ h.combine_input,
+ p.topk_idx,
+ p.topk_weights,
+ h.handle,
+ async_finish=False,
+ return_recv_hook=False,
+ )
+ return combined_x
+ combined_x, _, _ = self.buffer.combine(h.combine_input, h.handle)
+ return combined_x
+
+ def inspect_dispatch(self, p, h):
+ valid = h.recv_topk_idx >= 0
+ expert_ids = torch.where(
+ valid,
+ h.recv_topk_idx + self.rank * (self.args.experts // self.world_size),
+ h.recv_topk_idx,
+ )
+ return types.SimpleNamespace(
+ payload=h.recv_x,
+ expert_ids=expert_ids,
+ weights=h.recv_topk_weights.masked_fill(~valid, 0),
+ local_expert_counts=torch.tensor(h.recv_counts, device=self.device, dtype=torch.int64),
+ ordering_contract="source-rank-major-stable-v1",
+ )
+
+ def inspect_expert_dispatch(self, p, h):
+ if self.mode != "low-latency":
+ raise RuntimeError("expert-packed inspection requires low-latency mode")
+ return types.SimpleNamespace(
+ payload=h.recv_x,
+ local_expert_counts=h.recv_counts,
+ source_info=h.handle[0],
+ layout_range=h.handle[1],
+ )
+
+ def combine_transformed(self, p, h, transformed):
+ if self.mode == "low-latency":
+ packed = torch.zeros_like(h.recv_x)
+ packed[h.oracle_local_expert_slots, h.oracle_packed_positions] = transformed.to(
+ h.recv_x.dtype
+ )
+ combined, _, _ = self.buffer.low_latency_combine(
+ packed,
+ p.topk_idx,
+ p.topk_weights,
+ h.handle,
+ async_finish=False,
+ return_recv_hook=False,
+ )
+ return combined
+ combined, _, _ = self.buffer.combine(transformed.to(h.recv_x.dtype), h.handle)
+ return combined
+
+ def recv_tokens(self, h):
+ if self.mode == "low-latency":
+ return int(h.recv_counts.to(torch.int64).sum().item())
+ return int(h.recv_x.shape[0])
+
+ def finalize(self, rc):
+ try:
+ dist.barrier()
+ if self.mode == "low-latency":
+ self.buffer.destroy()
+ dist.destroy_process_group()
+ except Exception:
+ pass
+ return rc
diff --git a/experimental/CollectiveX/tests/ep_deepep_hybrid.py b/experimental/CollectiveX/tests/ep_deepep_hybrid.py
new file mode 100644
index 0000000000..70cd17b005
--- /dev/null
+++ b/experimental/CollectiveX/tests/ep_deepep_hybrid.py
@@ -0,0 +1,457 @@
+#!/usr/bin/env python3
+"""CollectiveX EP backend adapter — DeepEP `hybrid-ep` branch (NVIDIA TMA-based HybridEPBuffer).
+
+The hybrid-ep branch (https://github.com/deepseek-ai/DeepEP/tree/hybrid-ep) is NVIDIA's TMA +
+warp-pipeline implementation of expert-parallel all-to-all, exposing `deep_ep.HybridEPBuffer`
+(distinct from the mainline `deep_ep.Buffer`). HybridEP is NVIDIA's MoE backend built for NVL72
+rack-scale (Megatron `moe_flex_dispatcher_backend="hybridep"`). This adapter binds the API's
+"ranks per node" field to active ranks per NVLink/MNNVL communication domain, not physical host
+GPUs: x86 EP16 is two 8-rank domains, while GB EP8/EP16 is one 8/16-rank MNNVL domain across hosts.
+The container build is done by runtime/run_in_container.sh `cx_build_deepep_hybrid` (CUDA-13 CCCL
+include path, without the V2 NVSHMEM overlay).
+
+API (pinned on B300, branch e0a5b1d):
+ HybridEPBuffer(group, hidden_dim, max_num_of_tokens_per_rank, num_local_experts, use_fp8=False, ...)
+ .dispatch(hidden, topk_idx=, topk_weights=, num_of_experts=) -> (recv_hidden, recv_x2, None, handle)
+ .combine(hidden, handle=) -> [T, hidden]
+
+CORRECTNESS: identity expert (no expert compute), combine WITHOUT probs -> each source token is
+reconstructed as x * (distinct ranks among its top_k experts) — verified: an 8-rank uniform top_k=8
+round trip gives relerr(combined, x) = 4.28, matching E[distinct ranks] ~ 5.26 exactly. So this uses
+the same per-rank-sum combine contract (no gate re-weight). BF16 tolerance is 5e-2.
+
+STATUS: bf16 / normal / layout-and-dispatch-v1. The v1 scope covers one MNNVL domain or x86
+scale-out between two eight-GPU NVLink domains; fp8 and cross-NVL72 scale-out remain out of scope.
+"""
+from __future__ import annotations
+
+import hashlib
+import importlib
+import json
+import os
+from pathlib import Path
+import re
+import shutil
+import sys
+import tempfile
+import types
+
+import torch
+import torch.distributed as dist
+import contracts
+
+try:
+ import deep_ep
+ HybridEPBuffer = deep_ep.HybridEPBuffer
+except Exception as exc: # pragma: no cover - needs the hybrid-ep build
+ print("ERROR: deep_ep.HybridEPBuffer import failed — the hybrid-ep branch must be built at job "
+ "setup (cx_build_deepep_hybrid). "
+ f"{exc!r}", file=sys.stderr)
+ raise
+
+
+def _deepep_hybrid_version() -> str:
+ return os.environ.get("DEEPEP_COMMIT", getattr(deep_ep, "__version__", "hybrid-ep"))
+
+
+def _hybrid_build_evidence() -> list[dict[str, str]]:
+ records = []
+ for module_name, role in (
+ ("deep_ep_cpp", "deepep-extension"),
+ ("hybrid_ep_cpp", "deepep-hybrid-extension"),
+ ):
+ module = importlib.import_module(module_name)
+ path = getattr(module, "__file__", None)
+ if not path:
+ raise RuntimeError(f"{module_name} has no loaded extension path")
+ records.append(contracts.content_manifest_evidence(
+ role=role,
+ name=module_name,
+ files=[(os.path.basename(path), path)],
+ ))
+ return sorted(records, key=lambda item: (item["role"], item["name"]))
+
+
+HYBRID_CONFIG_FIELDS = (
+ "hidden_dim", "max_num_of_tokens_per_rank", "num_of_experts_per_rank",
+ "num_of_ranks_per_node", "num_of_nodes", "pad_multiple",
+ "num_of_tokens_per_chunk_preprocessing_api",
+ "num_of_threads_per_block_preprocessing_api", "num_of_blocks_preprocessing_api",
+ "num_of_blocks_permute", "num_of_blocks_unpermute", "token_data_type",
+ "num_of_stages_dispatch_api", "num_of_stages_permute_block_dispatch_api",
+ "num_of_in_flight_s2g_dispatch_api",
+ "num_of_in_flight_s2g_permute_block_dispatch_api",
+ "num_of_additional_in_flight_s2g_dispatch_api",
+ "num_of_tokens_per_chunk_dispatch_api", "num_of_blocks_dispatch_api",
+ "forward_dispatch_api", "device_side_sync_dispatch_api",
+ "num_of_stages_g2s_combine_api", "num_of_stages_s2g_combine_api",
+ "num_of_tokens_per_chunk_combine_api", "num_of_tokens_per_group_combine_api",
+ "num_of_blocks_combine_api", "num_of_additional_in_flight_s2g_combine_api",
+ "backward_combine_api", "device_side_sync_combine_api",
+)
+
+
+def _hybrid_realized_config(config) -> dict[str, str | int | bool]:
+ """Project the Python-visible, post-autotune HybridEP config to JSON scalars."""
+ realized = {}
+ for field in HYBRID_CONFIG_FIELDS:
+ try:
+ value = getattr(config, field)
+ except AttributeError as exc:
+ raise RuntimeError(f"HybridEP realized config omits {field}") from exc
+ if field == "token_data_type":
+ token_type = getattr(value, "name", None)
+ if token_type not in {"UINT8", "UINT16"}:
+ token_type = {"uint8_t": "UINT8", "uint16_t": "UINT16"}.get(str(value))
+ if token_type is None:
+ raise RuntimeError("HybridEP realized token_data_type is invalid")
+ realized[field] = token_type
+ continue
+ if type(value) is bool:
+ realized[field] = value
+ continue
+ try:
+ realized[field] = int(value)
+ except (TypeError, ValueError) as exc:
+ raise RuntimeError(f"HybridEP realized config {field} is not integral") from exc
+ return realized
+
+
+def _sha256_with_size(path: Path) -> tuple[str, int]:
+ digest = hashlib.sha256()
+ size = 0
+ with path.open("rb") as handle:
+ for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+ digest.update(chunk)
+ size += len(chunk)
+ return digest.hexdigest(), size
+
+
+def _hybrid_jit_evidence(root: Path) -> list[dict[str, str | int]]:
+ """Hash final JIT libraries without exposing rank-specific cache paths."""
+ if not root.is_dir():
+ raise RuntimeError("DeepEP Hybrid produced no JIT cache directory")
+ artifacts = []
+ for path in sorted(root.iterdir(), key=lambda item: item.name):
+ if path.suffix != ".so":
+ continue
+ if path.is_symlink() or not path.is_file():
+ raise RuntimeError("DeepEP Hybrid JIT artifact is not a regular file")
+ kernel_key = path.stem
+ if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.+-]{0,511}", kernel_key):
+ raise RuntimeError("DeepEP Hybrid JIT kernel key is invalid")
+ digest, size = _sha256_with_size(path)
+ if size <= 0:
+ raise RuntimeError("DeepEP Hybrid JIT artifact is empty")
+ artifacts.append({
+ "bytes": size,
+ "kernel_key": kernel_key,
+ "sha256": digest,
+ })
+ if len(artifacts) != 3:
+ raise RuntimeError(
+ f"DeepEP Hybrid expected 3 final JIT libraries, found {len(artifacts)}"
+ )
+ return artifacts
+
+
+def _require_cross_rank_equal(value, label: str) -> None:
+ gathered = [None] * dist.get_world_size()
+ dist.all_gather_object(gathered, value)
+ canonical = {json.dumps(item, sort_keys=True, separators=(",", ":")) for item in gathered}
+ if len(canonical) != 1:
+ raise RuntimeError(f"DeepEP Hybrid {label} differs across ranks")
+
+
+def _hybrid_topology(args, world_size: int) -> dict[str, int | str]:
+ """Translate physical placement into HybridEP communication-domain geometry."""
+ gpus_per_node = int(args.gpus_per_node or world_size)
+ scale_up_domain = int(args.scale_up_domain or gpus_per_node)
+ key = (
+ world_size, gpus_per_node, scale_up_domain, args.scope,
+ args.scale_up_transport, args.scale_out_transport or None, args.transport,
+ )
+ fixed = {
+ (8, 8, 8, "scale-up", "nvlink", None, "nvlink"): (8, 1),
+ (16, 8, 8, "scale-out", "nvlink", "rdma", "nvlink-rdma"): (8, 2),
+ (8, 4, 72, "scale-up", "mnnvl", None, "mnnvl"): (8, 1),
+ (16, 4, 72, "scale-up", "mnnvl", None, "mnnvl"): (16, 1),
+ }
+ if key not in fixed:
+ raise RuntimeError("DeepEP Hybrid topology is outside the fixed v1 matrix")
+ domain_ranks, communication_domains = fixed[key]
+
+ return {
+ "communication_domains": communication_domains,
+ "domain_ranks": domain_ranks,
+ "physical_nodes": world_size // gpus_per_node,
+ "transport": str(args.transport),
+ }
+
+
+class DeepEPHybridBackend:
+ name = "deepep-hybrid"
+ # HybridEPBuffer.combine consumes the recv payload + the dispatch handle (no re-dispatch needed
+ # before a timed combine); the harness times dispatch and combine separately (like ep_deepep).
+ combine_needs_redispatch = False
+ combine_weight_semantics = "unweighted-rank-sum"
+
+ def __init__(self, args, rank, world_size, local_rank, device):
+ self.args = args
+ self.rank = rank
+ self.world_size = world_size
+ self.device = device
+ self.mode = "normal"
+ self.group = dist.group.WORLD
+ self.tolerance = 5e-2
+ self.top_k = int(args.topk)
+ self.num_experts = int(args.experts)
+ self.hidden = int(args.hidden)
+ self.local_experts = max(1, self.num_experts // world_size)
+ topology = _hybrid_topology(args, world_size)
+ self.domain_ranks = int(topology["domain_ranks"])
+ self.communication_domains = int(topology["communication_domains"])
+ build_mode = os.environ.get("DEEPEP_HYBRID_BUILD_MODE", "")
+ if self.communication_domains > 1:
+ if (
+ os.environ.get("HYBRID_EP_MULTINODE") != "1"
+ or build_mode != "multinode-doca"
+ or os.environ.get("USE_NIXL", "0") != "0"
+ ):
+ raise RuntimeError("DeepEP Hybrid scale-out build mode is not realized")
+ elif build_mode != "intradomain":
+ raise RuntimeError("DeepEP Hybrid scale-up requires the intradomain build")
+ if args.scale_up_transport == "mnnvl" and any(
+ os.environ.get(name) != "1"
+ for name in ("NCCL_CUMEM_ENABLE", "NCCL_MNNVL_ENABLE", "MC_FORCE_MNNVL")
+ ):
+ raise RuntimeError("DeepEP Hybrid MNNVL runtime enablement is incomplete")
+ # Token cap (per rank) for the symmetric buffer; the sweep is capped here (buffer_cap).
+ self.max_tokens = 4096
+ dev_sms = torch.cuda.get_device_properties(device).multi_processor_count
+ ver = _deepep_hybrid_version()
+ loaded_libraries = _hybrid_build_evidence()
+ _require_cross_rank_equal(loaded_libraries, "loaded extension identities")
+
+ # HybridEP's compiler uses a process-specific child of HYBRID_EP_CACHE_DIR. Give every
+ # rank a fresh private base so stale kernels cannot enter this attempt's evidence.
+ self._previous_jit_cache_dir = os.environ.get("HYBRID_EP_CACHE_DIR")
+ self._previous_domain_ranks = os.environ.get(
+ "NUM_OF_HYBRID_EP_RANKS_PER_NVLINK_DOMAIN"
+ )
+ self._jit_cache_dir = tempfile.mkdtemp(prefix=f"collectivex-hybrid-r{rank}-")
+ os.environ["HYBRID_EP_CACHE_DIR"] = self._jit_cache_dir
+ os.environ["NUM_OF_HYBRID_EP_RANKS_PER_NVLINK_DOMAIN"] = str(self.domain_ranks)
+ self._jit_root = (
+ Path(self._jit_cache_dir) / ".deepep" / "hybrid_ep" / "jit"
+ / f"proc-{os.getpid()}"
+ )
+ self._realized_config = None
+ self._deferred_semantic_snapshot = None
+ self._deferred_jit_diagnostics = None
+
+ try:
+ self.buffer = HybridEPBuffer(
+ self.group, hidden_dim=self.hidden,
+ max_num_of_tokens_per_rank=self.max_tokens,
+ num_local_experts=self.local_experts, use_fp8=False)
+ realized_geometry = (
+ int(self.buffer.num_of_hybrid_ep_ranks_per_nvlink_domain),
+ int(self.buffer.num_of_nodes),
+ int(self.buffer.local_rank),
+ int(self.buffer.node_rank),
+ )
+ expected_geometry = (
+ self.domain_ranks,
+ self.communication_domains,
+ rank % self.domain_ranks,
+ rank // self.domain_ranks,
+ )
+ buffer_config = self.buffer.configurer.buffer_config
+ if realized_geometry != expected_geometry or (
+ int(buffer_config.num_of_ranks_per_node) != self.domain_ranks
+ or int(buffer_config.num_of_nodes) != self.communication_domains
+ ):
+ raise RuntimeError(
+ "HybridEPBuffer communication-domain geometry differs from the case"
+ )
+ except Exception as exc:
+ shutil.rmtree(self._jit_cache_dir, ignore_errors=True)
+ if self._previous_jit_cache_dir is None:
+ os.environ.pop("HYBRID_EP_CACHE_DIR", None)
+ else:
+ os.environ["HYBRID_EP_CACHE_DIR"] = self._previous_jit_cache_dir
+ if self._previous_domain_ranks is None:
+ os.environ.pop("NUM_OF_HYBRID_EP_RANKS_PER_NVLINK_DOMAIN", None)
+ else:
+ os.environ[
+ "NUM_OF_HYBRID_EP_RANKS_PER_NVLINK_DOMAIN"
+ ] = self._previous_domain_ranks
+ raise RuntimeError(
+ f"HybridEPBuffer construction failed (hidden={self.hidden} max_tokens={self.max_tokens} "
+ f"local_experts={self.local_experts} world={world_size}): {exc!r}") from exc
+ update_template_config = self.buffer.update_template_config
+
+ def tracked_update_template_config(*call_args, **call_kwargs):
+ config = update_template_config(*call_args, **call_kwargs)
+ realized = _hybrid_realized_config(config)
+ if (
+ realized["num_of_ranks_per_node"] != self.domain_ranks
+ or realized["num_of_nodes"] != self.communication_domains
+ ):
+ raise RuntimeError("DeepEP Hybrid realized topology changed within one case")
+ if self._realized_config is not None and realized != self._realized_config:
+ raise RuntimeError("DeepEP Hybrid realized autotune config changed within one case")
+ self._realized_config = realized
+ return config
+
+ self.buffer.update_template_config = tracked_update_template_config
+ self.domain_rank = int(self.buffer.local_rank)
+ if rank == 0:
+ print(
+ "[deepep-hybrid] HybridEPBuffer constructed "
+ f"(domains={self.communication_domains} ranks_per_domain={self.domain_ranks} "
+ f"world={world_size} local_experts={self.local_experts} hidden={self.hidden})",
+ file=sys.stderr,
+ )
+
+ self.backend_provenance = {
+ "deepep_commit": ver, "branch": "hybrid-ep",
+ "deepep_tree": os.environ.get("DEEPEP_TREE"),
+ "backend_lineage": "deepep-hybrid",
+ "loaded_libraries": loaded_libraries,
+ "impl": "deep_ep.HybridEPBuffer (NVIDIA TMA + warp-pipeline)",
+ "mode": "normal", "transport": topology["transport"],
+ "resource_mode": "tuned",
+ "num_sms": None, "device_sms": dev_sms,
+ "tuned_source": "deepep-hybrid-configurer-autotune-v1",
+ "realized_config": None, "jit_kernel_keys": [], "jit_shared_objects": [],
+ "max_num_tokens": self.max_tokens, "top_k": self.top_k,
+ "num_experts": self.num_experts, "local_experts": self.local_experts,
+ "routing_factor": "ranks",
+ }
+
+ def buffer_cap(self, args):
+ return self.max_tokens
+
+ def make_problem(self, T, idx, weights, x):
+ return types.SimpleNamespace(
+ T=int(T), x=x,
+ topk_idx=idx.to(torch.int64),
+ topk_weights=weights.to(torch.float32),
+ )
+
+ def dispatch(self, p):
+ recv, recv_probs, _scales, handle = self.buffer.dispatch(
+ p.x,
+ topk_idx=p.topk_idx,
+ topk_weights=p.topk_weights,
+ num_of_experts=self.num_experts,
+ )
+ return types.SimpleNamespace(
+ recv=recv,
+ recv_payload=recv,
+ recv_probs=recv_probs,
+ handle=handle,
+ combine_input=None,
+ )
+
+ def stage(self, p, h):
+ # Identity expert: the recv hidden IS the "expert output". combine reduces it per source token.
+ h.combine_input = h.recv_payload
+ return None
+
+ def combine(self, p, h):
+ # combine(hidden, handle=) -> [T, H] per-source-token reduction (no gate re-weight: "ranks").
+ comb = self.buffer.combine(h.combine_input, handle=h.handle)
+ return comb[0] if isinstance(comb, (tuple, list)) else comb
+
+ def capture_deferred_provenance(self):
+ torch.cuda.synchronize()
+ dist.barrier()
+ if self._realized_config is None:
+ raise RuntimeError("DeepEP Hybrid autotune config was not materialized")
+ local_artifacts = _hybrid_jit_evidence(self._jit_root)
+ semantic = {
+ "jit_kernel_keys": [item["kernel_key"] for item in local_artifacts],
+ "realized_config": dict(self._realized_config),
+ }
+ # NVCC may embed each rank's timestamped source basename in its ELF, so raw .so hashes are
+ # diagnostics rather than a cross-rank identity. Stable kernel keys encode every codegen
+ # input, including HybridEpConfigInstance fields that the Python binding does not expose.
+ _require_cross_rank_equal(semantic, "realized config/JIT kernel keys")
+ gathered_artifacts = [None] * dist.get_world_size()
+ dist.all_gather_object(gathered_artifacts, local_artifacts)
+ diagnostics = []
+ for artifact_index, kernel_key in enumerate(semantic["jit_kernel_keys"]):
+ diagnostics.append({
+ "kernel_key": kernel_key,
+ "rank_artifacts": [
+ {
+ "bytes": rank_artifacts[artifact_index]["bytes"],
+ "rank": artifact_rank,
+ "sha256": rank_artifacts[artifact_index]["sha256"],
+ }
+ for artifact_rank, rank_artifacts in enumerate(gathered_artifacts)
+ ],
+ })
+ if self._deferred_semantic_snapshot is not None and semantic != self._deferred_semantic_snapshot:
+ raise RuntimeError("DeepEP Hybrid config/JIT kernel set changed after measurement")
+ if self._deferred_jit_diagnostics is not None and diagnostics != self._deferred_jit_diagnostics:
+ raise RuntimeError("DeepEP Hybrid rank-local JIT artifacts changed after measurement")
+ self._deferred_semantic_snapshot = semantic
+ self._deferred_jit_diagnostics = diagnostics
+ self.backend_provenance.update(semantic)
+ self.backend_provenance["jit_shared_objects"] = diagnostics
+
+ def inspect_dispatch(self, p, h):
+ count = self.recv_tokens(h)
+ routing_map = h.handle[4][:count]
+ rows, local_expert_ids = routing_map.nonzero(as_tuple=True)
+ positions = routing_map.to(torch.int64).cumsum(dim=1)[rows, local_expert_ids] - 1
+ probability_columns = self.domain_rank * self.local_experts + local_expert_ids
+ if h.recv_probs.shape[1] < (self.domain_rank + 1) * self.local_experts:
+ raise RuntimeError("HybridEPBuffer probability tensor omits this NVLink-domain rank")
+ expert_ids = torch.full(
+ (count, self.top_k), -1, dtype=torch.int64, device=self.device
+ )
+ weights = torch.zeros(
+ (count, self.top_k), dtype=torch.float32, device=self.device
+ )
+ expert_ids[rows, positions] = local_expert_ids + self.rank * self.local_experts
+ weights[rows, positions] = h.recv_probs[:count][rows, probability_columns]
+ return types.SimpleNamespace(
+ payload=h.recv_payload[:count],
+ expert_ids=expert_ids,
+ weights=weights,
+ local_expert_counts=routing_map.sum(dim=0, dtype=torch.int64),
+ ordering_contract="global-source-filter-stable-v1",
+ )
+
+ def combine_transformed(self, p, h, transformed):
+ combined = self.buffer.combine(
+ transformed.to(h.recv_payload.dtype), handle=h.handle
+ )
+ return combined[0] if isinstance(combined, (tuple, list)) else combined
+
+ def recv_tokens(self, h):
+ return int(h.handle[3].item())
+
+ def finalize(self, rc):
+ try:
+ dist.barrier()
+ dist.destroy_process_group()
+ except Exception:
+ pass
+ shutil.rmtree(self._jit_cache_dir, ignore_errors=True)
+ if self._previous_jit_cache_dir is None:
+ os.environ.pop("HYBRID_EP_CACHE_DIR", None)
+ else:
+ os.environ["HYBRID_EP_CACHE_DIR"] = self._previous_jit_cache_dir
+ if self._previous_domain_ranks is None:
+ os.environ.pop("NUM_OF_HYBRID_EP_RANKS_PER_NVLINK_DOMAIN", None)
+ else:
+ os.environ[
+ "NUM_OF_HYBRID_EP_RANKS_PER_NVLINK_DOMAIN"
+ ] = self._previous_domain_ranks
+ return rc
diff --git a/experimental/CollectiveX/tests/ep_deepep_v2.py b/experimental/CollectiveX/tests/ep_deepep_v2.py
new file mode 100644
index 0000000000..c629b0294e
--- /dev/null
+++ b/experimental/CollectiveX/tests/ep_deepep_v2.py
@@ -0,0 +1,528 @@
+#!/usr/bin/env python3
+"""DeepEP PR #605 adapter with PR #630's pure scale-up initialization fix."""
+
+from __future__ import annotations
+
+import ctypes
+import hashlib
+import importlib.metadata
+import inspect
+import json
+import os
+import re
+import sys
+import types
+from pathlib import Path
+
+import torch
+import torch.distributed as dist
+import contracts
+import ep_harness
+
+try:
+ import deep_ep
+ from deep_ep import ElasticBuffer # type: ignore
+except Exception as exc: # pragma: no cover - requires the benchmark image
+ print(f"ERROR: DeepEP V2 import failed: {exc!r}", file=sys.stderr)
+ raise
+
+
+DEEPEP_V2_PR = 605
+DEEPEP_V2_FIX_PR = 630
+DEEPEP_V2_COMMIT = "fa8a9b16898204afd347c663b89e65ef87dc6ce6"
+DEEPEP_V2_TREE = "29809e75c5874e6609dac4804e7b651d5226959f"
+DEEPEP_V2_FMT_COMMIT = "a4c7e17133ee9cb6a2f45545f6e974dd3c393efa"
+DEEPEP_V2_VERSION = "2.0.0"
+DEEPEP_V2_DISTRIBUTION = "2.0.0+fa8a9b1"
+DEEPEP_V2_JIT_RANDOM_SEED = "collectivex-deepep-v2-fa8a9b1"
+TORCH_VERSION = "2.10.0+cu130"
+NCCL_VERSION = "2.30.4"
+NVSHMEM_VERSION = "3.3.9"
+DEEPEP_V2_JIT_KERNELS = contracts.DEEPEP_V2_JIT_KERNELS
+
+
+def _sha256(path: str) -> str:
+ digest = hashlib.sha256()
+ with open(path, "rb") as handle:
+ for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+ digest.update(chunk)
+ return digest.hexdigest()
+
+
+def _api_sha256() -> str:
+ signatures = {
+ "ElasticBuffer.__init__": str(inspect.signature(ElasticBuffer.__init__)),
+ "ElasticBuffer.dispatch": str(inspect.signature(ElasticBuffer.dispatch)),
+ "ElasticBuffer.combine": str(inspect.signature(ElasticBuffer.combine)),
+ }
+ return hashlib.sha256(
+ json.dumps(signatures, sort_keys=True, separators=(",", ":")).encode()
+ ).hexdigest()
+
+
+def _loaded_library_paths() -> set[str]:
+ extension = getattr(getattr(deep_ep, "_C", None), "__file__", None)
+ if not extension or not os.path.isfile(extension):
+ raise RuntimeError("DeepEP V2 extension library is not loaded")
+ paths = {os.path.realpath(extension)}
+ try:
+ with open("/proc/self/maps", encoding="utf-8") as handle:
+ for line in handle:
+ path = line.rstrip().split()[-1]
+ name = os.path.basename(path)
+ if ("libnccl.so" in name or "libnvshmem_host.so" in name) and os.path.isfile(path):
+ paths.add(os.path.realpath(path))
+ except OSError as exc: # pragma: no cover - benchmark runtime is Linux
+ raise RuntimeError("cannot inspect loaded communication libraries") from exc
+ return paths
+
+
+def _loaded_nccl_version() -> str:
+ matches = [
+ path for path in _loaded_library_paths()
+ if "libnccl.so" in os.path.basename(path)
+ ]
+ if len(matches) != 1:
+ raise RuntimeError("expected exactly one loaded NCCL library")
+ version = ctypes.c_int()
+ if ctypes.CDLL(matches[0]).ncclGetVersion(ctypes.byref(version)) != 0:
+ raise RuntimeError("loaded NCCL version query failed")
+ return ep_harness.format_collective_version(version.value)
+
+
+def _loaded_library_evidence() -> list[dict[str, str]]:
+ """Return content identities, never private library paths."""
+ paths = _loaded_library_paths()
+ required = {
+ "nccl": [path for path in paths if "libnccl.so" in os.path.basename(path)],
+ "nvshmem": [path for path in paths if "libnvshmem_host.so" in os.path.basename(path)],
+ }
+ mismatches = [f"{name}={len(matches)}" for name, matches in required.items() if len(matches) != 1]
+ if mismatches:
+ raise RuntimeError("expected one loaded library for each dependency: " + ", ".join(mismatches))
+
+ def role(path: str) -> str:
+ name = os.path.basename(path)
+ if "libnccl.so" in name:
+ return "nccl"
+ if "libnvshmem_host.so" in name:
+ return "nvshmem"
+ return "deepep-extension"
+
+ def label(path: str) -> str:
+ return "deep_ep._C" if role(path) == "deepep-extension" else os.path.basename(path)
+
+ return sorted(
+ ({"role": role(path), "name": label(path), "sha256": _sha256(path)} for path in paths),
+ key=lambda item: (item["role"], item["name"], item["sha256"]),
+ )
+
+
+def _jit_artifact_evidence() -> list[dict[str, str]]:
+ root = Path(os.environ["EP_JIT_CACHE_DIR"]) / "cache"
+ if root.is_symlink() or not root.is_dir():
+ raise RuntimeError("DeepEP V2 produced no JIT cache evidence")
+ artifacts = []
+ kernel_names = set()
+ for directory in sorted(root.iterdir(), key=lambda item: item.name):
+ match = re.fullmatch(r"kernel\.([A-Za-z0-9_+-]+)\.([0-9a-f]{32})", directory.name)
+ if directory.is_symlink() or not directory.is_dir() or match is None:
+ raise RuntimeError("DeepEP V2 JIT cache contains an invalid entry")
+ if {path.name for path in directory.iterdir()} != {
+ "kernel.cu", "kernel.cubin", "kernel.sass",
+ }:
+ raise RuntimeError("DeepEP V2 JIT kernel evidence is incomplete")
+ source = directory / "kernel.cu"
+ cubin = directory / "kernel.cubin"
+ sass = directory / "kernel.sass"
+ if any(path.is_symlink() or not path.is_file() for path in (source, cubin, sass)):
+ raise RuntimeError("DeepEP V2 JIT evidence is not a regular file")
+ if any(path.stat().st_size <= 0 for path in (source, cubin, sass)):
+ raise RuntimeError("DeepEP V2 JIT evidence is empty")
+ kernel_names.add(match.group(1))
+ artifacts.append({
+ "cache_key": directory.name,
+ "source_sha256": _sha256(str(source)),
+ "sass_sha256": _sha256(str(sass)),
+ "cubin_sha256": _sha256(str(cubin)),
+ })
+ if (
+ len(artifacts) != len(DEEPEP_V2_JIT_KERNELS)
+ or kernel_names != DEEPEP_V2_JIT_KERNELS
+ ):
+ raise RuntimeError("DeepEP V2 JIT kernel set differs from the v1 contract")
+ return sorted(artifacts, key=lambda item: item["cache_key"])
+
+
+def _jit_cache_key(
+ args,
+ world_size: int,
+ max_tokens: int,
+ allow_hybrid_mode: bool,
+ realized: dict[str, int | bool],
+) -> str:
+ """Key generated kernels by codegen inputs, not routing data or case identity."""
+ payload = {
+ "contract": "deepep-v2-jit-config-v3",
+ "runner": args.runner,
+ "world_size": world_size,
+ "hidden": args.hidden,
+ "topk": args.topk,
+ "physical_experts": args.experts,
+ "tuning_experts": getattr(args, "num_logical_experts", args.experts),
+ "max_tokens": max_tokens,
+ "dispatch_dtype": "bf16",
+ "combine_dtype": "bf16",
+ "input_layout": "bf16-no-sf",
+ "expert_alignment": 1,
+ "do_cpu_sync": True,
+ "cached_mode": False,
+ "do_expand": False,
+ "use_expanded_layout": False,
+ "allow_hybrid_mode": allow_hybrid_mode,
+ "allow_multiple_reduction": True,
+ "prefer_overlap_with_compute": True,
+ "deterministic": False,
+ **realized,
+ }
+ return "jitcfg-v3-" + hashlib.sha256(
+ json.dumps(payload, sort_keys=True, separators=(",", ":")).encode()
+ ).hexdigest()
+
+
+def _require_cross_rank_equal(value, label: str) -> None:
+ gathered = [None] * dist.get_world_size()
+ dist.all_gather_object(gathered, value)
+ canonical = {json.dumps(item, sort_keys=True, separators=(",", ":")) for item in gathered}
+ if len(canonical) != 1:
+ raise RuntimeError(f"DeepEP V2 {label} differs across ranks")
+
+
+def _configure_gin_mode(args, world_size: int) -> bool:
+ scale_up_domain = int(
+ getattr(args, "scale_up_domain", None)
+ or getattr(args, "gpus_per_node", None)
+ or world_size
+ )
+ allow_hybrid_mode = world_size > scale_up_domain
+ if allow_hybrid_mode:
+ os.environ.pop("EP_DISABLE_GIN", None)
+ else:
+ os.environ["EP_DISABLE_GIN"] = "1"
+ return allow_hybrid_mode
+
+
+def _lsa_topology_is_valid(
+ gin_enabled: bool,
+ world_size: int,
+ scale_up_domain: int,
+ config: dict[str, int | bool],
+) -> bool:
+ if gin_enabled:
+ domains = world_size // scale_up_domain
+ return (
+ world_size % scale_up_domain == 0
+ and domains > 1
+ and config["physical_rdma_ranks"] == domains
+ and config["physical_nvlink_ranks"] == scale_up_domain
+ and config["logical_scaleout_ranks"] == domains
+ and config["logical_scaleup_ranks"] == scale_up_domain
+ and config["is_scaleup_nvlink"] is True
+ )
+ return (
+ config["physical_rdma_ranks"] == 1
+ and config["physical_nvlink_ranks"] == world_size
+ and config["logical_scaleout_ranks"] == 1
+ and config["logical_scaleup_ranks"] == world_size
+ and config["is_scaleup_nvlink"] is True
+ )
+
+
+def _require_runtime() -> tuple[str, str]:
+ expected = {
+ "DEEPEP_V2_PR": str(DEEPEP_V2_PR),
+ "DEEPEP_V2_FIX_PR": str(DEEPEP_V2_FIX_PR),
+ "DEEPEP_V2_COMMIT": DEEPEP_V2_COMMIT,
+ "DEEPEP_V2_TREE": DEEPEP_V2_TREE,
+ "DEEPEP_V2_FMT_COMMIT": DEEPEP_V2_FMT_COMMIT,
+ "DEEPEP_V2_JIT_RANDOM_SEED": DEEPEP_V2_JIT_RANDOM_SEED,
+ "EP_JIT_DUMP_SASS": "1",
+ }
+ mismatches = [
+ f"{name}={os.environ.get(name)!r}, expected {value!r}"
+ for name, value in expected.items()
+ if os.environ.get(name) != value
+ ]
+ torch_version = str(torch.__version__)
+ nccl_package_version = importlib.metadata.version("nvidia-nccl-cu13")
+ nvshmem_package_version = importlib.metadata.version("nvidia-nvshmem-cu12")
+ actual = {
+ "deep_ep": str(getattr(deep_ep, "__version__", "")),
+ "deep_ep distribution": importlib.metadata.version("deep_ep"),
+ "torch": torch_version,
+ "nvidia-nccl-cu13": nccl_package_version,
+ "nvidia-nvshmem-cu12": nvshmem_package_version,
+ }
+ required = {
+ "deep_ep": DEEPEP_V2_VERSION,
+ "deep_ep distribution": DEEPEP_V2_DISTRIBUTION,
+ "torch": TORCH_VERSION,
+ "nvidia-nccl-cu13": NCCL_VERSION,
+ "nvidia-nvshmem-cu12": NVSHMEM_VERSION,
+ }
+ mismatches.extend(
+ f"{name}={actual[name]!r}, expected {value!r}"
+ for name, value in required.items()
+ if actual[name] != value
+ )
+ if not inspect.isclass(ElasticBuffer) or ElasticBuffer.__name__ != "ElasticBuffer":
+ mismatches.append("deep_ep.ElasticBuffer is absent")
+ if os.environ.get("EP_SUPPRESS_NCCL_CHECK"):
+ mismatches.append("EP_SUPPRESS_NCCL_CHECK must be unset")
+ nccl_runtime_version = _loaded_nccl_version()
+ if nccl_runtime_version != NCCL_VERSION:
+ mismatches.append(
+ f"loaded NCCL={nccl_runtime_version!r}, expected {NCCL_VERSION!r}"
+ )
+ if mismatches:
+ raise RuntimeError("invalid DeepEP V2 runtime: " + "; ".join(mismatches))
+ return torch_version, nccl_runtime_version
+
+
+class DeepEPV2Backend:
+ name = "deepep-v2"
+ combine_needs_redispatch = False
+ combine_weight_semantics = "unweighted-rank-sum"
+
+ def __init__(self, args, rank, world_size, local_rank, device):
+ self.args = args
+ self.rank = rank
+ self.world_size = world_size
+ self.device = device
+ self.mode = "normal"
+ self.group = dist.group.WORLD
+ torch_version, nccl_runtime_version = _require_runtime()
+ ladder, _ = ep_harness.token_ladder(args.tokens_ladder, args.phase, None)
+ conditioning = ep_harness.CONDITIONING_LADDERS[args.phase]
+ self.max_tokens = max([*ladder, *conditioning])
+ jit_root = Path(os.environ["EP_JIT_CACHE_DIR"])
+ scale_up_domain = int(
+ getattr(args, "scale_up_domain", None)
+ or getattr(args, "gpus_per_node", None)
+ or world_size
+ )
+ allow_hybrid_mode = _configure_gin_mode(args, world_size)
+ gin_enabled = allow_hybrid_mode
+ communication_backend = "nccl-gin" if gin_enabled else "nccl-device-lsa"
+ self._deferred_jit_snapshot = None
+ self.buffer = ElasticBuffer(
+ self.group,
+ num_max_tokens_per_rank=self.max_tokens,
+ hidden=args.hidden,
+ num_topk=args.topk,
+ use_fp8_dispatch=False,
+ deterministic=False,
+ allow_hybrid_mode=allow_hybrid_mode,
+ allow_multiple_reduction=True,
+ prefer_overlap_with_compute=True,
+ num_gpu_timeout_secs=100,
+ explicitly_destroy=True,
+ )
+ tuning_num_experts = int(getattr(args, "num_logical_experts", args.experts))
+ self.num_sms = int(
+ self.buffer.get_theoretical_num_sms(tuning_num_experts, args.topk)
+ )
+ self.num_qps = int(self.buffer.get_theoretical_num_qps(self.num_sms))
+ properties = torch.cuda.get_device_properties(device)
+ device_sms = int(properties.multi_processor_count)
+ jit_config = {
+ "num_sms": self.num_sms,
+ "num_qps": self.num_qps,
+ "allocated_qps": int(self.buffer.num_allocated_qps),
+ "logical_scaleout_ranks": int(self.buffer.num_scaleout_ranks),
+ "logical_scaleup_ranks": int(self.buffer.num_scaleup_ranks),
+ "physical_rdma_ranks": int(self.buffer.num_rdma_ranks),
+ "physical_nvlink_ranks": int(self.buffer.num_nvlink_ranks),
+ "is_scaleup_nvlink": self.buffer.num_scaleup_ranks == self.buffer.num_nvlink_ranks,
+ "device_arch_major": int(properties.major),
+ "device_arch_minor": int(properties.minor),
+ "device_sms": device_sms,
+ "device_smem_bytes": int(properties.shared_memory_per_block_optin),
+ "gpu_timeout_cycles": 100 * int(properties.clock_rate) * 1000,
+ }
+ _require_cross_rank_equal(jit_config, "JIT configuration")
+ if not _lsa_topology_is_valid(
+ gin_enabled, world_size, scale_up_domain, jit_config
+ ):
+ raise RuntimeError("DeepEP V2 realized communication domains differ from topology")
+ self.jit_cache_key = _jit_cache_key(
+ args, world_size, self.max_tokens, allow_hybrid_mode, jit_config
+ )
+ os.environ["EP_JIT_CACHE_DIR"] = str(jit_root / self.jit_cache_key)
+ realized_config = {
+ "jit_cache_key": self.jit_cache_key,
+ "num_max_tokens_per_rank": self.max_tokens,
+ **jit_config,
+ }
+ _require_cross_rank_equal(realized_config, "realized tuning/topology")
+ comm = getattr(self.buffer, "nccl_comm_handle", None)
+ communicator = (
+ "deepep-managed" if getattr(comm, "managed", True) else "pytorch-reused"
+ )
+
+ loaded_libraries = _loaded_library_evidence()
+ _require_cross_rank_equal(loaded_libraries, "loaded libraries")
+ self.backend_provenance = {
+ "deepep_version": DEEPEP_V2_VERSION,
+ "deepep_distribution_version": importlib.metadata.version("deep_ep"),
+ "deepep_commit": DEEPEP_V2_COMMIT,
+ "deepep_tree": DEEPEP_V2_TREE,
+ "deepep_pr": DEEPEP_V2_PR,
+ "deepep_fix_pr": DEEPEP_V2_FIX_PR,
+ "fmt_commit": DEEPEP_V2_FMT_COMMIT,
+ "api": "deep_ep.ElasticBuffer",
+ "api_signature_sha256": _api_sha256(),
+ "communication_backend": communication_backend,
+ "gin_enabled": gin_enabled,
+ "nccl_communicator": communicator,
+ "torch_version": torch_version,
+ "torch_git_version": str(torch.version.git_version),
+ "cuda_version": str(torch.version.cuda),
+ "nccl_package_version": importlib.metadata.version("nvidia-nccl-cu13"),
+ "nccl_version": nccl_runtime_version,
+ "nvshmem_package_version": importlib.metadata.version("nvidia-nvshmem-cu12"),
+ "loaded_libraries": loaded_libraries,
+ "jit_cache_key": self.jit_cache_key,
+ "jit_cubins": [],
+ "jit_random_seed": DEEPEP_V2_JIT_RANDOM_SEED,
+ "num_experts": int(args.experts),
+ "mode": "normal",
+ "dispatch_dtype": "bf16",
+ "combine_dtype": "bf16",
+ "deterministic": False,
+ "resource_mode": "tuned",
+ "requested_num_sms": self.num_sms,
+ "tuning_num_experts": tuning_num_experts,
+ "num_sms": self.num_sms,
+ "num_qps": self.num_qps,
+ "allocated_qps": int(self.buffer.num_allocated_qps),
+ "device_sms": device_sms,
+ "sm_fraction": self.num_sms / device_sms,
+ "tuned_source": "deepep-v2-analytical-sm-qp-logical-experts-v1",
+ "num_max_tokens_per_rank": self.max_tokens,
+ "allow_hybrid_mode": bool(self.buffer.allow_hybrid_mode),
+ "allow_multiple_reduction": bool(self.buffer.allow_multiple_reduction),
+ "prefer_overlap_with_compute": bool(
+ self.buffer.prefer_overlap_with_compute
+ ),
+ "logical_scaleout_ranks": int(self.buffer.num_scaleout_ranks),
+ "logical_scaleup_ranks": int(self.buffer.num_scaleup_ranks),
+ "physical_rdma_ranks": int(self.buffer.num_rdma_ranks),
+ "physical_nvlink_ranks": int(self.buffer.num_nvlink_ranks),
+ }
+
+ def buffer_cap(self, args):
+ return self.max_tokens
+
+ def make_problem(self, T, idx, weights, x):
+ return types.SimpleNamespace(
+ T=T,
+ x=x,
+ topk_idx=idx.to(deep_ep.topk_idx_t),
+ topk_weights=weights.to(torch.float32),
+ )
+
+ def dispatch(self, p):
+ recv_x, recv_topk_idx, recv_topk_weights, handle, _ = self.buffer.dispatch(
+ p.x,
+ topk_idx=p.topk_idx,
+ topk_weights=p.topk_weights,
+ num_experts=self.args.experts,
+ num_max_tokens_per_rank=self.max_tokens,
+ expert_alignment=1,
+ num_sms=self.num_sms,
+ num_qps=self.num_qps,
+ async_with_compute_stream=False,
+ do_handle_copy=True,
+ do_cpu_sync=True,
+ do_expand=False,
+ )
+ return types.SimpleNamespace(
+ recv_x=recv_x,
+ recv_topk_idx=recv_topk_idx,
+ recv_topk_weights=recv_topk_weights,
+ handle=handle,
+ )
+
+ def stage(self, p, h):
+ h.combine_input = h.recv_x
+
+ def combine(self, p, h):
+ combined_x, _, _ = self.buffer.combine(
+ h.combine_input,
+ handle=h.handle,
+ num_sms=self.num_sms,
+ num_qps=self.num_qps,
+ async_with_compute_stream=False,
+ )
+ return combined_x
+
+ def capture_deferred_provenance(self):
+ # destroy() uses this same barrier. Materialize its JIT kernel before hashing the
+ # implementation so the first and later routing cases see identical evidence.
+ self.buffer.barrier(use_comm_stream=True, with_cpu_sync=True)
+ torch.cuda.synchronize()
+ jit_cubins = _jit_artifact_evidence()
+ _require_cross_rank_equal(jit_cubins, "JIT CUBINs")
+ if (
+ self._deferred_jit_snapshot is not None
+ and jit_cubins != self._deferred_jit_snapshot
+ ):
+ raise RuntimeError("DeepEP V2 JIT CUBIN set changed after measurement")
+ self._deferred_jit_snapshot = jit_cubins
+ self.backend_provenance["jit_cubins"] = jit_cubins
+
+ def inspect_dispatch(self, p, h):
+ count = self.recv_tokens(h)
+ local_idx = h.recv_topk_idx[:count]
+ valid = local_idx >= 0
+ expert_ids = torch.where(
+ valid,
+ local_idx + self.rank * (self.args.experts // self.world_size),
+ local_idx,
+ )
+ local = local_idx[valid].to(torch.int64)
+ return types.SimpleNamespace(
+ payload=h.recv_x[:count],
+ expert_ids=expert_ids,
+ weights=h.recv_topk_weights[:count].masked_fill(~valid, 0),
+ local_expert_counts=torch.bincount(
+ local, minlength=self.args.experts // self.world_size
+ ),
+ ordering_contract="elastic-source-metadata-v1",
+ )
+
+ def combine_transformed(self, p, h, transformed):
+ combine_input = torch.zeros_like(h.recv_x)
+ combine_input[: transformed.shape[0]].copy_(transformed.to(combine_input.dtype))
+ combined, _, _ = self.buffer.combine(
+ combine_input,
+ handle=h.handle,
+ num_sms=self.num_sms,
+ num_qps=self.num_qps,
+ async_with_compute_stream=False,
+ )
+ return combined
+
+ def recv_tokens(self, h):
+ return int(h.handle.psum_num_recv_tokens_per_scaleup_rank[-1].item())
+
+ def finalize(self, rc):
+ try:
+ dist.barrier()
+ self.buffer.destroy()
+ dist.barrier()
+ dist.destroy_process_group()
+ except Exception:
+ return 1
+ return rc
diff --git a/experimental/CollectiveX/tests/ep_harness.py b/experimental/CollectiveX/tests/ep_harness.py
new file mode 100644
index 0000000000..c9a022a484
--- /dev/null
+++ b/experimental/CollectiveX/tests/ep_harness.py
@@ -0,0 +1,1780 @@
+#!/usr/bin/env python3
+"""CollectiveX — shared EP (expert-parallel) dispatch/combine benchmark harness.
+
+Backend-agnostic core. The per-backend adapters (`ep_deepep.py`, `ep_mori.py`)
+implement a small duck-typed protocol; this module owns the source-tokens-per-rank
+sweep, the timing, the correctness gate, and the provenance-tagged JSON doc.
+
+Fair-comparison contract (see docs/methodology.md):
+ * **Deterministic shared routing trace** (`routing.py`): the per-token expert IDs +
+ gate weights are generated once from a fixed seed over the *global* batch and are
+ identical on every SKU; each rank materializes its slice. So every platform runs
+ the *same* problem (no per-rank/per-platform RNG in the adapters).
+ * **Explicit measurement contract**: layout-and-dispatch-v1 includes routing-layout
+ generation in dispatch timing. Combine excludes staging.
+ Isolated sum is derived independently at each percentile and is not a measured chained op.
+ * **Correct collective percentile**: each iteration's latency is reduced MAX across
+ ranks first (a collective finishes with its slowest rank), THEN percentiled —
+ `median_i(max_r)`, not `max_r(median_i)`.
+ * **One line = one fixed config**; only T varies. Both `tokens_per_rank` and
+ `global_tokens = T * ep_size` are recorded as explicit chart coordinates.
+
+stdlib-only at module top (torch is passed in by the entrypoint; `routing` is imported
+lazily inside run_sweep) so this file `py_compile`s without torch.
+
+Backend protocol:
+ name, mode, combine_needs_redispatch, backend_provenance(dict)
+ buffer_cap(args) -> int|None
+ make_problem(T, idx, weights, x) -> problem # materialize this rank's trace slice
+ dispatch(problem) -> handle # pure dispatch comm (timed)
+ stage(problem, handle) # untimed expert-output placement
+ combine(problem, handle) -> tensor # pure combine comm (timed)
+ inspect_dispatch(problem, handle) -> view # normalized payload/expert/weight metadata
+ combine_transformed(problem, handle, tensor) -> tensor
+ recv_tokens(handle) -> int # realized tokens received this rank
+ finalize(rc) -> int|NoReturn
+"""
+from __future__ import annotations
+
+import argparse
+import datetime as _dt
+import hashlib
+import json
+import math
+import os
+import types
+
+import contracts
+import identity
+import workload as workload_contract
+
+# Raw v1 result emitted by one benchmark case. Publication uses a separate contract.
+SCHEMA_VERSION = 1
+
+# Every comparison-grade EP point uses the same literal timing profile on every SKU/backend.
+# Eight timed iterations keep each MoRI burst well below its sustained-iteration wedge, 64 trials
+# provide 512 observations per operation, and 32 warmups meet Blackwell's measured clock-ramp floor.
+SAMPLING_CONTRACT = identity.V1_CASE_PROFILE["sampling_contract"]
+TIMED_SAMPLES_PER_POINT = 512
+TIMED_ITERS_PER_TRIAL = 8
+TRIALS_PER_POINT = 64
+WARMUP_ITERS_PER_TRIAL = 32
+WARMUP_SEMANTICS = "full-roundtrip-before-each-component-trial-point-v1"
+ROUTING_SEED = 67
+ROUTING_GENERATOR = workload_contract.GENERATOR_VERSION
+ACTIVATION_PROFILE = "canonical-counter-source-v3"
+ACTIVATION_GENERATOR = workload_contract.ACTIVATION_GENERATOR
+PLACEMENT = "packed"
+COMPONENT_ORDER_CONTRACT = "roundtrip-dispatch-activation-only-combine-v2"
+LOW_LATENCY_MODE = "low-latency"
+LOW_LATENCY_MAX_TOKENS_PER_RANK = 128
+LOW_LATENCY_MEASUREMENT_CONTRACT = "expert-packed-weighted-combine-v1"
+LOW_LATENCY_COMPONENT_ORDER_CONTRACT = "roundtrip-dispatch-gate-weighted-combine-v1"
+LOW_LATENCY_ORACLE_CONTRACT = "expert-assignment-transform-v1"
+LOW_LATENCY_CORRECTNESS_SCOPE = "expert-assignment-and-weighted-combine"
+
+# Phase-default sweeps — token-size regimes, NOT distinct kernels (both run normal
+# mode; "decode"/"prefill" name the small/large-token regime). Powers of two for a
+# clean log x-axis; clamped to the backend buffer ceiling (MoRI's registerable heap).
+DECODE_LADDER = [1, 2, 4, 8, 16, 32, 64, 128]
+PREFILL_LADDER = [128, 256, 512, 1024, 2048, 4096]
+CONDITIONING_LADDERS = {
+ phase: list(ladder) for phase, ladder in contracts.V1_CONDITIONING_LADDERS.items()
+}
+CONDITIONING_ROUNDS_PER_SHAPE = contracts.V1_CONDITIONING_ROUNDS_PER_SHAPE
+CONDITIONING_CONTRACT = identity.V1_CASE_PROFILE["conditioning_contract"]
+ORACLE_CONTRACT = identity.V1_CASE_PROFILE["oracle_contract"]
+ORACLE_RTOL = 5e-2
+ORACLE_ATOL = 2e-2
+
+BF16_BYTES = 2
+EPLB_REDUNDANT_EXPERTS = 32
+EPLB_REFERENCE_TOKENS_PER_RANK = 2048
+EPLB_PLANNER = "greedy-rank-major-v1"
+V1_PROFILE = {
+ "dispatch_dtype": "bf16",
+ "combine_dtype": "bf16",
+ "combine_quant_mode": "none",
+ "mode": "normal",
+ "measurement_contract": "layout-and-dispatch-v1",
+ "resource_mode": "tuned",
+ "placement": PLACEMENT,
+ "activation_profile": ACTIVATION_PROFILE,
+ "activation_generator": ACTIVATION_GENERATOR,
+ "routing_generator": ROUTING_GENERATOR,
+ "component_order_contract": COMPONENT_ORDER_CONTRACT,
+ "conditioning_contract": CONDITIONING_CONTRACT,
+ "eplb_reference_tokens_per_rank": EPLB_REFERENCE_TOKENS_PER_RANK,
+ "eplb_redundant_experts": EPLB_REDUNDANT_EXPERTS,
+ "eplb_planner": EPLB_PLANNER,
+ # DeepEP/UCCL use this only as the fallback when their tuned default is not exported.
+ "num_sms": 24,
+}
+
+def format_collective_version(raw) -> str:
+ """Normalize PyTorch's tuple or packed NCCL/RCCL version representation."""
+ if isinstance(raw, int):
+ if raw < 10_000:
+ return f"{raw // 1000}.{raw // 100 % 10}.{raw % 100}"
+ return f"{raw // 10_000}.{raw // 100 % 100}.{raw % 100}"
+ if isinstance(raw, (tuple, list)):
+ return ".".join(map(str, raw))
+ return str(raw) if raw not in (None, "") else "unknown"
+
+
+def add_common_args(ap: argparse.ArgumentParser) -> None:
+ """Add the varying v1 inputs; fixed profile values are not CLI axes."""
+ ap.set_defaults(**V1_PROFILE)
+ ap.add_argument("--mode", default="normal", choices=["normal", LOW_LATENCY_MODE])
+ ap.add_argument("--phase", default="decode", choices=["decode", "prefill"],
+ help="token-size regime: decode (small T) / prefill (large T) — picks the default ladder")
+ ap.add_argument("--tokens-ladder", default="",
+ help="space/comma-separated source-tokens-per-rank sweep; blank = phase default")
+ ap.add_argument("--hidden", type=int, default=7168)
+ ap.add_argument("--topk", type=int, default=8)
+ ap.add_argument("--experts", type=int, default=256, help="TOTAL experts (fixed across EP degrees)")
+ ap.add_argument("--routing", default="uniform", choices=["uniform", "zipf"])
+ # EPLB (Expert-Parallel Load Balancer): replicate hot experts onto redundant physical
+ # slots + balanced-place so per-rank load equalizes. A pure routing-trace transform
+ # (tests/eplb.py); experts becomes num_logical+redundant. The remedy for `zipf` skew.
+ ap.add_argument("--eplb", action="store_true",
+ help="apply EPLB expert replication/placement to the routing trace")
+ # Canonical workloads consume pre-generated trace bytes instead of the
+ # seeded runtime generator, so a result is provably the SAME workload as another machine's
+ # (checksum match). Points at a dir of .npz/.manifest.json (make_workloads.py).
+ ap.add_argument("--workload-dir", default="",
+ help="dir of canonical workload traces; empty = seeded runtime generation (dev)")
+ ap.add_argument("--case-id", default="")
+ ap.add_argument("--suite", default="")
+ ap.add_argument("--workload-name", default="")
+ ap.add_argument("--required-publication", default="")
+ ap.add_argument("--seed", type=int, default=ROUTING_SEED)
+ # 32: B300/Blackwell needs ~30 untimed iters to reach steady-state GPU clocks +
+ # establish NVLink/NVSHMEM connections — at warmup=8 its dispatch read ~1787us
+ # (cold), at warmup>=30 it settles to ~85us (faster than H100, reproducible within
+ # ~2.5%). H100/MI355X reach steady state much sooner; the extra iters are harmless.
+ ap.add_argument("--warmup", type=int, default=WARMUP_ITERS_PER_TRIAL,
+ help=f"untimed full roundtrips before each trial/point; fixed by "
+ f"{SAMPLING_CONTRACT} to {WARMUP_ITERS_PER_TRIAL}")
+ ap.add_argument("--iters", type=int, default=TIMED_ITERS_PER_TRIAL,
+ help=f"timed iterations per trial; fixed by {SAMPLING_CONTRACT} to "
+ f"{TIMED_ITERS_PER_TRIAL}")
+ ap.add_argument("--trials", type=int, default=TRIALS_PER_POINT,
+ help=f"timed trials; fixed by {SAMPLING_CONTRACT} to {TRIALS_PER_POINT}")
+ # provenance / output
+ ap.add_argument("--runner", required=True)
+ ap.add_argument("--topology-class", required=True)
+ ap.add_argument("--transport", default="")
+ ap.add_argument("--scope", required=True, choices=["scale-up", "scale-out"])
+ ap.add_argument("--scale-up-transport", required=True)
+ ap.add_argument("--scale-out-transport", default="")
+ # gpus-per-node=0 means one node containing the whole EP group.
+ ap.add_argument("--gpus-per-node", type=int, default=0)
+ ap.add_argument("--scale-up-domain", type=int, default=0, help="0 = gpus_per_node*ep (one domain)")
+ ap.add_argument("--timestamp")
+ ap.add_argument("--out", required=True)
+
+
+def token_ladder(spec: str, phase: str, cap: int | None) -> tuple[list[int], list[int]]:
+ """Return (ladder, dropped): explicit spec else the phase default; positive ints;
+ clamped to `cap` with dropped points reported (never silently truncated)."""
+ if spec and spec.strip():
+ want = [int(t) for t in spec.replace(",", " ").split() if t]
+ else:
+ want = DECODE_LADDER if phase == "decode" else PREFILL_LADDER
+ want = sorted({t for t in want if t > 0})
+ if cap is not None:
+ return [t for t in want if t <= cap], [t for t in want if t > cap]
+ return want, []
+
+
+def sampling_contract_error(iters: int, trials: int, warmup: int) -> str | None:
+ """Return a user-facing error unless the exact cross-SKU timing profile is used."""
+ expected = (TIMED_ITERS_PER_TRIAL, TRIALS_PER_POINT, WARMUP_ITERS_PER_TRIAL)
+ observed = (iters, trials, warmup)
+ if observed != expected:
+ return (f"{SAMPLING_CONTRACT} requires exactly iters:trials:warmup="
+ f"{expected[0]}:{expected[1]}:{expected[2]} on every SKU/backend; got "
+ f"{observed[0]}:{observed[1]}:{observed[2]} "
+ f"({iters * trials if iters > 0 and trials > 0 else 'invalid'} timed samples)")
+ return None
+
+
+def _stats_vec(xs: list[int]) -> dict:
+ """min/mean/max/CV (+ empty count) of a per-rank count vector — self-describing source-token
+ or load summary without dumping the full vector."""
+ n = len(xs) or 1
+ mean = sum(xs) / n
+ var = sum((x - mean) ** 2 for x in xs) / n
+ cv = (var ** 0.5 / mean) if mean > 0 else 0.0
+ return {"min": min(xs) if xs else 0, "mean": round(mean, 3),
+ "max": max(xs) if xs else 0, "cv": round(cv, 4),
+ "empty_ranks": sum(1 for x in xs if x == 0), "total": sum(xs), "ranks": n}
+
+
+def percentile(xs: list[float], q: float) -> float:
+ if not xs:
+ return float("nan")
+ s = sorted(xs)
+ i = max(0, min(len(s) - 1, math.ceil(q / 100.0 * len(s)) - 1))
+ return s[i]
+
+
+def _sha256_json(value) -> str:
+ payload = json.dumps(
+ value, allow_nan=False, ensure_ascii=False, sort_keys=True, separators=(",", ":")
+ ).encode()
+ return hashlib.sha256(payload).hexdigest()
+
+
+def _series_provenance(provenance: dict) -> dict:
+ """Retain stable semantic build identity while keeping raw binaries diagnostic."""
+ return contracts.series_provenance(provenance)
+
+
+def _write_bytes_atomic(path: str, payload: bytes) -> tuple[str, int]:
+ os.makedirs(os.path.dirname(os.path.abspath(path)), exist_ok=True)
+ temporary = f"{path}.tmp-{os.getpid()}"
+ try:
+ with open(temporary, "wb") as handle:
+ handle.write(payload)
+ handle.flush()
+ os.fsync(handle.fileno())
+ os.replace(temporary, path)
+ finally:
+ try:
+ os.unlink(temporary)
+ except FileNotFoundError:
+ pass
+ return hashlib.sha256(payload).hexdigest(), len(payload)
+
+
+def _write_json_atomic(path: str, value) -> tuple[str, int]:
+ payload = (
+ json.dumps(value, allow_nan=False, ensure_ascii=False, indent=2) + "\n"
+ ).encode()
+ return _write_bytes_atomic(path, payload)
+
+
+def time_us(torch, fn, warmup: int, iters: int, pre=None) -> list[float]:
+ """Per-iteration CUDA-event latencies (µs) for THIS rank.
+
+ Without `pre`: times `fn()`. With `pre`: runs `pre()` UNTIMED each iteration (sync
+ before the start event so its GPU work can't bleed in), then times `fn(pre_result)`
+ — how combine is isolated when it consumes the dispatch state and needs a fresh
+ untimed dispatch+stage before every sample. Returns the raw per-iteration series;
+ the caller reduces across ranks per iteration before percentiling.
+ """
+ def sample():
+ arg = pre() if pre is not None else None
+ if pre is not None:
+ torch.cuda.synchronize()
+ s = torch.cuda.Event(enable_timing=True)
+ e = torch.cuda.Event(enable_timing=True)
+ s.record()
+ fn(arg) if pre is not None else fn()
+ e.record()
+ torch.cuda.synchronize()
+ return s.elapsed_time(e) * 1000.0 # ms -> us
+
+ for _ in range(max(0, warmup)):
+ if pre is not None:
+ a = pre()
+ torch.cuda.synchronize()
+ fn(a)
+ else:
+ fn()
+ # sync EACH warmup iteration, not just once after the loop: the measured-roundtrip fn
+ # interleaves dispatch+combine on a backend's persistent comm buffer, so back-to-back
+ # un-synced warmup iterations let iter N+1's dispatch race iter N's combine (CUDA abort
+ # on a rank -> NCCL-watchdog SIGABRT). Cheap (warmup is small); timed samples already sync.
+ torch.cuda.synchronize()
+ return [sample() for _ in range(iters)]
+
+
+def kernel_generation(backend) -> str:
+ """Return the adapter's explicit kernel family when one exists."""
+ declared = getattr(backend, "kernel_generation", None)
+ if declared:
+ return declared
+ return {
+ "deepep": "v1",
+ "deepep-v2": "v2-elastic-buffer",
+ "deepep-hybrid": "hybrid",
+ }.get(backend.name, "n-a")
+
+
+def _reduce_vec(torch, dist, device, vals, op):
+ t = torch.tensor(vals, device=device, dtype=torch.float64)
+ dist.all_reduce(t, op=op)
+ return [float(x) for x in t.tolist()]
+
+
+def _reduce_int(torch, dist, device, v: int, op) -> int:
+ t = torch.tensor([int(v)], device=device, dtype=torch.int64)
+ dist.all_reduce(t, op=op)
+ return int(t.item())
+
+
+def _same_hash_across_ranks(torch, dist, device, digest: str) -> bool:
+ parts = [int(digest[offset:offset + 8], 16) for offset in range(0, 64, 8)]
+ low = torch.tensor(parts, device=device, dtype=torch.int64)
+ high = low.clone()
+ dist.all_reduce(low, op=dist.ReduceOp.MIN)
+ dist.all_reduce(high, op=dist.ReduceOp.MAX)
+ return bool(torch.equal(low, high))
+
+
+def _tensor_sha256(*tensors) -> str:
+ digest = hashlib.sha256()
+ for tensor in tensors:
+ digest.update(tensor.detach().contiguous().cpu().numpy().tobytes())
+ return digest.hexdigest()
+
+
+def _normalized_expert_metadata(torch, expert_ids, weights):
+ """Sort each row by global expert ID while keeping -1 sentinels last."""
+ valid = expert_ids >= 0
+ keys = torch.where(valid, expert_ids.to(torch.int64), torch.full_like(expert_ids, 1 << 30))
+ order = torch.argsort(keys, dim=1, stable=True)
+ sorted_ids = torch.gather(expert_ids.to(torch.int64), 1, order)
+ sorted_weights = torch.gather(weights.to(torch.float32), 1, order)
+ sorted_valid = sorted_ids >= 0
+ return (
+ torch.where(sorted_valid, sorted_ids, torch.full_like(sorted_ids, -1)),
+ sorted_weights.masked_fill(~sorted_valid, 0),
+ )
+
+
+def expert_packed_slot_map(
+ counts,
+ src_info,
+ layout_range,
+ *,
+ tokens_per_rank: int,
+ experts_per_rank: int,
+ world_size: int,
+) -> list[tuple[int, int, int]]:
+ """Decode and validate DeepEP's expert-packed receive metadata.
+
+ ``src_info`` stores a source-local token index. The source rank is carried by
+ the corresponding packed ``layout_range`` interval, so neither field is
+ independently sufficient to identify a source token.
+ """
+ if tokens_per_rank <= 0 or experts_per_rank <= 0 or world_size <= 0:
+ raise ValueError("expert-packed dimensions must be positive")
+ if len(counts) != experts_per_rank:
+ raise ValueError("expert-packed count shape differs from local experts")
+ if len(src_info) != experts_per_rank or len(layout_range) != experts_per_rank:
+ raise ValueError("expert-packed metadata shape differs from local experts")
+
+ mask = (1 << 32) - 1
+ slots: list[tuple[int, int, int]] = []
+ pairs: set[tuple[int, int]] = set()
+ for local_expert in range(experts_per_rank):
+ count = counts[local_expert]
+ if type(count) is not int or count < 0:
+ raise ValueError("expert-packed receive count is invalid")
+ if len(layout_range[local_expert]) != world_size:
+ raise ValueError("expert-packed layout rank dimension is invalid")
+ if len(src_info[local_expert]) < count:
+ raise ValueError("expert-packed source metadata is truncated")
+
+ covered = [False] * count
+ for source_rank, encoded in enumerate(layout_range[local_expert]):
+ if type(encoded) is not int or encoded < 0:
+ raise ValueError("expert-packed layout range is invalid")
+ begin, span = encoded >> 32, encoded & mask
+ if begin > count or begin + span > count:
+ raise ValueError("expert-packed layout range exceeds valid slots")
+ for packed_position in range(begin, begin + span):
+ if covered[packed_position]:
+ raise ValueError("expert-packed layout ranges overlap")
+ covered[packed_position] = True
+ local_source = src_info[local_expert][packed_position]
+ if (
+ type(local_source) is not int
+ or local_source < 0
+ or local_source >= tokens_per_rank
+ ):
+ raise ValueError("expert-packed source token index is invalid")
+ source_id = source_rank * tokens_per_rank + local_source
+ pair = (source_id, local_expert)
+ if pair in pairs:
+ raise ValueError("expert-packed source/expert assignment is duplicated")
+ pairs.add(pair)
+ slots.append((local_expert, packed_position, source_id))
+ if not all(covered):
+ raise ValueError("expert-packed layout ranges omit valid receive slots")
+ return slots
+
+
+def expert_packed_dispatch_view(
+ torch,
+ packed_payload,
+ packed_counts,
+ packed_src_info,
+ packed_layout_range,
+ *,
+ rank: int,
+ tokens_per_rank: int,
+ experts_per_rank: int,
+ world_size: int,
+):
+ """Return the valid expert-packed rows with exact global source identities."""
+ if packed_payload.ndim != 3:
+ raise ValueError("expert-packed payload must have shape [experts, slots, hidden]")
+ if packed_payload.shape[0] != experts_per_rank:
+ raise ValueError("expert-packed payload expert dimension is invalid")
+ if tuple(packed_counts.shape) != (experts_per_rank,):
+ raise ValueError("expert-packed count tensor shape is invalid")
+ if tuple(packed_src_info.shape[:1]) != (experts_per_rank,):
+ raise ValueError("expert-packed source tensor shape is invalid")
+ if tuple(packed_layout_range.shape) != (experts_per_rank, world_size):
+ raise ValueError("expert-packed layout tensor shape is invalid")
+ if packed_src_info.ndim != 2 or packed_src_info.shape[1] < packed_payload.shape[1]:
+ raise ValueError("expert-packed source tensor capacity is invalid")
+
+ counts = [int(value) for value in packed_counts.detach().cpu().tolist()]
+ if any(count > packed_payload.shape[1] for count in counts):
+ raise ValueError("expert-packed receive count exceeds payload capacity")
+ slots = expert_packed_slot_map(
+ counts,
+ packed_src_info.detach().cpu().tolist(),
+ packed_layout_range.detach().cpu().tolist(),
+ tokens_per_rank=tokens_per_rank,
+ experts_per_rank=experts_per_rank,
+ world_size=world_size,
+ )
+ device = packed_payload.device
+ local_expert_slots = torch.tensor(
+ [slot[0] for slot in slots], device=device, dtype=torch.int64
+ )
+ packed_positions = torch.tensor(
+ [slot[1] for slot in slots], device=device, dtype=torch.int64
+ )
+ source_ids = torch.tensor(
+ [slot[2] for slot in slots], device=device, dtype=torch.int64
+ )
+ expert_ids = local_expert_slots + rank * experts_per_rank
+ payload = packed_payload[local_expert_slots, packed_positions]
+ return types.SimpleNamespace(
+ payload=payload,
+ source_ids=source_ids,
+ expert_ids=expert_ids,
+ local_expert_counts=packed_counts.to(torch.int64),
+ local_expert_slots=local_expert_slots,
+ packed_positions=packed_positions,
+ ordering_contract="expert-major/layout-addressed-packed-slot-v1",
+ )
+
+
+def _expert_transform(torch, payload, expert_ids, weights, combine_weight_semantics):
+ """Build one local expert aggregate for the v1 unweighted combine contract."""
+ if combine_weight_semantics != "unweighted-rank-sum":
+ raise ValueError("v1 requires unweighted rank-sum combine")
+ valid = expert_ids >= 0
+ expert = expert_ids.clamp(min=0).to(torch.int64)
+ gate = weights.to(torch.float32).masked_fill(~valid, 0)
+ scale = ((expert * 17 + 5) % 31 + 1).to(torch.float32) / 32
+ offset_a = (((expert * 29 + 7) % 37) - 18).to(torch.float32) / 64
+ offset_b = (((expert * 43 + 11) % 41) - 20).to(torch.float32) / 128
+ scale_sum = (gate * scale).sum(dim=1, keepdim=True)
+ offset_a_sum = (gate * offset_a).sum(dim=1, keepdim=True)
+ offset_b_sum = (gate * offset_b).sum(dim=1, keepdim=True)
+ columns = torch.arange(payload.shape[1], device=payload.device, dtype=torch.int64)
+ pattern = (((columns * 13) % 17) - 8).to(torch.float32) / 8
+ transformed = (
+ payload.float() * scale_sum + offset_a_sum + offset_b_sum * pattern.unsqueeze(0)
+ )
+ return transformed.to(payload.dtype)
+
+
+def _expert_transform_expanded(torch, payload, expert_ids):
+ """Apply the oracle transform to one row per token/expert assignment."""
+ expert = expert_ids.to(torch.int64)
+ scale = (((expert * 17 + 5) % 31 + 1).to(torch.float32) / 32).unsqueeze(1)
+ offset_a = ((((expert * 29 + 7) % 37) - 18).to(torch.float32) / 64).unsqueeze(1)
+ offset_b = ((((expert * 43 + 11) % 41) - 20).to(torch.float32) / 128).unsqueeze(1)
+ columns = torch.arange(payload.shape[1], device=payload.device, dtype=torch.int64)
+ pattern = (((columns * 13) % 17) - 8).to(torch.float32) / 8
+ transformed = payload.float() * scale + offset_a + offset_b * pattern.unsqueeze(0)
+ return transformed.to(payload.dtype)
+
+
+def _expected_transformed_combine(torch, problem):
+ """Independently derive sum_i gate_i * expert_i(x) for each source token."""
+ expected = torch.zeros_like(problem.x, dtype=torch.float32)
+ expert_ids = problem.topk_idx.to(torch.int64)
+ weights = problem.topk_weights.to(torch.float32)
+ columns = torch.arange(problem.x.shape[1], device=problem.x.device, dtype=torch.int64)
+ pattern = (((columns * 13) % 17) - 8).to(torch.float32) / 8
+ for slot in range(expert_ids.shape[1]):
+ expert = expert_ids[:, slot]
+ gate = weights[:, slot].unsqueeze(1)
+ scale = (((expert * 17 + 5) % 31 + 1).to(torch.float32) / 32).unsqueeze(1)
+ offset_a = ((((expert * 29 + 7) % 37) - 18).to(torch.float32) / 64).unsqueeze(1)
+ offset_b = ((((expert * 43 + 11) % 41) - 20).to(torch.float32) / 128).unsqueeze(1)
+ expert_output = problem.x.float() * scale + offset_a + offset_b * pattern.unsqueeze(0)
+ expected.add_(gate * expert_output)
+ return expected
+
+
+def _run_expert_packed_oracle(
+ torch,
+ routing,
+ backend,
+ problem,
+ global_idx,
+ global_weights,
+ rank: int,
+ experts_per_rank: int,
+ seed: int,
+):
+ """Verify an expert-packed dispatch and native gate-weighted combine."""
+ contract = LOW_LATENCY_ORACLE_CONTRACT
+ handle = backend.dispatch(problem)
+ torch.cuda.synchronize()
+ try:
+ packed = backend.inspect_expert_dispatch(problem, handle)
+ view = expert_packed_dispatch_view(
+ torch,
+ packed.payload,
+ packed.local_expert_counts,
+ packed.source_info,
+ packed.layout_range,
+ rank=rank,
+ tokens_per_rank=problem.T,
+ experts_per_rank=experts_per_rank,
+ world_size=backend.world_size,
+ )
+ decoded_source_ids = routing.decode_source_ids(view.payload, seed)
+ except Exception as inspection_error:
+ try:
+ problem.recv_tokens = backend.recv_tokens(handle)
+ backend.stage(problem, handle)
+ backend.combine(problem, handle)
+ torch.cuda.synchronize()
+ except Exception as cleanup_error:
+ raise inspection_error from cleanup_error
+ return {
+ "contract": contract,
+ "passed": False,
+ "ordering_contract": "adapter-inspection-failed",
+ "order_sha256": None,
+ "dispatch_sha256": None,
+ "combine_weight_semantics": getattr(
+ backend, "combine_weight_semantics", "undeclared"
+ ),
+ "receive_count": 0,
+ "atol": ORACLE_ATOL,
+ "max_absolute_error": None,
+ "max_elementwise_relative_error": None,
+ "max_relative_error": None,
+ "max_weight_error": None,
+ "rtol": ORACLE_RTOL,
+ "checks": {
+ "combine_values": False,
+ "counts": False,
+ "metadata": False,
+ "multiplicity": False,
+ "payload": False,
+ "source_set": False,
+ "weights": False,
+ },
+ }
+
+ device = problem.x.device
+ world_size = backend.world_size
+ total_experts = experts_per_rank * world_size
+ global_idx_device = global_idx.to(device=device, dtype=torch.int64)
+ global_weights_device = global_weights.to(device=device, dtype=torch.float32)
+ source_grid = torch.arange(
+ global_idx_device.shape[0], device=device, dtype=torch.int64
+ ).unsqueeze(1).expand_as(global_idx_device)
+ local_mask = (global_idx_device // experts_per_rank) == rank
+ expected_sources = source_grid[local_mask]
+ expected_experts = global_idx_device[local_mask]
+ expected_pair_weights = global_weights_device[local_mask]
+
+ receive_count = int(view.payload.shape[0])
+ shape_ok = (
+ view.payload.ndim == 2
+ and view.source_ids.shape == (receive_count,)
+ and view.expert_ids.shape == (receive_count,)
+ and view.local_expert_counts.shape == (experts_per_rank,)
+ )
+ source_range = bool(
+ receive_count == 0
+ or (
+ (view.source_ids >= 0)
+ & (view.source_ids < global_idx_device.shape[0])
+ ).all().item()
+ )
+ expected_payload = (
+ routing.activations_for_source_ids(
+ view.source_ids, problem.x.shape[1], seed, problem.x.dtype
+ )
+ if source_range
+ else torch.empty_like(view.payload)
+ )
+ payload_ok = bool(
+ source_range
+ and torch.equal(decoded_source_ids.to(torch.int64), view.source_ids)
+ and torch.equal(view.payload, expected_payload)
+ )
+
+ actual_keys = view.source_ids * total_experts + view.expert_ids
+ expected_keys = expected_sources * total_experts + expected_experts
+ actual_order = torch.argsort(actual_keys, stable=True)
+ expected_order = torch.argsort(expected_keys, stable=True)
+ canonical_sources = view.source_ids.index_select(0, actual_order)
+ canonical_experts = view.expert_ids.index_select(0, actual_order)
+ canonical_expected_weights = expected_pair_weights.index_select(0, expected_order)
+ expected_local_idx = global_idx_device[
+ rank * problem.T:(rank + 1) * problem.T
+ ]
+ metadata_ok = bool(
+ shape_ok
+ and torch.equal(problem.topk_idx.to(torch.int64), expected_local_idx)
+ and torch.equal(
+ actual_keys.index_select(0, actual_order),
+ expected_keys.index_select(0, expected_order),
+ )
+ )
+ expected_counts = torch.bincount(
+ expected_experts - rank * experts_per_rank, minlength=experts_per_rank
+ )
+ counts_ok = torch.equal(
+ view.local_expert_counts.to(torch.int64), expected_counts.to(torch.int64)
+ )
+ actual_multiplicity = torch.bincount(
+ view.source_ids, minlength=global_idx_device.shape[0]
+ )
+ expected_multiplicity = torch.bincount(
+ expected_sources, minlength=global_idx_device.shape[0]
+ )
+ multiplicity_ok = torch.equal(actual_multiplicity, expected_multiplicity)
+ source_set_ok = torch.equal(
+ torch.sort(torch.unique(view.source_ids)).values,
+ torch.sort(torch.unique(expected_sources)).values,
+ )
+
+ expected_local_weights = global_weights_device[
+ rank * problem.T:(rank + 1) * problem.T
+ ]
+ if problem.topk_weights.shape == expected_local_weights.shape:
+ max_weight_error = (
+ float((problem.topk_weights.float() - expected_local_weights).abs().max().item())
+ if expected_local_weights.numel()
+ else 0.0
+ )
+ else:
+ max_weight_error = None
+ weights_ok = max_weight_error == 0.0
+ ordering_contract = f"canonical-source-expert-v1/{view.ordering_contract}"
+ order_sha256 = _tensor_sha256(canonical_sources, canonical_experts)
+ dispatch_sha256 = _tensor_sha256(
+ canonical_sources, canonical_experts, canonical_expected_weights
+ )
+
+ handle.oracle_local_expert_slots = view.local_expert_slots
+ handle.oracle_packed_positions = view.packed_positions
+ problem.recv_tokens = receive_count
+ transformed = _expert_transform_expanded(torch, view.payload, view.expert_ids)
+ combined = backend.combine_transformed(problem, handle, transformed)
+ torch.cuda.synchronize()
+ expected_combined = _expected_transformed_combine(torch, problem)
+ if combined.shape == expected_combined.shape and combined.numel():
+ absolute_error = (combined.float() - expected_combined).abs()
+ max_absolute_error = float(absolute_error.max().item())
+ max_relative_error = max_absolute_error / (
+ float(expected_combined.abs().max().item()) + 1e-6
+ )
+ max_elementwise_relative_error = float(
+ (absolute_error / expected_combined.abs().clamp_min(ORACLE_ATOL)).max().item()
+ )
+ combine_values_ok = bool(torch.allclose(
+ combined.float(), expected_combined, rtol=ORACLE_RTOL, atol=ORACLE_ATOL
+ ))
+ elif combined.shape == expected_combined.shape:
+ max_absolute_error = 0.0
+ max_elementwise_relative_error = 0.0
+ max_relative_error = 0.0
+ combine_values_ok = True
+ else:
+ max_absolute_error = None
+ max_elementwise_relative_error = None
+ max_relative_error = None
+ combine_values_ok = False
+ tolerance = float(getattr(backend, "tolerance", ORACLE_RTOL))
+ checks = {
+ "combine_values": combine_values_ok,
+ "counts": counts_ok,
+ "metadata": metadata_ok,
+ "multiplicity": multiplicity_ok,
+ "payload": payload_ok,
+ "source_set": source_set_ok,
+ "weights": weights_ok,
+ }
+ return {
+ "contract": contract,
+ "passed": bool(
+ all(checks.values())
+ and ordering_contract
+ and max_relative_error is not None
+ and max_relative_error < tolerance
+ ),
+ "atol": ORACLE_ATOL,
+ "combine_weight_semantics": backend.combine_weight_semantics,
+ "ordering_contract": ordering_contract,
+ "order_sha256": order_sha256,
+ "dispatch_sha256": dispatch_sha256,
+ "receive_count": receive_count,
+ "max_absolute_error": max_absolute_error,
+ "max_elementwise_relative_error": max_elementwise_relative_error,
+ "max_relative_error": max_relative_error,
+ "max_weight_error": max_weight_error,
+ "rtol": ORACLE_RTOL,
+ "checks": checks,
+ }
+
+
+def _run_expert_oracle(
+ torch,
+ routing,
+ backend,
+ problem,
+ global_idx,
+ global_weights,
+ rank: int,
+ experts_per_rank: int,
+ seed: int,
+):
+ """Verify one real dispatch/transform/combine without entering a timed region."""
+ if getattr(backend, "oracle_layout", "token-rank") == "expert-packed":
+ return _run_expert_packed_oracle(
+ torch,
+ routing,
+ backend,
+ problem,
+ global_idx,
+ global_weights,
+ rank,
+ experts_per_rank,
+ seed,
+ )
+ handle = backend.dispatch(problem)
+ torch.cuda.synchronize()
+ try:
+ view = backend.inspect_dispatch(problem, handle)
+ source_ids = routing.decode_source_ids(view.payload, seed)
+ except Exception as inspection_error:
+ try:
+ problem.recv_tokens = backend.recv_tokens(handle)
+ backend.stage(problem, handle)
+ backend.combine(problem, handle)
+ torch.cuda.synchronize()
+ except Exception as cleanup_error:
+ raise inspection_error from cleanup_error
+ return {
+ "contract": ORACLE_CONTRACT,
+ "passed": False,
+ "ordering_contract": "adapter-inspection-failed",
+ "order_sha256": None,
+ "dispatch_sha256": None,
+ "combine_weight_semantics": getattr(
+ backend, "combine_weight_semantics", "undeclared"
+ ),
+ "receive_count": 0,
+ "atol": ORACLE_ATOL,
+ "max_absolute_error": None,
+ "max_elementwise_relative_error": None,
+ "max_relative_error": None,
+ "max_weight_error": None,
+ "rtol": ORACLE_RTOL,
+ "checks": {
+ "combine_values": False,
+ "counts": False,
+ "metadata": False,
+ "multiplicity": False,
+ "payload": False,
+ "source_set": False,
+ "weights": False,
+ },
+ }
+
+ receive_count = int(view.payload.shape[0])
+ shape_ok = (
+ view.payload.ndim == 2
+ and view.expert_ids.shape == (receive_count, problem.topk_idx.shape[1])
+ and view.weights.shape == view.expert_ids.shape
+ )
+ source_range = bool(
+ receive_count == 0
+ or ((source_ids >= 0) & (source_ids < global_idx.shape[0])).all().item()
+ )
+ if source_range:
+ expected_idx = global_idx.to(problem.x.device).index_select(0, source_ids)
+ expected_weights = global_weights.to(problem.x.device).index_select(0, source_ids)
+ local = (expected_idx // experts_per_rank) == rank
+ expected_ids = torch.where(local, expected_idx, torch.full_like(expected_idx, -1))
+ expected_weights = expected_weights.masked_fill(~local, 0)
+ expected_payload = routing.activations_for_source_ids(
+ source_ids, problem.x.shape[1], seed, problem.x.dtype
+ )
+ else:
+ expected_ids = torch.full_like(view.expert_ids, -1)
+ expected_weights = torch.zeros_like(view.weights)
+ expected_payload = torch.empty_like(view.payload)
+ actual_ids, actual_weights = _normalized_expert_metadata(
+ torch, view.expert_ids, view.weights
+ )
+ expected_ids, expected_weights = _normalized_expert_metadata(
+ torch, expected_ids, expected_weights
+ )
+ expected_sources = (
+ ((global_idx // experts_per_rank) == rank).any(dim=1).nonzero(as_tuple=True)[0]
+ ).to(problem.x.device)
+ source_set_ok = (
+ source_range
+ and source_ids.numel() == torch.unique(source_ids).numel()
+ and torch.equal(torch.sort(source_ids).values, expected_sources)
+ )
+ payload_ok = source_range and torch.equal(view.payload, expected_payload)
+ metadata_ok = shape_ok and torch.equal(actual_ids, expected_ids)
+ max_weight_error = (
+ float((actual_weights - expected_weights).abs().max().item())
+ if actual_weights.numel()
+ else 0.0
+ )
+ weights_ok = max_weight_error == 0.0
+ valid_expected = expected_ids >= 0
+ expected_local = expected_ids[valid_expected] - rank * experts_per_rank
+ expected_counts = torch.bincount(expected_local, minlength=experts_per_rank)
+ counts_ok = torch.equal(
+ view.local_expert_counts.to(torch.int64), expected_counts.to(torch.int64)
+ )
+ multiplicity_ok = torch.equal(
+ (actual_ids >= 0).sum(dim=1), (expected_ids >= 0).sum(dim=1)
+ )
+ # Receive-slot assignment may use atomics and is not a semantic EP guarantee. Compare
+ # pre/post dispatch evidence in canonical source-token order without changing the native path.
+ canonical_order = torch.argsort(source_ids.to(torch.int64), stable=True)
+ canonical_sources = source_ids.to(torch.int64).index_select(0, canonical_order)
+ canonical_ids = actual_ids.to(torch.int64).index_select(0, canonical_order)
+ canonical_weights = actual_weights.index_select(0, canonical_order)
+ ordering_contract = f"canonical-source-id-v1/{view.ordering_contract}"
+ order_sha256 = _tensor_sha256(canonical_sources)
+ dispatch_sha256 = _tensor_sha256(
+ canonical_sources, canonical_ids, canonical_weights
+ )
+
+ problem.recv_tokens = receive_count
+ combine_weight_semantics = backend.combine_weight_semantics
+ transformed = _expert_transform(
+ torch, view.payload, actual_ids, actual_weights, combine_weight_semantics
+ )
+ combined = backend.combine_transformed(problem, handle, transformed)
+ torch.cuda.synchronize()
+ expected_combined = _expected_transformed_combine(torch, problem)
+ if combined.shape == expected_combined.shape and combined.numel():
+ absolute_error = (combined.float() - expected_combined).abs()
+ max_absolute_error = float(absolute_error.max().item())
+ max_relative_error = max_absolute_error / (
+ float(expected_combined.abs().max().item()) + 1e-6
+ )
+ max_elementwise_relative_error = float(
+ (absolute_error / expected_combined.abs().clamp_min(ORACLE_ATOL)).max().item()
+ )
+ combine_values_ok = bool(torch.allclose(
+ combined.float(), expected_combined, rtol=ORACLE_RTOL, atol=ORACLE_ATOL
+ ))
+ elif combined.shape == expected_combined.shape:
+ max_absolute_error = 0.0
+ max_elementwise_relative_error = 0.0
+ max_relative_error = 0.0
+ combine_values_ok = True
+ else:
+ max_absolute_error = None
+ max_elementwise_relative_error = None
+ max_relative_error = None
+ combine_values_ok = False
+ tolerance = float(getattr(backend, "tolerance", 5e-2))
+ checks = {
+ "combine_values": combine_values_ok,
+ "counts": counts_ok,
+ "metadata": metadata_ok,
+ "multiplicity": multiplicity_ok,
+ "payload": payload_ok,
+ "source_set": source_set_ok,
+ "weights": weights_ok,
+ }
+ return {
+ "contract": ORACLE_CONTRACT,
+ "passed": bool(
+ all(checks.values())
+ and ordering_contract
+ and max_relative_error is not None
+ and max_relative_error < tolerance
+ ),
+ "atol": ORACLE_ATOL,
+ "combine_weight_semantics": combine_weight_semantics,
+ "ordering_contract": ordering_contract,
+ "order_sha256": order_sha256,
+ "dispatch_sha256": dispatch_sha256,
+ "receive_count": receive_count,
+ "max_absolute_error": max_absolute_error,
+ "max_elementwise_relative_error": max_elementwise_relative_error,
+ "max_relative_error": max_relative_error,
+ "max_weight_error": max_weight_error,
+ "rtol": ORACLE_RTOL,
+ "checks": checks,
+ }
+
+
+def _histogram(xs: list[float], nbins: int = 40) -> dict:
+ """Compact equal-width summary of the exact private cross-rank-max samples."""
+ if not xs:
+ return {"n": 0}
+ lo, hi = min(xs), max(xs)
+ if hi <= lo:
+ return {"n": len(xs), "min": lo, "max": hi, "bins": nbins, "counts": [len(xs)]}
+ counts = [0] * nbins
+ span = hi - lo
+ for x in xs:
+ b = min(nbins - 1, int((x - lo) / span * nbins))
+ counts[b] += 1
+ return {"n": len(xs), "min": round(lo, 3), "max": round(hi, 3), "bins": nbins, "counts": counts}
+
+
+def _derive_publication_status(v: dict) -> str:
+ """Classify raw attempts; only the isolated coverage publisher may promote evidence."""
+ if v["execution_status"] != "complete":
+ return "failed"
+ if v["semantic_correctness"] != "pass" or v["measurement_conformance"] != "conformant" \
+ or v["workload_identity"] == "inconsistent":
+ return "invalid"
+ # Per-case producers cannot prove exact matrix coverage, repeat stability, or controlled
+ # cohorts. Keep even sound attempts diagnostic until the isolated publisher validates them.
+ return "diagnostic"
+
+
+def run_sweep(args, backend, torch, dist, device, rank: int, world_size: int) -> int:
+ """Drive the source-tokens-per-rank sweep for one fully-specified line."""
+ mode = getattr(args, "mode", "normal")
+ try:
+ case_profile = identity.case_profile(mode)
+ except identity.IdentityError as exc:
+ if rank == 0:
+ print(f"ERROR: {exc}")
+ return 2
+ sampling_error = sampling_contract_error(args.iters, args.trials, args.warmup)
+ if sampling_error:
+ if rank == 0:
+ print(f"ERROR: {sampling_error}")
+ return 2
+ import routing # torch-based; imported lazily so the module byte-compiles without torch
+ import eplb # stdlib planner + torch remap (the EPLB transform)
+
+ ep_size = world_size
+ # EPLB (if on): run_ep.py already bumped args.experts to the PHYSICAL count and stashed the
+ # logical count, so experts_per_rank below is physical. The trace is built over LOGICAL
+ # experts then remapped to physical (build_trace), so the whole sweep runs over the
+ # balanced physical placement with no adapter change.
+ eplb_on = getattr(args, "eplb", False)
+ num_logical = getattr(args, "num_logical_experts", args.experts)
+ if args.experts % ep_size != 0:
+ if rank == 0:
+ print(f"ERROR: experts ({args.experts}) must divide ep_size ({ep_size})")
+ return 2
+ experts_per_rank = args.experts // ep_size
+ if getattr(backend, "mode", None) != mode:
+ if rank == 0:
+ print(f"ERROR: backend mode {getattr(backend, 'mode', None)!r} != {mode!r}")
+ return 2
+ expected_weight_semantics = (
+ "gate-weighted-sum"
+ if case_profile["combine_semantics"] == "gate-weighted"
+ else "unweighted-rank-sum"
+ )
+ if getattr(backend, "combine_weight_semantics", None) != expected_weight_semantics:
+ if rank == 0:
+ print(
+ f"ERROR: {mode} requires combine semantics {expected_weight_semantics}"
+ )
+ return 2
+ if mode == LOW_LATENCY_MODE and (
+ args.phase != "decode"
+ or getattr(backend, "oracle_layout", None) != "expert-packed"
+ or getattr(backend, "payload_unit", None) != "token-expert"
+ ):
+ if rank == 0:
+ print("ERROR: low-latency requires decode expert-packed token-expert execution")
+ return 2
+
+ cap = backend.buffer_cap(args)
+ conditioning_ladder = CONDITIONING_LADDERS[args.phase]
+ if cap is not None and cap < conditioning_ladder[-1]:
+ if rank == 0:
+ print(f"ERROR: {backend.name} buffer cap {cap} cannot run the v1 conditioning ladder")
+ return 2
+ ladder, dropped = token_ladder(args.tokens_ladder, args.phase, cap)
+ if rank == 0 and dropped:
+ print(f"NOTE: dropped tokens/rank {dropped} — exceed {backend.name} buffer cap {cap} "
+ f"(hidden={args.hidden}); not silently truncated.")
+ if not ladder:
+ if rank == 0:
+ print(f"ERROR: empty token ladder (phase={args.phase}, cap={cap})")
+ return 2
+ MAX, MIN, SUM = dist.ReduceOp.MAX, dist.ReduceOp.MIN, dist.ReduceOp.SUM
+
+ # EPLB plan (once): estimate logical load from the global logical trace at the largest
+ # ladder T (most samples), then replicate+place. Held fixed across all T (as real EPLB
+ # plans from an observed load estimate). build_trace builds the LOGICAL trace and remaps
+ # to physical when the plan is present; otherwise it's the identity (logical == physical).
+ eplb_plan = None
+ if eplb_on:
+ ref_idx, _ = routing.build_global_routing(
+ EPLB_REFERENCE_TOKENS_PER_RANK * ep_size,
+ num_logical,
+ args.topk,
+ args.routing,
+ args.seed,
+ )
+ load = torch.bincount(ref_idx.reshape(-1), minlength=num_logical).float().tolist()
+ eplb_plan = eplb.build_plan(load, args.experts, ep_size)
+ if rank == 0:
+ print(f"NOTE: EPLB {num_logical}->{args.experts} experts ({ep_size}x{experts_per_rank}); "
+ f"per-rank load imbalance {eplb_plan['imbalance_before']:.2f}x -> "
+ f"{eplb_plan['imbalance_after']:.2f}x; {eplb_plan['replicated_experts']} experts "
+ f"replicated (hottest {eplb_plan['max_replicas']}x)")
+
+ canonical = bool(getattr(args, "workload_dir", ""))
+ loaded_workload_ids, loaded_checksums = [], {}
+ if canonical:
+ import workload as _wl
+
+ def build_trace(gt):
+ # canonical: load pre-serialized trace bytes (verified by checksum) so this run is
+ # provably the SAME workload as any other consuming the same files. else: seeded gen.
+ if canonical:
+ wid = _wl.compute_workload_id(
+ args.routing, args.hidden, args.topk, num_logical, ep_size, gt, args.seed
+ )
+ idx_np, w_np, man = _wl.load_workload(os.path.join(args.workload_dir, f"{wid}.npz"), verify=True)
+ idx_l = torch.from_numpy(idx_np).to(torch.int64)
+ w = torch.from_numpy(w_np).to(torch.float32)
+ if wid not in loaded_workload_ids:
+ loaded_workload_ids.append(wid)
+ loaded_checksums[wid] = man.get("checksums")
+ else:
+ idx_l, w = routing.build_global_routing(
+ gt, num_logical, args.topk, args.routing, args.seed
+ )
+ return (eplb.remap_idx(idx_l, eplb_plan) if eplb_plan is not None else idx_l), w
+
+ # Fabric/clock warm-up BEFORE any timed point (review: H200 had an anomalous cold
+ # first point and a 40% decode-vs-prefill mismatch at the shared T=128). Gradually
+ # ramp through the small ladder shapes untimed — warms clocks/fabric for everyone
+ # and is also cold-jump-safe for MoRI.
+ def warm_roundtrips(problem, count):
+ for _ in range(count):
+ handle = backend.dispatch(problem)
+ if not hasattr(problem, "recv_tokens"):
+ # Dynamic receive cardinality is stable for this fixed routing trace. Cache it
+ # during untimed conditioning so adapters never read a device scalar in timing.
+ problem.recv_tokens = backend.recv_tokens(handle)
+ backend.stage(problem, handle)
+ backend.combine(problem, handle)
+ torch.cuda.synchronize()
+
+ for wt in conditioning_ladder:
+ # Warm-only shapes need not have canonical manifests: they are never measured or emitted.
+ wi, ww = routing.build_global_routing(
+ wt * ep_size, num_logical, args.topk, args.routing, args.seed,
+ )
+ if eplb_plan is not None:
+ wi = eplb.remap_idx(wi, eplb_plan)
+ wsi, wsw = routing.rank_slice(wi, ww, rank, wt)
+ wx = routing.rank_activations(wt, args.hidden, args.seed, rank, device, torch.bfloat16)
+ wp = backend.make_problem(wt, wsi.to(device), wsw.to(device), wx)
+ warm_roundtrips(wp, CONDITIONING_ROUNDS_PER_SHAPE)
+ torch.cuda.synchronize()
+ dist.barrier()
+ # Setup may materialize deferred provenance such as DeepEP V2 JIT CUBINs.
+ # Resolve it after conditioning but before correctness or timed measurements.
+ capture_deferred_provenance = getattr(backend, "capture_deferred_provenance", None)
+ if capture_deferred_provenance is not None:
+ capture_deferred_provenance()
+ provenance_issues = contracts.backend_provenance_issues(
+ backend.name, backend.backend_provenance
+ )
+ if provenance_issues:
+ if rank == 0:
+ print(
+ f"ERROR: unpinned provenance {provenance_issues} "
+ f"in {backend.backend_provenance}"
+ )
+ return 4
+ elem_dispatch = BF16_BYTES
+
+ # ---- Pass 1: build each deterministic problem and run the expert oracle. ----
+ problems, gate, gts, global_traces, input_snapshots = {}, {}, {}, {}, {}
+ routing_hashes = set()
+ for T in ladder:
+ counts = [T] * ep_size
+ gt = T * ep_size
+ gts[T] = gt
+ idx_g, w_g = build_trace(gt)
+ rstats = routing.routing_stats(idx_g, args.experts, experts_per_rank, weights=w_g)
+ gpn = args.gpus_per_node or ep_size
+ rstats["locality"] = routing.routing_locality(idx_g, experts_per_rank, ep_size, max(1, T),
+ gpn, args.scale_up_domain or None)
+ rstats["source_token_stats"] = _stats_vec(counts)
+ routing_hashes.add(rstats["routing_hash"])
+ my_off, my_cnt = rank * T, T
+ idx_s = idx_g[my_off:my_off + my_cnt].contiguous()
+ w_s = w_g[my_off:my_off + my_cnt].contiguous()
+ x = routing.rank_activations(my_cnt, args.hidden, args.seed, rank, device, torch.bfloat16)
+ problem = backend.make_problem(my_cnt, idx_s.to(device), w_s.to(device), x)
+ input_snapshots[T] = (
+ problem.x.clone(), problem.topk_idx.clone(), problem.topk_weights.clone()
+ )
+ oracle = _run_expert_oracle(
+ torch, routing, backend, problem, idx_g, w_g, rank, experts_per_rank,
+ args.seed,
+ )
+ before_x, before_idx, before_weights = input_snapshots[T]
+ pre_input_unchanged = (
+ torch.equal(problem.x, before_x)
+ and torch.equal(problem.topk_idx, before_idx)
+ and torch.equal(problem.topk_weights, before_weights)
+ )
+ problems[T] = problem
+ global_traces[T] = (idx_g, w_g)
+ gate[T] = {
+ "rstats": rstats,
+ "recv_local": oracle["receive_count"],
+ "max_rel": oracle["max_relative_error"] or 0.0,
+ "local_ok": int(oracle["passed"]),
+ "oracle_pre": oracle,
+ "pre_input_unchanged": pre_input_unchanged,
+ }
+
+ # ---- Pass 2: every backend uses the same ascending point order and conditioning ramp.
+ # Per-iteration cross-rank MAX samples are pooled across trials. ----
+ disp_pool = {T: [] for T in ladder} # pooled per-iteration cross-rank MAX (dispatch)
+ comb_pool = {T: [] for T in ladder} # ... combine
+ rt_pool = {T: [] for T in ladder} # independently measured round trip
+ disp_trials = {T: [] for T in ladder}
+ comb_trials = {T: [] for T in ladder}
+ rt_trials = {T: [] for T in ladder}
+ order = list(ladder)
+ for _trial in range(args.trials):
+ for T in order:
+ problem = problems[T]
+ # Stateful paired APIs may expose only a measured round trip.
+ # Do not synthesize component latency from that measurement.
+ roundtrip_only = getattr(backend, "roundtrip_only", False)
+
+ def rt_once(p=problem):
+ hh = backend.dispatch(p)
+ backend.stage(p, hh)
+ return backend.combine(p, hh)
+
+ # Every available component starts after the same synchronized full-roundtrip warmup.
+ # Roundtrip is first on every backend because it is the comparison headline.
+ warm_roundtrips(problem, args.warmup)
+ rt_iters = time_us(torch, lambda p=problem: rt_once(p), 0, args.iters)
+ if roundtrip_only:
+ disp_iters = comb_iters = []
+ else:
+ warm_roundtrips(problem, args.warmup)
+ disp_iters = time_us(torch, lambda p=problem: backend.dispatch(p),
+ 0, args.iters)
+
+ def prep(p=problem):
+ hh = backend.dispatch(p)
+ backend.stage(p, hh)
+ return hh
+ warm_roundtrips(problem, args.warmup)
+ if backend.combine_needs_redispatch:
+ comb_iters = time_us(torch, lambda hh, p=problem: backend.combine(p, hh),
+ 0, args.iters, pre=prep)
+ else:
+ hh = prep()
+ torch.cuda.synchronize()
+ comb_iters = time_us(torch, lambda p=problem, hx=hh: backend.combine(p, hx),
+ 0, args.iters)
+ # per-iteration cross-rank MAX (the distributed-op latency per iter), pooled.
+ if disp_iters:
+ reduced_dispatch = _reduce_vec(torch, dist, device, disp_iters, MAX)
+ reduced_combine = _reduce_vec(torch, dist, device, comb_iters, MAX)
+ disp_trials[T].append(reduced_dispatch)
+ comb_trials[T].append(reduced_combine)
+ disp_pool[T] += reduced_dispatch
+ comb_pool[T] += reduced_combine
+ reduced_roundtrip = _reduce_vec(torch, dist, device, rt_iters, MAX)
+ rt_trials[T].append(reduced_roundtrip)
+ rt_pool[T] += reduced_roundtrip
+
+ # ---- Pass 3: prove timed inputs were immutable and repeat the full oracle. ----
+ for T in ladder:
+ problem = problems[T]
+ before_x, before_idx, before_weights = input_snapshots[T]
+ input_unchanged = gate[T]["pre_input_unchanged"] and (
+ torch.equal(problem.x, before_x)
+ and torch.equal(problem.topk_idx, before_idx)
+ and torch.equal(problem.topk_weights, before_weights)
+ )
+ idx_g, w_g = global_traces[T]
+ post = _run_expert_oracle(
+ torch, routing, backend, problem, idx_g, w_g, rank, experts_per_rank,
+ args.seed,
+ )
+ pre = gate[T]["oracle_pre"]
+ order_stable = (
+ pre["ordering_contract"] == post["ordering_contract"]
+ and pre["order_sha256"] == post["order_sha256"]
+ and pre["dispatch_sha256"] == post["dispatch_sha256"]
+ )
+ gate[T].update({
+ "input_unchanged": input_unchanged,
+ "local_ok": int(pre["passed"] and post["passed"] and input_unchanged and order_stable),
+ "max_rel": max(pre["max_relative_error"] or 0.0, post["max_relative_error"] or 0.0),
+ "oracle_post": post,
+ "order_stable": order_stable,
+ })
+
+ # ---- Pass 4: percentiles (p50/p90/p95/p99, nearest-rank) from pooled samples + bytes + row ----
+ def pcts(xs):
+ return ({"p50": percentile(xs, 50), "p90": percentile(xs, 90),
+ "p95": percentile(xs, 95), "p99": percentile(xs, 99)} if xs else None)
+
+ def component(percentiles, count, *, derived=False):
+ if percentiles is None:
+ return {"availability": "unavailable", "origin": None,
+ "percentiles_us": None, "sample_count": 0}
+ return {
+ "availability": "derived" if derived else "measured",
+ "origin": "derived-percentile-sum" if derived else "measured",
+ "percentiles_us": percentiles,
+ "sample_count": 0 if derived else count,
+ }
+ rows = []
+ all_anomalies = []
+ thr_rt = 3.0
+ for T in ladder:
+ gt = gts[T]
+ g = gate[T]
+ rstats = g["rstats"]
+ d, c, rt = disp_pool[T], comb_pool[T], rt_pool[T]
+ dp, cp, rtp = pcts(d), pcts(c), pcts(rt)
+ # isolated_sum = SUM of the isolated dispatch+combine percentiles. NOT a measured op
+ # (can't reveal shared sync / launch amortization / overlap) — do NOT use for throughput
+ # or SLO capacity. The MEASURED round trip (rtp) is the real chained latency.
+ isum = {key: dp[key] + cp[key] for key in dp} if dp and cp else None
+ recv_total = _reduce_int(torch, dist, device, g["recv_local"], SUM)
+ recv_max = _reduce_int(torch, dist, device, g["recv_local"], MAX)
+ recv_min = _reduce_int(torch, dist, device, g["recv_local"], MIN)
+ global_ok = _reduce_int(torch, dist, device, g["local_ok"], MIN)
+ max_rel = _reduce_vec(torch, dist, device, [g["max_rel"]], MAX)[0]
+ point_ok = bool(global_ok) and recv_total > 0
+ rank_evidence = [None] * world_size
+ dist.all_gather_object(
+ rank_evidence,
+ {
+ "input_unchanged": g["input_unchanged"],
+ "order_stable": g["order_stable"],
+ "post_timing": g["oracle_post"],
+ "pre_timing": g["oracle_pre"],
+ "rank": rank,
+ },
+ )
+ # Canonical LOGICAL payload byte contracts (from the routing trace, NOT backend recv
+ # tensors): token-rank = one copy per unique (token,dest-rank); token-expert = one copy
+ # per routed (token,expert). routed_copies = token-rank copies; gt*topk = token-expert.
+ token_rank_copies = rstats["routed_copies"]
+ logical_copies = (
+ sum(rstats["expert_assignments_per_rank"])
+ if case_profile["payload_unit"] == "token-expert"
+ else token_rank_copies
+ )
+ H = args.hidden
+ throughput = {
+ percentile_name: gt / (latency_us * 1e-6)
+ for percentile_name, latency_us in rtp.items()
+ }
+ disp_bytes_l = logical_copies * H * elem_dispatch
+ comb_bytes_l = logical_copies * H * BF16_BYTES
+ # Contract-level anomalies are attached to the row and rolled into validity.
+ # roundtrip_gt_isolated_sum: measured RT p99 >> Σ(isolated dispatch+combine) p99.
+ # roundtrip_lt_component_floor: measured RT p50 < max(dispatch,combine) p50 — a chained
+ # op can't finish faster than its slowest required component (sync semantics violated).
+ row_anoms = []
+ if isum and isum["p99"] > 0 and rtp["p99"] > thr_rt * isum["p99"]:
+ row_anoms.append({"type": "roundtrip_gt_isolated_sum", "T": T,
+ "roundtrip_p99": round(rtp["p99"], 2), "isolated_sum_p99": round(isum["p99"], 2),
+ "ratio": round(rtp["p99"] / isum["p99"], 2), "threshold": thr_rt})
+ floor = max(dp["p50"], cp["p50"]) if dp and cp else None
+ if floor and rtp["p50"] > 0 and rtp["p50"] < 0.95 * floor:
+ row_anoms.append({"type": "roundtrip_lt_component_floor", "T": T,
+ "roundtrip_p50": round(rtp["p50"], 2), "component_floor_p50": round(floor, 2)})
+ all_anomalies.extend(row_anoms)
+ rows.append({
+ "anomalies": row_anoms,
+ "components": {
+ "combine": component(cp, len(c)),
+ "dispatch": component(dp, len(d)),
+ "isolated_sum": component(isum, 0, derived=True),
+ "roundtrip": component(rtp, len(rt)),
+ },
+ "correctness": {
+ "contract": case_profile["oracle_contract"],
+ "max_relative_error": max_rel,
+ "passed": point_ok,
+ "rank_evidence": rank_evidence,
+ "scope": case_profile["correctness_scope"],
+ },
+ "global_tokens": gt,
+ "logical_bytes": {
+ "combine": comb_bytes_l,
+ "dispatch": disp_bytes_l,
+ "roundtrip": disp_bytes_l + comb_bytes_l,
+ },
+ "receive": {
+ "max": recv_max,
+ "mean": recv_total / world_size,
+ "min": recv_min,
+ "total": recv_total,
+ },
+ "routing": {
+ "empty_expert_count": rstats["empty_expert_count"],
+ "empty_rank_count": rstats["empty_rank_count"],
+ "expert_assignment_rank_cv": rstats["expert_assignment_rank_cv"],
+ "expert_assignments_per_rank": rstats["expert_assignments_per_rank"],
+ "expert_load_cv": rstats["expert_load_cv"],
+ "expert_load_max": rstats["expert_load_max"],
+ "expert_load_mean": rstats["expert_load_mean"],
+ "expert_load_min": rstats["expert_load_min"],
+ "fanout_histogram": rstats["fanout_hist"],
+ "fanout_max": rstats["fanout_max"],
+ "fanout_mean": rstats["fanout_mean"],
+ "fanout_min": rstats["fanout_min"],
+ "hash": rstats["routing_hash"],
+ "hotspot_ratio": rstats["hotspot_ratio"],
+ "locality": rstats.get("locality"),
+ "payload_copies_per_rank": rstats["payload_copies_per_rank"],
+ "payload_rank_cv": rstats["payload_rank_cv"],
+ "routed_copies": rstats["routed_copies"],
+ "source_token_stats": rstats.get("source_token_stats"),
+ },
+ "sample_histograms": {
+ "dispatch": _histogram(d) if d else None,
+ "combine": _histogram(c) if c else None,
+ "roundtrip": _histogram(rt),
+ },
+ "token_rate_at_latency_percentile": throughput,
+ "tokens_per_rank": T,
+ })
+ if rank == 0:
+ component_log = (f"disp p50/p99={dp['p50']:7.1f}/{dp['p99']:7.1f} "
+ f"comb {cp['p50']:6.1f}/{cp['p99']:6.1f} " if dp and cp
+ else "components=unavailable ")
+ print(f" T={T:<5} {component_log}"
+ f"RT p50/p99={rtp['p50']:7.1f}/{rtp['p99']:7.1f}us n={len(rt)} fanout={rstats['fanout_mean']:.2f} "
+ f"recv[min/mean/max]={recv_min}/{recv_total // world_size}/{recv_max} "
+ f"correct={point_ok}")
+
+ # Cross-rank workload-identity proof: every rank must have built the SAME global routing
+ # (one hash per T here); confirm all ranks agree by hashing the per-T hash set and
+ # MIN/MAX-reducing it — a mismatch means NVIDIA and AMD did NOT run identical routing.
+ trace_sig = hashlib.sha256("|".join(sorted(routing_hashes)).encode()).hexdigest()
+ routing_consistent = _same_hash_across_ranks(torch, dist, device, trace_sig)
+
+ # Capture again after correctness and timing so no lazily generated kernel can escape
+ # the implementation identity recorded in the artifact.
+ if capture_deferred_provenance is not None:
+ capture_deferred_provenance()
+
+ if rank != 0:
+ return 0
+
+ # status=valid requires correctness AND a proven-identical routing trace across ranks.
+ all_ok = bool(rows) and all(r["correctness"]["passed"] for r in rows) and routing_consistent
+
+ # Adapters never self-label official; status is derived from these gates.
+ prov = backend.backend_provenance
+ provenance_complete = contracts.provenance_complete(
+ prov,
+ backend.name,
+ getattr(args, "git_run", None),
+ image_digest=getattr(args, "image_digest", None),
+ image_verified=getattr(args, "image_digest_verified", False),
+ squash_sha256=getattr(args, "squash_sha256", None),
+ )
+ resource_profile = contracts.project_resource_profile(prov)
+ resource_conformance = resource_profile["conformance_class"]
+ # record the canonical workload identity consumed (one trace per T -> set of ids/checksums).
+ if canonical and loaded_workload_ids:
+ args.workload_id = identity.workload_id(
+ {
+ "members": [
+ {"checksums": loaded_checksums[member], "workload_id": member}
+ for member in sorted(loaded_workload_ids)
+ ]
+ }
+ )
+ args.workload_members = sorted(loaded_workload_ids)
+ args.workload_checksums = loaded_checksums
+ canonical_workload = bool(getattr(args, "workload_id", None))
+ activation_identity = workload_contract.compute_activation_identity(args.seed, args.hidden)
+ # EPLB identity covers replica placement, not only counts.
+ eplb_mapping_hash = None
+ if eplb_plan is not None:
+ eplb_mapping_hash = eplb.mapping_hash(eplb_plan)
+ anomaly_free = len(all_anomalies) == 0
+ validity = {
+ "execution_status": "complete" if rows else "failed",
+ "semantic_correctness": (
+ "pass" if rows and all(r["correctness"]["passed"] for r in rows) else "fail"
+ ),
+ "workload_identity": "consistent-across-ranks" if routing_consistent else "inconsistent",
+ "workload_source": "canonical-serialized" if canonical_workload else "seeded-runtime",
+ "measurement_conformance": "conformant", # run_ep gate rejects nonconformant pre-run
+ "sampling_conformance": "conformant", # fixed-512-v1 gate rejects any other profile
+ "resource_conformance": resource_conformance,
+ "provenance_complete": provenance_complete,
+ # anomaly-free unless a contract-level timing anomaly fired (then diagnostic, see above).
+ "anomaly_free": anomaly_free,
+ }
+ publication_status = _derive_publication_status(validity)
+
+ shape = { # FIXED line identity (no T, no per-backend resource knobs)
+ "hidden": args.hidden, "topk": args.topk, "experts": args.experts,
+ "experts_per_rank": experts_per_rank, "dispatch_dtype": "bf16",
+ "routing": args.routing, "eplb": bool(eplb_plan), "num_logical_experts": num_logical,
+ # V2 is reserved for the PR #605 ElasticBuffer adapter; package versions never imply it.
+ "kernel_gen": kernel_generation(backend),
+ "activation_profile": ACTIVATION_PROFILE,
+ "quant": {
+ "combine_input_dtype": "bf16",
+ "combine_accum_dtype": getattr(backend, "combine_accum_dtype", "fp32"),
+ "combine_output_dtype": "bf16", "combine_quant_mode": "none",
+ "scale_layout": None,
+ },
+ }
+ generated_at = args.timestamp or _dt.datetime.now().astimezone().isoformat()
+ realized_placement = getattr(args, "realized_placement", None)
+ nodes = (
+ realized_placement["nodes"]
+ if realized_placement is not None
+ else int(os.environ.get("SLURM_NNODES", "1"))
+ )
+ case_factors = {
+ "case": {
+ "backend": backend.name,
+ "canonical": canonical,
+ "eplb": bool(eplb_plan),
+ "ep": ep_size,
+ "experts": num_logical,
+ "gpus_per_node": args.gpus_per_node or ep_size,
+ "hidden": args.hidden,
+ "ladder": " ".join(map(str, ladder)),
+ "mode": mode,
+ "nodes": nodes,
+ "phase": args.phase,
+ "required_publication": args.required_publication or "diagnostic",
+ "routing": args.routing,
+ "samples_per_point": TIMED_SAMPLES_PER_POINT,
+ "scale_up_domain": args.scale_up_domain or (args.gpus_per_node or ep_size),
+ "scale_up_transport": args.scale_up_transport,
+ "scale_out_transport": args.scale_out_transport or None,
+ "scope": args.scope,
+ "suite": args.suite or "manual",
+ "timing": f"{args.iters}:{args.trials}:{args.warmup}",
+ "topk": args.topk,
+ "topology_class": args.topology_class,
+ "transport": args.transport,
+ "warmup_semantics": WARMUP_SEMANTICS,
+ "workload": args.workload_name or "manual",
+ },
+ "profile": case_profile,
+ "sku": args.runner,
+ }
+ computed_case_id = identity.digest("case", case_factors)
+ if args.case_id and args.case_id != computed_case_id:
+ raise ValueError(
+ f"scheduled case ID does not match realized factors: {args.case_id} != {computed_case_id}"
+ )
+ case_identifier = args.case_id or computed_case_id
+ git_run = getattr(args, "git_run", None) or {}
+ allocation_factors = {
+ "artifact": git_run.get("artifact"),
+ "execution_id": getattr(args, "allocation_execution_id", None),
+ "job": git_run.get("job"),
+ "repo": git_run.get("repo"),
+ "run_attempt": git_run.get("run_attempt"),
+ "run_id": git_run.get("run_id"),
+ "runner": args.runner,
+ "source_sha": git_run.get("source_sha"),
+ }
+ allocation_identifier = identity.allocation_id(allocation_factors)
+ try:
+ attempt_ordinal = int(os.environ.get("CX_ATTEMPT_ID", "1"))
+ except ValueError:
+ attempt_ordinal = 0
+ if attempt_ordinal <= 0:
+ raise ValueError("CX_ATTEMPT_ID must be a positive integer")
+ attempt_identifier = identity.attempt_id(
+ allocation=allocation_identifier, case=case_identifier, ordinal=attempt_ordinal
+ )
+ runtime_fingerprint = getattr(args, "runtime_fingerprint", None) or {}
+ implementation_contract = {
+ "kernel_generation": kernel_generation(backend),
+ "name": backend.name,
+ "provenance": _series_provenance(backend.backend_provenance),
+ "resource_profile": resource_profile,
+ }
+ public_config = contracts.public_series_config(
+ kernel_generation=implementation_contract["kernel_generation"],
+ provenance=backend.backend_provenance,
+ resource_profile=resource_profile,
+ resource_mode=args.resource_mode,
+ device_product=getattr(args, "runtime_device_product", None),
+ )
+ series_factors = {
+ "backend": backend.name,
+ "implementation_contract_sha256": _sha256_json(implementation_contract),
+ "public_config_sha256": contracts.public_series_config_sha256(public_config),
+ "routing_control_sha256": contracts.routing_implementation_control_sha256(
+ implementation_contract
+ ),
+ "case_id": case_identifier,
+ "image_digest": getattr(args, "image_digest", None),
+ "runtime_fingerprint_sha256": _sha256_json(runtime_fingerprint),
+ "source_sha": git_run.get("source_sha"),
+ "squash_sha256": getattr(args, "squash_sha256", None),
+ "workload_id": getattr(args, "workload_id", None) or trace_sig,
+ }
+ series_identifier = identity.series_id(series_factors)
+
+ sample_points = []
+ for row in rows:
+ token_count = row["tokens_per_rank"]
+
+ def sampled_component(trials):
+ return {
+ "availability": "measured" if trials else "unavailable",
+ "sample_count": sum(len(trial) for trial in trials),
+ "trials": trials if trials else None,
+ }
+
+ sample_point = {
+ "components": {
+ "combine": sampled_component(comb_trials[token_count]),
+ "dispatch": sampled_component(disp_trials[token_count]),
+ "roundtrip": sampled_component(rt_trials[token_count]),
+ },
+ "tokens_per_rank": token_count,
+ }
+ sample_sha256 = _sha256_json(sample_point)
+ point_identifier = identity.point_id(
+ series=series_identifier, tokens_per_rank=token_count
+ )
+ evidence_identifier = identity.evidence_id(
+ point=point_identifier,
+ allocation=allocation_identifier,
+ attempt=attempt_identifier,
+ sample_sha256=sample_sha256,
+ )
+ sample_point.update(
+ {
+ "evidence_id": evidence_identifier,
+ "point_id": point_identifier,
+ "sample_sha256": sample_sha256,
+ }
+ )
+ sample_points.append(sample_point)
+ row.update({
+ "evidence_id": evidence_identifier,
+ "point_id": point_identifier,
+ "sample_sha256": sample_sha256,
+ })
+
+ samples_path = args.out[:-5] + ".samples.json" if args.out.endswith(".json") else args.out + ".samples.json"
+ samples_document = {
+ "allocation_id": allocation_identifier,
+ "attempt_id": attempt_identifier,
+ "case_id": case_identifier,
+ "format": "collectivex.samples.v1",
+ "points": sample_points,
+ "sampling": {
+ "iterations_per_trial": args.iters,
+ "reduction": case_profile["rank_reduction"],
+ "trials": args.trials,
+ },
+ "schema_version": 1,
+ "series_id": series_identifier,
+ }
+ samples_payload = contracts.canonical_json_bytes(samples_document)
+ samples_sha256 = hashlib.sha256(samples_payload).hexdigest()
+ samples_bytes = len(samples_payload)
+ sample_artifact = {
+ "bytes": samples_bytes,
+ "format": "collectivex.samples.v1",
+ "path": os.path.basename(samples_path),
+ "sha256": samples_sha256,
+ }
+ headline = next((r for r in rows if r["tokens_per_rank"] == 64), rows[len(rows) // 2])
+ eplb_record = (
+ {
+ "enabled": True,
+ "imbalance_after": eplb_plan["imbalance_after"],
+ "imbalance_before": eplb_plan["imbalance_before"],
+ "mapping_hash": eplb_mapping_hash,
+ "max_replicas": eplb_plan["max_replicas"],
+ "num_logical_experts": num_logical,
+ "num_physical_experts": args.experts,
+ "num_redundant": args.experts - num_logical,
+ "planner": EPLB_PLANNER,
+ "reference_tokens_per_rank": EPLB_REFERENCE_TOKENS_PER_RANK,
+ "replicated_experts": eplb_plan["replicated_experts"],
+ }
+ if eplb_plan
+ else {
+ "enabled": False,
+ "imbalance_after": None,
+ "imbalance_before": None,
+ "mapping_hash": None,
+ "max_replicas": None,
+ "num_logical_experts": num_logical,
+ "num_physical_experts": args.experts,
+ "num_redundant": 0,
+ "planner": None,
+ "reference_tokens_per_rank": None,
+ "replicated_experts": 0,
+ }
+ )
+ doc = {
+ "format": "collectivex.ep.v1",
+ "schema_version": SCHEMA_VERSION,
+ "record_type": "case-attempt",
+ "generated_at": generated_at,
+ "identity": {
+ "allocation_factors": allocation_factors,
+ "allocation_id": allocation_identifier,
+ "attempt_id": attempt_identifier,
+ "attempt_ordinal": attempt_ordinal,
+ "case_factors": case_factors,
+ "case_id": case_identifier,
+ "series_factors": series_factors,
+ "series_id": series_identifier,
+ },
+ "case": {
+ "attempt_ordinal": attempt_ordinal,
+ "backend": backend.name,
+ "eplb": eplb_record,
+ "ep_size": ep_size,
+ "mode": mode,
+ "phase": args.phase,
+ "required_publication": args.required_publication or "diagnostic",
+ "resource_mode": "tuned",
+ "runner": args.runner,
+ "shape": shape,
+ "suite": args.suite or "manual",
+ "workload_name": args.workload_name or "manual",
+ },
+ "workload": {
+ "activation_generator": ACTIVATION_GENERATOR,
+ "activation_identity": activation_identity,
+ "activation_profile": ACTIVATION_PROFILE,
+ "cross_rank_consistent": routing_consistent,
+ "manifest_checksums": getattr(args, "workload_checksums", None),
+ "members": getattr(args, "workload_members", None),
+ "routing_generator": ROUTING_GENERATOR,
+ "source": validity["workload_source"],
+ "trace_hashes": sorted(routing_hashes),
+ "trace_signature": trace_sig,
+ "workload_id": getattr(args, "workload_id", None),
+ },
+ "measurement": {
+ "component_order_contract": case_profile["component_order_contract"],
+ "conditioning": {
+ "contract": CONDITIONING_CONTRACT,
+ "ladder": conditioning_ladder,
+ "roundtrips_per_shape": CONDITIONING_ROUNDS_PER_SHAPE,
+ },
+ "contract": case_profile["contract"],
+ "rows": rows,
+ "sampling": {
+ "contract": SAMPLING_CONTRACT,
+ "iterations_per_trial": args.iters,
+ "percentile_method": case_profile["percentile_method"],
+ "reduction": case_profile["rank_reduction"],
+ "samples_per_component": TIMED_SAMPLES_PER_POINT,
+ "trials": args.trials,
+ "warmup_iterations": args.warmup,
+ "warmup_semantics": WARMUP_SEMANTICS,
+ },
+ "source_allocation": "even",
+ },
+ "implementation": {
+ "kernel_generation": kernel_generation(backend),
+ "name": backend.name,
+ "provenance": backend.backend_provenance,
+ "resource_profile": resource_profile,
+ },
+ "topology": {
+ "device_count": getattr(args, "runtime_device_count", None),
+ "device_product": getattr(args, "runtime_device_product", None),
+ "gpus_per_node": args.gpus_per_node or ep_size,
+ "nodes": nodes,
+ "placement": "packed",
+ "realized_placement": realized_placement,
+ "scale_up_domain": args.scale_up_domain or (args.gpus_per_node or ep_size),
+ "scale_up_transport": args.scale_up_transport,
+ "scale_out_transport": args.scale_out_transport or None,
+ "scope": args.scope,
+ "topology_class": args.topology_class,
+ "transport": args.transport,
+ "world_size": world_size,
+ },
+ "runtime_fingerprint": runtime_fingerprint,
+ "provenance": {
+ "command": getattr(args, "reproduction_command", ""),
+ "distributed_launcher": getattr(args, "distributed_launcher", None),
+ "git_run": getattr(args, "git_run", None),
+ "image": {
+ "arch": getattr(args, "image_arch", None),
+ "digest": getattr(args, "image_digest", "") or None,
+ "digest_verified": getattr(args, "image_digest_verified", False),
+ "reference": getattr(args, "image", "") or None,
+ "squash_sha256": getattr(args, "squash_sha256", None),
+ },
+ "redaction": "sanitized-v1",
+ },
+ "sample_artifact": sample_artifact,
+ "outcome": {
+ "publication_status": publication_status,
+ "reasons": [] if all_ok else ["semantic correctness or routing identity failed"],
+ "status": "success" if all_ok else "invalid",
+ "validity": validity,
+ },
+ }
+ contracts.validate_raw_document(doc, samples_document)
+ _write_bytes_atomic(samples_path, samples_payload)
+ _write_json_atomic(args.out, doc)
+ dispatch_percentiles = headline["components"]["dispatch"]["percentiles_us"]
+ dispatch_p99 = dispatch_percentiles["p99"] if dispatch_percentiles else None
+ component_summary = (f"disp_p99={dispatch_p99:.1f}us "
+ if dispatch_p99 is not None
+ else "components=unavailable ")
+ print(f"{backend.name} ep-dispatch-combine [{args.phase}/{mode}/{case_profile['contract']}]: "
+ f"status={doc['outcome']['status']} {len(rows)} pts, routing_consistent={routing_consistent}, "
+ f"headline T={headline['tokens_per_rank']} {component_summary}"
+ f"-> {args.out}")
+ # A complete invalid document is still a successfully captured terminal outcome. Launchers
+ # inspect its status to fail the case without conflating it with an execution failure.
+ return 0
diff --git a/experimental/CollectiveX/tests/ep_mori.py b/experimental/CollectiveX/tests/ep_mori.py
new file mode 100644
index 0000000000..936425c5af
--- /dev/null
+++ b/experimental/CollectiveX/tests/ep_mori.py
@@ -0,0 +1,359 @@
+#!/usr/bin/env python3
+"""CollectiveX MoRI adapter for the v1 BF16 normal-mode workload."""
+from __future__ import annotations
+
+import os
+from pathlib import Path
+import re
+import sys
+import types
+
+# MoRI registers the whole symmetric heap at import time. The pinned upstream
+# inter-node benchmark uses 6 GiB for its InterNodeV1 staging and signal buffers.
+os.environ["MORI_SHMEM_HEAP_SIZE"] = "6G"
+
+import torch
+import torch.distributed as dist
+
+try:
+ import mori # type: ignore
+except Exception as exc: # pragma: no cover - requires the benchmark image
+ print(f"ERROR: mori import failed: {exc!r}", file=sys.stderr)
+ raise
+
+
+def _project_local_metadata(torch_module, raw_expert_ids, raw_weights, rank, experts_per_rank):
+ local_start = rank * experts_per_rank
+ local = (raw_expert_ids >= local_start) & (
+ raw_expert_ids < local_start + experts_per_rank
+ )
+ expert_ids = torch_module.where(
+ local, raw_expert_ids, torch_module.full_like(raw_expert_ids, -1)
+ )
+ weights = torch_module.where(local, raw_weights, torch_module.zeros_like(raw_weights))
+ return expert_ids, weights, raw_expert_ids[local] - local_start
+
+
+def _mori_source_commit() -> str:
+ module_path = Path(mori.__file__).resolve()
+ for root in module_path.parents:
+ head = root / ".git" / "HEAD"
+ if not head.is_symlink() and head.is_file() and head.stat().st_size <= 128:
+ value = head.read_text(encoding="ascii").strip()
+ if re.fullmatch(r"[0-9a-f]{40}", value):
+ return value
+ raise RuntimeError("MoRI image source is not pinned to a detached commit")
+ raise RuntimeError("MoRI image source revision is unavailable")
+
+
+class MoRIBackend:
+ name = "mori"
+ combine_needs_redispatch = True
+ combine_weight_semantics = "unweighted-rank-sum"
+
+ def __init__(self, args, rank, world_size, local_rank, device):
+ self.args = args
+ self.rank = rank
+ self.world_size = world_size
+ self.device = device
+ self.mode = "normal"
+
+ self.ep_size = world_size
+ self.experts_per_rank = args.experts // self.ep_size
+ device_cus = torch.cuda.get_device_properties(device).multi_processor_count
+ gpus_per_node = int(args.gpus_per_node or world_size)
+ scale_up_domain = int(args.scale_up_domain or gpus_per_node)
+ scale_out = world_size > scale_up_domain
+ if (
+ gpus_per_node <= 0
+ or scale_up_domain <= 0
+ or world_size % gpus_per_node
+ or world_size % scale_up_domain
+ ):
+ raise RuntimeError("MoRI placement is not divisible into complete domains")
+ if scale_out != (args.scope == "scale-out"):
+ raise RuntimeError("MoRI requested scope differs from the EP topology")
+ if not scale_out and (
+ world_size != 8
+ or gpus_per_node != 8
+ or scale_up_domain != 8
+ or args.scale_up_transport != "xgmi"
+ or args.scale_out_transport
+ or args.transport != "xgmi"
+ ):
+ raise RuntimeError("MoRI scale-up is pinned to EP8 over one XGMI domain")
+ if scale_out and (
+ world_size != 16
+ or gpus_per_node != 8
+ or scale_up_domain != 8
+ or args.scale_up_transport != "xgmi"
+ or args.scale_out_transport != "rdma"
+ or args.transport != "xgmi-rdma"
+ ):
+ raise RuntimeError(
+ "MoRI InterNodeV1 is pinned to EP16 over two 8-GPU XGMI/RDMA nodes"
+ )
+ self.block_num = self._block_target = 80
+ self.rdma_block_num = 0
+ self.num_qps = 1
+ self._block_floored = False
+ self._tuned_source = "default-80"
+ self.dispatch_warps = 16
+ self.combine_warps = 8
+
+ # MI355X uses the direct intranode kernel. MI325X uses MoRI's split
+ # AsyncLL send/receive kernel as its normal-mode XGMI transport.
+ kernel_request = os.environ.get("CX_MORI_KERNEL_TYPE", "intranode").strip().lower()
+ self._kernel_type = None
+ self._kernel_type_label = "IntraNode"
+ self._async_ll = False
+ self._inter_node = False
+ if kernel_request in ("asyncll", "async_ll", "async-ll"):
+ if scale_out:
+ raise RuntimeError("MoRI EP16 must use InterNodeV1, not AsyncLL")
+ kernel_enum = getattr(mori.ops, "EpDispatchCombineKernelType", None)
+ if kernel_enum is None or not hasattr(kernel_enum, "AsyncLL"):
+ raise RuntimeError(
+ "CX_MORI_KERNEL_TYPE=asyncll requires "
+ "EpDispatchCombineKernelType.AsyncLL"
+ )
+ self._kernel_type = kernel_enum.AsyncLL
+ self._kernel_type_label = "AsyncLL"
+ self._async_ll = True
+ self.block_num = self._block_target = 64
+ self.dispatch_warps = self.combine_warps = 8
+ self._tuned_source = "upstream-asyncll-64x8-external-input"
+ elif kernel_request in ("internode-v1", "internode_v1", "internodev1"):
+ if not scale_out:
+ raise RuntimeError("MoRI InterNodeV1 is valid only for scale-out EP16")
+ kernel_enum = getattr(mori.ops, "EpDispatchCombineKernelType", None)
+ if kernel_enum is None or not hasattr(kernel_enum, "InterNodeV1"):
+ raise RuntimeError(
+ "CX_MORI_KERNEL_TYPE=internode-v1 requires "
+ "EpDispatchCombineKernelType.InterNodeV1"
+ )
+ self._kernel_type = kernel_enum.InterNodeV1
+ self._kernel_type_label = "InterNodeV1"
+ self._inter_node = True
+ self.block_num = self._block_target = 96
+ self.rdma_block_num = 64
+ self.dispatch_warps = self.combine_warps = 8
+ self._tuned_source = "upstream-internode-v1-96-64x8-qps1"
+ elif kernel_request not in ("intranode", "intra_node", "intra-node", ""):
+ raise RuntimeError(
+ f"unknown CX_MORI_KERNEL_TYPE={kernel_request!r} "
+ "(expected intranode|asyncll|internode-v1)"
+ )
+ elif scale_out:
+ raise RuntimeError("MoRI scale-out EP16 requires CX_MORI_KERNEL_TYPE=internode-v1")
+ self.kernel_generation = (
+ "inter-node-v1" if self._inter_node
+ else "async-ll" if self._async_ll
+ else "intranode"
+ )
+ self._external_input = self._async_ll or self._inter_node
+
+ world_group = torch.distributed.group.WORLD
+ torch._C._distributed_c10d._register_process_group("default", world_group)
+ mori.shmem.shmem_torch_process_group_init("default")
+ realized_qps = int(mori.shmem.shmem_num_qp_per_pe())
+ if realized_qps < self.num_qps:
+ raise RuntimeError(
+ f"MoRI realized {realized_qps} QPs per PE; {self.num_qps} required"
+ )
+
+ self._cap = self.buffer_cap(args)
+ config_kwargs = {
+ "data_type": torch.bfloat16,
+ "rank": rank,
+ "world_size": world_size,
+ "hidden_dim": args.hidden,
+ "scale_dim": 0,
+ "scale_type_size": 1,
+ "max_token_type_size": (
+ torch.tensor([], dtype=torch.bfloat16).element_size()
+ if self._inter_node
+ else torch.tensor([], dtype=torch.float32).element_size()
+ ),
+ "max_num_inp_token_per_rank": max(512, self._cap),
+ "num_experts_per_rank": self.experts_per_rank,
+ "num_experts_per_token": args.topk,
+ "use_external_inp_buf": self._external_input,
+ "quant_type": "none",
+ }
+ if self._kernel_type is not None:
+ config_kwargs["kernel_type"] = self._kernel_type
+ if self._async_ll:
+ config_kwargs["max_total_recv_tokens"] = 0
+ if self._async_ll or self._inter_node:
+ config_kwargs["block_num"] = self.block_num
+ config_kwargs["warp_num_per_block"] = self.dispatch_warps
+ if self._inter_node:
+ config_kwargs.update({
+ "gpu_per_node": gpus_per_node,
+ "rdma_block_num": self.rdma_block_num,
+ "num_qp_per_pe": self.num_qps,
+ })
+ self.config = mori.ops.EpDispatchCombineConfig(**config_kwargs)
+ expected_config = {"use_external_inp_buf": self._external_input}
+ if self._async_ll or self._inter_node:
+ expected_config.update({
+ "block_num": self.block_num,
+ "warp_num_per_block": self.dispatch_warps,
+ })
+ if self._inter_node:
+ expected_config.update({
+ "gpu_per_node": 8,
+ "rdma_block_num": 64,
+ "num_qp_per_pe": 1,
+ })
+ if any(getattr(self.config, key, None) != value for key, value in expected_config.items()):
+ raise RuntimeError("MoRI requested launch/topology configuration was not realized")
+ # The newer pinned MoRI revision can otherwise replace explicit values
+ # with token-dependent tuning rules from the image.
+ os.environ["MORI_EP_LAUNCH_CONFIG_MODE"] = "MANUAL"
+ self.op = mori.ops.EpDispatchCombineOp(self.config)
+ if getattr(self.op, "launch_config_mode", None) != "MANUAL":
+ raise RuntimeError("MoRI explicit launch configuration was not applied")
+
+ expected_mori_commit = os.environ.get("MORI_COMMIT")
+ mori_commit = _mori_source_commit()
+ if expected_mori_commit and mori_commit != expected_mori_commit:
+ raise RuntimeError("MoRI image source revision differs from canonical provenance")
+ self.backend_provenance = {
+ "mori_commit": mori_commit,
+ "api": (
+ "mori.ops.EpDispatchCombineOp/external-input"
+ if self._external_input
+ else "mori.ops.EpDispatchCombineOp/registered-input"
+ ),
+ "mode": "normal",
+ "dispatch_dtype": "bf16",
+ "combine_dtype": "bf16",
+ "kernel_type": self._kernel_type_label,
+ "enable_sdma": os.environ.get("MORI_ENABLE_SDMA"),
+ "heap_size": os.environ.get("MORI_SHMEM_HEAP_SIZE"),
+ "max_num_inp_token_per_rank": max(512, self._cap),
+ "max_total_recv_tokens": config_kwargs.get("max_total_recv_tokens"),
+ "gpus_per_node": gpus_per_node,
+ "rdma_block_num": self.rdma_block_num,
+ "use_external_inp_buf": self._external_input,
+ "num_qps": self.num_qps,
+ "resource_mode": "tuned",
+ "block_num": self.block_num,
+ "block_num_target": self._block_target,
+ "block_num_floored": self._block_floored,
+ "dispatch_warps": self.dispatch_warps,
+ "combine_warps": self.combine_warps,
+ "device_cus": device_cus,
+ "sm_fraction": None if self._async_ll else self.block_num / device_cus,
+ "tuned_source": self._tuned_source,
+ }
+
+ def buffer_cap(self, args):
+ return 512
+
+ def make_problem(self, T, idx, weights, x):
+ indices = idx.to(torch.int32)
+ gate_weights = weights.to(torch.float32)
+ return types.SimpleNamespace(
+ T=T,
+ x=x,
+ topk_idx=indices,
+ topk_weights=gate_weights,
+ indices=indices,
+ weights=gate_weights,
+ scales=torch.empty((T, 0), dtype=torch.uint8, device=self.device),
+ )
+
+ def dispatch(self, p):
+ dispatch_output, dispatch_weights, _scales, dispatch_indices, recv_num = (
+ self.op.dispatch(
+ p.x,
+ p.weights,
+ p.scales,
+ p.indices,
+ block_num=self.block_num,
+ rdma_block_num=self.rdma_block_num,
+ warp_per_block=self.dispatch_warps,
+ )
+ )
+ if self._async_ll:
+ self.op.dispatch_recv(warp_per_block=self.dispatch_warps)
+ return types.SimpleNamespace(
+ dispatch_output=dispatch_output,
+ dispatch_weights=dispatch_weights,
+ dispatch_indices=dispatch_indices,
+ recv_num=recv_num[0],
+ combine_input=dispatch_output.to(torch.bfloat16),
+ )
+
+ def stage(self, p, h):
+ rows = getattr(p, "recv_tokens", None)
+ if not isinstance(rows, int) or rows < 0 or rows > h.combine_input.size(0):
+ raise RuntimeError("MoRI receive count was not validated before staging")
+ if self._external_input:
+ return None
+ buffer = self.op.get_registered_combine_input_buffer(
+ torch.bfloat16, hidden_dim=h.combine_input.size(1)
+ )
+ buffer[:rows, :].copy_(h.combine_input[:rows, :])
+ h.combine_input = buffer
+
+ def combine(self, p, h):
+ combine_indices = p.indices if self._async_ll else h.dispatch_indices
+ combined, _weights = self.op.combine(
+ h.combine_input,
+ None,
+ combine_indices,
+ block_num=self.block_num,
+ rdma_block_num=self.rdma_block_num,
+ warp_per_block=self.combine_warps,
+ )
+ if self._async_ll:
+ self.op.combine_recv(warp_per_block=self.combine_warps)
+ return combined[:p.T]
+
+ def inspect_dispatch(self, p, h):
+ count = self.recv_tokens(h)
+ if h.dispatch_weights is None:
+ raise RuntimeError("MoRI dispatch did not expose gate weights")
+ if count < 0 or any(
+ tensor.ndim == 0 or count > tensor.size(0)
+ for tensor in (h.dispatch_output, h.dispatch_indices, h.dispatch_weights)
+ ):
+ raise RuntimeError("MoRI receive count exceeds dispatch metadata")
+ raw_expert_ids = h.dispatch_indices[:count].to(torch.int64)
+ expert_ids, weights, local_expert_ids = _project_local_metadata(
+ torch,
+ raw_expert_ids,
+ h.dispatch_weights[:count].to(torch.float32),
+ self.rank,
+ self.experts_per_rank,
+ )
+ return types.SimpleNamespace(
+ payload=h.dispatch_output[:count],
+ expert_ids=expert_ids,
+ weights=weights,
+ local_expert_counts=torch.bincount(
+ local_expert_ids, minlength=self.experts_per_rank
+ ),
+ ordering_contract="mori-global-topk-masked-v1",
+ )
+
+ def combine_transformed(self, p, h, transformed):
+ h.combine_input = transformed.to(torch.bfloat16)
+ self.stage(p, h)
+ return self.combine(p, h)
+
+ def recv_tokens(self, h):
+ return int(h.recv_num.item())
+
+ def finalize(self, rc):
+ try:
+ dist.barrier()
+ except Exception:
+ pass
+ sys.stdout.flush()
+ sys.stderr.flush()
+ os._exit(rc if 0 <= rc <= 255 else 1)
diff --git a/experimental/CollectiveX/tests/ep_nccl.py b/experimental/CollectiveX/tests/ep_nccl.py
new file mode 100644
index 0000000000..cab9cb8238
--- /dev/null
+++ b/experimental/CollectiveX/tests/ep_nccl.py
@@ -0,0 +1,186 @@
+"""CollectiveX NCCL all-to-all expert-parallel reference backend.
+
+The canonical "token-shuffle" EP built on torch.distributed's NCCL ``all_to_all_single``. Like the
+DeepEP-family APIs, dispatch sends one hidden-state copy to each distinct destination rank, even when
+multiple selected experts live on that rank. Combine reverses the shuffle and sums those rank copies.
+
+Why this exists alongside DeepEP/UCCL/MoRI: it is the portable collective reference baseline for the
+same rank-deduplicated payload and routing metadata. It keeps the library comparison anchored to the
+platform collective stack without claiming the custom fused kernels use the same transport algorithm.
+
+Scope: BF16, normal mode, layout-and-dispatch-v1. The timed dispatch includes layout, count exchange,
+payload, rank-masked expert indices, gate weights, and source-token metadata; combine returns only
+the activation payload. RCCL exposes the same API. The v1 AMD matrix uses this backend at EP8 and EP16.
+"""
+
+import os
+import re
+import types
+
+import torch
+import torch.distributed as dist
+import contracts
+
+
+def _runtime_collective(args, torch_module) -> tuple[str, str]:
+ expected = "rccl" if torch_module.version.hip else "nccl"
+ fingerprint = getattr(args, "runtime_fingerprint", None)
+ collective = fingerprint.get("collective_library") if isinstance(fingerprint, dict) else None
+ if (
+ not isinstance(collective, dict)
+ or collective.get("kind") != expected
+ or not isinstance(collective.get("version"), str)
+ or not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", collective["version"])
+ ):
+ raise RuntimeError("loaded collective runtime identity is unavailable")
+ return expected, collective["version"]
+
+
+class NCCLBackend:
+ name = "nccl-ep"
+ combine_needs_redispatch = False # dispatch saves the permutation + splits
+ combine_weight_semantics = "unweighted-rank-sum"
+
+ def __init__(self, args, rank, world_size, local_rank, device):
+ self.args = args
+ self.rank = rank
+ self.world_size = world_size
+ self.device = device
+ self.experts = args.experts
+ if args.experts % world_size:
+ raise ValueError(f"experts({args.experts}) must divide world_size({world_size})")
+ self.experts_per_rank = args.experts // world_size
+ self.tolerance = 5e-2 # bf16 round-trip
+ _library, _version = _runtime_collective(args, torch)
+ if args.scale_out_transport:
+ hcas = os.environ.get("NCCL_IB_HCA", "")
+ if os.environ.get("NCCL_NET") != "IB" or not re.fullmatch(
+ r"=[A-Za-z][A-Za-z0-9_.-]{0,31}(?::[1-9][0-9]*)?"
+ r"(?:,[A-Za-z][A-Za-z0-9_.-]{0,31}(?::[1-9][0-9]*)?)*",
+ hcas,
+ ):
+ raise RuntimeError("scale-out collective transport is not pinned to RDMA")
+ self.kernel_generation = contracts.collective_kernel_generation(_library)
+ self.backend_provenance = {
+ "backend": f"{_library}-all2all",
+ "backend_lineage": _library,
+ "collective_library": _library,
+ "nccl_version": _version,
+ "transport": f"{_library}-all_to_all_single",
+ "resource_mode": "tuned",
+ "num_sms": None,
+ "device_sms": torch.cuda.get_device_properties(device).multi_processor_count,
+ "tuned_source": "nccl-collective",
+ "reference_semantics": "rank-deduplicated-payload-plus-routing-metadata-v2",
+ "routing_metadata": "expert-index-gate-weight-source-token",
+ }
+
+ def buffer_cap(self, args):
+ return None # no fixed pre-allocated buffer; all-to-all sizes itself per step
+
+ def make_problem(self, T, idx, weights, x):
+ # idx[T,topk] int64, weights[T,topk] f32, x[T,hidden] bf16 — the shared routing-trace slice.
+ return types.SimpleNamespace(T=T, x=x, topk_idx=idx.to(torch.int64),
+ topk_weights=weights.to(torch.float32), layout=None)
+
+ def dispatch(self, p):
+ ws = self.world_size
+ x = p.x # [T, H] bf16
+ idx = p.topk_idx # [T, topk]
+ T, H = int(x.shape[0]), int(x.shape[1])
+ dev = x.device
+ # DeepEP dispatches one token per destination rank, not one copy per expert. Build the same
+ # rank-deduplicated routing map so NCCL traffic and combine semantics are comparable.
+ destinations = (idx // self.experts_per_rank).clamp_(0, ws - 1)
+ present = torch.zeros((T, ws), dtype=torch.bool, device=dev)
+ present.scatter_(1, destinations, True)
+ flat_token, flat_dest = present.nonzero(as_tuple=True)
+ # Group rank copies by destination (stable -> deterministic, invertible permutation).
+ order = torch.argsort(flat_dest, stable=True)
+ ordered_token = flat_token.index_select(0, order)
+ ordered_dest = flat_dest.index_select(0, order)
+ send_counts = torch.bincount(flat_dest, minlength=ws) # [ws]
+ send_x = x.index_select(0, ordered_token).contiguous()
+ send_topk_idx = idx.index_select(0, ordered_token).contiguous()
+ expert_start = ordered_dest.unsqueeze(1) * self.experts_per_rank
+ local_mask = ((send_topk_idx >= expert_start)
+ & (send_topk_idx < expert_start + self.experts_per_rank))
+ send_topk_idx = torch.where(
+ local_mask, send_topk_idx - expert_start, torch.full_like(send_topk_idx, -1)
+ )
+ send_topk_weights = p.topk_weights.index_select(0, ordered_token).contiguous()
+ send_topk_weights.masked_fill_(~local_mask, 0)
+ send_src_metadata = (ordered_token.to(torch.int64) | (self.rank << 32)).contiguous()
+ # Exchange per-rank counts so every rank can size its receive buffer.
+ recv_counts = torch.empty_like(send_counts)
+ dist.all_to_all_single(recv_counts, send_counts)
+ sc = send_counts.tolist()
+ rc = recv_counts.tolist()
+ total_recv = int(sum(rc))
+ recv_x = torch.empty((total_recv, H), dtype=x.dtype, device=dev)
+ recv_topk_idx = torch.empty((total_recv, int(idx.shape[1])), dtype=idx.dtype, device=dev)
+ recv_topk_weights = torch.empty((total_recv, int(idx.shape[1])),
+ dtype=p.topk_weights.dtype, device=dev)
+ recv_src_metadata = torch.empty((total_recv,), dtype=torch.int64, device=dev)
+ # Dispatch the uneven per-rank splits over the configured collective transport.
+ dist.all_to_all_single(recv_x, send_x, rc, sc)
+ dist.all_to_all_single(recv_topk_idx, send_topk_idx, rc, sc)
+ dist.all_to_all_single(recv_topk_weights, send_topk_weights, rc, sc)
+ dist.all_to_all_single(recv_src_metadata, send_src_metadata, rc, sc)
+ return types.SimpleNamespace(
+ recv_x=recv_x, combine_input=None, order=order, flat_token=flat_token,
+ recv_topk_idx=recv_topk_idx,
+ recv_topk_weights=recv_topk_weights, recv_src_rank=recv_src_metadata >> 32,
+ recv_src_token=recv_src_metadata & ((1 << 32) - 1), send_counts=sc, recv_counts=rc,
+ T=T, H=H, topk=int(idx.shape[1]), total_recv=total_recv)
+
+ def stage(self, p, h):
+ # No expert compute: the expert "output" is the received tokens as-is (the round-trip identity).
+ h.combine_input = h.recv_x
+ return None
+
+ def combine(self, p, h):
+ # Reverse all-to-all: ship expert outputs back to their origin ranks (swap the split lists).
+ send_back = torch.empty((int(h.order.shape[0]), h.H), dtype=h.combine_input.dtype,
+ device=h.combine_input.device)
+ dist.all_to_all_single(send_back, h.combine_input.contiguous(),
+ h.send_counts, h.recv_counts)
+ # send_back is in send (sorted) order; invert the argsort to token-copy order.
+ copies = torch.empty_like(send_back)
+ copies[h.order] = send_back
+ # Sum one copy per destination rank under this reference's explicit unweighted contract.
+ out = torch.zeros((h.T, h.H), dtype=torch.float32, device=send_back.device)
+ out.index_add_(0, h.flat_token, copies.float())
+ return out.to(p.x.dtype)
+
+ def inspect_dispatch(self, p, h):
+ valid = h.recv_topk_idx >= 0
+ expert_ids = torch.where(
+ valid,
+ h.recv_topk_idx + self.rank * self.experts_per_rank,
+ h.recv_topk_idx,
+ )
+ return types.SimpleNamespace(
+ payload=h.recv_x,
+ expert_ids=expert_ids,
+ weights=h.recv_topk_weights.masked_fill(~valid, 0),
+ local_expert_counts=torch.bincount(
+ h.recv_topk_idx[valid], minlength=self.experts_per_rank
+ ),
+ ordering_contract="source-rank-major-stable-v1",
+ )
+
+ def combine_transformed(self, p, h, transformed):
+ h.combine_input = transformed.to(h.recv_x.dtype)
+ return self.combine(p, h)
+
+ def recv_tokens(self, h):
+ return int(h.total_recv)
+
+ def finalize(self, rc):
+ try:
+ dist.barrier()
+ dist.destroy_process_group()
+ except Exception:
+ pass
+ return rc
diff --git a/experimental/CollectiveX/tests/ep_uccl.py b/experimental/CollectiveX/tests/ep_uccl.py
new file mode 100644
index 0000000000..7d02bb2991
--- /dev/null
+++ b/experimental/CollectiveX/tests/ep_uccl.py
@@ -0,0 +1,405 @@
+#!/usr/bin/env python3
+"""CollectiveX UCCL adapter for the v1 BF16 normal-mode workload."""
+from __future__ import annotations
+
+import importlib.metadata as metadata
+import json
+import os
+from pathlib import Path
+from pathlib import PurePosixPath
+import sys
+import types
+
+import torch
+import torch.distributed as dist
+import contracts
+
+try:
+ import uccl
+ import uccl_deepep
+ from uccl_deepep import Buffer # type: ignore
+except Exception as exc: # pragma: no cover - requires the benchmark image
+ print(f"ERROR: uccl.ep import failed: {exc!r}", file=sys.stderr)
+ raise
+
+
+def _uccl_version() -> str:
+ try:
+ return metadata.version("uccl")
+ except Exception:
+ return getattr(uccl, "__version__", "unknown")
+
+
+def _uccl_dependency_versions() -> dict[str, str]:
+ versions = {
+ package: metadata.version(package)
+ for package in contracts.UCCL_DEPENDENCY_VERSIONS
+ }
+ if versions != contracts.UCCL_DEPENDENCY_VERSIONS:
+ raise RuntimeError(
+ "UCCL runtime dependency versions differ from the v1 contract"
+ )
+ return versions
+
+
+def _is_uccl_runtime_payload(name: str) -> bool:
+ path = PurePosixPath(name)
+ return (
+ bool(path.parts)
+ and path.parts[0] in {"uccl", "uccl.libs"}
+ and "__pycache__" not in path.parts
+ and path.suffix != ".pyc"
+ )
+
+
+def _python_dependency_evidence(package: str, version: str) -> dict[str, str]:
+ distribution = metadata.distribution(package)
+ runtime_files = []
+ for entry in distribution.files or ():
+ logical = PurePosixPath(entry.as_posix())
+ path = Path(distribution.locate_file(entry))
+ if (
+ logical.parts
+ and logical.parts[0] == package
+ and "__pycache__" not in logical.parts
+ and logical.suffix != ".pyc"
+ and path.is_file()
+ ):
+ runtime_files.append((entry.as_posix(), path))
+ return contracts.content_manifest_evidence(
+ role=f"{package}-distribution",
+ name=f"{package}-{version}",
+ files=runtime_files,
+ )
+
+
+def _loaded_libcudart_evidence(
+ version: str, maps_path: Path = Path("/proc/self/maps")
+) -> dict[str, str]:
+ distribution = metadata.distribution("nvidia-cuda-runtime-cu12")
+ candidates = {
+ Path(distribution.locate_file(entry)).resolve()
+ for entry in distribution.files or ()
+ if PurePosixPath(entry.as_posix()).name.startswith("libcudart.so")
+ and Path(distribution.locate_file(entry)).is_file()
+ }
+ candidate_names = {path.name for path in candidates}
+ if not candidates or not candidate_names:
+ raise RuntimeError("pinned CUDA runtime distribution has no libcudart payload")
+
+ loaded: set[Path] = set()
+ try:
+ mappings = maps_path.read_text().splitlines()
+ except OSError as exc:
+ raise RuntimeError("cannot inspect mapped UCCL runtime libraries") from exc
+ for mapping in mappings:
+ columns = mapping.split(maxsplit=5)
+ if len(columns) != 6:
+ continue
+ raw_path = columns[5]
+ deleted = raw_path.endswith(" (deleted)")
+ if deleted:
+ raw_path = raw_path.removesuffix(" (deleted)")
+ mapped = Path(raw_path)
+ if mapped.name not in candidate_names:
+ continue
+ if deleted or not mapped.is_file():
+ raise RuntimeError(
+ "mapped libcudart is unavailable for content verification"
+ )
+ resolved = mapped.resolve()
+ if resolved not in candidates:
+ raise RuntimeError(
+ "mapped libcudart is not owned by the pinned CUDA runtime package"
+ )
+ loaded.add(resolved)
+ if len(loaded) != 1:
+ raise RuntimeError(
+ "expected exactly one mapped libcudart from the pinned CUDA runtime"
+ )
+ return contracts.content_manifest_evidence(
+ role="cuda-runtime",
+ name=f"nvidia-cuda-runtime-cu12-{version}",
+ files=[("libcudart.so", loaded.pop())],
+ )
+
+
+def _uccl_build_evidence(
+ version: str, dependency_versions: dict[str, str]
+) -> list[dict[str, str]]:
+ distribution = metadata.distribution("uccl")
+ distribution_files = [
+ (entry.as_posix(), distribution.locate_file(entry))
+ for entry in distribution.files or ()
+ if _is_uccl_runtime_payload(entry.as_posix())
+ and Path(distribution.locate_file(entry)).is_file()
+ ]
+ wrapper_root = Path(uccl_deepep.__file__).resolve().parent
+ wrapper_files = [
+ (path.relative_to(wrapper_root).as_posix(), path)
+ for path in wrapper_root.rglob("*.py")
+ if path.is_file()
+ ]
+ return [
+ contracts.content_manifest_evidence(
+ role="uccl-distribution",
+ name=f"uccl-{version}",
+ files=distribution_files,
+ ),
+ contracts.content_manifest_evidence(
+ role="uccl-wrapper",
+ name="uccl-deepep-wrapper",
+ files=wrapper_files,
+ ),
+ _python_dependency_evidence("intervaltree", dependency_versions["intervaltree"]),
+ _python_dependency_evidence(
+ "sortedcontainers", dependency_versions["sortedcontainers"]
+ ),
+ _loaded_libcudart_evidence(dependency_versions["nvidia-cuda-runtime-cu12"]),
+ ]
+
+
+def _require_cross_rank_equal(value, label: str) -> None:
+ gathered = [None] * dist.get_world_size()
+ dist.all_gather_object(gathered, value)
+ canonical = {json.dumps(item, sort_keys=True, separators=(",", ":")) for item in gathered}
+ if len(canonical) != 1:
+ raise RuntimeError(f"UCCL {label} differs across ranks")
+
+
+def _normal_buffer_sizes(hidden: int, world_size: int) -> tuple[int, int]:
+ """Apply the wrapped DeepEP dispatch/combine sizing contract for this EP world."""
+ hidden_bytes = hidden * torch.tensor([], dtype=torch.bfloat16).element_size()
+ configs = (Buffer.get_dispatch_config(world_size), Buffer.get_combine_config(world_size))
+ num_nvl_bytes = max(
+ int(config.get_nvl_buffer_size_hint(hidden_bytes, world_size)) for config in configs
+ )
+ num_rdma_bytes = max(
+ int(config.get_rdma_buffer_size_hint(hidden_bytes, world_size)) for config in configs
+ )
+ if num_nvl_bytes <= 0 or num_rdma_bytes < 0:
+ raise RuntimeError("UCCL returned invalid normal-mode buffer size hints")
+ return num_nvl_bytes, num_rdma_bytes
+
+
+class UCCLBackend:
+ name = "uccl"
+ combine_needs_redispatch = False
+ combine_weight_semantics = "unweighted-rank-sum"
+ oracle_layout = "token-rank"
+ payload_unit = "token-rank"
+
+ def __init__(self, args, rank, world_size, local_rank, device):
+ self.args = args
+ self.rank = rank
+ self.world_size = world_size
+ self.device = device
+ self.mode = getattr(args, "mode", "normal")
+ if self.mode not in {"normal", "low-latency"}:
+ raise ValueError(f"unsupported UCCL mode {self.mode!r}")
+
+ self.group = dist.group.WORLD
+ device_sms = torch.cuda.get_device_properties(device).multi_processor_count
+ if self.mode == "low-latency":
+ if args.phase != "decode":
+ raise ValueError("UCCL low-latency mode only supports the decode ladder")
+ if args.experts % world_size:
+ raise ValueError("UCCL low-latency experts must divide the EP group")
+ self.combine_needs_redispatch = True
+ self.combine_weight_semantics = "gate-weighted-sum"
+ self.oracle_layout = "expert-packed"
+ self.payload_unit = "token-expert"
+ self.max_tokens_per_rank = 128
+ num_qps_per_rank = args.experts // world_size
+ num_rdma_bytes = Buffer.get_low_latency_rdma_size_hint(
+ self.max_tokens_per_rank, args.hidden, world_size, args.experts
+ )
+ self.buffer = Buffer(
+ self.group,
+ num_nvl_bytes=0,
+ num_rdma_bytes=num_rdma_bytes,
+ low_latency_mode=True,
+ num_qps_per_rank=num_qps_per_rank,
+ allow_nvlink_for_low_latency_mode=True,
+ )
+ self.buffer.clean_low_latency_buffer(
+ self.max_tokens_per_rank, args.hidden, args.experts
+ )
+ resource_provenance = {
+ "requested_num_sms": None,
+ "num_sms": None,
+ "sm_fraction": None,
+ "tuned_source": "uccl-low-latency-fixed-kernel",
+ "num_max_tokens_per_rank": self.max_tokens_per_rank,
+ "num_nvl_bytes": 0,
+ "num_rdma_bytes": num_rdma_bytes,
+ }
+ else:
+ num_nvl_bytes, num_rdma_bytes = _normal_buffer_sizes(args.hidden, world_size)
+ if world_size > args.scale_up_domain and num_rdma_bytes == 0:
+ raise RuntimeError("UCCL scale-out configuration returned no RDMA buffer")
+ self.buffer = Buffer(self.group, num_nvl_bytes, num_rdma_bytes)
+ num_sms = int(getattr(Buffer, "num_sms", args.num_sms))
+ try:
+ Buffer.set_num_sms(num_sms)
+ except Exception as exc: # pragma: no cover - version dependent
+ raise RuntimeError(
+ f"UCCL did not apply requested num_sms={num_sms}: {exc!r}"
+ ) from exc
+ applied_num_sms = int(getattr(Buffer, "num_sms", num_sms))
+ if applied_num_sms != num_sms:
+ raise RuntimeError(
+ f"UCCL num_sms mismatch: requested={num_sms} applied={applied_num_sms}"
+ )
+ resource_provenance = {
+ "requested_num_sms": num_sms,
+ "num_sms": applied_num_sms,
+ "sm_fraction": applied_num_sms / device_sms,
+ "tuned_source": "uccl-default-num_sms",
+ "num_nvl_bytes": num_nvl_bytes,
+ "num_rdma_bytes": num_rdma_bytes,
+ }
+ version = _uccl_version()
+ dependency_versions = _uccl_dependency_versions()
+ loaded_libraries = _uccl_build_evidence(version, dependency_versions)
+ _require_cross_rank_equal(loaded_libraries, "installed content identities")
+ self.backend_provenance = {
+ "uccl_version": version,
+ "uccl_commit": os.environ.get("UCCL_COMMIT") or f"pkg-{version}",
+ "uccl_wrapper_commit": os.environ.get("UCCL_WRAPPER_COMMIT"),
+ "backend_lineage": "uccl",
+ "uccl_dependency_versions": dependency_versions,
+ "loaded_libraries": loaded_libraries,
+ "mode": self.mode,
+ "dispatch_dtype": "bf16",
+ "combine_dtype": "bf16",
+ "resource_mode": "tuned",
+ "device_sms": device_sms,
+ **resource_provenance,
+ }
+
+ def buffer_cap(self, args):
+ return self.max_tokens_per_rank if self.mode == "low-latency" else None
+
+ def make_problem(self, T, idx, weights, x):
+ return types.SimpleNamespace(
+ T=T,
+ x=x,
+ topk_idx=idx.to(torch.int64),
+ topk_weights=weights.to(torch.float32),
+ )
+
+ def dispatch(self, p):
+ if self.mode == "low-latency":
+ recv_x, recv_counts, handle, _, _ = self.buffer.low_latency_dispatch(
+ p.x,
+ p.topk_idx,
+ self.max_tokens_per_rank,
+ self.args.experts,
+ use_fp8=False,
+ async_finish=False,
+ return_recv_hook=False,
+ )
+ return types.SimpleNamespace(
+ recv_x=recv_x,
+ recv_counts=recv_counts,
+ handle=handle,
+ )
+ (
+ num_tokens_per_rank,
+ num_tokens_per_rdma_rank,
+ num_tokens_per_expert,
+ is_token_in_rank,
+ _,
+ ) = self.buffer.get_dispatch_layout(p.topk_idx, self.args.experts)
+ recv_x, recv_topk_idx, recv_topk_weights, recv_counts, handle, _ = self.buffer.dispatch(
+ p.x,
+ topk_idx=p.topk_idx,
+ topk_weights=p.topk_weights,
+ num_tokens_per_rank=num_tokens_per_rank,
+ num_tokens_per_rdma_rank=num_tokens_per_rdma_rank,
+ is_token_in_rank=is_token_in_rank,
+ num_tokens_per_expert=num_tokens_per_expert,
+ )
+ return types.SimpleNamespace(
+ recv_x=recv_x,
+ recv_topk_idx=recv_topk_idx,
+ recv_topk_weights=recv_topk_weights,
+ recv_counts=recv_counts,
+ handle=handle,
+ )
+
+ def stage(self, p, h):
+ h.combine_input = h.recv_x
+
+ def combine(self, p, h):
+ if self.mode == "low-latency":
+ combined_x, _, _ = self.buffer.low_latency_combine(
+ h.combine_input,
+ p.topk_idx,
+ p.topk_weights,
+ h.handle,
+ async_finish=False,
+ return_recv_hook=False,
+ )
+ return combined_x
+ combined_x, _, _ = self.buffer.combine(h.combine_input, h.handle)
+ return combined_x
+
+ def inspect_dispatch(self, p, h):
+ valid = h.recv_topk_idx >= 0
+ expert_ids = torch.where(
+ valid,
+ h.recv_topk_idx + self.rank * (self.args.experts // self.world_size),
+ h.recv_topk_idx,
+ )
+ return types.SimpleNamespace(
+ payload=h.recv_x,
+ expert_ids=expert_ids,
+ weights=h.recv_topk_weights.masked_fill(~valid, 0),
+ local_expert_counts=torch.tensor(h.recv_counts, device=self.device, dtype=torch.int64),
+ ordering_contract="source-rank-major-stable-v1",
+ )
+
+ def inspect_expert_dispatch(self, p, h):
+ if self.mode != "low-latency":
+ raise RuntimeError("expert-packed inspection requires low-latency mode")
+ return types.SimpleNamespace(
+ payload=h.recv_x,
+ local_expert_counts=h.recv_counts,
+ source_info=h.handle[0],
+ layout_range=h.handle[1],
+ )
+
+ def combine_transformed(self, p, h, transformed):
+ if self.mode == "low-latency":
+ packed = torch.zeros_like(h.recv_x)
+ packed[h.oracle_local_expert_slots, h.oracle_packed_positions] = transformed.to(
+ h.recv_x.dtype
+ )
+ combined, _, _ = self.buffer.low_latency_combine(
+ packed,
+ p.topk_idx,
+ p.topk_weights,
+ h.handle,
+ async_finish=False,
+ return_recv_hook=False,
+ )
+ return combined
+ combined, _, _ = self.buffer.combine(transformed.to(h.recv_x.dtype), h.handle)
+ return combined
+
+ def recv_tokens(self, h):
+ if self.mode == "low-latency":
+ return int(h.recv_counts.to(torch.int64).sum().item())
+ return int(h.recv_x.shape[0])
+
+ def finalize(self, rc):
+ # UCCL's proxy teardown can crash after results are written; preserve the real rc.
+ try:
+ dist.barrier()
+ except Exception:
+ pass
+ sys.stdout.flush()
+ sys.stderr.flush()
+ os._exit(rc if 0 <= rc <= 255 else 1)
diff --git a/experimental/CollectiveX/tests/eplb.py b/experimental/CollectiveX/tests/eplb.py
new file mode 100644
index 0000000000..b1479da9f1
--- /dev/null
+++ b/experimental/CollectiveX/tests/eplb.py
@@ -0,0 +1,199 @@
+#!/usr/bin/env python3
+"""CollectiveX — EPLB (Expert-Parallel Load Balancer), the DeepSeek-style remedy for
+skewed (zipf) expert load.
+
+Under skewed routing, the ranks hosting hot logical experts receive far more token-copies
+than the rest; dispatch/combine latency is gated by that busiest rank (the cross-rank MAX
+the harness measures), so the whole collective stalls on it. EPLB REPLICATES hot experts
+onto extra physical slots and PLACES the slots so every rank carries ~equal load.
+
+This module is backend-agnostic: it is purely a transform of the deterministic routing
+trace. The trick that keeps every adapter unchanged — DeepEP/MoRI both route expert i to
+rank `i // experts_per_rank` (contiguous block placement) — is to number the physical slots
+RANK-MAJOR (rank r owns physical ids [r*spp, (r+1)*spp)), so the standard contiguous mapping
+reproduces EPLB's balanced placement. The harness then runs with `experts = num_physical`
+and the remapped (physical) trace; nothing else changes.
+
+ num_physical = num_logical + redundant (redundant rounded up to a multiple of ep_size)
+ build_plan(): greedy replicate-by-load + equal-cardinality balanced packing onto ep_size ranks
+ remap_idx(): each token's logical targets -> physical replicas, spread by global token id
+
+Pure-Python planner (no torch) so it unit-tests on a login node; remap_idx needs torch.
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+
+
+def physical_count(num_logical: int, num_redundant: int, ep_size: int) -> int:
+ """num_logical + redundant, with redundant rounded UP to a multiple of ep_size so the
+ physical experts divide evenly across ranks (symmetric dispatch)."""
+ r = ((max(0, num_redundant) + ep_size - 1) // ep_size) * ep_size
+ return num_logical + r
+
+
+def _contiguous_rank_load(logical_load, ep_size):
+ """Per-rank received load WITHOUT EPLB: logical experts placed contiguously
+ (experts_per_rank = num_logical/ep_size), so rank r carries its block's total."""
+ n = len(logical_load)
+ per = n // ep_size
+ return [sum(logical_load[r * per:(r + 1) * per]) for r in range(ep_size)]
+
+
+def build_plan(logical_load, num_physical: int, ep_size: int) -> dict:
+ """logical_load: list[float] length num_logical (token-copies per logical expert).
+ Returns the replication+placement plan (all pure-Python lists) + before/after balance."""
+ num_logical = len(logical_load)
+ assert num_physical >= num_logical, "num_physical must be >= num_logical"
+ assert num_physical % ep_size == 0, "num_physical must divide ep_size"
+ assert num_logical % ep_size == 0, "num_logical must divide ep_size"
+ spp = num_physical // ep_size # physical slots per rank (fixed)
+
+ # 1) Replica allocation — start one slot per logical expert, then hand each redundant
+ # slot to the expert with the highest CURRENT per-replica load (greedy min-max).
+ replicas = [1] * num_logical
+ for _ in range(num_physical - num_logical):
+ best, best_lps = 0, -1.0
+ for e in range(num_logical):
+ lps = logical_load[e] / replicas[e]
+ if lps > best_lps:
+ best, best_lps = e, lps
+ replicas[best] += 1
+
+ # 2) Slots = (per-replica load, logical expert), one per replica.
+ slots = []
+ for e in range(num_logical):
+ lps = logical_load[e] / replicas[e]
+ slots.extend((lps, e) for _ in range(replicas[e]))
+
+ # 3) Balanced packing into ep_size bins of EQUAL cardinality (spp each), minimizing the
+ # max per-rank load: heaviest slot first -> least-loaded rank that still has capacity.
+ slots.sort(reverse=True)
+ rank_slots = [[] for _ in range(ep_size)]
+ rank_load = [0.0] * ep_size
+ for lps, e in slots:
+ r = min((r for r in range(ep_size) if len(rank_slots[r]) < spp),
+ key=lambda r: rank_load[r])
+ rank_slots[r].append(e)
+ rank_load[r] += lps
+
+ # 4) Rank-major physical numbering -> contiguous placement == this balanced placement.
+ phys2log, rank_of_phys = [], []
+ for r in range(ep_size):
+ for e in rank_slots[r]:
+ phys2log.append(e)
+ rank_of_phys.append(r)
+ log2phys = [[] for _ in range(num_logical)]
+ for pid, e in enumerate(phys2log):
+ log2phys[e].append(pid)
+
+ before = _contiguous_rank_load(logical_load, ep_size)
+ total = sum(logical_load) or 1.0
+ mean = total / ep_size
+ return {
+ "num_logical": num_logical, "num_physical": num_physical, "ep_size": ep_size,
+ "slots_per_rank": spp, "replicas": replicas, "max_replicas": max(replicas),
+ "phys2log": phys2log, "rank_of_phys": rank_of_phys, "log2phys": log2phys,
+ "rank_load_after": rank_load, "rank_load_before": before,
+ # imbalance = busiest rank / mean (1.0 = perfect). This is the number EPLB cuts.
+ "imbalance_before": max(before) / mean, "imbalance_after": max(rank_load) / mean,
+ "replicated_experts": sum(1 for r in replicas if r > 1),
+ }
+
+
+def mapping_hash(plan: dict) -> str:
+ """Hash the placement fields that fully determine the logical-to-physical remap."""
+ payload = {
+ "phys2log": plan["phys2log"],
+ "rank_of_phys": plan["rank_of_phys"],
+ "replicas": plan["replicas"],
+ }
+ return hashlib.sha256(json.dumps(payload, sort_keys=True).encode()).hexdigest()
+
+
+def remap_rows(indices: list[list[int]], plan: dict) -> list[list[int]]:
+ """Pure-Python equivalent of remap_idx for contract verification."""
+ replicas = plan["log2phys"]
+ return [
+ [replicas[expert][token % len(replicas[expert])] for expert in row]
+ for token, row in enumerate(indices)
+ ]
+
+
+def remap_idx(idx_logical, plan):
+ """idx_logical: torch [gt, topk] int64 logical-expert ids (global trace).
+ Returns idx_physical [gt, topk]: each token's logical target -> one of that expert's
+ physical replicas, SPREAD by global token id (row) so a hot expert's tokens fan out
+ across its replicas (= across ranks). Replicas of distinct logical experts are disjoint,
+ so a token's top-k physical ids stay distinct (dispatch invariant preserved)."""
+ import torch
+ replicas = plan["replicas"]
+ num_logical = len(replicas)
+ max_rc = plan["max_replicas"]
+ rc = torch.tensor(replicas, dtype=torch.int64)
+ # padded [num_logical, max_rc] table of physical ids (pad with replica 0; never indexed
+ # past rc[e] because the replica index is taken mod rc[e]).
+ padded = torch.zeros(num_logical, max_rc, dtype=torch.int64)
+ for e, phys in enumerate(plan["log2phys"]):
+ for k in range(max_rc):
+ padded[e, k] = phys[k] if k < len(phys) else phys[0]
+ gt = idx_logical.shape[0]
+ rows = torch.arange(gt, dtype=torch.int64).unsqueeze(1) # [gt,1] global token id
+ e = idx_logical.to(torch.int64) # [gt,topk]
+ ridx = rows % rc[e] # [gt,topk] replica index
+ return padded[e, ridx] # [gt,topk] physical ids
+
+
+# --------------------------------------------------------------------------- self-test
+if __name__ == "__main__":
+ # Synthetic zipf load (popularity ∝ 1/(e+1)) — the case EPLB targets. No torch needed.
+ import sys
+ NUM_LOGICAL, EP, REDUNDANT = 256, 8, 32
+ load = [1.0 / (e + 1) for e in range(NUM_LOGICAL)]
+ nphys = physical_count(NUM_LOGICAL, REDUNDANT, EP)
+ plan = build_plan(load, nphys, EP)
+ print(f"num_logical={NUM_LOGICAL} ep={EP} num_physical={nphys} slots/rank={plan['slots_per_rank']}")
+ print(f"replicated experts={plan['replicated_experts']} max_replicas={plan['max_replicas']} "
+ f"(hottest expert 0 replicas={plan['replicas'][0]})")
+ print(f"per-rank load BEFORE (contiguous): {[round(x,3) for x in plan['rank_load_before']]}")
+ print(f"per-rank load AFTER (EPLB): {[round(x,3) for x in plan['rank_load_after']]}")
+ print(f"imbalance (max/mean) BEFORE={plan['imbalance_before']:.2f}x AFTER={plan['imbalance_after']:.2f}x")
+ # Gates: equal slot cardinality, every logical expert placed, big imbalance cut.
+ assert all(plan["replicas"][e] >= 1 for e in range(NUM_LOGICAL))
+ assert sum(plan["replicas"]) == nphys
+ assert len(plan["phys2log"]) == nphys
+ assert all(len(plan["log2phys"][e]) == plan["replicas"][e] for e in range(NUM_LOGICAL))
+ # rank-major numbering => contiguous block per rank => rank_of_phys is non-decreasing
+ assert plan["rank_of_phys"] == sorted(plan["rank_of_phys"])
+ assert plan["imbalance_after"] < plan["imbalance_before"], "EPLB must reduce imbalance"
+ assert plan["imbalance_after"] < 1.30, f"EPLB should get within ~30% of perfect, got {plan['imbalance_after']:.2f}"
+ # remap (if torch present): distinctness + balanced receive on a sampled zipf trace.
+ try:
+ import torch
+ g = torch.Generator().manual_seed(0)
+ p = torch.tensor(load)
+ p = (p / p.sum()).expand(4096, NUM_LOGICAL)
+ idx_l = torch.multinomial(p, 8, replacement=False, generator=g).to(torch.int64)
+ idx_p = remap_idx(idx_l, plan)
+ assert idx_p.shape == idx_l.shape
+ # top-k physical ids distinct per token
+ assert all(len(set(row.tolist())) == 8 for row in idx_p), "physical top-k must stay distinct"
+ spp = plan["slots_per_rank"]
+ recv_before = [0] * EP
+ recv_after = [0] * EP
+ per_log = NUM_LOGICAL // EP
+ for row_l, row_p in zip(idx_l.tolist(), idx_p.tolist()):
+ for e in row_l:
+ recv_before[e // per_log] += 1
+ for pid in row_p:
+ recv_after[pid // spp] += 1
+ ib = max(recv_before) / (sum(recv_before) / EP)
+ ia = max(recv_after) / (sum(recv_after) / EP)
+ print(f"sampled-trace receive imbalance BEFORE={ib:.2f}x AFTER={ia:.2f}x")
+ assert ia < ib and ia < 1.35, "remap must balance per-rank receive load"
+ print("remap self-test: OK")
+ except ImportError:
+ print("(torch absent — skipped remap self-test; planner gates passed)")
+ print("EPLB self-test: PASS")
+ sys.exit(0)
diff --git a/experimental/CollectiveX/tests/make_workloads.py b/experimental/CollectiveX/tests/make_workloads.py
new file mode 100644
index 0000000000..862c3d0375
--- /dev/null
+++ b/experimental/CollectiveX/tests/make_workloads.py
@@ -0,0 +1,119 @@
+#!/usr/bin/env python3
+"""Generate canonical serialized workloads. Runs the stdlib counter generator for
+each (routing, global_tokens) in a ladder and writes .npz + .manifest.json into a
+dir that runs then consume via `run_ep.py --workload-dir`. One trace is emitted per global-token
+count because global token count is part of workload identity.
+
+ python3 tests/make_workloads.py --out-dir /path/to/cx_workloads \\
+ --routing uniform --ep 8 --hidden 7168 --topk 8 --experts 256 --seed 67 \\
+ --tokens-ladder "1 2 4 8 16 32 64 128 256 512"
+
+Or by the named v1 workload in configs/workloads.yaml. Explicit dimension flags still override it:
+
+ python3 tests/make_workloads.py --out-dir /path/to/cx_workloads --workload deepseek-v3-v1 --routing uniform --ep 8
+
+--id-only prints the content-bound workload_id per ladder point without torch/numpy:
+
+ python3 tests/make_workloads.py --workload deepseek-v3-v1 --ep 8 --id-only
+
+Generate every routing the suites need by running once per --routing. Idempotent (same id => same
+file). The dir is the cross-hardware artifact: copy it to each cluster so all consume identical bytes.
+"""
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+import workload as wl # noqa: E402
+
+# Repo root holds configs/ (this file is in tests/). Used only for --workload name resolution.
+_REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+
+def resolve_manifest(name):
+ """Look a workload name up in configs/workloads.yaml and return (hidden, topk, experts).
+ Searches synthetic + model_derived; expert count = `experts` or (for model-derived) `routed_experts`.
+ Raises SystemExit with the known names if the manifest is absent. Pure PyYAML + stdlib."""
+ import yaml
+ path = os.path.join(_REPO, "configs", "workloads.yaml")
+ with open(path) as handle:
+ cfg = yaml.safe_load(handle)
+ known = []
+ for section in ("synthetic", "model_derived"):
+ sec = cfg.get(section) or {}
+ known += list(sec)
+ m = sec.get(name)
+ if m is None:
+ continue
+ experts = m.get("experts", m.get("routed_experts"))
+ if m.get("hidden") is None or m.get("topk") is None or experts is None:
+ raise SystemExit(f"workload '{name}' is missing hidden/topk/experts in {path}")
+ return int(m["hidden"]), int(m["topk"]), int(experts)
+ raise SystemExit(f"unknown --workload '{name}'; known: {sorted(known)}")
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(description="Generate canonical CollectiveX workloads")
+ ap.add_argument("--out-dir", help="required unless --id-only")
+ ap.add_argument("--workload", help="named manifest in configs/workloads.yaml (sets hidden/topk/experts)")
+ ap.add_argument("--routing", default="uniform", choices=["uniform", "zipf"])
+ ap.add_argument("--ep", type=int, required=True, help="ep_size (global_tokens = T * ep)")
+ ap.add_argument("--hidden", type=int, help="override (default 7168, or the --workload's hidden)")
+ ap.add_argument("--topk", type=int, help="override (default 8, or the --workload's topk)")
+ ap.add_argument("--experts", type=int, help="override (default 256, or the --workload's experts)")
+ ap.add_argument("--seed", type=int, default=67)
+ ap.add_argument("--tokens-ladder", default="1 2 4 8 16 32 64 128 256 512")
+ ap.add_argument("--id-only", action="store_true",
+ help="print content-bound workload_id per point without torch/numpy")
+ a = ap.parse_args()
+
+ # Resolve dims: a named --workload supplies defaults; explicit --hidden/--topk/--experts override
+ # per field. With neither, fall back to the v1 DeepSeek dimensions (7168/8/256).
+ base_h, base_t, base_e = (7168, 8, 256)
+ if a.workload:
+ base_h, base_t, base_e = resolve_manifest(a.workload)
+ hidden = a.hidden if a.hidden is not None else base_h
+ topk = a.topk if a.topk is not None else base_t
+ experts = a.experts if a.experts is not None else base_e
+
+ if not a.id_only and not a.out_dir:
+ ap.error("--out-dir is required unless --id-only")
+
+ raw_ladder = [int(token) for token in a.tokens_ladder.replace(",", " ").split()]
+ if (a.ep <= 0 or min(hidden, topk, experts) <= 0 or topk > experts or experts % a.ep
+ or not raw_ladder or any(token <= 0 for token in raw_ladder)
+ or len(raw_ladder) != len(set(raw_ladder))):
+ ap.error("shape, EP, and token ladder must be positive, divisible, and unique")
+ ladder = sorted(raw_ladder)
+ epr = experts // a.ep
+ label = f"workload={a.workload} " if a.workload else ""
+
+ if a.id_only:
+ # The stdlib counter generator derives the same content-bound ID on every runtime.
+ made = []
+ for T in ladder:
+ gt = T * a.ep
+ wid = wl.compute_workload_id(a.routing, hidden, topk, experts, a.ep, gt, a.seed)
+ made.append((T, gt, wid))
+ print(f" T={T:<5} gt={gt:<6} routing={a.routing} -> {wid}")
+ print(f"{label}id-only: {len(made)} workload_id(s) "
+ f"(hidden={hidden} topk={topk} experts={experts} ep={a.ep} routing={a.routing} seed={a.seed})")
+ return 0
+
+ os.makedirs(a.out_dir, exist_ok=True)
+ made = []
+ for T in ladder:
+ gt = T * a.ep
+ idx, w, man = wl.build_workload(hidden, topk, experts, a.routing, gt, a.seed, epr)
+ wid = wl.save_workload(a.out_dir, idx, w, man)
+ made.append((T, gt, wid))
+ print(f" T={T:<5} gt={gt:<6} routing={a.routing} -> {wid} "
+ f"(trace sha {man['checksums']['trace'][:12]})")
+ print(f"{label}wrote {len(made)} canonical workloads to {a.out_dir} (routing={a.routing}, ep={a.ep})")
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/experimental/CollectiveX/tests/routing.py b/experimental/CollectiveX/tests/routing.py
new file mode 100644
index 0000000000..6065a06e43
--- /dev/null
+++ b/experimental/CollectiveX/tests/routing.py
@@ -0,0 +1,191 @@
+#!/usr/bin/env python3
+"""CollectiveX — deterministic, platform-independent MoE routing trace.
+
+Fair-comparison fix #1: routing (per-token expert IDs + gate weights) is generated
+ONCE from a fixed seed over the *global* token batch, indexed by global token id, and
+is identical on every SKU for the same (seed, routing, global_tokens, experts, top-k).
+Each rank materializes its slice `[rank*T,(rank+1)*T)`. Activations
+are per-rank (same rank ⇒ same x on any platform), so a given global token id has
+identical activation everywhere without materializing a global activation tensor.
+
+The v1 suite keeps two routing distributions:
+
+ * uniform — top-k distinct experts drawn uniformly per token. The DEFAULT.
+ Expected fan-out for top-k=8, 256 experts, EP8 (32 experts/rank) ≈
+ 8·(1 − C(224,8)/C(256,8)) ≈ 5.3 ranks/token. Load ~ Poisson.
+ * zipf — expert popularity proportional to 1/rank, producing expert/rank load skew.
+
+Always publish the realized fan-out so the workload is never misread again
+(`routing_stats`).
+"""
+from __future__ import annotations
+
+import hashlib
+
+import torch
+
+ACTIVATION_GENERATOR = "collectivex-activation-counter-v3"
+SOURCE_ID_BASE = 128
+SOURCE_ID_COLUMNS = 4
+
+
+def build_global_routing(
+ global_tokens: int, experts: int, topk: int, routing: str, seed: int
+):
+ """Return byte-stable counter-generated routing tensors on CPU."""
+ import workload
+
+ indices, weights = workload.canonical_routing_rows(
+ int(global_tokens), int(experts), int(topk), routing, int(seed)
+ )
+ return (
+ torch.tensor(indices, dtype=torch.int64),
+ torch.tensor(weights, dtype=torch.float32),
+ )
+
+
+def rank_slice(idx, weights, rank: int, tokens_per_rank: int):
+ lo = rank * tokens_per_rank
+ return idx[lo:lo + tokens_per_rank].contiguous(), weights[lo:lo + tokens_per_rank].contiguous()
+
+
+def rank_activations(tokens: int, hidden: int, seed: int, rank: int, device,
+ dtype=torch.bfloat16):
+ """Exact counter-derived inputs with a reversible global source-token prefix."""
+ source = torch.arange(tokens, device=device, dtype=torch.int64) + rank * tokens
+ return activations_for_source_ids(source, hidden, seed, dtype)
+
+
+def activations_for_source_ids(source, hidden: int, seed: int, dtype=torch.bfloat16):
+ """Materialize canonical activations for arbitrary global source-token IDs."""
+ if hidden < SOURCE_ID_COLUMNS:
+ raise ValueError(f"hidden must be at least {SOURCE_ID_COLUMNS}")
+ source = source.to(torch.int64)
+ column = torch.arange(hidden, device=source.device, dtype=torch.int64)
+ values = (source[:, None] * 131 + column[None, :] * 17 + int(seed) * 19) % 257 - 128
+ output = values.to(dtype).mul_(1 / 64)
+ output[:, 0] = source % SOURCE_ID_BASE
+ output[:, 1] = (source // SOURCE_ID_BASE) % SOURCE_ID_BASE
+ output[:, 2] = (source // (SOURCE_ID_BASE**2)) % SOURCE_ID_BASE
+ output[:, 3] = (source * 29 + int(seed) * 7) % SOURCE_ID_BASE
+ return output
+
+
+def decode_source_ids(payload, seed: int):
+ """Decode and validate source IDs carried by rank_activations."""
+ if payload.ndim != 2 or payload.shape[1] < SOURCE_ID_COLUMNS:
+ raise ValueError("received payload cannot carry the source-token prefix")
+ prefix = payload[:, :SOURCE_ID_COLUMNS].float()
+ digits = prefix.round().to(torch.int64)
+ if not torch.equal(prefix, digits.float()):
+ raise ValueError("received source-token prefix is not exact")
+ if bool(((digits < 0) | (digits >= SOURCE_ID_BASE)).any().item()):
+ raise ValueError("received source-token prefix is out of range")
+ source = digits[:, 0] + SOURCE_ID_BASE * digits[:, 1] + SOURCE_ID_BASE**2 * digits[:, 2]
+ checksum = (source * 29 + int(seed) * 7) % SOURCE_ID_BASE
+ if not torch.equal(checksum, digits[:, 3]):
+ raise ValueError("received source-token checksum differs")
+ return source
+
+
+def routing_locality(idx, experts_per_rank: int, ep_size: int, tokens_per_rank: int,
+ gpus_per_node: int, scale_up_domain: int = None) -> dict:
+ """Locality of rank-deduplicated payload copies under packed placement."""
+ import torch as _t
+ gt = idx.shape[0]
+ assignments = (idx // experts_per_rank).clamp(max=ep_size - 1)
+ destinations = _t.zeros((gt, ep_size), dtype=_t.bool)
+ destinations.scatter_(1, assignments, True)
+ token, dest = destinations.nonzero(as_tuple=True)
+ src = (token // max(1, tokens_per_rank)).clamp(max=ep_size - 1)
+ sud = scale_up_domain or (gpus_per_node * ep_size) # default: all one domain
+ phys = _t.arange(ep_size, dtype=_t.int64)
+ pd, ps = phys[dest], phys[src]
+ local = (dest == src)
+ same_node = (pd // gpus_per_node) == (ps // gpus_per_node)
+ same_dom = (pd // sud) == (ps // sud)
+ n = dest.numel()
+ return {
+ "placement": "packed",
+ "local_rank_fraction": float(local.float().mean()),
+ "same_node_fraction": float(same_node.float().mean()),
+ "same_scaleup_domain_fraction": float(same_dom.float().mean()),
+ "cross_node_fraction": float((~same_node).float().mean()),
+ "cross_domain_fraction": float((~same_dom).float().mean()),
+ "gpus_per_node": gpus_per_node, "scale_up_domain": sud, "copies": int(n),
+ }
+
+
+def routing_stats(idx, experts: int, experts_per_rank: int, weights=None) -> dict:
+ """Realized routing properties for the GLOBAL trace — published per point so the
+ fan-out / load can never be silently misread. idx is the global [gt, topk] tensor;
+ weights the matching [gt, topk] gate weights (hashed too for workload identity).
+ """
+ ep = max(1, experts // max(1, experts_per_rank))
+ ranks = (idx // experts_per_rank) # [gt, topk] destination rank per assignment
+ # unique destination ranks per token (fan-out)
+ onehot = torch.zeros(idx.shape[0], ep, dtype=torch.bool)
+ onehot.scatter_(1, ranks.clamp(max=ep - 1), True)
+ fanout = onehot.sum(dim=1) # [gt]
+ hist = torch.bincount(fanout, minlength=ep + 1)[1:ep + 1].tolist() # counts for fan-out 1..ep
+ load = torch.bincount(idx.reshape(-1), minlength=experts).float()
+ # Keep expert assignments (compute load) separate from rank-deduplicated payload copies
+ # (network load). Conflating them overstates traffic when two experts share a rank.
+ assignment_load = torch.bincount(
+ ranks.reshape(-1).clamp(max=ep - 1), minlength=ep
+ ).float()
+ payload_load = onehot.sum(dim=0).float()
+ # One-number imbalance summaries so a row is self-describing for the distribution-sensitivity
+ # suite (no need to read the full histograms): CV = std/mean of the load; hotspot_ratio =
+ # worst expert load over the mean. Zipf should be more concentrated than uniform.
+ def _cv(t):
+ m = float(t.mean())
+ return float(t.std(unbiased=False) / m) if m > 0 else 0.0
+ expert_load_cv = _cv(load)
+ assignment_rank_cv = _cv(assignment_load)
+ payload_rank_cv = _cv(payload_load)
+ hotspot_ratio = float(load.max() / load.mean()) if float(load.mean()) > 0 else 0.0
+ # Empty experts capture compute skew; empty destination ranks capture network skew.
+ empty_expert_count = int((load == 0).sum())
+ empty_rank_count = int((payload_load == 0).sum())
+ # SHA-256 workload identity over both topk_idx and gate weights: a chart
+ # point's routing is provably identical across SKUs only if both hashes match.
+ idx_bytes = idx.to(torch.int32).cpu().numpy().tobytes()
+ idx_hash = hashlib.sha256(idx_bytes).hexdigest()
+ if weights is not None:
+ w_bytes = weights.to(torch.float32).cpu().numpy().tobytes()
+ w_hash = hashlib.sha256(w_bytes).hexdigest()
+ routing_hash = hashlib.sha256(idx_bytes + w_bytes).hexdigest()
+ else:
+ w_hash, routing_hash = None, idx_hash
+ return {
+ "fanout_mean": float(fanout.float().mean()),
+ "fanout_min": int(fanout.min()), "fanout_max": int(fanout.max()),
+ "fanout_hist": hist, # index k-1 = #tokens with fan-out k
+ "expert_assignments_per_rank": [int(x) for x in assignment_load.tolist()],
+ "payload_copies_per_rank": [int(x) for x in payload_load.tolist()],
+ "routed_copies": int(fanout.sum()), # total (token, dest-rank) pairs
+ "expert_load_min": int(load.min()), "expert_load_max": int(load.max()),
+ "expert_load_mean": float(load.mean()), "expert_load_cv": expert_load_cv,
+ "expert_assignment_rank_cv": assignment_rank_cv,
+ "payload_rank_cv": payload_rank_cv, "hotspot_ratio": hotspot_ratio,
+ "empty_expert_count": empty_expert_count, "empty_rank_count": empty_rank_count,
+ "routing_hash": routing_hash, "idx_hash": idx_hash, "weights_hash": w_hash,
+ }
+
+
+# --------------------------------------------------------------------------- self-test
+if __name__ == "__main__":
+ import sys
+ E, TOPK, EPR, GT = 256, 8, 32, 4096
+ ui, _ = build_global_routing(GT, E, TOPK, "uniform", 67)
+ zi, _ = build_global_routing(GT, E, TOPK, "zipf", 67)
+ assert all(len(set(row.tolist())) == TOPK for row in ui[:16])
+ uniform, zipf = routing_stats(ui, E, EPR), routing_stats(zi, E, EPR)
+ assert uniform["hotspot_ratio"] < zipf["hotspot_ratio"]
+ dev = torch.device("cpu")
+ first = rank_activations(8, 256, 67, 0, dev, dtype=torch.float32)
+ second = rank_activations(8, 256, 67, 0, dev, dtype=torch.float32)
+ assert torch.equal(first, second) and torch.isfinite(first).all()
+ print("routing self-test: PASS")
+ sys.exit(0)
diff --git a/experimental/CollectiveX/tests/run_ep.py b/experimental/CollectiveX/tests/run_ep.py
new file mode 100644
index 0000000000..45d7dd60c9
--- /dev/null
+++ b/experimental/CollectiveX/tests/run_ep.py
@@ -0,0 +1,427 @@
+#!/usr/bin/env python3
+"""CollectiveX v1 EP benchmark entrypoint for torchrun or rank environments."""
+
+from __future__ import annotations
+
+import argparse
+import ctypes
+import json
+import os
+import platform
+import re
+import shlex
+import socket
+import subprocess
+import sys
+
+# Make the sibling tests/ modules importable when run as `tests/run_ep.py` under
+# torchrun (it executes the file as __main__, not as a package).
+HERE = os.path.dirname(os.path.abspath(__file__))
+sys.path[:0] = [HERE, os.path.dirname(HERE)]
+
+import ep_harness # noqa: E402 (stdlib-only; safe before torch)
+import identity # noqa: E402
+
+
+def _numeric_version(command: list[str]) -> str | None:
+ try:
+ result = subprocess.run(
+ command, capture_output=True, check=False, text=True, timeout=10
+ )
+ except (OSError, subprocess.TimeoutExpired):
+ return None
+ if result.returncode != 0:
+ return None
+ match = re.search(r"\b[0-9]+(?:\.[0-9]+){1,3}\b", result.stdout)
+ return match.group(0) if match else None
+
+
+def _loaded_collective_version() -> str | None:
+ try:
+ with open("/proc/self/maps", encoding="utf-8") as handle:
+ paths = {
+ os.path.realpath(line.rstrip().split()[-1])
+ for line in handle
+ if any(name in line for name in ("libnccl.so", "librccl.so"))
+ and os.path.isfile(line.rstrip().split()[-1])
+ }
+ if len(paths) != 1:
+ return None
+ version = ctypes.c_int()
+ library = ctypes.CDLL(paths.pop())
+ if library.ncclGetVersion(ctypes.byref(version)) != 0:
+ return None
+ return ep_harness.format_collective_version(version.value)
+ except (AttributeError, OSError):
+ return None
+
+
+def _runtime_fingerprint(
+ torch, device, *, machine: str, vendor: str, arch: str
+) -> dict:
+ """Return strict runtime facts without hosts, addresses, UUIDs, or paths."""
+ properties = torch.cuda.get_device_properties(device)
+ if vendor == "nvidia":
+ driver = _numeric_version(
+ ["nvidia-smi", "--query-gpu=driver_version", "--format=csv,noheader"]
+ )
+ runtime_kind, runtime_version, collective_kind = (
+ "cuda",
+ torch.version.cuda,
+ "nccl",
+ )
+ else:
+ driver = _numeric_version(["rocm-smi", "--showdriverversion"])
+ runtime_kind, runtime_version, collective_kind = (
+ "hip",
+ torch.version.hip,
+ "rccl",
+ )
+ return {
+ "accelerator_runtime": {"kind": runtime_kind, "version": runtime_version},
+ "collective_library": {
+ "kind": collective_kind,
+ "version": _loaded_collective_version(),
+ },
+ "device": {
+ "arch": arch,
+ "compute_units": int(properties.multi_processor_count),
+ "memory_bytes": int(properties.total_memory),
+ "product": torch.cuda.get_device_name(device),
+ "warp_size": int(properties.warp_size),
+ },
+ "driver_version": driver,
+ "framework": {"kind": "torch", "version": str(torch.__version__)},
+ "machine": machine,
+ "python_version": platform.python_version(),
+ "vendor": vendor,
+ }
+
+
+def _summarize_realized_placement(
+ records: list[tuple[str, int]],
+ *,
+ expected_nodes: int,
+ expected_gpus_per_node: int,
+ expected_world_size: int,
+) -> dict:
+ """Validate private host/rank records and return only publication-safe aggregates."""
+ if expected_nodes < 1 or expected_gpus_per_node < 1:
+ raise ValueError("requested placement dimensions must be positive")
+ if expected_nodes * expected_gpus_per_node != expected_world_size:
+ raise ValueError("requested nodes x GPUs per node differs from world size")
+ if len(records) != expected_world_size:
+ raise ValueError("realized rank count differs from world size")
+
+ by_host: dict[str, list[int]] = {}
+ for host, local_rank in records:
+ if not isinstance(host, str) or not host or type(local_rank) is not int:
+ raise ValueError("realized placement record has invalid types")
+ by_host.setdefault(host, []).append(local_rank)
+
+ counts = sorted(len(local_ranks) for local_ranks in by_host.values())
+ complete_local_ranks = all(
+ sorted(local_ranks) == list(range(expected_gpus_per_node))
+ for local_ranks in by_host.values()
+ )
+ unique_pairs = len(set(records)) == len(records)
+ if len(by_host) != expected_nodes:
+ raise ValueError(
+ f"realized node count {len(by_host)} differs from requested {expected_nodes}"
+ )
+ if counts != [expected_gpus_per_node] * expected_nodes:
+ raise ValueError("realized ranks per node differ from requested GPUs per node")
+ if not complete_local_ranks or not unique_pairs:
+ raise ValueError("realized local ranks are incomplete or duplicated")
+ return {
+ "gpus_per_node": expected_gpus_per_node,
+ "nodes": expected_nodes,
+ "ranks_per_node": expected_gpus_per_node,
+ "unique_local_ranks": True,
+ "valid": True,
+ }
+
+
+def _common_runtime_fingerprint(records: list[dict]) -> dict:
+ """Return the shared sanitized fingerprint, rejecting heterogeneous ranks."""
+ if not records:
+ raise ValueError("runtime fingerprint evidence is empty")
+ canonical = {
+ json.dumps(record, allow_nan=False, sort_keys=True, separators=(",", ":"))
+ for record in records
+ }
+ if len(canonical) != 1:
+ raise ValueError("runtime fingerprint differs across distributed ranks")
+ return records[0]
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(description="CollectiveX EP dispatch/combine sweep")
+ ap.add_argument(
+ "--backend",
+ required=True,
+ choices=[
+ "deepep",
+ "deepep-v2",
+ "deepep-hybrid",
+ "mori",
+ "uccl",
+ "nccl-ep",
+ ],
+ )
+ ep_harness.add_common_args(ap)
+ args = ap.parse_args()
+
+ if args.mode == ep_harness.LOW_LATENCY_MODE:
+ if args.backend not in {"deepep", "uccl"}:
+ print(
+ "ERROR: low-latency mode is supported only by deepep and uccl",
+ file=sys.stderr,
+ )
+ return 2
+ if args.phase != "decode":
+ print("ERROR: low-latency mode requires --phase decode", file=sys.stderr)
+ return 2
+ if args.case_id and not identity.is_typed_id(args.case_id, "case"):
+ print(f"ERROR: invalid native case ID {args.case_id!r}", file=sys.stderr)
+ return 2
+ if args.case_id and args.seed != ep_harness.ROUTING_SEED:
+ print(
+ f"ERROR: scheduled v1 cases require seed={ep_harness.ROUTING_SEED}; got {args.seed}",
+ file=sys.stderr,
+ )
+ return 2
+
+ sampling_error = ep_harness.sampling_contract_error(
+ args.iters, args.trials, args.warmup
+ )
+ if sampling_error:
+ print(f"ERROR: {sampling_error}", file=sys.stderr)
+ return 2
+
+ try:
+ import torch
+ import torch.distributed as dist
+ except Exception as exc: # pragma: no cover
+ print(f"ERROR: torch unavailable: {exc!r}", file=sys.stderr)
+ return 3
+
+ rank = int(os.environ.get("RANK", "0"))
+ world_size = int(os.environ.get("WORLD_SIZE", "1"))
+ local_rank = int(os.environ.get("LOCAL_RANK", "0"))
+ torch.cuda.set_device(local_rank)
+ device = torch.device(f"cuda:{local_rank}")
+ os.environ.setdefault("MASTER_ADDR", "localhost")
+ os.environ.setdefault("MASTER_PORT", "12355")
+
+ import capability
+
+ sku = capability.PLATFORMS.get(args.runner)
+ if sku is None:
+ print(f"ERROR: unknown runner identity {args.runner!r}", file=sys.stderr)
+ return 5
+ machine = {"x86_64": "amd64", "aarch64": "arm64"}.get(
+ platform.machine(), platform.machine()
+ )
+ props = torch.cuda.get_device_properties(device)
+ if torch.version.hip:
+ vendor = "amd"
+ accelerator = str(getattr(props, "gcnArchName", "")).split(":", 1)[0]
+ else:
+ vendor = "nvidia"
+ major, minor = torch.cuda.get_device_capability(device)
+ accelerator = f"sm{major}{minor}"
+ device_name = torch.cuda.get_device_name(device)
+ device_count = torch.cuda.device_count()
+ identity_issues = capability.runtime_identity_issues(
+ args.runner,
+ vendor=vendor,
+ arch=accelerator,
+ machine=machine,
+ device_name=device_name,
+ device_count=device_count,
+ world_size=world_size,
+ )
+ if identity_issues:
+ print(
+ f"ERROR: runtime identity does not match {args.runner}: "
+ + "; ".join(identity_issues),
+ file=sys.stderr,
+ )
+ return 5
+ observed_gpus_per_node = args.gpus_per_node or device_count
+ if observed_gpus_per_node != sku["gpus_per_node"]:
+ print(
+ f"ERROR: {args.runner} requires {sku['gpus_per_node']} GPUs per node",
+ file=sys.stderr,
+ )
+ return 5
+ if world_size % observed_gpus_per_node:
+ print("ERROR: distributed world is not divisible by GPUs per node", file=sys.stderr)
+ return 5
+ observed_nodes = world_size // observed_gpus_per_node
+ topology = capability.topology_for(args.runner, world_size)
+ observed_topology = {
+ "nodes": observed_nodes,
+ "gpus_per_node": observed_gpus_per_node,
+ "scale_up_domain": args.scale_up_domain or observed_gpus_per_node,
+ "scope": args.scope,
+ "scale_up_transport": args.scale_up_transport,
+ "scale_out_transport": args.scale_out_transport or None,
+ "transport": args.transport,
+ "topology_class": args.topology_class,
+ }
+ if topology is None or any(
+ observed_topology[field] != topology[field] for field in observed_topology
+ ):
+ print(
+ f"ERROR: runtime topology does not match {args.runner} EP{world_size}",
+ file=sys.stderr,
+ )
+ return 5
+ schedulable, reason = capability.resolve(
+ args.runner,
+ args.backend,
+ ep=world_size,
+ nodes=observed_nodes,
+ routing=args.routing,
+ eplb=args.eplb,
+ mode=args.mode,
+ )
+ if not schedulable:
+ print(f"ERROR: scheduled case is unsupported: {reason}", file=sys.stderr)
+ return 5
+ args.runtime_device_product = device_name
+ args.runtime_device_count = device_count
+ args.allocation_execution_id = os.environ.get("COLLECTIVEX_EXECUTION_ID")
+
+ # EPLB bumps the expert count to PHYSICAL (logical + redundant) BEFORE backend construction
+ # so the backend sizes its buffers for the replicated set; ep_harness builds the LOGICAL
+ # routing trace and remaps it to the balanced physical placement (a pure routing transform,
+ # tests/eplb.py — no adapter change). Deterministic, so every rank agrees on the count.
+ if getattr(args, "eplb", False):
+ import eplb
+
+ args.num_logical_experts = args.experts
+ args.experts = eplb.physical_count(
+ args.experts, ep_harness.EPLB_REDUNDANT_EXPERTS, world_size
+ )
+
+ # Reproduction provenance (recorded in the artifact). Rack launchers provide ranks directly
+ # through srun, while single-node launchers use torchrun; do not claim torchrun for both.
+ if os.environ.get("TORCHELASTIC_RUN_ID"):
+ args.distributed_launcher = "torchrun"
+ prefix = f"torchrun --nproc_per_node={world_size}"
+ else:
+ args.distributed_launcher = "rank-environment"
+ prefix = f"RANK={rank} WORLD_SIZE={world_size} LOCAL_RANK={local_rank} python3"
+ args.reproduction_command = f"{prefix} tests/run_ep.py {shlex.join(sys.argv[1:])}"
+ args.image = os.environ.get("COLLECTIVEX_IMAGE", "")
+ args.image_digest = os.environ.get("COLLECTIVEX_IMAGE_DIGEST", "")
+ args.image_digest_verified = (
+ os.environ.get("COLLECTIVEX_IMAGE_DIGEST_VERIFIED") == "1"
+ )
+ # Container architecture and local squash hash for Enroot/Pyxis.
+ args.image_arch = machine
+ args.squash_sha256 = os.environ.get("COLLECTIVEX_SQUASH_SHA256")
+ # GitHub provenance: repo, run ID, attempt, ref, source SHA, job,
+ # artifact. A result is only publication-'official' when these are present (validity gate).
+ _run = {
+ "run_id": os.environ.get("GITHUB_RUN_ID"),
+ "run_attempt": os.environ.get("GITHUB_RUN_ATTEMPT"),
+ "ref": os.environ.get("GITHUB_REF_NAME") or os.environ.get("GITHUB_REF"),
+ "source_sha": os.environ.get("COLLECTIVEX_SOURCE_SHA")
+ or os.environ.get("GITHUB_SHA"),
+ "repo": os.environ.get("GITHUB_REPOSITORY"),
+ "job": os.environ.get("GITHUB_JOB"),
+ "artifact": os.environ.get("COLLECTIVEX_ARTIFACT_NAME"),
+ }
+ args.git_run = _run if any(_run.values()) else None
+
+ # Import the backend class only after torch initializes. The selected mode is an
+ # explicit case dimension; adapters do not infer it from the token ladder.
+ if args.backend == "mori":
+ from ep_mori import MoRIBackend as Backend
+ elif args.backend == "nccl-ep":
+ from ep_nccl import NCCLBackend as Backend
+ elif args.backend == "uccl":
+ from ep_uccl import UCCLBackend as Backend
+ elif args.backend == "deepep-hybrid":
+ from ep_deepep_hybrid import DeepEPHybridBackend as Backend
+ elif args.backend == "deepep-v2":
+ from ep_deepep_v2 import DeepEPV2Backend as Backend
+ else:
+ from ep_deepep import DeepEPBackend as Backend
+
+ # MoRI uses the gloo+NCCL group shape from its reference; other adapters use NCCL/RCCL.
+ if not dist.is_initialized():
+ if args.backend == "mori":
+ dist.init_process_group(
+ backend="cpu:gloo,cuda:nccl",
+ rank=rank,
+ world_size=world_size,
+ device_id=device,
+ )
+ elif args.backend == "deepep-v2":
+ # PR #605 reuses PyTorch's NCCL communicator through ``_comm_ptr``. Supplying
+ # device_id eagerly forms it before ElasticBuffer construction.
+ dist.init_process_group("nccl", device_id=device)
+ else:
+ dist.init_process_group("nccl")
+
+ args.runtime_fingerprint = _runtime_fingerprint(
+ torch, device, machine=machine, vendor=vendor, arch=accelerator
+ )
+
+ gpus_per_node = args.gpus_per_node or sku["gpus_per_node"]
+ try:
+ expected_nodes = int(
+ os.environ.get("SLURM_NNODES", str(world_size // gpus_per_node))
+ )
+ except ValueError as exc:
+ raise ValueError("SLURM_NNODES must be a positive integer") from exc
+ realized_records: list[tuple[str, int, dict] | None] = [None] * world_size
+ dist.all_gather_object(
+ realized_records,
+ (socket.gethostname(), local_rank, args.runtime_fingerprint),
+ )
+ complete_records = [record for record in realized_records if record is not None]
+ args.realized_placement = _summarize_realized_placement(
+ [(record[0], record[1]) for record in complete_records],
+ expected_nodes=expected_nodes,
+ expected_gpus_per_node=gpus_per_node,
+ expected_world_size=world_size,
+ )
+ args.runtime_fingerprint = _common_runtime_fingerprint(
+ [record[2] for record in complete_records]
+ )
+
+ # Construct + run inside a try so a backend exception (esp. a new adapter on GPU) prints its
+ # FULL traceback to STDOUT — torchrun captures per-rank stdout but only summarizes stderr, so an
+ # uncaught exception is otherwise invisible in CI. Print on every rank (prefixed) then re-raise.
+ try:
+ backend = Backend(args, rank, world_size, local_rank, device)
+ if rank == 0:
+ print(
+ f"[run_ep] backend={args.backend} phase={args.phase} mode={args.mode} "
+ f"world={world_size} ep_size={world_size} hidden={args.hidden} "
+ f"topk={args.topk} experts={args.experts} dtype=bf16 "
+ f"routing={args.routing} seed={args.seed}"
+ )
+ rc = ep_harness.run_sweep(args, backend, torch, dist, device, rank, world_size)
+ except Exception:
+ import traceback
+
+ print(
+ f"[run_ep][rank{rank}] backend={args.backend} FAILED:\n"
+ + traceback.format_exc(),
+ flush=True,
+ )
+ raise
+ # finalize() handles backend-specific teardown: DeepEP returns rc cleanly;
+ # MoRI hard-exits past its post-shmem_finalize teardown assertion.
+ return backend.finalize(rc)
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/experimental/CollectiveX/tests/test_deepep_v2_contract.py b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
new file mode 100644
index 0000000000..082cd491a6
--- /dev/null
+++ b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
@@ -0,0 +1,2151 @@
+#!/usr/bin/env python3
+"""CPU-only structural and registry tests for the pinned DeepEP V2 path."""
+from __future__ import annotations
+
+import ast
+import argparse
+import copy
+import ctypes
+import hashlib
+import json
+import os
+from pathlib import Path
+from pathlib import PurePosixPath
+import shutil
+import stat
+import subprocess
+import sys
+import tempfile
+import types
+import unittest
+
+HERE = Path(__file__).resolve().parent
+ROOT = HERE.parent
+sys.path.insert(0, str(ROOT))
+
+import capability # noqa: E402
+import contracts # noqa: E402
+import ep_harness # noqa: E402
+import identity # noqa: E402
+import run_ep # noqa: E402
+
+
+COMMIT = "fa8a9b16898204afd347c663b89e65ef87dc6ce6"
+TREE = "29809e75c5874e6609dac4804e7b651d5226959f"
+FMT_COMMIT = "a4c7e17133ee9cb6a2f45545f6e974dd3c393efa"
+
+
+def deepep_v2_jit_provenance() -> list[dict[str, str]]:
+ return [
+ {
+ "cache_key": f"kernel.{name}.{index:032x}",
+ "cubin_sha256": f"{index + 1:x}" * 64,
+ "sass_sha256": f"{index + 2:x}" * 64,
+ "source_sha256": f"{index + 3:x}" * 64,
+ }
+ for index, name in enumerate(sorted(contracts.DEEPEP_V2_JIT_KERNELS))
+ ]
+
+
+def hybrid_realized_config() -> dict[str, object]:
+ config = {field: 1 for field in contracts.HYBRID_REALIZED_CONFIG_FIELDS}
+ for field in contracts.HYBRID_REALIZED_BOOL_FIELDS:
+ config[field] = True
+ config["token_data_type"] = "UINT16"
+ return config
+
+
+def hybrid_jit_provenance(ranks: int = 2) -> tuple[list[str], list[dict[str, object]]]:
+ keys = ["combine-key", "dispatch-key", "preprocess-key"]
+ artifacts = [
+ {
+ "kernel_key": key,
+ "rank_artifacts": [
+ {"bytes": 10 + index, "rank": rank, "sha256": f"{index + 1:x}" * 64}
+ for rank in range(ranks)
+ ],
+ }
+ for index, key in enumerate(keys)
+ ]
+ return keys, artifacts
+
+
+def load_uccl_function(name: str, namespace: dict[str, object]):
+ path = HERE / "ep_uccl.py"
+ function = next(
+ node
+ for node in ast.parse(path.read_text()).body
+ if isinstance(node, ast.FunctionDef) and node.name == name
+ )
+ exec(compile(ast.Module(body=[function], type_ignores=[]), str(path), "exec"), namespace)
+ return namespace[name]
+
+
+def operator_config(root: Path) -> dict[str, object]:
+ path = str(root)
+ network = {"socket_ifname": "eth0", "rdma_devices": "mlx5_0:1"}
+ runners = {
+ "h100-dgxc": {
+ "partition": "test", "account": "test", "squash_dir": path,
+ "stage_dir": path, **network,
+ },
+ "h200-dgxc": {
+ "partition": "test", "squash_dir": path, "stage_dir": path, **network,
+ },
+ "b200-dgxc": {
+ "partition": "test", "account": "test", "squash_dir": path,
+ "stage_dir": path, **network,
+ },
+ "b300": {
+ "partition": "test", "account": "test", "squash_dir": path, "stage_dir": path,
+ **network,
+ },
+ "gb200": {"partition": "test", "account": "test", "storage_roots": [path]},
+ "gb300": {
+ "partition": "test", "account": "test", "squash_dir": path,
+ "stage_dir": path, "enroot_cache_path": path,
+ },
+ "mi325x": {
+ "partition": "test", "squash_dir": path, "stage_dir": path, **network,
+ },
+ "mi355x": {
+ "partition": "test", "squash_dir": path, "stage_dir": path, **network,
+ },
+ }
+ return {"schema_version": 1, "runners": runners}
+
+
+class DeepEPV2ContractTests(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls) -> None:
+ cls.path = HERE / "ep_deepep_v2.py"
+ cls.tree = ast.parse(cls.path.read_text(), str(cls.path))
+
+ def test_capability_is_explicit_for_every_sku(self) -> None:
+ backend = capability.BACKENDS["deepep-v2"]
+ self.assertEqual(
+ (backend["implementation"], backend["commit"], backend["torch"], backend["nccl"]),
+ ("deep_ep.ElasticBuffer", COMMIT, "2.10.0+cu130", "2.30.4"),
+ )
+ self.assertEqual(backend["source"], "deepseek-ai/DeepEP#605+#630")
+ self.assertEqual(backend["communication_backend"], "nccl-device-lsa")
+ self.assertEqual(set(backend["sku_capabilities"]), set(capability.PLATFORMS))
+ for sku, platform in capability.PLATFORMS.items():
+ ok, _ = capability.resolve(sku, "deepep-v2")
+ self.assertEqual(ok, platform["vendor"] == "nvidia" and sku != "h100-dgxc")
+ self.assertEqual(
+ set(backend["sku_capabilities"][sku]), {"basis", "schedulable"}
+ )
+ self.assertEqual(
+ backend["sku_capabilities"]["h100-dgxc"],
+ {
+ "schedulable": False,
+ "basis": "current-runner-nccl-device-api-symmetric-memory-unavailable",
+ },
+ )
+
+ def test_adapter_ast_pins_elastic_api_and_weight_semantics(self) -> None:
+ imports = {
+ alias.name
+ for node in ast.walk(self.tree)
+ if isinstance(node, ast.ImportFrom) and node.module == "deep_ep"
+ for alias in node.names
+ }
+ self.assertEqual(imports, {"ElasticBuffer"})
+ constants = {
+ node.targets[0].id: ast.literal_eval(node.value)
+ for node in self.tree.body
+ if isinstance(node, ast.Assign)
+ and len(node.targets) == 1
+ and isinstance(node.targets[0], ast.Name)
+ and isinstance(node.value, ast.Constant)
+ }
+ self.assertEqual(constants["DEEPEP_V2_COMMIT"], COMMIT)
+ self.assertEqual(constants["DEEPEP_V2_TREE"], TREE)
+ self.assertEqual(constants["DEEPEP_V2_FMT_COMMIT"], FMT_COMMIT)
+ self.assertEqual(constants["DEEPEP_V2_PR"], 605)
+ self.assertEqual(constants["DEEPEP_V2_FIX_PR"], 630)
+ self.assertEqual(
+ constants["DEEPEP_V2_JIT_RANDOM_SEED"],
+ "collectivex-deepep-v2-fa8a9b1",
+ )
+ self.assertEqual(constants["NCCL_VERSION"], "2.30.4")
+ self.assertEqual(constants["NVSHMEM_VERSION"], "3.3.9")
+ backend = next(
+ node for node in self.tree.body
+ if isinstance(node, ast.ClassDef) and node.name == "DeepEPV2Backend"
+ )
+ assignments = {
+ node.targets[0].id: ast.literal_eval(node.value)
+ for node in backend.body
+ if isinstance(node, ast.Assign)
+ and isinstance(node.targets[0], ast.Name)
+ and isinstance(node.value, ast.Constant)
+ }
+ self.assertEqual(assignments["combine_weight_semantics"], "unweighted-rank-sum")
+ methods = {node.name for node in backend.body if isinstance(node, ast.FunctionDef)}
+ self.assertTrue({
+ "dispatch", "inspect_dispatch", "combine_transformed", "capture_deferred_provenance",
+ "finalize",
+ } <= methods)
+ self.assertNotIn("expected", methods)
+ constructor = next(
+ node for node in ast.walk(backend)
+ if isinstance(node, ast.Call)
+ and isinstance(node.func, ast.Name)
+ and node.func.id == "ElasticBuffer"
+ )
+ deterministic = next(
+ keyword for keyword in constructor.keywords if keyword.arg == "deterministic"
+ )
+ self.assertIs(ast.literal_eval(deterministic.value), False)
+ self.assertIn("deterministic", contracts.REQUIRED_BACKEND_PROVENANCE["deepep-v2"])
+ self.assertIn("num_experts", contracts.REQUIRED_BACKEND_PROVENANCE["deepep-v2"])
+ self.assertIn("tuning_num_experts", contracts.REQUIRED_BACKEND_PROVENANCE["deepep-v2"])
+ self.assertIn("jit_random_seed", contracts.REQUIRED_BACKEND_PROVENANCE["deepep-v2"])
+ self.assertIn("gin_enabled", contracts.REQUIRED_BACKEND_PROVENANCE["deepep-v2"])
+ self.assertIn("communication_backend", contracts.REQUIRED_BACKEND_PROVENANCE["deepep-v2"])
+ self.assertIn("deepep_pr", contracts.REQUIRED_BACKEND_PROVENANCE["deepep-v2"])
+ self.assertIn("deepep_fix_pr", contracts.REQUIRED_BACKEND_PROVENANCE["deepep-v2"])
+ source = self.path.read_text()
+ self.assertIn('getattr(args, "num_logical_experts", args.experts)', source)
+ self.assertIn('"use_expanded_layout": False', source)
+ self.assertIn("allow_hybrid_mode = _configure_gin_mode(args, world_size)", source)
+ self.assertIn("get_theoretical_num_sms(tuning_num_experts, args.topk)", source)
+
+ jit_function = next(
+ node for node in self.tree.body
+ if isinstance(node, ast.FunctionDef) and node.name == "_jit_cache_key"
+ )
+ namespace = {"hashlib": __import__("hashlib"), "json": json}
+ exec(compile(ast.Module(body=[jit_function], type_ignores=[]), str(self.path), "exec"), namespace)
+ key = namespace["_jit_cache_key"]
+ baseline = types.SimpleNamespace(
+ runner="h100-dgxc", hidden=7168, topk=8, experts=256,
+ routing="uniform", eplb=False, case_id="uniform",
+ )
+ zipf = types.SimpleNamespace(**{**vars(baseline), "routing": "zipf", "case_id": "zipf"})
+ eplb = types.SimpleNamespace(
+ **{**vars(zipf), "experts": 288, "num_logical_experts": 256, "eplb": True}
+ )
+ realized = {
+ "num_sms": 24,
+ "num_qps": 9,
+ "allocated_qps": 17,
+ "logical_scaleout_ranks": 1,
+ "logical_scaleup_ranks": 8,
+ "physical_rdma_ranks": 2,
+ "physical_nvlink_ranks": 4,
+ "is_scaleup_nvlink": False,
+ "device_arch_major": 9,
+ "device_arch_minor": 0,
+ "device_sms": 132,
+ "device_smem_bytes": 232448,
+ "gpu_timeout_cycles": 198000000000,
+ }
+ direct = key(baseline, 8, 128, False, realized)
+ self.assertTrue(direct.startswith("jitcfg-v3-"))
+ self.assertEqual(direct, key(zipf, 8, 128, False, realized))
+ self.assertNotEqual(direct, key(zipf, 8, 128, True, realized))
+ self.assertNotEqual(direct, key(eplb, 8, 128, False, realized))
+ for field, value in realized.items():
+ changed = not value if type(value) is bool else value + 1
+ self.assertNotEqual(
+ direct,
+ key(baseline, 8, 128, False, {**realized, field: changed}),
+ field,
+ )
+ init = next(
+ node for node in backend.body
+ if isinstance(node, ast.FunctionDef) and node.name == "__init__"
+ )
+ buffer_call = next(
+ node for node in ast.walk(init)
+ if isinstance(node, ast.Call)
+ and isinstance(node.func, ast.Name)
+ and node.func.id == "ElasticBuffer"
+ )
+ jit_config_check = next(
+ node for node in ast.walk(init)
+ if isinstance(node, ast.Call)
+ and isinstance(node.func, ast.Name)
+ and node.func.id == "_require_cross_rank_equal"
+ and ast.literal_eval(node.args[1]) == "JIT configuration"
+ )
+ cache_assignment = next(
+ node for node in ast.walk(init)
+ if isinstance(node, ast.Assign)
+ and isinstance(node.targets[0], ast.Subscript)
+ and ast.unparse(node.targets[0].value) == "os.environ"
+ and ast.literal_eval(node.targets[0].slice) == "EP_JIT_CACHE_DIR"
+ )
+ self.assertLess(buffer_call.lineno, jit_config_check.lineno)
+ self.assertLess(jit_config_check.lineno, cache_assignment.lineno)
+ capture = next(
+ node for node in backend.body
+ if isinstance(node, ast.FunctionDef)
+ and node.name == "capture_deferred_provenance"
+ )
+ calls = [node for node in ast.walk(capture) if isinstance(node, ast.Call)]
+ barrier = next(
+ node for node in calls
+ if isinstance(node.func, ast.Attribute) and node.func.attr == "barrier"
+ )
+ self.assertEqual(
+ {keyword.arg: ast.literal_eval(keyword.value) for keyword in barrier.keywords},
+ {"use_comm_stream": True, "with_cpu_sync": True},
+ )
+ scan = next(
+ node for node in calls
+ if isinstance(node.func, ast.Name) and node.func.id == "_jit_artifact_evidence"
+ )
+ self.assertLess(barrier.lineno, scan.lineno)
+ realized_check = next(
+ node for node in ast.walk(backend)
+ if isinstance(node, ast.Call)
+ and isinstance(node.func, ast.Name)
+ and node.func.id == "_require_cross_rank_equal"
+ and len(node.args) > 1
+ and isinstance(node.args[1], ast.Constant)
+ and node.args[1].value == "realized tuning/topology"
+ )
+ self.assertIsInstance(realized_check, ast.Call)
+ self.assertEqual(
+ (ROOT / "tests" / "ep_harness.py").read_text().count(
+ "capture_deferred_provenance()"
+ ),
+ 2,
+ )
+ schema = json.loads((ROOT / "schemas" / "raw-case-v1.schema.json").read_text())
+ provenance = schema["properties"]["implementation"]["properties"]["provenance"]
+ self.assertEqual(provenance["properties"]["deterministic"], {"type": "boolean"})
+ self.assertEqual(
+ provenance["properties"]["num_experts"],
+ {"minimum": 1, "type": "integer"},
+ )
+ self.assertEqual(
+ provenance["properties"]["tuning_num_experts"],
+ {"minimum": 1, "type": "integer"},
+ )
+ self.assertEqual(
+ provenance["properties"]["jit_cubins"]["items"],
+ {"$ref": "#/$defs/deepep_v2_jit_cubin"},
+ )
+ self.assertEqual(
+ (
+ provenance["properties"]["jit_cubins"]["minItems"],
+ provenance["properties"]["jit_cubins"]["maxItems"],
+ ),
+ (5, 5),
+ )
+ self.assertEqual(
+ provenance["properties"]["jit_random_seed"],
+ {"const": "collectivex-deepep-v2-fa8a9b1"},
+ )
+ self.assertEqual(provenance["properties"]["allow_hybrid_mode"], {"type": "boolean"})
+ self.assertEqual(provenance["properties"]["gin_enabled"], {"type": "boolean"})
+ self.assertEqual(provenance["properties"]["deepep_pr"], {"const": 605})
+ self.assertEqual(provenance["properties"]["deepep_fix_pr"], {"const": 630})
+ self.assertEqual(
+ provenance["properties"]["communication_backend"],
+ {"enum": ["nccl-device-lsa", "nccl-gin"]},
+ )
+ self.assertEqual(
+ provenance["properties"]["num_rdma_bytes"],
+ {"minimum": 0, "type": "integer"},
+ )
+ self.assertEqual(
+ provenance["properties"]["num_qps_per_rank"],
+ {"minimum": 1, "type": "integer"},
+ )
+ for field, value in (
+ ("num_experts", "288"),
+ ("tuning_num_experts", "not-an-integer"),
+ ("tuning_num_experts", 0),
+ ):
+ with self.subTest(provenance_field=field, value=value):
+ self.assertIn(
+ field,
+ contracts.backend_provenance_issues(
+ "deepep-v2", {field: value}
+ ),
+ )
+
+ def test_v2_gin_mode_uses_the_scale_up_domain_and_safe_fallbacks(self) -> None:
+ functions = {
+ node.name: node for node in self.tree.body if isinstance(node, ast.FunctionDef)
+ }
+ namespace = {"os": os}
+ exec(
+ compile(
+ ast.Module(
+ body=[
+ functions["_configure_gin_mode"],
+ functions["_lsa_topology_is_valid"],
+ ],
+ type_ignores=[],
+ ),
+ str(self.path),
+ "exec",
+ ),
+ namespace,
+ )
+ configure = namespace["_configure_gin_mode"]
+ topology_is_valid = namespace["_lsa_topology_is_valid"]
+ original = os.environ.get("EP_DISABLE_GIN")
+ try:
+ args = types.SimpleNamespace(scale_up_domain=72, gpus_per_node=4)
+ self.assertFalse(configure(args, 8))
+ self.assertEqual(os.environ.get("EP_DISABLE_GIN"), "1")
+
+ os.environ["EP_DISABLE_GIN"] = "stale"
+ args = types.SimpleNamespace(scale_up_domain=8, gpus_per_node=4)
+ self.assertTrue(configure(args, 16))
+ self.assertNotIn("EP_DISABLE_GIN", os.environ)
+
+ args = types.SimpleNamespace(gpus_per_node=4)
+ self.assertTrue(configure(args, 8))
+ self.assertNotIn("EP_DISABLE_GIN", os.environ)
+
+ self.assertFalse(configure(types.SimpleNamespace(), 8))
+ self.assertEqual(os.environ.get("EP_DISABLE_GIN"), "1")
+
+ topology = {
+ "physical_rdma_ranks": 1,
+ "physical_nvlink_ranks": 8,
+ "logical_scaleout_ranks": 1,
+ "logical_scaleup_ranks": 8,
+ "is_scaleup_nvlink": True,
+ }
+ self.assertTrue(topology_is_valid(False, 8, 8, topology))
+ topology["physical_rdma_ranks"] = 2
+ topology["logical_scaleout_ranks"] = 2
+ self.assertTrue(topology_is_valid(True, 16, 8, topology))
+ topology["physical_nvlink_ranks"] = 4
+ self.assertFalse(topology_is_valid(False, 8, 8, topology))
+ finally:
+ if original is None:
+ os.environ.pop("EP_DISABLE_GIN", None)
+ else:
+ os.environ["EP_DISABLE_GIN"] = original
+
+ def test_ep_adapters_declare_unweighted_rank_sum(self) -> None:
+ adapters = {
+ "ep_deepep.py": "DeepEPBackend",
+ "ep_deepep_v2.py": "DeepEPV2Backend",
+ "ep_deepep_hybrid.py": "DeepEPHybridBackend",
+ "ep_mori.py": "MoRIBackend",
+ "ep_nccl.py": "NCCLBackend",
+ "ep_uccl.py": "UCCLBackend",
+ }
+ for filename, class_name in adapters.items():
+ with self.subTest(adapter=filename):
+ tree = ast.parse((HERE / filename).read_text())
+ backend = next(
+ node for node in tree.body
+ if isinstance(node, ast.ClassDef) and node.name == class_name
+ )
+ assignment = next(
+ node for node in backend.body
+ if isinstance(node, ast.Assign)
+ and isinstance(node.targets[0], ast.Name)
+ and node.targets[0].id == "combine_weight_semantics"
+ )
+ self.assertEqual(ast.literal_eval(assignment.value), "unweighted-rank-sum")
+ combine_methods = [
+ item for item in backend.body
+ if isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef))
+ and item.name in {"combine", "combine_transformed"}
+ ]
+ self.assertEqual(len(combine_methods), 2)
+ for method in combine_methods:
+ source = ast.unparse(method)
+ if filename in {"ep_deepep.py", "ep_uccl.py"}:
+ self.assertIn("self.mode == 'low-latency'", source)
+ else:
+ self.assertNotIn("topk_weights", source)
+ self.assertNotIn("combine_topk_weights", source)
+
+ def test_low_latency_mode_parser_and_profile_are_explicit(self) -> None:
+ parser = argparse.ArgumentParser()
+ ep_harness.add_common_args(parser)
+ required = [
+ "--runner", "test", "--topology-class", "test",
+ "--scope", "scale-up", "--scale-up-transport", "nvlink",
+ "--out", "test.json",
+ ]
+ self.assertEqual(parser.parse_args(required).mode, "normal")
+ self.assertEqual(
+ parser.parse_args([*required, "--mode", "low-latency"]).mode,
+ "low-latency",
+ )
+ profile = identity.case_profile("low-latency")
+ self.assertEqual(profile["contract"], "expert-packed-weighted-combine-v1")
+ self.assertEqual(
+ profile["component_order_contract"],
+ "roundtrip-dispatch-gate-weighted-combine-v1",
+ )
+ self.assertEqual(
+ profile["correctness_scope"],
+ "expert-assignment-and-weighted-combine",
+ )
+ self.assertEqual(profile["payload_unit"], "token-expert")
+
+ def test_expert_packed_slot_map_reconstructs_exact_sources(self) -> None:
+ pack = lambda begin, count: (begin << 32) | count
+ slots = ep_harness.expert_packed_slot_map(
+ [2, 1],
+ [[1, 0, 0, 0], [1, 0, 0, 0]],
+ [[pack(0, 1), pack(1, 1)], [pack(0, 0), pack(0, 1)]],
+ tokens_per_rank=2,
+ experts_per_rank=2,
+ world_size=2,
+ )
+ self.assertEqual(slots, [(0, 0, 1), (0, 1, 2), (1, 0, 3)])
+
+ invalid = (
+ ([1], [[0]], [[pack(1, 1), pack(0, 0)]]),
+ ([1], [[2]], [[pack(0, 1), pack(1, 0)]]),
+ ([2], [[1, 1]], [[pack(0, 2), pack(2, 0)]]),
+ )
+ for counts, source, layout in invalid:
+ with self.subTest(counts=counts, source=source, layout=layout):
+ with self.assertRaises(ValueError):
+ ep_harness.expert_packed_slot_map(
+ counts,
+ source,
+ layout,
+ tokens_per_rank=2,
+ experts_per_rank=1,
+ world_size=2,
+ )
+
+ def test_deepep_and_uccl_expose_genuine_low_latency_calls(self) -> None:
+ required_fragments = (
+ "Buffer.get_low_latency_rdma_size_hint(",
+ "low_latency_mode=True",
+ "num_qps_per_rank=num_qps_per_rank",
+ "self.buffer.clean_low_latency_buffer(",
+ "self.buffer.low_latency_dispatch(",
+ "use_fp8=False",
+ "self.buffer.low_latency_combine(",
+ "p.topk_weights",
+ 'self.combine_weight_semantics = "gate-weighted-sum"',
+ "self.combine_needs_redispatch = True",
+ "def inspect_expert_dispatch(",
+ )
+ for filename in ("ep_deepep.py", "ep_uccl.py"):
+ source = (HERE / filename).read_text()
+ with self.subTest(adapter=filename):
+ for fragment in required_fragments:
+ self.assertIn(fragment, source)
+ self.assertIn("self.max_tokens_per_rank = 128", source)
+ self.assertIn("async_finish=False", source)
+ self.assertIn("return_recv_hook=False", source)
+
+ run_ep_source = (HERE / "run_ep.py").read_text()
+ self.assertIn('args.backend not in {"deepep", "uccl"}', run_ep_source)
+ self.assertIn('args.phase != "decode"', run_ep_source)
+
+ def test_deepep_v2_jit_evidence_is_strict_and_stable(self) -> None:
+ valid = deepep_v2_jit_provenance()
+ self.assertTrue(contracts._deepep_v2_jit_cubins_are_valid(valid))
+ for invalid in (
+ [],
+ [{**valid[0], "path": "/private/kernel.cubin"}],
+ [{**item, "cache_key": "dispatch"} for item in valid],
+ [{**item, "cubin_sha256": "invalid"} for item in valid],
+ valid[:-1],
+ [*valid, valid[0]],
+ [
+ *valid,
+ {
+ **valid[0],
+ "cache_key": valid[0]["cache_key"][:-32] + "f" * 32,
+ },
+ ],
+ ):
+ with self.subTest(invalid=invalid):
+ self.assertFalse(contracts._deepep_v2_jit_cubins_are_valid(invalid))
+
+ backend = next(
+ node for node in self.tree.body
+ if isinstance(node, ast.ClassDef) and node.name == "DeepEPV2Backend"
+ )
+ capture = next(
+ node for node in backend.body
+ if isinstance(node, ast.FunctionDef)
+ and node.name == "capture_deferred_provenance"
+ )
+ artifacts = copy.deepcopy(valid)
+
+ class FakeBuffer:
+ @staticmethod
+ def barrier(*, use_comm_stream: bool, with_cpu_sync: bool) -> None:
+ self.assertTrue(use_comm_stream)
+ self.assertTrue(with_cpu_sync)
+
+ namespace = {
+ "torch": types.SimpleNamespace(
+ cuda=types.SimpleNamespace(synchronize=lambda: None)
+ ),
+ "_jit_artifact_evidence": lambda: copy.deepcopy(artifacts),
+ "_require_cross_rank_equal": lambda _value, _label: None,
+ }
+ exec(
+ compile(ast.Module(body=[capture], type_ignores=[]), str(self.path), "exec"),
+ namespace,
+ )
+ state = types.SimpleNamespace(
+ buffer=FakeBuffer(),
+ _deferred_jit_snapshot=None,
+ backend_provenance={"jit_cubins": []},
+ )
+ namespace["capture_deferred_provenance"](state)
+ namespace["capture_deferred_provenance"](state)
+ artifacts[0]["cubin_sha256"] = "f" * 64
+ with self.assertRaisesRegex(RuntimeError, "changed after measurement"):
+ namespace["capture_deferred_provenance"](state)
+
+ def test_deepep_v2_jit_files_are_complete_regular_and_content_bound(self) -> None:
+ functions = [
+ node for node in self.tree.body
+ if isinstance(node, ast.FunctionDef)
+ and node.name in {"_sha256", "_jit_artifact_evidence"}
+ ]
+ namespace = {
+ "hashlib": hashlib,
+ "os": os,
+ "Path": Path,
+ "re": __import__("re"),
+ "DEEPEP_V2_JIT_KERNELS": contracts.DEEPEP_V2_JIT_KERNELS,
+ }
+ exec(compile(ast.Module(body=functions, type_ignores=[]), str(self.path), "exec"), namespace)
+ with tempfile.TemporaryDirectory() as temporary:
+ cache = Path(temporary) / "cache"
+ cache.mkdir()
+ for index, name in enumerate(sorted(contracts.DEEPEP_V2_JIT_KERNELS)):
+ kernel = cache / f"kernel.{name}.{index:032x}"
+ kernel.mkdir()
+ for suffix in ("cu", "cubin", "sass"):
+ (kernel / f"kernel.{suffix}").write_bytes(f"{name}-{suffix}".encode())
+ old_cache = os.environ.get("EP_JIT_CACHE_DIR")
+ os.environ["EP_JIT_CACHE_DIR"] = temporary
+ try:
+ evidence = namespace["_jit_artifact_evidence"]()
+ self.assertEqual(len(evidence), len(contracts.DEEPEP_V2_JIT_KERNELS))
+ self.assertEqual(
+ set(evidence[0]),
+ {"cache_key", "cubin_sha256", "sass_sha256", "source_sha256"},
+ )
+ first = cache / evidence[0]["cache_key"]
+ duplicate = cache / (evidence[0]["cache_key"][:-32] + "f" * 32)
+ duplicate.mkdir()
+ for suffix in ("cu", "cubin", "sass"):
+ (duplicate / f"kernel.{suffix}").write_bytes(b"duplicate")
+ with self.assertRaisesRegex(RuntimeError, "kernel set"):
+ namespace["_jit_artifact_evidence"]()
+ shutil.rmtree(duplicate)
+ (first / "kernel.sass").unlink()
+ with self.assertRaisesRegex(RuntimeError, "incomplete"):
+ namespace["_jit_artifact_evidence"]()
+ (first / "kernel.sass").symlink_to(first / "kernel.cubin")
+ with self.assertRaisesRegex(RuntimeError, "regular file"):
+ namespace["_jit_artifact_evidence"]()
+ finally:
+ if old_cache is None:
+ os.environ.pop("EP_JIT_CACHE_DIR", None)
+ else:
+ os.environ["EP_JIT_CACHE_DIR"] = old_cache
+
+ def test_runtime_and_shared_version_formatter_are_valid(self) -> None:
+ subprocess.run(
+ ["bash", "-n", str(ROOT / "runtime" / "run_in_container.sh")],
+ check=True,
+ )
+ self.assertEqual(ep_harness.format_collective_version(23004), "2.30.4")
+ self.assertEqual(ep_harness.format_collective_version((2, 30, 4)), "2.30.4")
+ source = self.path.read_text()
+ version_function = next(
+ node for node in self.tree.body
+ if isinstance(node, ast.FunctionDef) and node.name == "_loaded_nccl_version"
+ )
+
+ class FakeNccl:
+ @staticmethod
+ def ncclGetVersion(pointer) -> int:
+ pointer._obj.value = 23004
+ return 0
+
+ namespace = {
+ "ctypes": types.SimpleNamespace(
+ CDLL=lambda _path: FakeNccl(), byref=ctypes.byref, c_int=ctypes.c_int,
+ ),
+ "ep_harness": ep_harness,
+ "os": os,
+ "_loaded_library_paths": lambda: {"/safe/libnccl.so.2"},
+ }
+ exec(
+ compile(ast.Module(body=[version_function], type_ignores=[]), str(self.path), "exec"),
+ namespace,
+ )
+ self.assertEqual(namespace["_loaded_nccl_version"](), "2.30.4")
+ for paths in (set(), {"/safe/libnccl.so.2", "/other/libnccl.so.2"}):
+ namespace["_loaded_library_paths"] = lambda paths=paths: paths
+ with self.assertRaisesRegex(RuntimeError, "exactly one"):
+ namespace["_loaded_nccl_version"]()
+ evidence_function = next(
+ node for node in self.tree.body
+ if isinstance(node, ast.FunctionDef) and node.name == "_loaded_library_evidence"
+ )
+ paths = {
+ "/safe/_C.cpython-310-x86_64-linux-gnu.so",
+ "/safe/libnccl.so.2",
+ "/safe/libnvshmem_host.so.3",
+ }
+ namespace.update(
+ _loaded_library_paths=lambda: paths,
+ _sha256=lambda _path: "a" * 64,
+ )
+ exec(
+ compile(ast.Module(body=[evidence_function], type_ignores=[]), str(self.path), "exec"),
+ namespace,
+ )
+ evidence = namespace["_loaded_library_evidence"]()
+ self.assertIn(
+ {"name": "deep_ep._C", "role": "deepep-extension", "sha256": "a" * 64},
+ evidence,
+ )
+ self.assertTrue(
+ contracts._content_evidence_is_valid(
+ evidence, {"deepep-extension", "nccl", "nvshmem"}
+ )
+ )
+ self.assertNotIn("torch.cuda.nccl.version()", source)
+ fingerprint = {"runtime": "cuda", "version": "13.0"}
+ self.assertIs(
+ run_ep._common_runtime_fingerprint([fingerprint, dict(fingerprint)]),
+ fingerprint,
+ )
+ with self.assertRaises(ValueError):
+ run_ep._common_runtime_fingerprint([fingerprint, {"runtime": "cuda", "version": "12.8"}])
+
+ def test_conditioning_contract_is_exact_for_each_phase(self) -> None:
+ expected = {
+ "decode": [1, 2, 4, 8, 16, 32, 64, 128],
+ "prefill": [1, 2, 4, 8, 16, 32, 64, 128, 256, 512],
+ }
+ for phase, ladder in expected.items():
+ valid = {
+ "contract": "fixed-phase-ramp-8-roundtrips-v1",
+ "ladder": ladder,
+ "roundtrips_per_shape": 8,
+ }
+ self.assertIs(contracts.validate_conditioning_contract(valid, phase), valid)
+ for mutate in (
+ lambda item: item["ladder"].reverse(),
+ lambda item: item["ladder"].pop(),
+ lambda item: item.update(ladder=[1.0, *item["ladder"][1:]]),
+ lambda item: item.update(roundtrips_per_shape=7),
+ lambda item: item.update(roundtrips_per_shape=8.0),
+ ):
+ changed = copy.deepcopy(valid)
+ mutate(changed)
+ with self.assertRaises(contracts.ContractError):
+ contracts.validate_conditioning_contract(changed, phase)
+ other = "prefill" if phase == "decode" else "decode"
+ with self.assertRaises(contracts.ContractError):
+ contracts.validate_conditioning_contract(valid, other)
+
+ def test_content_manifest_evidence_is_stable_and_content_sensitive(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary)
+ first, second = root / "first", root / "second"
+ first.write_bytes(b"first")
+ second.write_bytes(b"second")
+ files = [("pkg/first", first), ("pkg/second", second)]
+ evidence = contracts.content_manifest_evidence(
+ role="test-content", name="test-build", files=files,
+ )
+ self.assertNotIn(temporary, json.dumps(evidence))
+ self.assertEqual(
+ evidence,
+ contracts.content_manifest_evidence(
+ role="test-content", name="test-build", files=reversed(files),
+ ),
+ )
+ self.assertRegex(evidence["sha256"], r"^[0-9a-f]{64}$")
+ second.write_bytes(b"changed")
+ self.assertNotEqual(
+ evidence,
+ contracts.content_manifest_evidence(
+ role="test-content", name="test-build", files=files,
+ ),
+ )
+ for invalid in (
+ [("../first", first)],
+ [("same", first), ("same", second)],
+ [("missing", root / "missing")],
+ ):
+ with self.assertRaises(contracts.ContractError):
+ contracts.content_manifest_evidence(
+ role="test-content", name="test-build", files=invalid,
+ )
+
+ def test_hybrid_realized_config_and_jit_evidence_are_path_free(self) -> None:
+ path = HERE / "ep_deepep_hybrid.py"
+ tree = ast.parse(path.read_text(), str(path))
+ selected = [
+ node for node in tree.body
+ if (
+ isinstance(node, ast.Assign)
+ and any(
+ isinstance(target, ast.Name) and target.id == "HYBRID_CONFIG_FIELDS"
+ for target in node.targets
+ )
+ )
+ or isinstance(node, ast.FunctionDef)
+ and node.name in {
+ "_hybrid_realized_config", "_sha256_with_size", "_hybrid_jit_evidence",
+ }
+ ]
+ namespace = {"Path": Path, "hashlib": hashlib, "re": __import__("re")}
+ exec(compile(ast.Module(body=selected, type_ignores=[]), str(path), "exec"), namespace)
+ fields = namespace["HYBRID_CONFIG_FIELDS"]
+ self.assertEqual(set(fields), contracts.HYBRID_REALIZED_CONFIG_FIELDS)
+
+ class TokenType:
+ def __init__(self, label: str, name: str | None = None) -> None:
+ self.label = label
+ if name is not None:
+ self.name = name
+
+ def __str__(self) -> str:
+ return self.label
+
+ values = {field: 1 for field in fields}
+ values.update({field: True for field in contracts.HYBRID_REALIZED_BOOL_FIELDS})
+ for raw, expected in (("uint16_t", "UINT16"), ("uint8_t", "UINT8")):
+ values["token_data_type"] = TokenType(raw)
+ config = types.SimpleNamespace(**values)
+ realized = namespace["_hybrid_realized_config"](config)
+ self.assertEqual(realized["token_data_type"], expected)
+ self.assertEqual(set(realized), contracts.HYBRID_REALIZED_CONFIG_FIELDS)
+ values["token_data_type"] = TokenType("opaque-enum", "UINT16")
+ self.assertEqual(
+ namespace["_hybrid_realized_config"](types.SimpleNamespace(**values))[
+ "token_data_type"
+ ],
+ "UINT16",
+ )
+ values["token_data_type"] = TokenType("UINT16")
+ with self.assertRaisesRegex(RuntimeError, "token_data_type is invalid"):
+ namespace["_hybrid_realized_config"](types.SimpleNamespace(**values))
+ values["token_data_type"] = TokenType("uint16_t")
+ config = types.SimpleNamespace(**values)
+ delattr(config, "hidden_dim")
+ with self.assertRaisesRegex(RuntimeError, "omits hidden_dim"):
+ namespace["_hybrid_realized_config"](config)
+
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary)
+ for key, payload in (
+ ("preprocess-key", b"pre"),
+ ("combine-key", b"combine"),
+ ("dispatch-key", b"dispatch"),
+ ):
+ (root / f"{key}.so").write_bytes(payload)
+ evidence = namespace["_hybrid_jit_evidence"](root)
+ self.assertEqual(
+ [item["kernel_key"] for item in evidence],
+ ["combine-key", "dispatch-key", "preprocess-key"],
+ )
+ self.assertNotIn(temporary, json.dumps(evidence))
+ (root / "dispatch-key.so").write_bytes(b"changed")
+ self.assertNotEqual(evidence, namespace["_hybrid_jit_evidence"](root))
+ (root / "extra-key.so").write_bytes(b"extra")
+ with self.assertRaisesRegex(RuntimeError, "expected 3"):
+ namespace["_hybrid_jit_evidence"](root)
+ (root / "extra-key.so").unlink()
+ (root / "bad key.so").write_bytes(b"bad")
+ with self.assertRaisesRegex(RuntimeError, "kernel key"):
+ namespace["_hybrid_jit_evidence"](root)
+ (root / "bad key.so").unlink()
+ (root / "combine-key.so").unlink()
+ (root / "combine-key.so").symlink_to(root / "dispatch-key.so")
+ with self.assertRaisesRegex(RuntimeError, "regular file"):
+ namespace["_hybrid_jit_evidence"](root)
+ empty = root / "empty"
+ empty.mkdir()
+ with self.assertRaisesRegex(RuntimeError, "expected 3"):
+ namespace["_hybrid_jit_evidence"](empty)
+
+ def test_hybrid_uses_communication_domains_not_physical_hosts(self) -> None:
+ path = HERE / "ep_deepep_hybrid.py"
+ function = next(
+ node for node in ast.parse(path.read_text(), str(path)).body
+ if isinstance(node, ast.FunctionDef) and node.name == "_hybrid_topology"
+ )
+ namespace: dict[str, object] = {}
+ exec(compile(ast.Module(body=[function], type_ignores=[]), str(path), "exec"), namespace)
+ resolve = namespace["_hybrid_topology"]
+ cases = (
+ (8, 8, 8, "scale-up", "nvlink", "", 8, 1, 1),
+ (16, 8, 8, "scale-out", "nvlink", "rdma", 8, 2, 2),
+ (8, 4, 72, "scale-up", "mnnvl", "", 8, 1, 2),
+ (16, 4, 72, "scale-up", "mnnvl", "", 16, 1, 4),
+ )
+ for world, gpn, domain, scope, up, out, ranks, domains, hosts in cases:
+ with self.subTest(world=world, gpus_per_node=gpn, transport=up):
+ topology = resolve(types.SimpleNamespace(
+ gpus_per_node=gpn,
+ scale_up_domain=domain,
+ scope=scope,
+ scale_up_transport=up,
+ scale_out_transport=out,
+ transport=up if not out else f"{up}-{out}",
+ ), world)
+ self.assertEqual(
+ (topology["domain_ranks"], topology["communication_domains"],
+ topology["physical_nodes"]),
+ (ranks, domains, hosts),
+ )
+ with self.assertRaisesRegex(RuntimeError, "outside the fixed v1 matrix"):
+ resolve(types.SimpleNamespace(
+ gpus_per_node=8, scale_up_domain=8, scope="scale-up",
+ scale_up_transport="nvlink", scale_out_transport="", transport="nvlink",
+ ), 16)
+
+ def test_mori_ep16_pins_upstream_internode_v1_resources(self) -> None:
+ source = (HERE / "ep_mori.py").read_text()
+ for fragment in (
+ 'kernel_enum.InterNodeV1',
+ 'self.block_num = self._block_target = 96',
+ 'self.rdma_block_num = 64',
+ 'self.dispatch_warps = self.combine_warps = 8',
+ 'self.num_qps = 1',
+ '"gpu_per_node": gpus_per_node',
+ '"rdma_block_num": self.rdma_block_num',
+ '"num_qp_per_pe": self.num_qps',
+ '"use_external_inp_buf": self._external_input',
+ 'os.environ["MORI_EP_LAUNCH_CONFIG_MODE"] = "MANUAL"',
+ 'rdma_block_num=self.rdma_block_num',
+ ):
+ self.assertIn(fragment, source)
+ self.assertGreaterEqual(source.count("rdma_block_num=self.rdma_block_num"), 2)
+
+ def test_hybrid_deferred_provenance_wraps_before_conditioning_and_recaptures(self) -> None:
+ path = HERE / "ep_deepep_hybrid.py"
+ source = path.read_text()
+ tree = ast.parse(source, str(path))
+ backend = next(
+ node for node in tree.body
+ if isinstance(node, ast.ClassDef) and node.name == "DeepEPHybridBackend"
+ )
+ methods = {node.name for node in backend.body if isinstance(node, ast.FunctionDef)}
+ self.assertIn("capture_deferred_provenance", methods)
+ constructor = next(node for node in backend.body if isinstance(node, ast.FunctionDef) and node.name == "__init__")
+ buffer_call = next(
+ node for node in ast.walk(constructor)
+ if isinstance(node, ast.Call) and isinstance(node.func, ast.Name)
+ and node.func.id == "HybridEPBuffer"
+ )
+ wrapper_install = next(
+ node for node in ast.walk(constructor)
+ if isinstance(node, ast.Assign)
+ and any(
+ isinstance(target, ast.Attribute)
+ and target.attr == "update_template_config"
+ for target in node.targets
+ )
+ )
+ cache_line = source[:source.index('os.environ["HYBRID_EP_CACHE_DIR"]')].count("\n") + 1
+ self.assertLess(cache_line, buffer_call.lineno)
+ self.assertLess(buffer_call.lineno, wrapper_install.lineno)
+
+ capture = next(
+ node for node in backend.body
+ if isinstance(node, ast.FunctionDef) and node.name == "capture_deferred_provenance"
+ )
+ called = {
+ node.func.id if isinstance(node.func, ast.Name) else node.func.attr
+ for node in ast.walk(capture) if isinstance(node, ast.Call)
+ and isinstance(node.func, (ast.Name, ast.Attribute))
+ }
+ self.assertTrue({"_hybrid_jit_evidence", "_require_cross_rank_equal", "all_gather_object"} <= called)
+ self.assertIn("changed after measurement", ast.get_source_segment(source, capture))
+
+ artifacts = [[
+ {"bytes": 1, "kernel_key": key, "sha256": digit * 64}
+ for key, digit in (("a", "1"), ("b", "2"), ("c", "3"))
+ ]]
+
+ class FakeCuda:
+ @staticmethod
+ def synchronize() -> None:
+ return None
+
+ class FakeDist:
+ @staticmethod
+ def barrier() -> None:
+ return None
+
+ @staticmethod
+ def get_world_size() -> int:
+ return 2
+
+ @staticmethod
+ def all_gather_object(output, value) -> None:
+ output[:] = [copy.deepcopy(value), copy.deepcopy(value)]
+
+ namespace = {
+ "torch": types.SimpleNamespace(cuda=FakeCuda),
+ "dist": FakeDist,
+ "_hybrid_jit_evidence": lambda _root: copy.deepcopy(artifacts[0]),
+ "_require_cross_rank_equal": lambda _value, _label: None,
+ }
+ exec(compile(ast.Module(body=[capture], type_ignores=[]), str(path), "exec"), namespace)
+ state = types.SimpleNamespace(
+ _deferred_jit_diagnostics=None,
+ _deferred_semantic_snapshot=None,
+ _jit_root=Path("private-cache"),
+ _realized_config=hybrid_realized_config(),
+ backend_provenance={},
+ )
+ namespace["capture_deferred_provenance"](state)
+ artifacts[0][0]["kernel_key"] = "changed"
+ with self.assertRaisesRegex(RuntimeError, "kernel set changed"):
+ namespace["capture_deferred_provenance"](state)
+ artifacts[0][0]["kernel_key"] = "a"
+ artifacts[0][0]["sha256"] = "f" * 64
+ with self.assertRaisesRegex(RuntimeError, "artifacts changed"):
+ namespace["capture_deferred_provenance"](state)
+
+ harness = (HERE / "ep_harness.py").read_text()
+ captures = [
+ index for index in range(len(harness))
+ if harness.startswith("capture_deferred_provenance()", index)
+ ]
+ self.assertEqual(len(captures), 2)
+ self.assertLess(harness.index("for wt in conditioning_ladder:"), captures[0])
+ self.assertLess(captures[0], harness.index("oracle = _run_expert_oracle("))
+ self.assertLess(harness.index("trace_sig = hashlib.sha256"), captures[1])
+
+ def test_hybrid_diagnostic_hashes_do_not_split_series_identity(self) -> None:
+ keys, artifacts = hybrid_jit_provenance()
+ provenance = {
+ "deepep_tree": "b" * 40,
+ "jit_kernel_keys": keys,
+ "jit_shared_objects": artifacts,
+ "loaded_libraries": [{
+ "name": "hybrid_ep_cpp", "role": "deepep-hybrid-extension",
+ "sha256": "a" * 64,
+ }],
+ "realized_config": hybrid_realized_config(),
+ }
+ baseline = ep_harness._series_provenance(provenance)
+ changed = copy.deepcopy(provenance)
+ changed["jit_shared_objects"][0]["rank_artifacts"][0]["sha256"] = "f" * 64
+ self.assertEqual(ep_harness._series_provenance(changed), baseline)
+ changed = copy.deepcopy(provenance)
+ changed["loaded_libraries"][0]["sha256"] = "f" * 64
+ self.assertEqual(ep_harness._series_provenance(changed), baseline)
+ changed = copy.deepcopy(provenance)
+ changed["jit_kernel_keys"][0] = "changed-key"
+ self.assertNotEqual(ep_harness._series_provenance(changed), baseline)
+ changed = copy.deepcopy(provenance)
+ changed["realized_config"]["num_of_blocks_dispatch_api"] += 1
+ self.assertNotEqual(ep_harness._series_provenance(changed), baseline)
+ changed = copy.deepcopy(provenance)
+ changed["deepep_tree"] = "c" * 40
+ self.assertNotEqual(ep_harness._series_provenance(changed), baseline)
+
+ def test_v2_series_identity_uses_source_and_sass_not_container_metadata(self) -> None:
+ provenance = {
+ "deepep_tree": "a" * 40,
+ "loaded_libraries": [
+ {"name": "deep_ep._C.so", "role": "deepep-extension", "sha256": "1" * 64},
+ {"name": "libnccl.so.2", "role": "nccl", "sha256": "2" * 64},
+ ],
+ "jit_cubins": deepep_v2_jit_provenance(),
+ "jit_random_seed": "collectivex-deepep-v2-fa8a9b1",
+ }
+ baseline = contracts.series_provenance(provenance)
+ changed = copy.deepcopy(provenance)
+ changed["loaded_libraries"][0]["sha256"] = "f" * 64
+ changed["jit_cubins"][0]["cubin_sha256"] = "e" * 64
+ self.assertEqual(contracts.series_provenance(changed), baseline)
+ for mutate in (
+ lambda item: item["loaded_libraries"][1].update(sha256="f" * 64),
+ lambda item: item["jit_cubins"][0].update(source_sha256="f" * 64),
+ lambda item: item["jit_cubins"][0].update(sass_sha256="f" * 64),
+ lambda item: item.update(deepep_tree="f" * 40),
+ ):
+ changed = copy.deepcopy(provenance)
+ mutate(changed)
+ self.assertNotEqual(contracts.series_provenance(changed), baseline)
+
+ def test_mnnvl_resolution_has_no_ambiguous_signature_fallback(self) -> None:
+ self.assertEqual(
+ contracts.resolve_deepep_mnnvl(
+ requested=False, signature_parameters=(), deepep_commit=None,
+ ),
+ ({}, "not-requested"),
+ )
+ self.assertEqual(
+ contracts.resolve_deepep_mnnvl(
+ requested=True, signature_parameters=("allow_mnnvl",),
+ deepep_commit="a" * 40,
+ ),
+ ({"allow_mnnvl": True}, "explicit-allow-mnnvl"),
+ )
+ with self.assertRaises(contracts.ContractError):
+ contracts.resolve_deepep_mnnvl(
+ requested=True, signature_parameters=(),
+ deepep_commit="814e508537c6ffc775d59f6f1b9ba43f3a65968c",
+ )
+
+ def test_backend_provenance_requires_lineage_and_content_hashes(self) -> None:
+ def record(role: str, name: str, digit: str) -> dict[str, str]:
+ return {"role": role, "name": name, "sha256": digit * 64}
+
+ hybrid_keys, hybrid_artifacts = hybrid_jit_provenance()
+ v2 = {
+ **contracts.DEEPEP_V2_V1_PROVENANCE,
+ "api_signature_sha256": "c" * 64,
+ "loaded_libraries": [
+ record("deepep-extension", "deep_ep._C", "1"),
+ record("nccl", "libnccl.so.2", "2"),
+ record("nvshmem", "libnvshmem_host.so.3", "3"),
+ ],
+ "jit_cubins": deepep_v2_jit_provenance(),
+ "jit_random_seed": "collectivex-deepep-v2-fa8a9b1",
+ "deterministic": False,
+ "num_experts": 256,
+ "tuning_num_experts": 256,
+ "allow_hybrid_mode": False,
+ "gin_enabled": False,
+ "communication_backend": "nccl-device-lsa",
+ }
+ deepep = {
+ "deepep_version": "1.1.0", "deepep_commit": "a" * 40,
+ "backend_lineage": "deepep-v1", "allow_mnnvl": False,
+ "mnnvl_comm": "not-requested", "mode": "normal",
+ "num_nvl_bytes": 1024, "num_rdma_bytes": 0,
+ }
+ hybrid = {
+ "deepep_commit": "a" * 40, "deepep_tree": "b" * 40,
+ "branch": "hybrid-ep", "backend_lineage": "deepep-hybrid",
+ "loaded_libraries": [
+ record("deepep-extension", "deep_ep_cpp", "1"),
+ record("deepep-hybrid-extension", "hybrid_ep_cpp", "2"),
+ ],
+ "jit_kernel_keys": hybrid_keys,
+ "jit_shared_objects": hybrid_artifacts,
+ "realized_config": hybrid_realized_config(),
+ }
+ uccl = {
+ "uccl_version": "0.1.1", "uccl_commit": "pkg-0.1.1",
+ "uccl_wrapper_commit": "c" * 40, "backend_lineage": "uccl",
+ "uccl_dependency_versions": dict(contracts.UCCL_DEPENDENCY_VERSIONS),
+ "loaded_libraries": [
+ record("uccl-distribution", "uccl-0.1.1", "3"),
+ record("uccl-wrapper", "uccl-deepep-wrapper", "4"),
+ record("intervaltree-distribution", "intervaltree-3.1.0", "5"),
+ record("sortedcontainers-distribution", "sortedcontainers-2.4.0", "6"),
+ record("cuda-runtime", "nvidia-cuda-runtime-cu12-12.9.79", "7"),
+ ],
+ "mode": "normal", "num_nvl_bytes": 1024, "num_rdma_bytes": 0,
+ }
+ reference = {
+ "nccl_version": "2.30.4", "collective_library": "nccl",
+ "backend_lineage": "nccl",
+ }
+ for backend, provenance in (
+ ("deepep", deepep), ("deepep-v2", v2), ("deepep-hybrid", hybrid),
+ ("uccl", uccl), ("nccl-ep", reference),
+ ):
+ self.assertEqual(contracts.backend_provenance_issues(backend, provenance), [])
+ changed = copy.deepcopy(provenance)
+ if "loaded_libraries" in changed:
+ changed["loaded_libraries"][0]["sha256"] = "invalid"
+ expected = "loaded_libraries"
+ else:
+ changed["backend_lineage"] = "wrong"
+ expected = "backend_lineage"
+ self.assertIn(expected, contracts.backend_provenance_issues(backend, changed))
+
+ changed = copy.deepcopy(uccl)
+ changed["uccl_dependency_versions"]["intervaltree"] = "3.2.0"
+ self.assertIn(
+ "uccl_dependency_versions",
+ contracts.backend_provenance_issues("uccl", changed),
+ )
+ changed = copy.deepcopy(uccl)
+ changed["loaded_libraries"] = [
+ item
+ for item in changed["loaded_libraries"]
+ if item["role"] != "sortedcontainers-distribution"
+ ]
+ self.assertIn(
+ "loaded_libraries", contracts.backend_provenance_issues("uccl", changed)
+ )
+
+ for field, mutate in (
+ ("realized_config", lambda item: item["realized_config"].pop("hidden_dim")),
+ ("jit_kernel_keys", lambda item: item["jit_kernel_keys"].reverse()),
+ (
+ "jit_shared_objects",
+ lambda item: item["jit_shared_objects"][0]["rank_artifacts"][0].update(
+ sha256="invalid"
+ ),
+ ),
+ ):
+ with self.subTest(hybrid_field=field):
+ changed = copy.deepcopy(hybrid)
+ mutate(changed)
+ self.assertIn(
+ field,
+ contracts.backend_provenance_issues("deepep-hybrid", changed),
+ )
+
+ for field, value in (
+ ("jit_cubins", [{"cache_key": "invalid", "cubin_sha256": "4" * 64}]),
+ ("jit_random_seed", "different-seed"),
+ ):
+ with self.subTest(v2_field=field):
+ changed = copy.deepcopy(v2)
+ changed[field] = value
+ self.assertIn(
+ field,
+ contracts.backend_provenance_issues("deepep-v2", changed),
+ )
+
+ changed = copy.deepcopy(v2)
+ changed["gin_enabled"] = True
+ self.assertIn("gin_enabled", contracts.backend_provenance_issues("deepep-v2", changed))
+ changed = copy.deepcopy(v2)
+ changed["communication_backend"] = "nccl-gin"
+ self.assertIn(
+ "communication_backend", contracts.backend_provenance_issues("deepep-v2", changed)
+ )
+ changed = copy.deepcopy(v2)
+ changed.update(
+ allow_hybrid_mode=True,
+ gin_enabled=True,
+ communication_backend="nccl-gin",
+ )
+ self.assertEqual(
+ contracts.backend_provenance_issues("deepep-v2", changed),
+ [],
+ )
+ changed["allow_hybrid_mode"] = False
+ self.assertEqual(
+ contracts.backend_provenance_issues("deepep-v2", changed),
+ ["allow_hybrid_mode", "communication_backend", "gin_enabled"],
+ )
+ for field, expected in contracts.DEEPEP_V2_V1_PROVENANCE.items():
+ with self.subTest(v2_pin_field=field):
+ changed = copy.deepcopy(v2)
+ changed[field] = not expected if type(expected) is bool else "wrong"
+ self.assertIn(
+ field,
+ contracts.backend_provenance_issues("deepep-v2", changed),
+ )
+
+ schema = json.loads((ROOT / "schemas" / "raw-case-v1.schema.json").read_text())
+ provenance_schema = schema["properties"]["implementation"]["properties"]["provenance"]
+ self.assertEqual(
+ provenance_schema["properties"]["realized_config"],
+ {"$ref": "#/$defs/hybrid_realized_config"},
+ )
+ self.assertFalse(schema["$defs"]["hybrid_realized_config"]["additionalProperties"])
+ self.assertEqual(provenance_schema["properties"]["jit_kernel_keys"]["minItems"], 3)
+ self.assertEqual(provenance_schema["properties"]["jit_shared_objects"]["minItems"], 3)
+
+ self.assertEqual(contracts.collective_kernel_generation("nccl"), "nccl")
+ self.assertEqual(contracts.collective_kernel_generation("rccl"), "rccl")
+ with self.assertRaises(contracts.ContractError):
+ contracts.collective_kernel_generation("unknown")
+
+ def test_transport_resource_provenance_is_exact(self) -> None:
+ self.assertEqual(contracts.hybrid_communication_domains(8, 8), (8, 1))
+ self.assertEqual(contracts.hybrid_communication_domains(16, 8), (8, 2))
+ self.assertEqual(contracts.hybrid_communication_domains(8, 72), (8, 1))
+ self.assertEqual(contracts.hybrid_communication_domains(16, 72), (16, 1))
+
+ profile = contracts.project_resource_profile({
+ "num_nvl_bytes": 1024, "num_rdma_bytes": 2048,
+ "num_qps_per_rank": 32, "heap_size": "6G",
+ })
+ self.assertEqual(profile["persistent_bytes"], 3072)
+ self.assertEqual(profile["qps_per_rank"], 32)
+ self.assertEqual(
+ contracts.project_resource_profile({
+ "num_nvl_bytes": 0, "num_rdma_bytes": 0, "heap_size": "6G",
+ })["persistent_bytes"],
+ 0,
+ )
+ self.assertEqual(
+ contracts.project_resource_profile({"heap_size": "6G"})[
+ "persistent_bytes"
+ ],
+ "6G",
+ )
+
+ mori = {
+ "mori_commit": "a" * 40, "kernel_type": "InterNodeV1",
+ "block_num": 96, "rdma_block_num": 64,
+ "dispatch_warps": 8, "combine_warps": 8, "num_qps": 1,
+ "use_external_inp_buf": True, "gpus_per_node": 8,
+ }
+ self.assertEqual(contracts.backend_provenance_issues("mori", mori), [])
+ for field in (
+ "block_num", "rdma_block_num", "dispatch_warps", "combine_warps",
+ "num_qps", "use_external_inp_buf", "gpus_per_node",
+ ):
+ changed = copy.deepcopy(mori)
+ changed[field] = False if field == "use_external_inp_buf" else 0
+ with self.subTest(mori_field=field):
+ self.assertIn(
+ field, contracts.backend_provenance_issues("mori", changed)
+ )
+
+ def test_routing_control_binds_binary_but_allows_treatment_configuration(self) -> None:
+ hybrid_keys, hybrid_artifacts = hybrid_jit_provenance()
+ implementation = {
+ "kernel_generation": "hybrid",
+ "name": "deepep-hybrid",
+ "provenance": {
+ "deepep_tree": "a" * 40,
+ "loaded_libraries": [{
+ "role": "deepep-extension", "name": "deep_ep_cpp", "sha256": "1" * 64,
+ }],
+ "local_experts": 32,
+ "num_experts": 256,
+ "num_sms": 24,
+ "jit_cache_key": "case-one",
+ "jit_cubins": [{"cache_key": "one", "cubin_sha256": "2" * 64}],
+ "jit_kernel_keys": hybrid_keys,
+ "jit_shared_objects": hybrid_artifacts,
+ "realized_config": hybrid_realized_config(),
+ },
+ "resource_profile": {"configured_units": 24},
+ }
+ baseline = contracts.routing_implementation_control_sha256(implementation)
+ treatment = copy.deepcopy(implementation)
+ treatment["provenance"].update({
+ "local_experts": 36,
+ "num_experts": 288,
+ "jit_cache_key": "case-two",
+ "jit_cubins": [{"cache_key": "two", "cubin_sha256": "3" * 64}],
+ "jit_kernel_keys": ["changed-a", "changed-b", "changed-c"],
+ "jit_shared_objects": hybrid_jit_provenance(3)[1],
+ "realized_config": {
+ **hybrid_realized_config(),
+ "num_of_experts_per_rank": 36,
+ },
+ })
+ self.assertEqual(
+ contracts.routing_implementation_control_sha256(treatment), baseline
+ )
+ changed = copy.deepcopy(implementation)
+ changed["provenance"]["loaded_libraries"][0]["sha256"] = "4" * 64
+ self.assertEqual(
+ contracts.routing_implementation_control_sha256(changed), baseline
+ )
+ changed = copy.deepcopy(implementation)
+ changed["provenance"]["deepep_tree"] = "b" * 40
+ self.assertNotEqual(
+ contracts.routing_implementation_control_sha256(changed), baseline
+ )
+ changed = copy.deepcopy(implementation)
+ changed["provenance"]["num_sms"] = 20
+ self.assertNotEqual(
+ contracts.routing_implementation_control_sha256(changed), baseline
+ )
+
+ def test_runtime_pins_uccl_wheel_and_hybrid_source_tree(self) -> None:
+ runtime = (ROOT / "runtime" / "run_in_container.sh").read_text()
+ common = (ROOT / "runtime" / "common.sh").read_text()
+ self.assertIn("cd /ix/experimental/CollectiveX", runtime)
+ for launcher_name in ("launch_single-slurm.sh", "launch_gb-nv.sh"):
+ launcher = (ROOT / "launchers" / launcher_name).read_text()
+ self.assertIn("$MOUNT_SRC:/ix", launcher)
+ self.assertIn("cx_prepare_backend_cache", launcher)
+ self.assertNotIn('$(cx_prepare_backend_cache', launcher)
+ self.assertIn("$CX_PREPARED_BACKEND_CACHE:/cx-cache", launcher)
+ self.assertIn("CX_BACKEND_CACHE_ROOT=/cx-cache", launcher)
+ self.assertIn("CX_BACKEND_SOURCE_ROOT=/ix/experimental/CollectiveX/.cx_sources", launcher)
+ self.assertIn('|| [ "$CX_BENCH" = deepep-hybrid ]', launcher)
+ self.assertIn("cx_prepare_backend_source", launcher)
+ cache_block = launcher[launcher.index('if [ "$CX_BENCH" = deepep-v2 ]'):]
+ self.assertLess(
+ cache_block.index("cx_set_failure_stage backend-setup"),
+ cache_block.index("cx_prepare_backend_cache"),
+ )
+ self.assertLess(
+ cache_block.index("cx_prepare_backend_source"),
+ cache_block.index("cx_set_failure_stage scheduler-allocation"),
+ )
+ self.assertIn("--frandom-seed=$seed", runtime)
+ self.assertIn("DEEPEP_V2_JIT_RANDOM_SEED", runtime)
+ persisted = runtime[runtime.index("cx_persist_backend_env()") :]
+ self.assertIn("CUDA_HOME CPATH NVCC_PREPEND_FLAGS", persisted)
+ self.assertIn(
+ "390c1320918972206546e44d79b132988f2818ec07e23afcd0595f7183916cec",
+ runtime,
+ )
+ self.assertIn("--require-hashes", runtime)
+ self.assertIn("d77aeab7f1bb52b615666fe178d26ced41fae08e", common)
+ self.assertIn("HEAD^{tree}", runtime)
+ self.assertIn("$PWD/.cx_backend/deepep-hybrid-", runtime)
+ self.assertIn("cx_materialize_backend_source deepep-hybrid", runtime)
+ self.assertIn("cx_materialize_backend_source deepep-v2", runtime)
+ self.assertIn("cx_deepep_hybrid_marker_content_sha256", runtime)
+ self.assertIn("cx_deepep_hybrid_cache_is_valid", runtime)
+ self.assertIn("cx_extension_pair_sha256", runtime)
+ self.assertIn(".collectivex-complete.tmp.", runtime)
+ self.assertNotIn("cx_fetch_revision", runtime)
+ self.assertIn("cx_fetch_revision", common)
+ self.assertIn("third-party/fmt", common)
+ hybrid = runtime[
+ runtime.index("cx_build_deepep_hybrid()"):
+ runtime.index("# UCCL EP")
+ ]
+ configure = runtime[
+ runtime.index("cx_configure_deepep_hybrid_build()"):
+ runtime.index("cx_deepep_hybrid_marker_content_sha256()")
+ ]
+ self.assertIn("cx_prepare_cuda_cccl", hybrid)
+ self.assertIn("unset NVSHMEM_DIR", hybrid)
+ self.assertIn(
+ "unset HYBRID_EP_MULTINODE USE_NIXL RDMA_CORE_HOME", configure
+ )
+ self.assertIn("cx_configure_deepep_hybrid_build || return 1", hybrid)
+ self.assertIn('[ "$(uname -m)" = x86_64 ]', configure)
+ self.assertIn('[ -n "${GLOO_SOCKET_IFNAME:-}" ]', configure)
+ self.assertIn('[ -d "/sys/class/infiniband/$rdma_name" ]', configure)
+ self.assertIn("command -v make", configure)
+ self.assertIn("/usr/include/infiniband/verbs.h", configure)
+ self.assertIn("export HYBRID_EP_MULTINODE=1 USE_NIXL=0", configure)
+ self.assertNotIn("cx_prepare_deepep_toolchain", hybrid)
+ toolchain = runtime[
+ runtime.index("cx_prepare_deepep_toolchain()"):
+ runtime.index("cx_probe_deepep()")
+ ]
+ self.assertIn('overlay="$root/nvshmem-overlay"', toolchain)
+ self.assertIn("flock 8 || exit 1", toolchain)
+ self.assertIn('mv "$temporary" "$overlay" || exit 1', toolchain)
+ self.assertNotIn("/tmp/collectivex-nvshmem", toolchain)
+ jit = runtime[
+ runtime.index("cx_enable_deepep_v2_jit_reproducibility()"):
+ runtime.index("cx_probe_deepep_v2()")
+ ]
+ self.assertIn('cccl="${CX_CUDA_CCCL:-}"', jit)
+ self.assertNotIn("/usr/local/cuda*", jit)
+ self.assertIn("deepep-v2-cache-v2|$cpu|sm${arch/./}", runtime)
+ self.assertNotIn("deepep-v2-cache-v1|", runtime)
+ self.assertIn('base="${CX_BACKEND_CACHE_ROOT:-}"', runtime)
+ self.assertNotIn("${CX_BACKEND_CACHE_ROOT:-$PWD/.cx_backend}", runtime)
+ self.assertIn(
+ "recipe=aot-persistent-nvshmem-active-cuda-maxjobs16-v2", runtime
+ )
+ self.assertNotIn("recipe=aot-source-date-epoch-arch-maxjobs16-v1", runtime)
+ self.assertNotIn("recipe=$source_sha", runtime)
+ self.assertIn("pip=26.1.2|setuptools=82.0.1|wheel=0.47.0|ninja=1.13.0", runtime)
+ self.assertIn("manual-unverified", runtime)
+ self.assertIn("cx_deepep_v2_content_sha256", runtime)
+ self.assertIn("DeepEP V2 cache validation failed", runtime)
+ probe = runtime[
+ runtime.index("cx_probe_deepep_v2()"):
+ runtime.index("cx_deepep_v2_content_sha256()")
+ ]
+ self.assertNotIn("torch.cuda.nccl.version", probe)
+ self.assertIn("ncclGetVersion", probe)
+ self.assertIn("runtime_version.value == 23004", probe)
+ self.assertIn("cx_nvidia_package_root nvidia-nccl-cu13 nccl", runtime)
+ self.assertIn("cx_nvidia_package_root nvidia-nvshmem-cu12 nvshmem", runtime)
+ self.assertNotIn("import os,nvidia.nccl", runtime)
+ self.assertNotIn("import os,nvidia.nvshmem", runtime)
+ self.assertIn(
+ 'export EP_JIT_CACHE_DIR="$stage_root/.cx_backend/deepep-v2-jit"', runtime
+ )
+ self.assertIn('stage_root="${CX_BACKEND_SOURCE_ROOT%/.cx_sources}"', runtime)
+ self.assertNotIn('export EP_JIT_CACHE_DIR="$root/jit"', runtime)
+ self.assertIn('EP_NVSHMEM_ROOT_DIR="$NVSHMEM_DIR"', runtime)
+ reference = (HERE / "ep_nccl.py").read_text()
+ self.assertIn("self.kernel_generation = contracts.collective_kernel_generation", reference)
+
+ def test_deepep_v2_cache_recovers_from_an_unpublished_partial_build(self) -> None:
+ runtime = ROOT / "runtime" / "run_in_container.sh"
+ with tempfile.TemporaryDirectory() as temporary:
+ cache_key = "a" * 64
+ content_hash = "b" * 64
+ root = Path(temporary) / f"deepep-v2-{cache_key}"
+ root.mkdir(mode=0o700)
+ marker = root / ".collectivex-complete"
+ stale = root / "stale-partial-build"
+ stale.write_text("partial\n")
+ command = r'''
+ set -euo pipefail
+ eval "$(sed -n '/^cx_build_deepep_v2()/,/^}/p' "$1")"
+ cache_root="$2"; expected_revision="$3"; expected_tree="$4"; expected_fmt="$5"
+ expected_content="$6"
+ cx_log() { :; }
+ cx_verify_backend_cache_mount() { return 0; }
+ cx_cuda_arch() { printf '9.0'; }
+ cx_deepep_v2_root() { printf '%s' "$cache_root"; }
+ cx_activate_deepep_v2() { export DEEPEP_V2_COMMIT="$expected_revision"; }
+ cx_prepare_deepep_toolchain() { export NVSHMEM_DIR=/tmp/cx-test-nvshmem; }
+ cx_probe_deepep_v2() { return 0; }
+ cx_deepep_v2_content_sha256() { printf '%s' "$expected_content"; }
+ cx_deepep_v2_cache_is_valid() {
+ test -f "$2" && test "$(wc -l < "$2" | tr -d ' ')" = 5
+ }
+ cx_enable_deepep_v2_jit_reproducibility() { return 0; }
+ cx_materialize_backend_source() { mkdir -p "$2/third-party/fmt"; }
+ flock() { return 0; }
+ python3() {
+ if [ "${1:-}" = -m ] && [ "${2:-}" = venv ]; then
+ mkdir -p "$3/bin"
+ printf '#!/bin/sh\nexit 0\n' > "$3/bin/python"
+ chmod 700 "$3/bin/python"
+ fi
+ return 0
+ }
+ git() {
+ case " $* " in
+ *' third-party/fmt rev-parse HEAD '*) printf '%s\n' "$expected_fmt" ;;
+ *' rev-parse HEAD^{tree} '*) printf '%s\n' "$expected_tree" ;;
+ *' show -s --format=%ct HEAD '*) printf '1\n' ;;
+ *) return 0 ;;
+ esac
+ }
+ cx_git_in_tree() { shift; git "$@"; }
+ cx_build_deepep_v2
+ '''
+ subprocess.run(
+ [
+ "bash", "-c", command, "_", str(runtime), str(root),
+ COMMIT, TREE, FMT_COMMIT, content_hash,
+ ],
+ check=True,
+ )
+ self.assertFalse(stale.exists())
+ self.assertEqual(
+ marker.read_text(),
+ f"{COMMIT}\n{TREE}\n{FMT_COMMIT}\n{cache_key}\n{content_hash}\n",
+ )
+ self.assertEqual(list(root.glob(".collectivex-complete.tmp.*")), [])
+
+ def test_deepep_v2_published_cache_is_never_deleted_after_probe_failure(self) -> None:
+ runtime = ROOT / "runtime" / "run_in_container.sh"
+ with tempfile.TemporaryDirectory() as temporary:
+ cache_key = "a" * 64
+ root = Path(temporary) / f"deepep-v2-{cache_key}"
+ root.mkdir(mode=0o700)
+ marker = root / ".collectivex-complete"
+ marker.write_text("published\n")
+ sentinel = root / "active-reader"
+ sentinel.write_text("active\n")
+ command = r'''
+ set -euo pipefail
+ eval "$(sed -n '/^cx_build_deepep_v2()/,/^}/p' "$1")"
+ cache_root="$2"
+ cx_log() { :; }
+ cx_verify_backend_cache_mount() { return 0; }
+ cx_cuda_arch() { printf '9.0'; }
+ cx_deepep_v2_root() { printf '%s' "$cache_root"; }
+ cx_deepep_v2_cache_is_valid() { return 0; }
+ cx_activate_deepep_v2() { return 0; }
+ cx_prepare_deepep_toolchain() { return 0; }
+ cx_enable_deepep_v2_jit_reproducibility() { return 0; }
+ cx_probe_deepep_v2() { return 1; }
+ ! cx_build_deepep_v2
+ '''
+ subprocess.run(
+ ["bash", "-c", command, "_", str(runtime), str(root)],
+ check=True,
+ )
+ self.assertEqual(sentinel.read_text(), "active\n")
+ self.assertEqual(marker.read_text(), "published\n")
+
+ def test_deepep_v2_corrupt_published_cache_fails_without_reset(self) -> None:
+ runtime = ROOT / "runtime" / "run_in_container.sh"
+ with tempfile.TemporaryDirectory() as temporary:
+ cache_key = "a" * 64
+ root = Path(temporary) / f"deepep-v2-{cache_key}"
+ root.mkdir(mode=0o700)
+ marker = root / ".collectivex-complete"
+ marker.write_text("corrupt\n")
+ sentinel = root / "active-reader"
+ sentinel.write_text("active\n")
+ command = r'''
+ set -euo pipefail
+ eval "$(sed -n '/^cx_build_deepep_v2()/,/^}/p' "$1")"
+ cache_root="$2"
+ cx_log() { :; }
+ cx_verify_backend_cache_mount() { return 0; }
+ cx_cuda_arch() { printf '9.0'; }
+ cx_deepep_v2_root() { printf '%s' "$cache_root"; }
+ cx_deepep_v2_cache_is_valid() { return 1; }
+ flock() { return 0; }
+ ! cx_build_deepep_v2
+ '''
+ subprocess.run(
+ ["bash", "-c", command, "_", str(runtime), str(root)],
+ check=True,
+ )
+ self.assertEqual(sentinel.read_text(), "active\n")
+ self.assertEqual(marker.read_text(), "corrupt\n")
+
+ def test_deepep_v2_marker_requires_private_owned_cache_objects(self) -> None:
+ runtime = ROOT / "runtime" / "run_in_container.sh"
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary) / "cache"
+ root.mkdir(mode=0o700)
+ (root / "source").mkdir(mode=0o700)
+ (root / "venv").mkdir(mode=0o700)
+ marker = root / ".collectivex-complete"
+ cache_key = "a" * 64
+ content_hash = "b" * 64
+ marker.write_text(
+ f"{COMMIT}\n{TREE}\n{FMT_COMMIT}\n{cache_key}\n{content_hash}\n"
+ )
+ root.chmod(0o2700)
+ marker.chmod(0o600)
+ command = r'''
+ set -euo pipefail
+ eval "$(sed -n '/^cx_deepep_v2_marker_content_sha256()/,/^}/p' "$1")"
+ cx_deepep_v2_marker_content_sha256 "$2" "$3" "$4" "$5" "$6" "$7"
+ '''
+ args = [
+ "bash", "-c", command, "_", str(runtime), str(root), str(marker),
+ COMMIT, TREE, FMT_COMMIT, cache_key,
+ ]
+ valid = subprocess.run(args, text=True, capture_output=True, check=True)
+ self.assertEqual(valid.stdout, content_hash)
+ marker.chmod(0o644)
+ self.assertNotEqual(subprocess.run(args).returncode, 0)
+
+ def test_deepep_hybrid_marker_requires_a_private_regular_file(self) -> None:
+ runtime = ROOT / "runtime" / "run_in_container.sh"
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary) / "cache"
+ root.mkdir(mode=0o700)
+ marker = root / ".collectivex-complete"
+ content_hash = "b" * 64
+ marker.write_text(f"{COMMIT}\n{TREE}\n{content_hash}\n")
+ root.chmod(0o2700)
+ marker.chmod(0o600)
+ command = r'''
+ set -euo pipefail
+ eval "$(sed -n '/^cx_deepep_hybrid_marker_content_sha256()/,/^}/p' "$1")"
+ cx_deepep_hybrid_marker_content_sha256 "$2" "$3" "$4" "$5"
+ '''
+ args = [
+ "bash", "-c", command, "_", str(runtime), str(root), str(marker),
+ COMMIT, TREE,
+ ]
+ valid = subprocess.run(args, text=True, capture_output=True, check=True)
+ self.assertEqual(valid.stdout, content_hash)
+ marker_contract = runtime.read_text()
+ marker_contract = marker_contract[
+ marker_contract.index("cx_deepep_hybrid_marker_content_sha256()"):
+ marker_contract.index("cx_deepep_hybrid_cache_is_valid()")
+ ]
+ self.assertIn("marker_item.st_uid != root_item.st_uid", marker_contract)
+ self.assertNotIn("st_uid != os.getuid()", marker_contract)
+ marker.chmod(0o644)
+ self.assertNotEqual(subprocess.run(args).returncode, 0)
+
+ def test_deepep_v2_installed_content_digest_binds_every_distribution_file(self) -> None:
+ runtime = ROOT / "runtime" / "run_in_container.sh"
+ with tempfile.TemporaryDirectory() as temporary:
+ site = Path(temporary) / "venv" / "lib" / "python3.11" / "site-packages"
+ package = site / "deep_ep"
+ info = site / "deep_ep-2.0.0.dist-info"
+ package.mkdir(parents=True)
+ info.mkdir()
+ (package / "__init__.py").write_text("__version__ = '2.0.0'\n")
+ extension = package / "_C.so"
+ extension.write_bytes(b"extension-one")
+ (info / "METADATA").write_text(
+ "Metadata-Version: 2.1\nName: deep_ep\nVersion: 2.0.0\n"
+ )
+ (info / "RECORD").write_text(
+ "deep_ep/__init__.py,,\n"
+ "deep_ep/_C.so,,\n"
+ "deep_ep-2.0.0.dist-info/METADATA,,\n"
+ "deep_ep-2.0.0.dist-info/RECORD,,\n"
+ )
+ command = r'''
+ set -euo pipefail
+ eval "$(sed -n '/^cx_deepep_v2_content_sha256()/,/^}/p' "$1")"
+ cx_deepep_v2_content_sha256
+ '''
+ env = {
+ **os.environ,
+ "PYTHONPATH": str(site),
+ "VIRTUAL_ENV": str(Path(temporary) / "venv"),
+ }
+ first = subprocess.run(
+ ["bash", "-c", command, "_", str(runtime)],
+ text=True, capture_output=True, check=True, env=env,
+ ).stdout
+ extension.write_bytes(b"extension-two")
+ second = subprocess.run(
+ ["bash", "-c", command, "_", str(runtime)],
+ text=True, capture_output=True, check=True, env=env,
+ ).stdout
+ self.assertRegex(first, r"^[0-9a-f]{64}$")
+ self.assertRegex(second, r"^[0-9a-f]{64}$")
+ self.assertNotEqual(first, second)
+ extension.unlink()
+ outside = Path(temporary) / "outside.so"
+ outside.write_bytes(b"outside")
+ extension.symlink_to(outside)
+ self.assertNotEqual(
+ subprocess.run(
+ ["bash", "-c", command, "_", str(runtime)], env=env,
+ ).returncode,
+ 0,
+ )
+
+ def test_uccl_content_identity_excludes_install_generated_files(self) -> None:
+ keep = load_uccl_function(
+ "_is_uccl_runtime_payload", {"PurePosixPath": PurePosixPath}
+ )
+ self.assertTrue(keep("uccl/ep.abi3.so"))
+ self.assertTrue(keep("uccl.libs/libnuma.so"))
+ self.assertFalse(keep("uccl/__pycache__/collective.cpython-312.pyc"))
+ self.assertFalse(keep("uccl-0.1.1.dist-info/RECORD"))
+
+ def test_uccl_dependency_versions_are_exact(self) -> None:
+ installed = dict(contracts.UCCL_DEPENDENCY_VERSIONS)
+ dependency_versions = load_uccl_function(
+ "_uccl_dependency_versions",
+ {
+ "contracts": contracts,
+ "metadata": types.SimpleNamespace(
+ version=lambda package: installed[package]
+ ),
+ },
+ )
+ self.assertEqual(dependency_versions(), contracts.UCCL_DEPENDENCY_VERSIONS)
+ installed["intervaltree"] = "3.2.0"
+ with self.assertRaisesRegex(RuntimeError, "differ from the v1 contract"):
+ dependency_versions()
+
+ schema = json.loads((ROOT / "schemas" / "raw-case-v1.schema.json").read_text())
+ dependency_schema = schema["properties"]["implementation"]["properties"][
+ "provenance"
+ ]["properties"]["uccl_dependency_versions"]
+ self.assertFalse(dependency_schema["additionalProperties"])
+ self.assertEqual(
+ {
+ package: definition["const"]
+ for package, definition in dependency_schema["properties"].items()
+ },
+ contracts.UCCL_DEPENDENCY_VERSIONS,
+ )
+
+ def test_uccl_support_dependency_content_is_path_free(self) -> None:
+ with tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ source_entry = PurePosixPath("intervaltree/__init__.py")
+ cache_entry = PurePosixPath("intervaltree/__pycache__/__init__.pyc")
+ metadata_entry = PurePosixPath("intervaltree-3.1.0.dist-info/RECORD")
+ for entry in (source_entry, cache_entry, metadata_entry):
+ path = root / entry
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_bytes(entry.as_posix().encode())
+ distribution = types.SimpleNamespace(
+ files=[source_entry, cache_entry, metadata_entry],
+ locate_file=lambda item: root / item,
+ )
+ evidence_for = load_uccl_function(
+ "_python_dependency_evidence",
+ {
+ "Path": Path,
+ "PurePosixPath": PurePosixPath,
+ "contracts": contracts,
+ "metadata": types.SimpleNamespace(
+ distribution=lambda package: distribution
+ ),
+ },
+ )
+ evidence = evidence_for("intervaltree", "3.1.0")
+ self.assertEqual(
+ evidence,
+ contracts.content_manifest_evidence(
+ role="intervaltree-distribution",
+ name="intervaltree-3.1.0",
+ files=[(source_entry.as_posix(), root / source_entry)],
+ ),
+ )
+ self.assertNotIn(str(root), json.dumps(evidence))
+
+ def test_uccl_hashes_the_mapped_pinned_libcudart_without_exposing_paths(
+ self,
+ ) -> None:
+ with tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ entry = PurePosixPath("nvidia/cuda_runtime/lib/libcudart.so.12")
+ library = root / entry
+ library.parent.mkdir(parents=True)
+ library.write_bytes(b"pinned CUDA 12 runtime")
+ distribution = types.SimpleNamespace(
+ files=[entry],
+ locate_file=lambda item: root / item,
+ )
+ evidence_for = load_uccl_function(
+ "_loaded_libcudart_evidence",
+ {
+ "Path": Path,
+ "PurePosixPath": PurePosixPath,
+ "contracts": contracts,
+ "metadata": types.SimpleNamespace(
+ distribution=lambda package: distribution
+ ),
+ },
+ )
+ maps = root / "maps"
+ maps.write_text(f"7f00-7f10 r-xp 00000000 00:00 0 {library}\n")
+ evidence = evidence_for("12.9.79", maps)
+ self.assertEqual(
+ evidence,
+ contracts.content_manifest_evidence(
+ role="cuda-runtime",
+ name="nvidia-cuda-runtime-cu12-12.9.79",
+ files=[("libcudart.so", library)],
+ ),
+ )
+ self.assertNotIn(str(root), json.dumps(evidence))
+
+ unowned = root / "unowned" / library.name
+ unowned.parent.mkdir()
+ unowned.write_bytes(library.read_bytes())
+ maps.write_text(f"7f00-7f10 r-xp 00000000 00:00 0 {unowned}\n")
+ with self.assertRaisesRegex(RuntimeError, "not owned") as raised:
+ evidence_for("12.9.79", maps)
+ self.assertNotIn(str(root), str(raised.exception))
+
+ def test_private_runtime_logs_are_not_public_artifacts(self) -> None:
+ path = subprocess.check_output(
+ [
+ "bash", "-c", 'source "$1"; cx_private_log_path test', "_",
+ str(ROOT / "runtime" / "common.sh"),
+ ],
+ text=True,
+ env={**os.environ, "COLLECTIVEX_EXECUTION_ID": "contract-test"},
+ ).strip()
+ try:
+ log = Path(path)
+ self.assertEqual(stat.S_IMODE(log.stat().st_mode), 0o600)
+ self.assertEqual(stat.S_IMODE(log.parent.stat().st_mode), 0o700)
+ self.assertFalse(log.is_relative_to(ROOT))
+ finally:
+ shutil.rmtree(Path(path).parent, ignore_errors=True)
+
+ def test_private_runtime_logs_reject_traversal_and_symlinks(self) -> None:
+ common = str(ROOT / "runtime" / "common.sh")
+ for variable, value in (
+ ("COLLECTIVEX_EXECUTION_ID", ".."),
+ ("CX_TEST_LABEL", ".."),
+ ):
+ environment = {
+ **os.environ,
+ "COLLECTIVEX_EXECUTION_ID": "contract-adversarial",
+ "CX_TEST_LABEL": "test",
+ variable: value,
+ }
+ result = subprocess.run(
+ ["bash", "-c", 'source "$1"; cx_private_log_path "$CX_TEST_LABEL"', "_", common],
+ text=True,
+ capture_output=True,
+ env=environment,
+ )
+ self.assertNotEqual(result.returncode, 0)
+ self.assertNotIn(value, result.stderr)
+
+ private_root = Path(f"/tmp/inferencex-collectivex-{os.getuid()}")
+ private_root.mkdir(mode=0o700, exist_ok=True)
+ self.assertFalse(private_root.is_symlink())
+ os.chmod(private_root, 0o700)
+ with tempfile.TemporaryDirectory() as temporary:
+ target = Path(temporary)
+ tag = f"contract-symlink-{os.getpid()}"
+ link = private_root / tag
+ link.symlink_to(target, target_is_directory=True)
+ try:
+ result = subprocess.run(
+ ["bash", "-c", 'source "$1"; cx_private_log_path test', "_", common],
+ text=True,
+ capture_output=True,
+ env={**os.environ, "COLLECTIVEX_EXECUTION_ID": tag},
+ )
+ self.assertNotEqual(result.returncode, 0)
+ self.assertEqual(list(target.iterdir()), [])
+ finally:
+ link.unlink(missing_ok=True)
+
+ tag = f"contract-log-symlink-{os.getpid()}"
+ directory = private_root / tag
+ directory.mkdir(mode=0o700)
+ target_file = target / "target"
+ target_file.write_text("unchanged")
+ log_link = directory / "test.log"
+ log_link.symlink_to(target_file)
+ try:
+ result = subprocess.run(
+ ["bash", "-c", 'source "$1"; cx_private_log_path test', "_", common],
+ text=True,
+ capture_output=True,
+ env={**os.environ, "COLLECTIVEX_EXECUTION_ID": tag},
+ )
+ self.assertNotEqual(result.returncode, 0)
+ self.assertEqual(target_file.read_text(), "unchanged")
+ finally:
+ log_link.unlink(missing_ok=True)
+ directory.rmdir()
+
+ def test_operator_config_failure_is_value_free(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ config = Path(temporary) / "operator.env"
+ config.write_text("printf 'private-config-token\\n' >&2\nfalse\n")
+ config.chmod(0o600)
+ result = subprocess.run(
+ ["bash", "-c",
+ 'export COLLECTIVEX_EXECUTION_ID="operator-failure-$$"; '
+ "trap 'cx_cleanup_private_logs 0' EXIT; source \"$1\"; "
+ "cx_load_operator_config", "_",
+ str(ROOT / "runtime" / "common.sh")],
+ text=True,
+ capture_output=True,
+ env={
+ **os.environ,
+ "CX_RUNNER": "h100-dgxc",
+ "COLLECTIVEX_OPERATOR_CONFIG": str(config),
+ },
+ )
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn("runner-local configuration failed", result.stderr)
+ self.assertNotIn("private-config-token", result.stderr)
+
+ def test_ephemeral_operator_config_is_removed_after_source(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ config = Path(temporary) / "operator.env"
+ decoy = Path(temporary) / "decoy"
+ decoy.write_text("keep")
+ config.write_text(json.dumps(operator_config(Path(temporary) / "storage")))
+ config.chmod(0o600)
+ result = subprocess.run(
+ [
+ "bash", "-c",
+ 'export COLLECTIVEX_EXECUTION_ID="operator-ephemeral-$$"; '
+ "trap 'cx_cleanup_private_logs 0' EXIT; "
+ 'config="$COLLECTIVEX_OPERATOR_CONFIG"; source "$1"; '
+ 'cx_load_operator_config; test ! -e "$config"; '
+ 'test "$CX_PARTITION" = test; '
+ 'test -z "${COLLECTIVEX_OPERATOR_CONFIG+x}"',
+ "_", str(ROOT / "runtime" / "common.sh"),
+ ],
+ text=True,
+ capture_output=True,
+ env={
+ **os.environ,
+ "CX_RUNNER": "h100-dgxc",
+ "COLLECTIVEX_OPERATOR_CONFIG": str(config),
+ "COLLECTIVEX_OPERATOR_CONFIG_EPHEMERAL": "1",
+ },
+ )
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertFalse(config.exists())
+ self.assertEqual(decoy.read_text(), "keep")
+
+ def test_operator_config_is_strict_per_runner_json(self) -> None:
+ command = (
+ 'source "$1"; export COLLECTIVEX_EXECUTION_ID="operator-config-$$"; '
+ "trap 'cx_cleanup_private_logs 0' EXIT; cx_load_operator_config; "
+ 'test "$CX_PARTITION" = test; '
+ 'test -z "${COLLECTIVEX_OPERATOR_CONFIG_CONTENT+x}"; '
+ 'test -z "${ENROOT_CACHE_PATH+x}"'
+ )
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary)
+ document = operator_config(root / "storage")
+ config = root / "operator.json"
+ config.write_text(json.dumps(document))
+ config.chmod(0o600)
+ for runner in capability.PLATFORMS:
+ with self.subTest(runner=runner):
+ result = subprocess.run(
+ ["bash", "-c", command, "_", str(ROOT / "runtime" / "common.sh")],
+ text=True,
+ capture_output=True,
+ env={
+ **os.environ,
+ "CX_RUNNER": runner,
+ "ENROOT_CACHE_PATH": "/private/stale-enroot-cache",
+ "COLLECTIVEX_OPERATOR_CONFIG": str(config),
+ },
+ )
+ self.assertEqual(result.returncode, 0, result.stderr)
+
+ lock_dir = root / "amd-locks"
+ document["runners"]["mi355x"]["lock_dir"] = str(lock_dir)
+ config.write_text(json.dumps(document))
+ config.chmod(0o600)
+ canonical = subprocess.run(
+ [
+ "bash",
+ "-c",
+ 'source "$1"; export COLLECTIVEX_EXECUTION_ID="canonical-lock-$$"; '
+ "trap 'cx_cleanup_private_logs 0' EXIT; cx_load_operator_config; "
+ 'cx_lock_canonical_gha_env mi355x; test "$CX_LOCK_DIR" = "$2"',
+ "_",
+ str(ROOT / "runtime" / "common.sh"),
+ str(lock_dir),
+ ],
+ text=True,
+ capture_output=True,
+ env={
+ **os.environ,
+ "CX_RUNNER": "mi355x",
+ "CX_SHARD_FILE": ".shards/test.json",
+ "CX_SHARD_SKU": "mi355x",
+ "CX_NODES": "1",
+ "CX_GPUS_PER_NODE": "8",
+ "COLLECTIVEX_CANONICAL_GHA": "1",
+ "COLLECTIVEX_OPERATOR_CONFIG": str(config),
+ "COLLECTIVEX_SOURCE_SHA": "a" * 40,
+ "GITHUB_ACTIONS": "true",
+ "GITHUB_RUN_ATTEMPT": "1",
+ "GITHUB_RUN_ID": "1",
+ "GITHUB_WORKSPACE": str(root.resolve()),
+ },
+ )
+ self.assertEqual(canonical.returncode, 0, canonical.stderr)
+
+ selected_only = {
+ "schema_version": 1,
+ "runners": {"h100-dgxc": document["runners"]["h100-dgxc"]},
+ }
+ result = subprocess.run(
+ [
+ "bash", "-c", command + '; test "$CX_STAGE_DIR" = "$2"', "_",
+ str(ROOT / "runtime" / "common.sh"), str(root / "storage"),
+ ],
+ text=True,
+ capture_output=True,
+ env={
+ **os.environ,
+ "CX_RUNNER": "h100-dgxc",
+ "CX_STAGE_DIR": "/private/stale-stage",
+ "ENROOT_CACHE_PATH": "/private/stale-enroot-cache",
+ "COLLECTIVEX_OPERATOR_CONFIG_LOADED": "1",
+ "COLLECTIVEX_OPERATOR_CONFIG_CONTENT": json.dumps(selected_only),
+ "COLLECTIVEX_OPERATOR_CONFIG_REQUIRED": "1",
+ },
+ )
+ self.assertEqual(result.returncode, 0, result.stderr)
+
+ rejected = json.loads(json.dumps(document))
+ rejected["runners"]["h100-dgxc"]["shell"] = "private-command"
+ boolean_version = {**document, "schema_version": True}
+ missing_socket = json.loads(json.dumps(document))
+ del missing_socket["runners"]["h100-dgxc"]["socket_ifname"]
+ missing_rdma = json.loads(json.dumps(document))
+ del missing_rdma["runners"]["mi355x"]["rdma_devices"]
+ missing_amd_stage = json.loads(json.dumps(document))
+ del missing_amd_stage["runners"]["mi325x"]["stage_dir"]
+ missing_nvidia_stage = json.loads(json.dumps(document))
+ del missing_nvidia_stage["runners"]["h100-dgxc"]["stage_dir"]
+ for invalid in (rejected, boolean_version, missing_nvidia_stage):
+ config.write_text(json.dumps(invalid))
+ config.chmod(0o600)
+ result = subprocess.run(
+ ["bash", "-c", command, "_", str(ROOT / "runtime" / "common.sh")],
+ text=True,
+ capture_output=True,
+ env={
+ **os.environ,
+ "CX_RUNNER": "h100-dgxc",
+ "COLLECTIVEX_OPERATOR_CONFIG": str(config),
+ },
+ )
+ self.assertNotEqual(result.returncode, 0)
+ self.assertNotIn("private-command", result.stderr)
+
+ for valid, runner in (
+ (missing_socket, "h100-dgxc"),
+ (missing_rdma, "h100-dgxc"),
+ (missing_amd_stage, "h100-dgxc"),
+ ):
+ config.write_text(json.dumps(valid))
+ config.chmod(0o600)
+ result = subprocess.run(
+ [
+ "bash", "-c", command + "; cx_apply_network_profile 1 nvlink",
+ "_", str(ROOT / "runtime" / "common.sh"),
+ ],
+ text=True,
+ capture_output=True,
+ env={
+ **os.environ,
+ "CX_RUNNER": runner,
+ "COLLECTIVEX_OPERATOR_CONFIG": str(config),
+ },
+ )
+ self.assertEqual(result.returncode, 0, result.stderr)
+
+ config.write_text(json.dumps(missing_socket))
+ config.chmod(0o600)
+ scaleout = subprocess.run(
+ [
+ "bash", "-c", command + "; cx_apply_network_profile 2 nvlink-rdma",
+ "_", str(ROOT / "runtime" / "common.sh"),
+ ],
+ text=True,
+ capture_output=True,
+ env={
+ **os.environ,
+ "CX_RUNNER": "h100-dgxc",
+ "COLLECTIVEX_OPERATOR_CONFIG": str(config),
+ },
+ )
+ self.assertNotEqual(scaleout.returncode, 0)
+
+ config.write_text(json.dumps(missing_amd_stage))
+ config.chmod(0o600)
+ selected_missing_stage = subprocess.run(
+ ["bash", "-c", command, "_", str(ROOT / "runtime" / "common.sh")],
+ text=True,
+ capture_output=True,
+ env={
+ **os.environ,
+ "CX_RUNNER": "mi325x",
+ "COLLECTIVEX_OPERATOR_CONFIG": str(config),
+ },
+ )
+ self.assertNotEqual(selected_missing_stage.returncode, 0)
+
+ config.write_text(json.dumps(document))
+ config.chmod(0o644)
+ result = subprocess.run(
+ ["bash", "-c", command, "_", str(ROOT / "runtime" / "common.sh")],
+ text=True,
+ capture_output=True,
+ env={
+ **os.environ,
+ "CX_RUNNER": "h100-dgxc",
+ "COLLECTIVEX_OPERATOR_CONFIG": str(config),
+ },
+ )
+ self.assertNotEqual(result.returncode, 0)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/experimental/CollectiveX/tests/test_publisher.py b/experimental/CollectiveX/tests/test_publisher.py
new file mode 100644
index 0000000000..5624fbcf68
--- /dev/null
+++ b/experimental/CollectiveX/tests/test_publisher.py
@@ -0,0 +1,2418 @@
+#!/usr/bin/env python3
+"""Focused end-to-end tests for the isolated CollectiveX publisher."""
+from __future__ import annotations
+
+import copy
+import hashlib
+import itertools
+import json
+import os
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+import types
+import unittest
+from unittest import mock
+import zipfile
+
+HERE = Path(__file__).resolve().parent
+ROOT = HERE.parent
+sys.path[:0] = [str(ROOT), str(HERE)]
+
+import contracts # noqa: E402
+import identity # noqa: E402
+import publisher # noqa: E402
+import summarize # noqa: E402
+import sweep_matrix # noqa: E402
+
+
+RUN = {
+ "repository": "SemiAnalysisAI/InferenceX",
+ "run_id": "12345",
+ "run_attempt": 1,
+ "source_sha": "a" * 40,
+}
+
+
+def _unsupported_delivery(
+ root: Path, ordinals: tuple[int, ...] = (1,), run: dict = RUN,
+) -> tuple[Path, Path]:
+ matrix = sweep_matrix.resolve_matrix(backends="all")
+ wrapper = next(item for item in matrix["requested_cases"] if item["disposition"] == "unsupported")
+ matrix = {
+ "format": "collectivex.matrix.v1",
+ "schema_version": 1,
+ "requested_cases": [wrapper],
+ "include": [],
+ }
+ case = {key: value for key, value in wrapper["case"].items() if key != "case_id"}
+ artifact_name = f"cxunsupported-{run['run_id']}-{run['run_attempt']}"
+ git_run = {
+ "artifact": artifact_name,
+ "job": "setup",
+ "ref": "collectivex",
+ "repo": run["repository"],
+ "run_attempt": str(run["run_attempt"]),
+ "run_id": run["run_id"],
+ "source_sha": run["source_sha"],
+ }
+ allocation = {
+ "artifact": artifact_name,
+ "execution_id": f"{run['run_id']}_{run['run_attempt']}_unsupported",
+ "job": "setup",
+ "repo": run["repository"],
+ "run_attempt": str(run["run_attempt"]),
+ "run_id": run["run_id"],
+ "runner": "capability-resolver",
+ "source_sha": run["source_sha"],
+ }
+ matrix_path = root / "matrix.json"
+ artifact = root / artifact_name
+ artifact.mkdir()
+ matrix_path.write_text(json.dumps(matrix))
+ control_sha256 = hashlib.sha256(matrix_path.read_bytes()).hexdigest()
+ for ordinal in ordinals:
+ terminal = contracts.make_terminal_document(
+ allocation_factors=allocation, attempt_ordinal=ordinal, case=case,
+ case_factors={"case": case, "profile": identity.V1_CASE_PROFILE,
+ "sku": wrapper["sku"]},
+ control_sha256=control_sha256, failure_mode="capability",
+ generated_at="2026-07-04T00:00:00Z", git_run=git_run,
+ reason=wrapper["reason"], return_code=5, source="matrix-capability-resolver",
+ status="unsupported", expected_case_id=wrapper["case"]["case_id"],
+ )
+ (artifact / f"unsupported-{ordinal}.json").write_text(json.dumps(terminal))
+ return matrix_path, artifact
+
+
+def _args(
+ store: Path, matrix: Path, artifact: Path, run: dict = RUN
+) -> types.SimpleNamespace:
+ return types.SimpleNamespace(
+ store_root=str(store),
+ matrix=str(matrix),
+ artifact=[str(artifact)],
+ repository=run["repository"],
+ run_id=run["run_id"],
+ run_attempt=run["run_attempt"],
+ source_sha=run["source_sha"],
+ )
+
+
+def _ids(seed: str) -> tuple[str, str, str, str, str, str]:
+ case = identity.digest("case", {"seed": seed})
+ allocation = identity.allocation_id({"seed": seed})
+ attempt = identity.attempt_id(allocation=allocation, case=case, ordinal=1)
+ series = identity.series_id({"seed": seed})
+ point = identity.point_id(series=series, tokens_per_rank=8)
+ evidence = identity.evidence_id(
+ point=point, allocation=allocation, attempt=attempt, sample_sha256="b" * 64
+ )
+ return case, allocation, attempt, series, point, evidence
+
+
+def _component(scale: float = 1.0) -> dict:
+ latency = {"p50": 10.0 * scale, "p90": 12.0 * scale,
+ "p95": 14.0 * scale, "p99": 20.0 * scale}
+ logical_bytes = 100_000
+ return {
+ "origin": "measured",
+ "latency_us": latency,
+ "logical_bytes": logical_bytes,
+ "logical_payload_rate_gbps_at_latency_percentile": {
+ name: logical_bytes / (value * 1000.0) for name, value in latency.items()
+ },
+ "sample_count": 512,
+ }
+
+
+def _hybrid_provenance(ep_size: int = 1) -> dict:
+ realized = {field: 1 for field in contracts.HYBRID_REALIZED_CONFIG_FIELDS}
+ for field in contracts.HYBRID_REALIZED_BOOL_FIELDS:
+ realized[field] = True
+ realized.update({
+ "num_of_experts_per_rank": 1,
+ "num_of_nodes": 1,
+ "num_of_ranks_per_node": ep_size,
+ "token_data_type": "UINT16",
+ })
+ kernel_keys = ["combine-key", "dispatch-key", "preprocess-key"]
+ return {
+ "backend_lineage": "deepep-hybrid", "branch": "hybrid-ep",
+ "deepep_commit": "a" * 40, "deepep_tree": "b" * 40,
+ "device_sms": 1,
+ "jit_kernel_keys": kernel_keys,
+ "jit_shared_objects": [
+ {
+ "kernel_key": key,
+ "rank_artifacts": [
+ {"bytes": 1, "rank": rank, "sha256": f"{index + 1:x}" * 64}
+ for rank in range(ep_size)
+ ],
+ }
+ for index, key in enumerate(kernel_keys)
+ ],
+ "loaded_libraries": [
+ {"name": "deep_ep_cpp", "role": "deepep-extension", "sha256": "4" * 64},
+ {"name": "hybrid_ep_cpp", "role": "deepep-hybrid-extension", "sha256": "5" * 64},
+ ],
+ "realized_config": realized,
+ "resource_mode": "tuned",
+ "tuned_source": "deepep-hybrid-configurer-autotune-v1",
+ }
+
+
+def _native_fixture(backend: str = "nccl-ep") -> tuple[dict, dict]:
+ def digest(value: object) -> str:
+ return hashlib.sha256(contracts.canonical_json_bytes(value)).hexdigest()
+
+ scheduled = {
+ "backend": backend, "canonical": True, "eplb": False, "ep": 1,
+ "experts": 1, "gpus_per_node": 1, "hidden": 1, "ladder": "1", "nodes": 1,
+ "mode": "normal", "phase": "decode", "required_publication": "official",
+ "routing": "uniform", "samples_per_point": 512,
+ "scale_out_transport": None, "scale_up_domain": 1,
+ "scale_up_transport": "nvlink", "scope": "scale-up", "suite": "ep-core-v1",
+ "timing": "8:64:32", "topk": 1,
+ "topology_class": "fixture", "transport": "nvlink",
+ "warmup_semantics": "full-roundtrip-before-each-component-trial-point-v1",
+ "workload": "deepseek-v3-v1",
+ }
+ case_factors = {
+ "case": scheduled, "profile": identity.V1_NORMAL_CASE_PROFILE, "sku": "fixture"
+ }
+ case_id = identity.digest("case", case_factors)
+ git_run = {
+ "artifact": "cxshard-fixture-999-1", "job": "sweep", "ref": "collectivex",
+ "repo": RUN["repository"], "run_attempt": "1", "run_id": "999",
+ "source_sha": RUN["source_sha"],
+ }
+ allocation_factors = {
+ "artifact": git_run["artifact"], "execution_id": "999_1_fixture",
+ "job": git_run["job"], "repo": git_run["repo"], "run_attempt": "1",
+ "run_id": "999", "runner": "fixture", "source_sha": git_run["source_sha"],
+ }
+ allocation_id = identity.allocation_id(allocation_factors)
+ attempt_id = identity.attempt_id(allocation=allocation_id, case=case_id, ordinal=1)
+ member_id, member_checksums, routing_hash, routing_rows, routing_weights = (
+ contracts._expected_canonical_trace(
+ "uniform", hidden=1, topk=1, logical_experts=1, physical_experts=1,
+ ep_size=1, tokens_per_rank=1, seed=67, eplb_enabled=False,
+ reference_tokens_per_rank=2048,
+ )
+ )
+ workload_id = identity.workload_id({
+ "members": [{"checksums": member_checksums, "workload_id": member_id}]
+ })
+ runtime = {
+ "accelerator_runtime": {"kind": "cuda", "version": "13.0"},
+ "collective_library": {"kind": "nccl", "version": "2.30.4"},
+ "device": {
+ "arch": "sm100", "compute_units": 1, "memory_bytes": 1,
+ "product": "Fixture GPU", "warp_size": 32,
+ },
+ "driver_version": "1", "framework": {"kind": "torch", "version": "2.10.0"},
+ "machine": "fixture", "python_version": "3.12", "vendor": "nvidia",
+ }
+ implementation_provenance = (
+ {
+ "backend": "nccl-ep", "backend_lineage": "nccl",
+ "collective_library": "nccl", "nccl_version": "2.30.4",
+ "reference_semantics": "fixture-v1",
+ }
+ if backend == "nccl-ep"
+ else _hybrid_provenance()
+ )
+ kernel_generation = "nccl" if backend == "nccl-ep" else "hybrid"
+ implementation = {
+ "kernel_generation": kernel_generation,
+ "name": backend,
+ "provenance": implementation_provenance,
+ "resource_profile": contracts.project_resource_profile(implementation_provenance),
+ }
+ public_config = contracts.public_series_config(
+ kernel_generation=implementation["kernel_generation"],
+ provenance=implementation_provenance,
+ resource_profile=implementation["resource_profile"],
+ resource_mode="tuned",
+ device_product=runtime["device"]["product"],
+ )
+ series_factors = {
+ "backend": backend, "case_id": case_id,
+ "image_digest": "sha256:" + "d" * 64,
+ "implementation_contract_sha256": digest({
+ **implementation,
+ "provenance": contracts.series_provenance(implementation_provenance),
+ }),
+ "public_config_sha256": contracts.public_series_config_sha256(public_config),
+ "routing_control_sha256": contracts.routing_implementation_control_sha256(
+ implementation
+ ),
+ "runtime_fingerprint_sha256": digest(runtime),
+ "source_sha": RUN["source_sha"], "squash_sha256": "e" * 64,
+ "workload_id": workload_id,
+ }
+ series_id = identity.series_id(series_factors)
+ point_id = identity.point_id(series=series_id, tokens_per_rank=1)
+ sample_components = {
+ name: {
+ "availability": "measured", "sample_count": 512,
+ "trials": [[latency] * 8 for _ in range(64)],
+ }
+ for name, latency in (("combine", 20.0), ("dispatch", 10.0), ("roundtrip", 40.0))
+ }
+ sample_sha = digest({"components": sample_components, "tokens_per_rank": 1})
+ evidence_id = identity.evidence_id(
+ point=point_id, allocation=allocation_id, attempt=attempt_id,
+ sample_sha256=sample_sha,
+ )
+ samples = {
+ "allocation_id": allocation_id, "attempt_id": attempt_id, "case_id": case_id,
+ "format": contracts.SAMPLES_FORMAT,
+ "points": [{
+ "components": sample_components, "evidence_id": evidence_id,
+ "point_id": point_id, "sample_sha256": sample_sha, "tokens_per_rank": 1,
+ }],
+ "sampling": {
+ "iterations_per_trial": 8, "reduction": "cross-rank-max-per-iteration",
+ "trials": 64,
+ },
+ "schema_version": 1, "series_id": series_id,
+ }
+ sample_bytes = contracts.canonical_json_bytes(samples)
+ oracle = {
+ "atol": 0.02,
+ "checks": {name: True for name in (
+ "combine_values", "counts", "metadata", "multiplicity", "payload",
+ "source_set", "weights",
+ )},
+ "combine_weight_semantics": "unweighted-rank-sum",
+ "contract": "expert-specific-transform-v1", "dispatch_sha256": "1" * 64,
+ "max_absolute_error": 0.0, "max_elementwise_relative_error": 0.0,
+ "max_relative_error": 0.0, "max_weight_error": 0.0,
+ "order_sha256": "2" * 64, "ordering_contract": "fixture-order-v1",
+ "passed": True, "receive_count": 1, "rtol": 0.05,
+ }
+ def pct(value: float) -> dict[str, float]:
+ return {name: value for name in ("p50", "p90", "p95", "p99")}
+
+ def measured(value: float) -> dict:
+ return {
+ "availability": "measured", "origin": "measured",
+ "percentiles_us": pct(value), "sample_count": 512,
+ }
+ row = {
+ "anomalies": [],
+ "components": {
+ "combine": measured(20.0), "dispatch": measured(10.0),
+ "isolated_sum": {
+ "availability": "derived", "origin": "derived-percentile-sum",
+ "percentiles_us": pct(30.0), "sample_count": 0,
+ },
+ "roundtrip": measured(40.0),
+ },
+ "correctness": {
+ "contract": "expert-specific-transform-v1", "max_relative_error": 0.0,
+ "passed": True,
+ "rank_evidence": [{
+ "input_unchanged": True, "order_stable": True,
+ "post_timing": copy.deepcopy(oracle), "pre_timing": copy.deepcopy(oracle),
+ "rank": 0,
+ }],
+ "scope": "dispatch-metadata-and-transformed-combine",
+ },
+ "evidence_id": evidence_id, "global_tokens": 1,
+ "logical_bytes": {"combine": 2, "dispatch": 2, "roundtrip": 4},
+ "point_id": point_id,
+ "receive": {"max": 1, "mean": 1.0, "min": 1, "total": 1},
+ "routing": contracts._expected_routing_summary(
+ routing_rows,
+ routing_weights,
+ physical_experts=1,
+ ep_size=1,
+ tokens_per_rank=1,
+ gpus_per_node=1,
+ scale_up_domain=1,
+ ),
+ "sample_histograms": {
+ name: contracts._expected_histogram([value] * 512)
+ for name, value in (("combine", 20.0), ("dispatch", 10.0), ("roundtrip", 40.0))
+ },
+ "sample_sha256": sample_sha,
+ "token_rate_at_latency_percentile": pct(25_000.0), "tokens_per_rank": 1,
+ }
+ raw = {
+ "case": {
+ "attempt_ordinal": 1, "backend": backend,
+ "eplb": {
+ "enabled": False, "imbalance_after": None, "imbalance_before": None,
+ "mapping_hash": None, "max_replicas": None, "num_logical_experts": 1,
+ "num_physical_experts": 1, "num_redundant": 0, "planner": None,
+ "reference_tokens_per_rank": None, "replicated_experts": 0,
+ },
+ "ep_size": 1, "mode": "normal", "phase": "decode",
+ "required_publication": "official", "resource_mode": "tuned", "runner": "fixture",
+ "shape": {
+ "activation_profile": "canonical-counter-source-v3", "dispatch_dtype": "bf16",
+ "eplb": False, "experts": 1, "experts_per_rank": 1, "hidden": 1,
+ "kernel_gen": kernel_generation, "num_logical_experts": 1,
+ "quant": {
+ "combine_accum_dtype": "fp32", "combine_input_dtype": "bf16",
+ "combine_output_dtype": "bf16", "combine_quant_mode": "none",
+ "scale_layout": None,
+ },
+ "routing": "uniform", "topk": 1,
+ },
+ "suite": "ep-core-v1", "workload_name": "deepseek-v3-v1",
+ },
+ "format": contracts.RAW_FORMAT, "generated_at": "2026-07-04T00:00:00Z",
+ "identity": {
+ "allocation_factors": allocation_factors, "allocation_id": allocation_id,
+ "attempt_id": attempt_id, "attempt_ordinal": 1, "case_factors": case_factors,
+ "case_id": case_id, "series_factors": series_factors, "series_id": series_id,
+ },
+ "implementation": implementation,
+ "measurement": {
+ "component_order_contract": "roundtrip-dispatch-activation-only-combine-v2",
+ "conditioning": {
+ "contract": "fixed-phase-ramp-8-roundtrips-v1",
+ "ladder": [1, 2, 4, 8, 16, 32, 64, 128],
+ "roundtrips_per_shape": 8,
+ },
+ "contract": "layout-and-dispatch-v1",
+ "rows": [row],
+ "sampling": {
+ "contract": "fixed-512-v1", "iterations_per_trial": 8,
+ "percentile_method": "nearest-rank",
+ "reduction": "cross-rank-max-per-iteration", "samples_per_component": 512,
+ "trials": 64, "warmup_iterations": 32,
+ "warmup_semantics": "full-roundtrip-before-each-component-trial-point-v1",
+ },
+ "source_allocation": "even",
+ },
+ "outcome": {
+ "publication_status": "diagnostic", "reasons": [], "status": "success",
+ "validity": {
+ "anomaly_free": True, "execution_status": "complete",
+ "measurement_conformance": "conformant", "provenance_complete": True,
+ "resource_conformance": implementation["resource_profile"]["conformance_class"],
+ "sampling_conformance": "conformant",
+ "semantic_correctness": "pass",
+ "workload_identity": "consistent-across-ranks",
+ "workload_source": "canonical-serialized",
+ },
+ },
+ "provenance": {
+ "command": "run_ep", "distributed_launcher": "torchrun", "git_run": git_run,
+ "image": {
+ "arch": "amd64", "digest": "sha256:" + "d" * 64,
+ "digest_verified": True, "reference": "fixture:1", "squash_sha256": "e" * 64,
+ },
+ "redaction": "sanitized-v1",
+ },
+ "record_type": "case-attempt",
+ "runtime_fingerprint": runtime,
+ "sample_artifact": {
+ "bytes": len(sample_bytes), "format": contracts.SAMPLES_FORMAT,
+ "path": "samples.json", "sha256": hashlib.sha256(sample_bytes).hexdigest(),
+ },
+ "schema_version": 1,
+ "topology": {
+ "device_count": 1, "device_product": "Fixture GPU", "gpus_per_node": 1,
+ "nodes": 1, "placement": "packed",
+ "realized_placement": {
+ "gpus_per_node": 1, "nodes": 1, "ranks_per_node": 1,
+ "unique_local_ranks": True, "valid": True,
+ },
+ "scale_out_transport": None, "scale_up_domain": 1,
+ "scale_up_transport": "nvlink", "scope": "scale-up",
+ "topology_class": "fixture", "transport": "nvlink",
+ "world_size": 1,
+ },
+ "workload": {
+ "activation_generator": "collectivex-activation-counter-v3",
+ "activation_identity": hashlib.sha256(
+ b"counter|seed=67|hidden=1|gen=collectivex-activation-counter-v3"
+ ).hexdigest(),
+ "activation_profile": "canonical-counter-source-v3", "cross_rank_consistent": True,
+ "manifest_checksums": {member_id: member_checksums}, "members": [member_id],
+ "routing_generator": "collectivex-routing-counter-v3", "source": "canonical-serialized",
+ "trace_hashes": [routing_hash],
+ "trace_signature": hashlib.sha256(routing_hash.encode()).hexdigest(),
+ "workload_id": workload_id,
+ },
+ }
+ return raw, samples
+
+
+def _series(seed: str, backend: str, *, decision_grade: bool = False) -> tuple[dict, dict]:
+ case, allocation, attempt, series_id, point_id, evidence = _ids(seed)
+ allocations = [identity.allocation_id({"seed": seed, "run": run}) for run in range(3)]
+ eligibility = publisher._eligibility_record(
+ allocations if decision_grade else [allocation],
+ complete=decision_grade,
+ correct=True,
+ measured=True,
+ stable_ordering=True,
+ p50_ratio=1.01 if decision_grade else None,
+ p99_ratio=1.02 if decision_grade else None,
+ )
+ component = _component(1.0 if backend == "deepep" else 1.2)
+ item = {
+ "series_id": series_id,
+ "label": f"H100 / {backend}",
+ "status": "decision-grade" if decision_grade else "diagnostic",
+ "case_ids": [case],
+ "allocation_ids": allocations if decision_grade else [allocation],
+ "model": "deepseek-v3-v1",
+ "suite": "ep-core-v1",
+ "mode": "normal",
+ "phase": "decode",
+ "publication_tier": "official",
+ "backend": {
+ "id": backend, "label": publisher.BACKEND_LABELS[backend],
+ "role": "reference" if backend == "nccl-ep" else "library",
+ "generation": "nccl" if backend == "nccl-ep" else None,
+ "version": "1.0"},
+ "build": {
+ "implementation_contract_sha256": hashlib.sha256(backend.encode()).hexdigest(),
+ "public_config_sha256": "0" * 64,
+ "routing_control_sha256": hashlib.sha256(backend.encode()).hexdigest(),
+ "runtime_fingerprint_sha256": "3" * 64,
+ "image_digest": "sha256:" + "1" * 64,
+ "source_sha": "a" * 40,
+ "squash_sha256": "2" * 64,
+ },
+ "system": {
+ "sku": "h100-dgxc", "label": "NVIDIA H100", "vendor": "nvidia",
+ "topology_class": "h100-nvlink-island", "transport": "nvlink",
+ "scale_up_transport": "nvlink", "scale_out_transport": None,
+ "scope": "scale-up", "nodes": 1, "gpus_per_node": 8,
+ "scale_up_domain": 8,
+ "world_size": 8, "ep_size": 8, "placement": "packed",
+ },
+ "workload": {
+ "workload_id": identity.workload_id({"shape": "shared"}),
+ "hidden": 7168, "top_k": 8, "experts": 256,
+ "routing": "uniform", "eplb": False,
+ "dispatch_dtype": "bf16", "combine_dtype": "bf16",
+ "activation_profile": "canonical-counter-source-v3",
+ },
+ "eplb": {
+ "enabled": False, "planner": None, "mapping_sha256": None,
+ "logical_experts": 256, "physical_experts": 256,
+ "redundant_experts": 0, "reference_tokens_per_rank": None,
+ "replicated_experts": 0, "max_replicas": None,
+ "imbalance_before": None, "imbalance_after": None,
+ },
+ "resource": {"mode": "tuned", "profile": "profile-1", "comm_units_kind": "sm", "configured_units": 24},
+ "measurement": {
+ "contract": "layout-and-dispatch-v1",
+ "component_order_contract": "roundtrip-dispatch-activation-only-combine-v2",
+ "combine_semantics": "activation-only", "payload_unit": "token-rank",
+ "sampling_contract": "fixed-512-v1",
+ "iters": 8, "trials": 64, "warmups": 32, "samples_per_component": 512,
+ "headline_component": "roundtrip", "headline_percentile": "p99",
+ },
+ "points": [{
+ "point_id": point_id, "tokens_per_rank": 8, "global_tokens": 64,
+ "correct": True,
+ "routing": {
+ "fanout_mean": 4.0, "recv_tokens_max": 64,
+ "expert_load_cv": 0.5, "payload_rank_cv": 0.25,
+ "hotspot_ratio": 2.0, "empty_expert_count": 0,
+ "empty_rank_count": 0, "routed_copies": 256,
+ },
+ "components": {"dispatch": None, "combine": None,
+ "roundtrip": component, "isolated_sum": None},
+ "roundtrip_token_rate_at_latency_percentile": {
+ name: 64 / (latency * 1e-6)
+ for name, latency in component["latency_us"].items()
+ },
+ "evidence_ids": [evidence],
+ }],
+ "eligibility": eligibility,
+ }
+ item["build"]["public_config_sha256"] = contracts.public_series_config_sha256(
+ publisher._public_series_config(item)
+ )
+ case = identity.digest("case", publisher._public_case_factors(item))
+ item["case_ids"] = [case]
+ build = item["build"]
+ series_id = identity.series_id({
+ "backend": item["backend"]["id"],
+ "case_id": case,
+ "image_digest": build["image_digest"],
+ "implementation_contract_sha256": build["implementation_contract_sha256"],
+ "public_config_sha256": build["public_config_sha256"],
+ "routing_control_sha256": build["routing_control_sha256"],
+ "runtime_fingerprint_sha256": build["runtime_fingerprint_sha256"],
+ "source_sha": build["source_sha"],
+ "squash_sha256": build["squash_sha256"],
+ "workload_id": item["workload"]["workload_id"],
+ })
+ item["series_id"] = series_id
+ point_id = identity.point_id(series=series_id, tokens_per_rank=8)
+ item["points"][0]["point_id"] = point_id
+ attempt = identity.attempt_id(allocation=allocation, case=case, ordinal=1)
+ evidence = identity.evidence_id(
+ point=point_id, allocation=allocation, attempt=attempt,
+ sample_sha256=hashlib.sha256(seed.encode()).hexdigest(),
+ )
+ item["points"][0]["evidence_ids"] = [evidence]
+ runs = {
+ str(run): {8: {
+ "latency_us": {
+ statistic: component["latency_us"][statistic] * (1 + run / 100)
+ for statistic in ("p50", "p99")
+ },
+ "logical_payload_rate_gbps_at_latency_percentile": {
+ statistic: component["logical_payload_rate_gbps_at_latency_percentile"][statistic] / (1 + run / 100)
+ for statistic in ("p50", "p99")
+ },
+ }}
+ for run in range(3)
+ }
+ internal = {"run_metrics": runs}
+ return item, internal
+
+
+def _dataset() -> dict:
+ item, _ = _series("one", "deepep")
+ case = item["case_ids"][0]
+ allocation = item["allocation_ids"][0]
+ attempt = identity.attempt_id(allocation=allocation, case=case, ordinal=1)
+ evidence = item["points"][0]["evidence_ids"][0]
+ return {
+ "format": "collectivex.public.v1", "schema_version": 1,
+ "generated_at": "2026-07-04T00:00:00Z", "source_bundle_ids": ["c" * 64],
+ "promotion": {
+ "status": "diagnostic", "reason": None, "matrix_id": "d" * 64,
+ "allocation_ids": [allocation], "required_allocations": 3,
+ "requested_cases": 1, "terminal_cases": 1,
+ "policy": "collectivex-decision-grade-v1",
+ },
+ "coverage": [{
+ "case_id": case, "label": "case", "required": True, "sku": "h100-dgxc",
+ "backend": "deepep", "mode": "normal", "phase": "decode",
+ "topology": publisher._coverage_topology(item["system"]),
+ "disposition": "runnable",
+ "selected_attempt_id": attempt,
+ "outcome": "success", "failure_mode": None, "reason": None,
+ "attempt_ids": [attempt],
+ }],
+ "attempts": [{
+ "attempt_id": attempt,
+ "evidence": [{"evidence_id": evidence,
+ "point_id": item["points"][0]["point_id"]}],
+ "case_id": case,
+ "allocation_id": allocation, "run_id": "1", "run_attempt": 1,
+ "attempt_index": 1,
+ "selected": True, "outcome": "success", "failure_mode": None, "reason": None,
+ "series_id": item["series_id"],
+ "completed_at": "2026-07-04T00:00:00Z",
+ }],
+ "series": [item], "cohorts": [], "rankings": [], "recommendations": [],
+ "sensitivities": [],
+ }
+
+
+def _promoted_dataset() -> dict:
+ specifications = (
+ ("library-fast", "deepep", None, False),
+ ("library-slow", "uccl", None, False),
+ ("chip-peer", "deepep", "h200-dgxc", False),
+ ("system-one", "nccl-ep", None, True),
+ ("system-two", "nccl-ep", "h200-dgxc", True),
+ ("routing-zipf", "deepep", None, False),
+ ("routing-zipf-eplb", "deepep", None, False),
+ )
+ series = []
+ internals = {}
+ attempts = []
+ coverage = []
+ for seed, backend, peer_sku, reference in specifications:
+ item, internal = _series(seed, backend, decision_grade=True)
+ if peer_sku:
+ platform = publisher.capability.PLATFORMS[peer_sku]
+ item["system"].update({
+ "sku": peer_sku,
+ "label": f"NVIDIA {platform['product'].upper()}",
+ "topology_class": platform["topology_class"],
+ "transport": platform["transport"],
+ })
+ if reference:
+ item["backend"]["role"] = "reference"
+ if seed.startswith("routing-zipf"):
+ item["suite"] = "ep-routing-v1"
+ item["publication_tier"] = "comparable-experimental"
+ item["workload"]["routing"] = "zipf"
+ if seed == "routing-zipf-eplb":
+ item["workload"]["eplb"] = True
+ plan = contracts._expected_eplb_plan(
+ "zipf", 8, 256, 288, item["system"]["ep_size"], 67, 2048
+ )
+ item["eplb"] = {
+ "enabled": True, "planner": "greedy-rank-major-v1",
+ "mapping_sha256": contracts.eplb_contract.mapping_hash(plan),
+ "logical_experts": 256, "physical_experts": 288,
+ "redundant_experts": 32, "reference_tokens_per_rank": 2048,
+ "replicated_experts": plan["replicated_experts"],
+ "max_replicas": plan["max_replicas"],
+ "imbalance_before": plan["imbalance_before"],
+ "imbalance_after": plan["imbalance_after"],
+ }
+ item["build"]["implementation_contract_sha256"] = "8" * 64
+ case_id = identity.digest("case", publisher._public_case_factors(item))
+ item["case_ids"] = [case_id]
+ build = item["build"]
+ build["public_config_sha256"] = contracts.public_series_config_sha256(
+ publisher._public_series_config(item)
+ )
+ item["series_id"] = identity.series_id({
+ "backend": item["backend"]["id"],
+ "case_id": case_id,
+ "image_digest": build["image_digest"],
+ "implementation_contract_sha256": build["implementation_contract_sha256"],
+ "public_config_sha256": build["public_config_sha256"],
+ "routing_control_sha256": build["routing_control_sha256"],
+ "runtime_fingerprint_sha256": build["runtime_fingerprint_sha256"],
+ "source_sha": build["source_sha"],
+ "squash_sha256": build["squash_sha256"],
+ "workload_id": item["workload"]["workload_id"],
+ })
+ point = item["points"][0]
+ point["point_id"] = identity.point_id(
+ series=item["series_id"], tokens_per_rank=point["tokens_per_rank"]
+ )
+ case_attempts = []
+ evidence_ids = []
+ for run_id, allocation_id in enumerate(item["allocation_ids"], 1):
+ attempt_id = identity.attempt_id(
+ allocation=allocation_id, case=case_id, ordinal=1
+ )
+ evidence_id = identity.evidence_id(
+ point=point["point_id"], allocation=allocation_id,
+ attempt=attempt_id,
+ sample_sha256=hashlib.sha256(f"{seed}-{run_id}".encode()).hexdigest(),
+ )
+ attempts.append({
+ "attempt_id": attempt_id,
+ "evidence": [{"evidence_id": evidence_id, "point_id": point["point_id"]}],
+ "case_id": case_id, "allocation_id": allocation_id,
+ "run_id": str(run_id), "run_attempt": 1,
+ "attempt_index": 1, "selected": True,
+ "outcome": "success", "failure_mode": None, "reason": None,
+ "series_id": item["series_id"],
+ "completed_at": "2026-07-04T00:00:00Z",
+ })
+ case_attempts.append(attempt_id)
+ evidence_ids.append(evidence_id)
+ point["evidence_ids"] = evidence_ids
+ coverage.append({
+ "case_id": case_id, "label": seed, "required": True,
+ "sku": item["system"]["sku"], "backend": backend,
+ "mode": item["mode"], "phase": item["phase"], "disposition": "runnable",
+ "topology": publisher._coverage_topology(item["system"]),
+ "selected_attempt_id": case_attempts[-1], "outcome": "success",
+ "failure_mode": None, "reason": None, "attempt_ids": case_attempts,
+ })
+ series.append(item)
+ internals[item["series_id"]] = internal
+
+ unsupported_case = (
+ "cxcase-v1-f5d0b1c21df4eea0f5b74da25a0b85da7388a0dd22c8d81564c4bfb4e4465a34"
+ )
+ unsupported_attempts = []
+ for run_id in range(1, 4):
+ allocation_id = identity.allocation_id(
+ {"seed": "planned-unsupported", "run": run_id}
+ )
+ attempt_id = identity.attempt_id(
+ allocation=allocation_id, case=unsupported_case, ordinal=1
+ )
+ attempts.append({
+ "attempt_id": attempt_id, "evidence": [], "case_id": unsupported_case,
+ "allocation_id": allocation_id, "run_id": str(run_id),
+ "run_attempt": 1,
+ "attempt_index": 1, "selected": True, "outcome": "unsupported",
+ "failure_mode": "capability", "reason": "backend-platform-unsupported",
+ "series_id": None, "completed_at": "2026-07-04T00:00:00Z",
+ })
+ unsupported_attempts.append(attempt_id)
+ coverage.append({
+ "case_id": unsupported_case, "label": "planned unsupported", "required": True,
+ "sku": "mi355x", "backend": "deepep", "mode": "normal", "phase": "decode",
+ "topology": {
+ "ep_size": 8, "nodes": 1, "gpus_per_node": 8, "scale_up_domain": 8,
+ "scope": "scale-up", "scale_up_transport": "xgmi",
+ "scale_out_transport": None, "transport": "xgmi",
+ "topology_class": "mi355x-xgmi",
+ },
+ "disposition": "unsupported", "selected_attempt_id": unsupported_attempts[-1],
+ "outcome": "unsupported", "failure_mode": "capability",
+ "reason": "backend-platform-unsupported", "attempt_ids": unsupported_attempts,
+ })
+ cohorts, rankings, recommendations, sensitivities = publisher.build_decisions(
+ series, internals
+ )
+ return {
+ "format": "collectivex.public.v1", "schema_version": 1,
+ "generated_at": "2026-07-04T00:00:00Z",
+ "source_bundle_ids": ["a" * 64, "b" * 64, "c" * 64],
+ "promotion": {
+ "status": "promoted", "reason": None,
+ "matrix_id": publisher.CANONICAL_FULL_V1_MATRIX_SHA256,
+ "allocation_ids": sorted({item["allocation_id"] for item in attempts}),
+ "required_allocations": 3, "requested_cases": len(coverage),
+ "terminal_cases": len(coverage), "policy": "collectivex-decision-grade-v1",
+ },
+ "coverage": sorted(coverage, key=lambda item: item["case_id"]),
+ "attempts": sorted(attempts, key=lambda item: item["attempt_id"]),
+ "series": sorted(series, key=lambda item: item["series_id"]),
+ "cohorts": cohorts, "rankings": rankings,
+ "recommendations": recommendations, "sensitivities": sensitivities,
+ }
+
+
+def _cohort_counts(dataset: dict) -> dict[str, int]:
+ return {
+ kind: sum(item["kind"] == kind for item in dataset["cohorts"])
+ for kind in ("library", "system", "routing")
+ }
+
+
+class PublisherTest(unittest.TestCase):
+ def test_terminal_allocation_and_source_status_are_bound(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary).resolve()
+ matrix, artifact = _unsupported_delivery(root)
+ path = next(artifact.glob("*.json"))
+ terminal = contracts.strict_load(path)
+ self.assertIs(contracts.validate_terminal_document(terminal), terminal)
+ self.assertEqual(
+ contracts.validate_delivery(
+ [str(path)], str(matrix), disposition="unsupported"
+ ),
+ 1,
+ )
+
+ for control_sha256 in (None, "0" * 64):
+ broken = copy.deepcopy(terminal)
+ broken["provenance"]["control_sha256"] = control_sha256
+ path.write_text(json.dumps(broken))
+ with self.assertRaisesRegex(contracts.ContractError, "exact control document"):
+ contracts.validate_delivery(
+ [str(path)], str(matrix), disposition="unsupported"
+ )
+ path.write_text(json.dumps(terminal))
+
+ for field in (
+ "artifact", "job", "repo", "run_attempt", "run_id", "source_sha", "runner"
+ ):
+ broken = copy.deepcopy(terminal)
+ broken["identity"]["allocation_factors"][field] = f"forged-{field}"
+ allocation_id = identity.allocation_id(
+ broken["identity"]["allocation_factors"]
+ )
+ broken["identity"]["allocation_id"] = allocation_id
+ broken["identity"]["attempt_id"] = identity.attempt_id(
+ allocation=allocation_id,
+ case=broken["identity"]["case_id"],
+ ordinal=broken["identity"]["attempt_ordinal"],
+ )
+ with self.assertRaisesRegex(
+ contracts.ContractError, "allocation factors differ"
+ ):
+ contracts.validate_terminal_document(broken)
+
+ broken = copy.deepcopy(terminal)
+ broken["outcome"]["status"] = "failed"
+ with self.assertRaisesRegex(contracts.ContractError, "source and outcome"):
+ contracts.validate_terminal_document(broken)
+ broken = copy.deepcopy(terminal)
+ broken["provenance"]["source"] = "runtime-emitter"
+ with self.assertRaisesRegex(contracts.ContractError, "source and outcome"):
+ contracts.validate_terminal_document(broken)
+
+ for path_parts, replacement in (
+ (("provenance", "source"), "unregistered-producer"),
+ (("outcome", "failure_mode"), "unsupported-capability"),
+ (("outcome", "reason"), "unregistered-capability"),
+ ):
+ with self.subTest(path=path_parts):
+ broken = copy.deepcopy(terminal)
+ broken[path_parts[0]][path_parts[1]] = replacement
+ with self.assertRaises(publisher.PublisherError):
+ publisher._schema("terminal-outcome-v1.schema.json", broken)
+ with self.assertRaises(contracts.ContractError):
+ contracts.validate_terminal_document(broken)
+
+ runtime_allocation = copy.deepcopy(
+ terminal["identity"]["allocation_factors"]
+ )
+ runtime_allocation["runner"] = terminal["identity"]["case_factors"]["sku"]
+ runtime = contracts.make_terminal_document(
+ allocation_factors=runtime_allocation,
+ attempt_ordinal=1,
+ case=terminal["case"],
+ case_factors=terminal["identity"]["case_factors"],
+ control_sha256=terminal["provenance"]["control_sha256"],
+ failure_mode="setup",
+ generated_at=terminal["generated_at"],
+ git_run=terminal["provenance"]["git_run"],
+ reason="launcher-setup-failed",
+ return_code=1,
+ source="runtime-emitter",
+ status="failed",
+ expected_case_id=terminal["identity"]["case_id"],
+ )
+ publisher._schema("terminal-outcome-v1.schema.json", runtime)
+ broken = copy.deepcopy(runtime)
+ broken["outcome"]["reason"] = "backend-setup-failed"
+ with self.assertRaises(publisher.PublisherError):
+ publisher._schema("terminal-outcome-v1.schema.json", broken)
+ with self.assertRaises(contracts.ContractError):
+ contracts.validate_terminal_document(broken)
+
+ def test_post_emit_demotion_uses_closed_failure_taxonomy(self) -> None:
+ raw, _ = _native_fixture()
+ expected = {
+ 5: "runtime-identity",
+ 6: "execution",
+ 124: "timeout",
+ 137: "timeout",
+ 134: "execution",
+ 9: "execution",
+ }
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary)
+ for return_code, failure_mode in expected.items():
+ with self.subTest(return_code=return_code):
+ path = root / f"attempt-{return_code}.json"
+ path.write_text(json.dumps(raw))
+ terminal = contracts.demote_raw_attempt(path, return_code)
+ self.assertEqual(
+ terminal["outcome"],
+ {
+ "failure_mode": failure_mode,
+ "reason": "post-emit-distributed-command-failed",
+ "return_code": return_code,
+ "status": "failed",
+ },
+ )
+ self.assertEqual(terminal["provenance"]["source"], "post-emit-command")
+ publisher._schema("terminal-outcome-v1.schema.json", terminal)
+
+ broken = copy.deepcopy(terminal)
+ broken["outcome"]["reason"] = "distributed-command-failed"
+ with self.assertRaises(publisher.PublisherError):
+ publisher._schema("terminal-outcome-v1.schema.json", broken)
+ with self.assertRaises(contracts.ContractError):
+ contracts.validate_terminal_document(broken)
+
+ def test_artifact_safety_accepts_current_v1_fixtures(self) -> None:
+ raw, samples = _native_fixture()
+ publisher.artifact_safety.assert_publication_safe([
+ sweep_matrix.resolve_matrix(backends="all"),
+ raw,
+ samples,
+ _dataset(),
+ _promoted_dataset(),
+ ])
+
+ def test_native_raw_and_sample_schema_match_semantic_validator(self) -> None:
+ raw, samples = _native_fixture()
+ publisher._schema("samples-v1.schema.json", samples)
+ publisher._schema("raw-case-v1.schema.json", raw)
+ self.assertIs(contracts.validate_raw_document(raw, samples), raw)
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary).resolve()
+ (root / "samples.json").write_bytes(contracts.canonical_json_bytes(samples))
+ (root / "raw.json").write_bytes(contracts.canonical_json_bytes(raw))
+ self.assertEqual(contracts.load_raw_attempt(root / "raw.json"), raw)
+ for target in ("raw", "samples"):
+ broken_raw, broken_samples = copy.deepcopy((raw, samples))
+ broken = broken_raw if target == "raw" else broken_samples
+ broken["unexpected"] = True
+ with self.assertRaises(publisher.PublisherError):
+ publisher._schema(
+ "raw-case-v1.schema.json" if target == "raw" else "samples-v1.schema.json",
+ broken,
+ )
+ with self.assertRaises(contracts.ContractError):
+ contracts.validate_raw_document(broken_raw, broken_samples)
+ tampered = copy.deepcopy(raw)
+ tampered["measurement"]["rows"][0]["token_rate_at_latency_percentile"]["p50"] *= 2
+ with self.assertRaisesRegex(contracts.ContractError, "token_rate_at_latency_percentile"):
+ contracts.validate_raw_document(tampered, samples)
+ tampered = copy.deepcopy(raw)
+ tampered["case"]["shape"]["hidden"] = 2
+ with self.assertRaises(contracts.ContractError):
+ contracts.validate_raw_document(tampered, samples)
+ tampered = copy.deepcopy(raw)
+ configured = tampered["implementation"]["resource_profile"]["configured_units"]
+ tampered["implementation"]["resource_profile"]["configured_units"] = (
+ 1 if configured is None else configured + 1
+ )
+ with self.assertRaisesRegex(contracts.ContractError, "resource profile"):
+ contracts.validate_raw_document(tampered, samples)
+ tampered = copy.deepcopy(raw)
+ oracle = tampered["measurement"]["rows"][0]["correctness"]["rank_evidence"][0]
+ oracle["pre_timing"]["checks"]["combine_values"] = False
+ with self.assertRaisesRegex(contracts.ContractError, "passed differs"):
+ contracts.validate_raw_document(tampered, samples)
+
+ def test_hybrid_raw_binds_realized_config_and_every_rank_artifact(self) -> None:
+ raw, samples = _native_fixture("deepep-hybrid")
+ publisher._schema("raw-case-v1.schema.json", raw)
+ self.assertIs(contracts.validate_raw_document(raw, samples), raw)
+
+ mutations = {
+ "hidden_dim": lambda provenance: provenance["realized_config"].update(
+ hidden_dim=2
+ ),
+ "experts_per_rank": lambda provenance: provenance["realized_config"].update(
+ num_of_experts_per_rank=2
+ ),
+ "ranks_per_node": lambda provenance: provenance["realized_config"].update(
+ num_of_ranks_per_node=2
+ ),
+ "num_nodes": lambda provenance: provenance["realized_config"].update(
+ num_of_nodes=2
+ ),
+ "token_data_type": lambda provenance: provenance["realized_config"].update(
+ token_data_type="UINT8"
+ ),
+ "rank_coverage": lambda provenance: [
+ artifact["rank_artifacts"].append({
+ "bytes": 1, "rank": 1, "sha256": "9" * 64,
+ })
+ for artifact in provenance["jit_shared_objects"]
+ ],
+ }
+ for name, mutate in mutations.items():
+ with self.subTest(name=name):
+ changed = copy.deepcopy(raw)
+ mutate(changed["implementation"]["provenance"])
+ with self.assertRaisesRegex(
+ contracts.ContractError,
+ "DeepEP Hybrid realized config/JIT evidence differs",
+ ):
+ contracts.validate_raw_document(changed, samples)
+
+ def test_native_contract_recomputes_routing_receive_histograms_and_anomalies(self) -> None:
+ raw, samples = _native_fixture()
+
+ tampered = copy.deepcopy(raw)
+ changed = tampered["measurement"]["rows"][0]
+ changed["routing"]["routed_copies"] *= 2
+ for name in ("combine", "dispatch", "roundtrip"):
+ changed["logical_bytes"][name] *= 2
+ with self.assertRaisesRegex(contracts.ContractError, "routing.routed_copies"):
+ contracts.validate_raw_document(tampered, samples)
+
+ tampered = copy.deepcopy(raw)
+ changed = tampered["measurement"]["rows"][0]
+ changed["routing"]["payload_copies_per_rank"] = [2]
+ changed["receive"] = {"max": 2, "mean": 2.0, "min": 2, "total": 2}
+ with self.assertRaisesRegex(contracts.ContractError, "payload_copies_per_rank"):
+ contracts.validate_raw_document(tampered, samples)
+
+ tampered = copy.deepcopy(raw)
+ tampered["measurement"]["rows"][0]["sample_histograms"]["roundtrip"][
+ "counts"
+ ] = [511]
+ with self.assertRaisesRegex(contracts.ContractError, "sample_histograms"):
+ contracts.validate_raw_document(tampered, samples)
+
+ tampered = copy.deepcopy(raw)
+ tampered["measurement"]["rows"][0]["anomalies"] = [{
+ "type": "roundtrip_gt_isolated_sum",
+ "T": 1,
+ "roundtrip_p99": 40.0,
+ "isolated_sum_p99": 30.0,
+ "ratio": 1.33,
+ "threshold": 3.0,
+ }]
+ tampered["outcome"]["validity"]["anomaly_free"] = False
+ with self.assertRaisesRegex(contracts.ContractError, "anomalies"):
+ contracts.validate_raw_document(tampered, samples)
+
+ anomalous_raw, anomalous_samples = copy.deepcopy((raw, samples))
+ sample_point = anomalous_samples["points"][0]
+ sample_point["components"]["roundtrip"]["trials"] = [
+ [100.0] * 8 for _ in range(64)
+ ]
+ sample_core = {
+ "components": sample_point["components"],
+ "tokens_per_rank": sample_point["tokens_per_rank"],
+ }
+ sample_sha = hashlib.sha256(
+ contracts.canonical_json_bytes(sample_core)
+ ).hexdigest()
+ point_id = sample_point["point_id"]
+ evidence_id = identity.evidence_id(
+ point=point_id,
+ allocation=anomalous_raw["identity"]["allocation_id"],
+ attempt=anomalous_raw["identity"]["attempt_id"],
+ sample_sha256=sample_sha,
+ )
+ sample_point.update({"sample_sha256": sample_sha, "evidence_id": evidence_id})
+ changed = anomalous_raw["measurement"]["rows"][0]
+ changed["sample_sha256"] = sample_sha
+ changed["evidence_id"] = evidence_id
+ changed["components"]["roundtrip"]["percentiles_us"] = {
+ name: 100.0 for name in ("p50", "p90", "p95", "p99")
+ }
+ changed["token_rate_at_latency_percentile"] = {
+ name: 10_000.0 for name in ("p50", "p90", "p95", "p99")
+ }
+ changed["sample_histograms"]["roundtrip"] = contracts._expected_histogram(
+ [100.0] * 512
+ )
+ changed["anomalies"] = contracts._expected_anomalies(1, changed["components"])
+ anomalous_raw["outcome"]["validity"]["anomaly_free"] = False
+ sample_bytes = contracts.canonical_json_bytes(anomalous_samples)
+ anomalous_raw["sample_artifact"].update({
+ "bytes": len(sample_bytes),
+ "sha256": hashlib.sha256(sample_bytes).hexdigest(),
+ })
+ self.assertIs(
+ contracts.validate_raw_document(anomalous_raw, anomalous_samples),
+ anomalous_raw,
+ )
+ changed["anomalies"] = []
+ anomalous_raw["outcome"]["validity"]["anomaly_free"] = True
+ with self.assertRaisesRegex(contracts.ContractError, "anomalies"):
+ contracts.validate_raw_document(anomalous_raw, anomalous_samples)
+
+ def test_native_contract_rejects_every_schema_only_nested_mutation(self) -> None:
+ raw, samples = _native_fixture()
+ self.assertIs(contracts.validate_raw_document(raw, samples), raw)
+
+ def locate(document: object, path: tuple[object, ...]) -> object:
+ value = document
+ for part in path:
+ value = value[part] # type: ignore[index]
+ return value
+
+ def reject_raw(document: dict) -> None:
+ with self.assertRaises(publisher.PublisherError):
+ publisher._schema("raw-case-v1.schema.json", document)
+ with self.assertRaises(contracts.ContractError):
+ contracts.validate_raw_document(document, samples)
+
+ required_fields = (
+ (("measurement", "rows", 0, "receive"), "total"),
+ (("measurement", "rows", 0, "routing"), "fanout_mean"),
+ (("measurement", "rows", 0, "routing", "source_token_stats"), "ranks"),
+ (("measurement", "rows", 0, "sample_histograms"), "roundtrip"),
+ (("measurement", "rows", 0, "sample_histograms", "roundtrip"), "n"),
+ (("runtime_fingerprint", "accelerator_runtime"), "kind"),
+ (("runtime_fingerprint", "collective_library"), "kind"),
+ (("runtime_fingerprint", "framework"), "kind"),
+ )
+ for path, required in required_fields:
+ with self.subTest(path=path, mutation="missing"):
+ broken = copy.deepcopy(raw)
+ del locate(broken, path)[required] # type: ignore[index]
+ reject_raw(broken)
+ with self.subTest(path=path, mutation="extra"):
+ broken = copy.deepcopy(raw)
+ locate(broken, path)["unexpected"] = None # type: ignore[index]
+ reject_raw(broken)
+
+ invalid_values = (
+ (("measurement", "rows", 0, "receive", "mean"), "one"),
+ (("measurement", "rows", 0, "routing", "fanout_mean"), "one"),
+ (("measurement", "rows", 0, "sample_histograms", "roundtrip", "bins"), 0),
+ (("provenance", "image", "arch"), "AMD64"),
+ (("runtime_fingerprint", "accelerator_runtime", "kind"), "rocm"),
+ )
+ for path, invalid in invalid_values:
+ with self.subTest(path=path, mutation="value"):
+ broken = copy.deepcopy(raw)
+ parent = locate(broken, path[:-1])
+ parent[path[-1]] = invalid # type: ignore[index]
+ reject_raw(broken)
+
+ def reject_samples(document: dict) -> None:
+ with self.assertRaises(publisher.PublisherError):
+ publisher._schema("samples-v1.schema.json", document)
+ with self.assertRaises(contracts.ContractError):
+ contracts.validate_samples_document(document)
+
+ for path, required in (
+ (("points", 0), "evidence_id"),
+ (("points", 0, "components"), "roundtrip"),
+ (("points", 0, "components", "roundtrip"), "trials"),
+ (("sampling",), "reduction"),
+ ):
+ with self.subTest(path=path, artifact="samples"):
+ broken = copy.deepcopy(samples)
+ del locate(broken, path)[required] # type: ignore[index]
+ reject_samples(broken)
+
+ def test_terminal_contract_and_schema_reject_the_same_shape_gaps(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ _, artifact = _unsupported_delivery(Path(temporary).resolve())
+ terminal = contracts.strict_load(next(artifact.glob("*.json")))
+ publisher._schema("terminal-outcome-v1.schema.json", terminal)
+ self.assertIs(contracts.validate_terminal_document(terminal), terminal)
+
+ def reject(document: dict) -> None:
+ with self.assertRaises(publisher.PublisherError):
+ publisher._schema("terminal-outcome-v1.schema.json", document)
+ with self.assertRaises(contracts.ContractError):
+ contracts.validate_terminal_document(document)
+
+ for path, invalid in (
+ (("outcome", "failure_mode"), "Not Safe"),
+ (("outcome", "reason"), "x" * 241),
+ (("provenance", "source"), "Not Safe"),
+ (("provenance", "git_run", "ref"), ""),
+ ):
+ with self.subTest(path=path):
+ broken = copy.deepcopy(terminal)
+ parent = broken
+ for part in path[:-1]:
+ parent = parent[part]
+ parent[path[-1]] = invalid
+ reject(broken)
+
+ def test_invalid_retry_is_quarantined_before_valid_retry_upload(self) -> None:
+ raw, samples = _native_fixture()
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary).resolve()
+ sample_bytes = contracts.canonical_json_bytes(samples)
+ bad = copy.deepcopy(raw)
+ bad["sample_artifact"].update({
+ "path": "a01.samples.json", "bytes": len(sample_bytes),
+ "sha256": hashlib.sha256(sample_bytes).hexdigest(),
+ })
+ bad["measurement"]["rows"][0]["token_rate_at_latency_percentile"]["p50"] *= 2
+ (root / "a01.samples.json").write_bytes(sample_bytes)
+ (root / "a01.json").write_bytes(contracts.canonical_json_bytes(bad))
+ self.assertTrue(contracts.quarantine_invalid_attempt(root / "a01.json"))
+ valid = copy.deepcopy(raw)
+ valid["sample_artifact"].update({
+ "path": "a02.samples.json", "bytes": len(sample_bytes),
+ "sha256": hashlib.sha256(sample_bytes).hexdigest(),
+ })
+ (root / "a02.samples.json").write_bytes(sample_bytes)
+ (root / "a02.json").write_bytes(contracts.canonical_json_bytes(valid))
+ paths = sorted(str(path) for path in root.glob("*.json"))
+ self.assertEqual(contracts.validate_attempt_paths(paths), 1)
+ self.assertTrue((root / "a01.json.quarantine").is_file())
+ self.assertTrue((root / "a01.samples.json.quarantine").is_file())
+
+ def test_ingest_archives_first_and_publishes_latest_attempt(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary).resolve()
+ matrix, artifact = _unsupported_delivery(root)
+ self.assertEqual(len(summarize.load_results(str(artifact), None, None)), 1)
+ result = publisher.ingest_command(_args(root / "store", matrix, artifact))
+ store = publisher.Store(root / "store")
+ pointer = store.verify_channel("latest-attempt")
+ self.assertEqual(result["status"], "accepted")
+ self.assertEqual(pointer["dataset"]["sha256"], result["dataset_sha256"])
+ self.assertTrue((store.incoming / result["incoming_id"] / "COMPLETE").is_file())
+ self.assertTrue((store.bundles / result["bundle_id"] / "COMPLETE").is_file())
+ self.assertFalse((store.channels / "dev-latest.json").exists())
+ self.assertEqual(os.stat(store.private).st_mode & 0o777, 0o700)
+ self.assertEqual(os.stat(store.public).st_mode & 0o777, 0o755)
+ self.assertEqual(os.stat(store.bundles / result["bundle_id"]).st_mode & 0o777, 0o500)
+ dataset_dir = store.datasets / result["dataset_sha256"]
+ self.assertEqual(os.stat(dataset_dir).st_mode & 0o777, 0o555)
+ self.assertEqual(os.stat(dataset_dir / "dataset.json").st_mode & 0o777, 0o444)
+
+ def test_repeated_ingest_is_content_idempotent(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary).resolve()
+ matrix, artifact = _unsupported_delivery(root)
+ args = _args(root / "store", matrix, artifact)
+ first = publisher.ingest_command(args)
+ store = publisher.Store(root / "store")
+ pointer_before = (store.channels / "latest-attempt.json").read_bytes()
+ second = publisher.ingest_command(args)
+ self.assertEqual(second, first)
+ self.assertEqual(
+ (store.channels / "latest-attempt.json").read_bytes(), pointer_before
+ )
+ self.assertEqual(len(list(store.incoming.iterdir())), 1)
+ self.assertEqual(len(list(store.bundles.iterdir())), 1)
+ self.assertEqual(len(list(store.datasets.iterdir())), 1)
+ bundle = publisher.strict_load(
+ store.bundles / first["bundle_id"] / "bundle.json"
+ )
+ terminal = publisher.strict_load(next(artifact.glob("*.json")))
+ self.assertEqual(bundle["created_at"], terminal["generated_at"])
+
+ def test_dataset_is_invariant_to_bundle_argument_order(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary).resolve()
+ store_root = root / "store"
+ bundle_ids = []
+ for run_id in (9, 11, 10):
+ run = {**RUN, "run_id": str(run_id)}
+ delivery = root / f"run-{run_id}"
+ delivery.mkdir()
+ matrix, artifact = _unsupported_delivery(delivery, run=run)
+ result = publisher.ingest_command(
+ _args(store_root, matrix, artifact, run=run)
+ )
+ bundle_ids.append(result["bundle_id"])
+ datasets = [
+ publisher.build_dataset(
+ publisher.Store(store_root), order, promote=False,
+ )
+ for order in itertools.permutations(bundle_ids)
+ ]
+ self.assertTrue(all(dataset == datasets[0] for dataset in datasets[1:]))
+ self.assertEqual(datasets[0]["generated_at"], "2026-07-04T00:00:00Z")
+ selected = datasets[0]["coverage"][0]["selected_attempt_id"]
+ selected_attempt = next(
+ item for item in datasets[0]["attempts"]
+ if item["attempt_id"] == selected
+ )
+ self.assertEqual(selected_attempt["run_id"], "11")
+
+ def test_diagnostic_dataset_orders_reruns_by_run_attempt(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary).resolve()
+ store_root = root / "store"
+ bundle_ids = []
+ for run_attempt in (1, 2):
+ run = {**RUN, "run_attempt": run_attempt}
+ delivery = root / f"attempt-{run_attempt}"
+ delivery.mkdir()
+ matrix, artifact = _unsupported_delivery(delivery, run=run)
+ result = publisher.ingest_command(
+ _args(store_root, matrix, artifact, run=run)
+ )
+ bundle_ids.append(result["bundle_id"])
+ dataset = publisher.build_dataset(
+ publisher.Store(store_root), bundle_ids, promote=False
+ )
+ selected_id = dataset["coverage"][0]["selected_attempt_id"]
+ selected = next(
+ item for item in dataset["attempts"]
+ if item["attempt_id"] == selected_id
+ )
+ self.assertEqual(selected["run_attempt"], 2)
+
+ def test_promotion_requires_every_runnable_case_to_succeed_in_every_bundle(self) -> None:
+ cases = {
+ "runnable": {"_disposition": "runnable"},
+ "planned-unsupported": {"_disposition": "unsupported"},
+ }
+ bundles = []
+ for _ in range(3):
+ runnable = {
+ "identity": {"case_id": "runnable"},
+ "outcome": {"status": "success"},
+ }
+ unsupported = {
+ "identity": {"case_id": "planned-unsupported"},
+ "outcome": {"status": "unsupported"},
+ }
+ bundles.append({
+ "selected": {"runnable": runnable, "planned-unsupported": unsupported},
+ "documents": {"runnable": runnable, "planned-unsupported": unsupported},
+ })
+ publisher._require_runnable_promotion_success(bundles, cases)
+
+ for status in ("failed", "invalid", "unsupported", "diagnostic"):
+ with self.subTest(status=status):
+ broken = copy.deepcopy(bundles)
+ broken[1]["selected"]["runnable"]["outcome"]["status"] = status
+ with self.assertRaisesRegex(
+ publisher.PublisherError, "every runnable matrix case"
+ ):
+ publisher._require_runnable_promotion_success(broken, cases)
+
+ broken = copy.deepcopy(bundles)
+ broken[1]["documents"]["retry"] = {
+ "identity": {"case_id": "runnable"},
+ "outcome": {"status": "failed"},
+ }
+ with self.assertRaisesRegex(publisher.PublisherError, "rejects runnable cases"):
+ publisher._require_runnable_promotion_success(broken, cases)
+
+ def test_promoted_public_dataset_rejects_failed_retry_history(self) -> None:
+ dataset = _promoted_dataset()
+ successful = next(
+ item for item in dataset["attempts"]
+ if item["outcome"] == "success"
+ )
+ failed = copy.deepcopy(successful)
+ old_attempt_id = successful["attempt_id"]
+ successful["attempt_index"] = 2
+ successful["attempt_id"] = identity.attempt_id(
+ allocation=successful["allocation_id"], case=successful["case_id"], ordinal=2
+ )
+ failed.update({
+ "attempt_id": old_attempt_id,
+ "attempt_index": 1,
+ "outcome": "failed",
+ "failure_mode": "execution",
+ "reason": "execution-failed",
+ "series_id": None,
+ "selected": False,
+ "evidence": [],
+ })
+ dataset["attempts"].append(failed)
+ dataset["attempts"].sort(key=lambda item: item["attempt_id"])
+ coverage = next(
+ item for item in dataset["coverage"]
+ if item["case_id"] == failed["case_id"]
+ )
+ coverage["attempt_ids"] = [
+ successful["attempt_id"] if value == old_attempt_id else value
+ for value in coverage["attempt_ids"]
+ ]
+ coverage["attempt_ids"].append(failed["attempt_id"])
+ coverage["attempt_ids"].sort()
+ if coverage["selected_attempt_id"] == old_attempt_id:
+ coverage["selected_attempt_id"] = successful["attempt_id"]
+
+ fixture_catalog = publisher._case_disposition_catalog_sha256(dataset["coverage"])
+ with mock.patch.object(
+ publisher, "CANONICAL_FULL_V1_CASE_CATALOG_SHA256", fixture_catalog
+ ), self.assertRaisesRegex(publisher.PublisherError, "rejects runnable cases"):
+ publisher.validate_public_dataset(dataset)
+
+ def test_unselected_success_does_not_reference_an_unpublished_series(self) -> None:
+ raw, _ = _native_fixture()
+ retained = publisher._public_attempt(raw, selected=False)
+ selected = publisher._public_attempt(raw, selected=True)
+ self.assertEqual(retained["outcome"], "success")
+ self.assertIsNone(retained["series_id"])
+ self.assertEqual(selected["series_id"], raw["identity"]["series_id"])
+
+ def test_public_dataset_selects_latest_derived_retry(self) -> None:
+ dataset = _dataset()
+ first = dataset["attempts"][0]
+ second = copy.deepcopy(first)
+ second.update({
+ "attempt_id": identity.attempt_id(
+ allocation=first["allocation_id"], case=first["case_id"], ordinal=2
+ ),
+ "attempt_index": 2,
+ "selected": False,
+ "series_id": None,
+ "evidence": [],
+ })
+ dataset["attempts"].append(second)
+ dataset["attempts"].sort(key=lambda item: item["attempt_id"])
+ dataset["coverage"][0]["attempt_ids"].append(second["attempt_id"])
+ dataset["coverage"][0]["attempt_ids"].sort()
+ with self.assertRaisesRegex(publisher.PublisherError, "select the latest retry"):
+ publisher.validate_public_dataset(dataset)
+
+ second["attempt_id"] = identity.digest("attempt", {"not": "derived"})
+ dataset["attempts"].sort(key=lambda item: item["attempt_id"])
+ dataset["coverage"][0]["attempt_ids"] = [
+ item["attempt_id"] for item in dataset["attempts"]
+ ]
+ with self.assertRaisesRegex(publisher.PublisherError, "retry identity differs"):
+ publisher.validate_public_dataset(dataset)
+
+ def test_promotion_requires_an_eligible_cohort_for_every_comparison_kind(self) -> None:
+ stable_fast, stable_fast_internal = _series(
+ "stable-fast", "deepep", decision_grade=True
+ )
+ stable_slow, stable_slow_internal = _series(
+ "stable-slow", "uccl", decision_grade=True
+ )
+ unstable_fast, unstable_fast_internal = _series(
+ "unstable-fast", "deepep", decision_grade=True
+ )
+ unstable_slow, unstable_slow_internal = _series(
+ "unstable-slow", "uccl", decision_grade=True
+ )
+ unstable_fast["phase"] = unstable_slow["phase"] = "prefill"
+ unstable_fast["series_id"] = identity.series_id({"test": "unstable-fast"})
+ unstable_slow["series_id"] = identity.series_id({"test": "unstable-slow"})
+ for statistic in ("p50", "p99"):
+ unstable_slow_internal["run_metrics"]["1"][8]["latency_us"][statistic] = (
+ unstable_fast_internal["run_metrics"]["1"][8]["latency_us"][statistic]
+ / 2
+ )
+ unstable_slow_internal["run_metrics"]["1"][8]["logical_payload_rate_gbps_at_latency_percentile"][statistic] = (
+ unstable_fast_internal["run_metrics"]["1"][8]["logical_payload_rate_gbps_at_latency_percentile"][statistic]
+ * 2
+ )
+ series = [stable_fast, stable_slow, unstable_fast, unstable_slow]
+ internals = {
+ stable_fast["series_id"]: stable_fast_internal,
+ stable_slow["series_id"]: stable_slow_internal,
+ unstable_fast["series_id"]: unstable_fast_internal,
+ unstable_slow["series_id"]: unstable_slow_internal,
+ }
+ cohorts, _, _, _ = publisher.build_decisions(series, internals)
+ eligible = [item for item in cohorts if item["eligibility"]["decision_grade"]]
+ ineligible = [item for item in cohorts if not item["eligibility"]["decision_grade"]]
+ self.assertEqual({item["kind"] for item in eligible}, {"library"})
+ self.assertTrue(ineligible)
+ anchor_series = [
+ {
+ "series_id": name,
+ "workload": {"routing": routing, "eplb": eplb},
+ "build": {"implementation_contract_sha256": "1" * 64},
+ }
+ for name, routing, eplb in (
+ ("uniform", "uniform", False),
+ ("zipf", "zipf", False),
+ ("zipf-eplb", "zipf", True),
+ )
+ ]
+ required = eligible + [
+ {
+ "kind": kind,
+ "eligibility": {"decision_grade": True},
+ **({"series_ids": [item["series_id"] for item in anchor_series]}
+ if kind == "routing" else {}),
+ }
+ for kind in publisher.REQUIRED_COHORT_KINDS
+ if kind != "library"
+ ]
+ with mock.patch.object(
+ publisher, "REQUIRED_PROMOTION_COHORT_COUNTS", {}
+ ), mock.patch.object(
+ publisher, "_expected_chip_cohort_count", return_value=1
+ ):
+ publisher._require_promotion_cohorts(
+ required + ineligible, anchor_series
+ )
+ for kind in publisher.REQUIRED_COHORT_KINDS:
+ with self.subTest(missing_kind=kind), self.assertRaisesRegex(
+ publisher.PublisherError, rf"cohort kinds:.*{kind}"
+ ):
+ publisher._require_promotion_cohorts([
+ item for item in required + ineligible
+ if item["kind"] != kind or not item["eligibility"]["decision_grade"]
+ ], anchor_series)
+
+ def test_promotion_requires_exact_counts_and_routing_anchors(self) -> None:
+ dataset = _promoted_dataset()
+ counts = _cohort_counts(dataset)
+ with mock.patch.object(
+ publisher, "REQUIRED_PROMOTION_COHORT_COUNTS", counts
+ ):
+ publisher._require_promotion_cohorts(
+ dataset["cohorts"], dataset["series"]
+ )
+ routing = next(
+ item for item in dataset["cohorts"] if item["kind"] == "routing"
+ )
+ eplb = next(
+ item for item in dataset["series"]
+ if item["series_id"] in routing["series_ids"]
+ and item["workload"]["eplb"]
+ )
+ eplb["workload"]["eplb"] = False
+ with self.assertRaisesRegex(publisher.PublisherError, "exact uniform"):
+ publisher._require_promotion_cohorts(
+ dataset["cohorts"], dataset["series"]
+ )
+
+ dataset = _promoted_dataset()
+ routing = next(item for item in dataset["cohorts"] if item["kind"] == "routing")
+ zipf = next(
+ item for item in dataset["series"]
+ if item["series_id"] in routing["series_ids"]
+ and item["workload"]["routing"] == "zipf"
+ and not item["workload"]["eplb"]
+ )
+ zipf["build"]["implementation_contract_sha256"] = "f" * 64
+ with mock.patch.object(
+ publisher, "REQUIRED_PROMOTION_COHORT_COUNTS", counts
+ ), self.assertRaisesRegex(publisher.PublisherError, "identical off-EPLB"):
+ publisher._require_promotion_cohorts(dataset["cohorts"], dataset["series"])
+
+ wrong_counts = {**counts, "library": counts["library"] + 1}
+ with mock.patch.object(
+ publisher, "REQUIRED_PROMOTION_COHORT_COUNTS", wrong_counts
+ ), self.assertRaisesRegex(publisher.PublisherError, "exactly"):
+ publisher._require_promotion_cohorts(
+ dataset["cohorts"], dataset["series"]
+ )
+
+ def test_promotion_requires_every_derived_chip_cohort_to_be_stable(self) -> None:
+ dataset = _promoted_dataset()
+ chip = next(item for item in dataset["cohorts"] if item["kind"] == "chip")
+ self.assertEqual(
+ publisher._expected_chip_cohort_count(dataset["series"]),
+ sum(item["kind"] == "chip" for item in dataset["cohorts"]),
+ )
+ with mock.patch.object(
+ publisher, "REQUIRED_PROMOTION_COHORT_COUNTS", _cohort_counts(dataset)
+ ):
+ missing = [item for item in dataset["cohorts"] if item is not chip]
+ with self.assertRaisesRegex(publisher.PublisherError, "derived chip cohorts"):
+ publisher._require_promotion_cohorts(missing, dataset["series"])
+
+ chip["eligibility"]["decision_grade"] = False
+ with self.assertRaisesRegex(publisher.PublisherError, "derived chip cohorts"):
+ publisher._require_promotion_cohorts(
+ dataset["cohorts"], dataset["series"]
+ )
+
+ def test_promotion_rejects_more_than_three_bundles(self) -> None:
+ bundles = {
+ str(run_id): {
+ "id": str(run_id), "cases": [],
+ "manifest": {
+ "matrix": {"sha256": publisher.CANONICAL_FULL_V1_MATRIX_SHA256},
+ "run": {"run_id": str(run_id), "run_attempt": 1},
+ },
+ }
+ for run_id in range(1, 5)
+ }
+ with mock.patch.object(
+ publisher, "load_bundle", side_effect=lambda _, bundle_id: bundles[bundle_id]
+ ), self.assertRaisesRegex(publisher.PublisherError, "three independent"):
+ publisher.build_dataset(object(), list(bundles), promote=True)
+
+ dataset = _promoted_dataset()
+ dataset["source_bundle_ids"].append("d" * 64)
+ counts = _cohort_counts(dataset)
+ with mock.patch.object(
+ publisher,
+ "CANONICAL_FULL_V1_CASE_CATALOG_SHA256",
+ publisher._case_disposition_catalog_sha256(dataset["coverage"]),
+ ), mock.patch.object(
+ publisher, "REQUIRED_PROMOTION_COHORT_COUNTS", counts
+ ), self.assertRaisesRegex(publisher.PublisherError, "complete coverage"):
+ publisher.validate_public_dataset(dataset)
+
+ def test_standalone_promotion_binds_matrix_and_requested_dispositions(self) -> None:
+ dataset = _promoted_dataset()
+ fixture_catalog = publisher._case_disposition_catalog_sha256(dataset["coverage"])
+ with self.assertRaisesRegex(
+ publisher.PublisherError, "canonical case/disposition catalog"
+ ):
+ publisher.validate_public_dataset(dataset)
+ with mock.patch.object(
+ publisher, "CANONICAL_FULL_V1_CASE_CATALOG_SHA256", fixture_catalog
+ ), mock.patch.object(
+ publisher,
+ "REQUIRED_PROMOTION_COHORT_COUNTS",
+ _cohort_counts(dataset),
+ ):
+ publisher.validate_public_dataset(dataset)
+
+ diagnostic = copy.deepcopy(dataset)
+ item = diagnostic["series"][0]
+ item["status"] = "diagnostic"
+ item["eligibility"].update({
+ "decision_grade": False,
+ "stable_p50": False,
+ "p50_max_min_ratio": 1.20,
+ "reasons": ["unstable-p50"],
+ })
+ with mock.patch.object(
+ publisher, "CANONICAL_FULL_V1_CASE_CATALOG_SHA256", fixture_catalog
+ ), mock.patch.object(
+ publisher,
+ "REQUIRED_PROMOTION_COHORT_COUNTS",
+ _cohort_counts(dataset),
+ ), self.assertRaisesRegex(
+ publisher.PublisherError, "unstable or incomplete required series"
+ ):
+ publisher.validate_public_dataset(diagnostic)
+
+ broken = copy.deepcopy(dataset)
+ broken["promotion"]["matrix_id"] = "d" * 64
+ with self.assertRaisesRegex(publisher.PublisherError, "canonical full-v1 matrix"):
+ publisher.validate_public_dataset(broken)
+
+ for original, replacement in (("runnable", "unsupported"),
+ ("unsupported", "runnable")):
+ with self.subTest(original=original):
+ broken = copy.deepcopy(dataset)
+ item = next(
+ coverage for coverage in broken["coverage"]
+ if coverage["disposition"] == original
+ )
+ item["disposition"] = replacement
+ with mock.patch.object(
+ publisher,
+ "CANONICAL_FULL_V1_CASE_CATALOG_SHA256",
+ publisher._case_disposition_catalog_sha256(broken["coverage"]),
+ ), self.assertRaisesRegex(
+ publisher.PublisherError, "requested dispositions"
+ ):
+ publisher.validate_public_dataset(broken)
+
+ def test_workflow_matrix_and_catalog_digests_do_not_drift(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ matrix_path = Path(temporary) / "matrix_full.json"
+ result = subprocess.run(
+ [
+ sys.executable, str(ROOT / "sweep_matrix.py"),
+ "--suites", "all", "--max-cases", "128",
+ "--backends", "all", "--out", str(matrix_path),
+ ],
+ text=True,
+ capture_output=True,
+ )
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertEqual(
+ hashlib.sha256(matrix_path.read_bytes()).hexdigest(),
+ publisher.CANONICAL_FULL_V1_MATRIX_SHA256,
+ )
+ matrix = contracts.strict_load(matrix_path)
+ coverage = [
+ {
+ "case_id": item["case"]["case_id"],
+ "disposition": item["disposition"],
+ }
+ for item in matrix["requested_cases"]
+ ]
+ self.assertEqual(
+ publisher._case_disposition_catalog_sha256(coverage),
+ publisher.CANONICAL_FULL_V1_CASE_CATALOG_SHA256,
+ )
+ self.assertEqual(
+ (
+ len(matrix["include"]), len(coverage),
+ sum(item["disposition"] == "runnable" for item in coverage),
+ sum(item["disposition"] == "unsupported" for item in coverage),
+ sum(
+ len(item["case"]["ladder"].split())
+ for item in matrix["requested_cases"]
+ ),
+ sum(
+ len(item["case"]["ladder"].split())
+ for item in matrix["requested_cases"]
+ if item["disposition"] == "runnable"
+ ),
+ sum(
+ len(item["case"]["ladder"].split())
+ for item in matrix["requested_cases"]
+ if item["disposition"] == "unsupported"
+ ),
+ ),
+ (58, 608, 364, 244, 1600, 940, 660),
+ )
+ library: dict[tuple, set[str]] = {}
+ system: dict[tuple, set[str]] = {}
+ routing: dict[tuple, list[tuple[str, bool]]] = {}
+ for requested in matrix["requested_cases"]:
+ if requested["disposition"] != "runnable":
+ continue
+ case = requested["case"]
+ shape = tuple(
+ case[field]
+ for field in (
+ "workload", "mode", "hidden", "topk", "experts", "ep", "phase"
+ )
+ )
+ route = (case["routing"], case["eplb"])
+ if case["backend"] != "nccl-ep":
+ library.setdefault((requested["sku"], shape, route), set()).add(
+ case["backend"]
+ )
+ else:
+ system.setdefault((shape, route), set()).add(requested["sku"])
+ routing.setdefault(
+ (requested["sku"], case["backend"], shape), []
+ ).append(route)
+ anchors = {("uniform", False), ("zipf", False), ("zipf", True)}
+ self.assertEqual(
+ {
+ "library": sum(len(variants) >= 2 for variants in library.values()),
+ "system": sum(len(variants) >= 2 for variants in system.values()),
+ "routing": sum(
+ len(variants) == 3 and set(variants) == anchors
+ for variants in routing.values()
+ ),
+ },
+ publisher.REQUIRED_PROMOTION_COHORT_COUNTS,
+ )
+
+ def test_build_promotion_requires_canonical_full_matrix(self) -> None:
+ bundles = {
+ str(run_id): {
+ "id": str(run_id), "cases": [],
+ "manifest": {
+ "matrix": {"sha256": "d" * 64},
+ "run": {"run_id": str(run_id), "run_attempt": 1},
+ },
+ }
+ for run_id in range(1, 4)
+ }
+ with mock.patch.object(
+ publisher, "load_bundle", side_effect=lambda _, bundle_id: bundles[bundle_id]
+ ), self.assertRaisesRegex(publisher.PublisherError, "canonical full-v1 matrix"):
+ publisher.build_dataset(object(), list(bundles), promote=True)
+
+ def test_rejection_updates_latest_but_never_dev_latest(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary).resolve()
+ matrix, artifact = _unsupported_delivery(root)
+ store = publisher.Store(root / "store")
+ sentinel = b"existing-promoted-pointer\n"
+ (store.channels / "dev-latest.json").write_bytes(sentinel)
+ (artifact / "unknown.json").write_text('{"format":"unknown"}')
+ with self.assertRaises(publisher.PublisherError):
+ publisher.ingest_command(_args(store.root, matrix, artifact))
+ self.assertEqual((store.channels / "dev-latest.json").read_bytes(), sentinel)
+ pointer = store.verify_channel("latest-attempt")
+ dataset = publisher.strict_load(store.public / pointer["dataset"]["path"])
+ self.assertEqual(dataset["promotion"]["status"], "quarantined")
+ self.assertTrue(any(store.quarantine.iterdir()))
+
+ def test_repeated_rejection_is_content_idempotent(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary).resolve()
+ matrix, artifact = _unsupported_delivery(root)
+ store = publisher.Store(root / "store")
+ (artifact / "unknown.json").write_text('{"format":"unknown"}')
+ with self.assertRaises(publisher.PublisherError):
+ publisher.ingest_command(_args(store.root, matrix, artifact))
+ pointer = (store.channels / "latest-attempt.json").read_bytes()
+ counts = tuple(
+ len(list(path.iterdir()))
+ for path in (store.incoming, store.quarantine, store.datasets)
+ )
+ with self.assertRaises(publisher.PublisherError):
+ publisher.ingest_command(_args(store.root, matrix, artifact))
+ self.assertEqual((store.channels / "latest-attempt.json").read_bytes(), pointer)
+ self.assertEqual(
+ tuple(
+ len(list(path.iterdir()))
+ for path in (store.incoming, store.quarantine, store.datasets)
+ ),
+ counts,
+ )
+
+ def test_distinct_rejections_advance_latest_attempt(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary).resolve()
+ matrix, artifact = _unsupported_delivery(root)
+ store = publisher.Store(root / "store")
+ unknown = artifact / "unknown.json"
+ unknown.write_text('{"format":"unknown-one"}')
+ with self.assertRaises(publisher.PublisherError):
+ publisher.ingest_command(_args(store.root, matrix, artifact))
+ first = (store.channels / "latest-attempt.json").read_bytes()
+ unknown.write_text('{"format":"unknown-two"}')
+ with self.assertRaises(publisher.PublisherError):
+ publisher.ingest_command(_args(store.root, matrix, artifact))
+ second = (store.channels / "latest-attempt.json").read_bytes()
+ self.assertNotEqual(second, first)
+ self.assertEqual(len(list(store.datasets.iterdir())), 2)
+
+ def test_zip_traversal_is_rejected(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary).resolve()
+ archive = root / "bad.zip"
+ with zipfile.ZipFile(archive, "w") as handle:
+ handle.writestr("../escape.json", "{}")
+ with self.assertRaisesRegex(publisher.PublisherError, "escapes"):
+ publisher.extract_archive(archive, root / "out")
+
+ def test_store_and_directory_archive_reject_symlinks(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary).resolve()
+ real = root / "real"
+ real.mkdir()
+ alias = root / "alias"
+ alias.symlink_to(real, target_is_directory=True)
+ with self.assertRaisesRegex(publisher.PublisherError, "symlinked parent"):
+ publisher.Store(alias / "store")
+ self.assertFalse((real / "store").exists())
+ artifact = root / f"cxunsupported-{RUN['run_id']}-{RUN['run_attempt']}"
+ artifact.mkdir()
+ target = root / "target.json"
+ target.write_text("{}")
+ (artifact / "linked.json").symlink_to(target)
+ with self.assertRaisesRegex(publisher.PublisherError, "symlink"):
+ publisher._archive_download_directory(artifact, root / "artifact.zip")
+
+ def test_offline_caller_metadata_is_validated_before_store_creation(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary).resolve()
+ matrix, artifact = _unsupported_delivery(root)
+ store_root = root / "store"
+ args = _args(store_root, matrix, artifact)
+ args.run_id = "0"
+ with self.assertRaisesRegex(publisher.PublisherError, "run-id"):
+ publisher.ingest_command(args)
+ self.assertFalse(store_root.exists())
+
+ promote = types.SimpleNamespace(
+ store_root=str(store_root), bundle=["not-a-digest"]
+ )
+ with self.assertRaisesRegex(publisher.PublisherError, "bundle IDs"):
+ publisher.promote_command(promote)
+ self.assertFalse(store_root.exists())
+ with self.assertRaisesRegex(publisher.PublisherError, "absolute path"):
+ publisher._store_from_args(types.SimpleNamespace(store_root="relative-store"))
+
+ def test_store_rejects_group_or_world_writable_root(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary).resolve() / "unsafe-store"
+ root.mkdir()
+ root.chmod(0o772)
+ with self.assertRaisesRegex(publisher.PublisherError, "group/world writable"):
+ publisher.Store(root)
+
+ def test_retry_ordinals_must_be_contiguous_from_one(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary).resolve()
+ matrix, artifact = _unsupported_delivery(root, (1, 3))
+ with self.assertRaisesRegex(publisher.PublisherError, "contiguous ordinals"):
+ publisher.ingest_command(_args(root / "store", matrix, artifact))
+
+ def test_delivery_rejects_extra_archive_and_non_native_member(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary).resolve()
+ matrix, artifact = _unsupported_delivery(root)
+ extra = root / f"cxshard-extra-{RUN['run_id']}-{RUN['run_attempt']}"
+ extra.mkdir()
+ (extra / "extra.json").write_text("{}")
+ args = _args(root / "store-extra", matrix, artifact)
+ args.artifact.append(str(extra))
+ with self.assertRaisesRegex(publisher.PublisherError, "archive set"):
+ publisher.ingest_command(args)
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary).resolve()
+ matrix, artifact = _unsupported_delivery(root)
+ (artifact / "notes.txt").write_text("not native evidence")
+ with self.assertRaisesRegex(publisher.PublisherError, "unconsumed"):
+ publisher.ingest_command(_args(root / "store-member", matrix, artifact))
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary).resolve()
+ matrix, artifact = _unsupported_delivery(root)
+ path = next(artifact.glob("*.json"))
+ terminal = json.loads(path.read_text())
+ terminal["outcome"]["reason"] = next(
+ reason for reason in contracts.CAPABILITY_FAILURE_REASONS
+ if reason != terminal["outcome"]["reason"]
+ )
+ path.write_text(json.dumps(terminal))
+ with self.assertRaisesRegex(publisher.PublisherError, "reason differs"):
+ publisher.ingest_command(_args(root / "store-reason", matrix, artifact))
+
+ def test_rates_invert_latency_and_global_tokens_use_ep_size(self) -> None:
+ dataset = _dataset()
+ publisher.validate_public_dataset(dataset)
+ rates = dataset["series"][0]["points"][0]["components"]["roundtrip"]["logical_payload_rate_gbps_at_latency_percentile"]
+ self.assertGreater(rates["p50"], rates["p99"])
+ broken = copy.deepcopy(dataset)
+ broken["series"][0]["points"][0]["global_tokens"] = 128
+ with self.assertRaisesRegex(publisher.PublisherError, "EP size"):
+ publisher.validate_public_dataset(broken)
+ broken = copy.deepcopy(dataset)
+ broken["series"][0]["points"][0]["roundtrip_token_rate_at_latency_percentile"]["p99"] *= 2
+ with self.assertRaisesRegex(publisher.PublisherError, "token throughput"):
+ publisher.validate_public_dataset(broken)
+ broken = copy.deepcopy(dataset)
+ broken["attempts"][0]["evidence"][0]["point_id"] = identity.point_id(
+ series=broken["series"][0]["series_id"], tokens_per_rank=16
+ )
+ with self.assertRaisesRegex(publisher.PublisherError, "point evidence"):
+ publisher.validate_public_dataset(broken)
+ broken = copy.deepcopy(dataset)
+ broken["attempts"][0]["series_id"] = None
+ with self.assertRaisesRegex(publisher.PublisherError, "present exactly for selected success"):
+ publisher.validate_public_dataset(broken)
+ broken = copy.deepcopy(dataset)
+ component = broken["series"][0]["points"][0]["components"]["roundtrip"]
+ component["logical_bytes"] = None
+ component["logical_payload_rate_gbps_at_latency_percentile"] = None
+ with self.assertRaisesRegex(publisher.PublisherError, "logical bandwidth is missing"):
+ publisher.validate_public_dataset(broken)
+
+ for mutate in (
+ lambda item: item.update({"model": "different-model"}),
+ lambda item: item["workload"].update({"hidden": 4096}),
+ lambda item: item["workload"].update({"top_k": 4}),
+ lambda item: item["workload"].update({"experts": 128}),
+ ):
+ broken = copy.deepcopy(dataset)
+ mutate(broken["series"][0])
+ with self.assertRaisesRegex(publisher.PublisherError, "frozen v1"):
+ publisher.validate_public_dataset(broken)
+
+ broken = copy.deepcopy(dataset)
+ broken["series"][0]["eplb"]["mapping_sha256"] = "f" * 64
+ with self.assertRaisesRegex(publisher.PublisherError, "claims a plan"):
+ publisher.validate_public_dataset(broken)
+
+ broken = copy.deepcopy(dataset)
+ broken["series"][0]["backend"].update({
+ "id": "nccl-ep", "label": publisher.BACKEND_LABELS["nccl-ep"],
+ "role": "reference", "generation": "rccl",
+ })
+ broken["coverage"][0]["backend"] = "nccl-ep"
+ with self.assertRaisesRegex(publisher.PublisherError, "configuration"):
+ publisher.validate_public_dataset(broken)
+
+ def test_public_coverage_binds_exact_topology_and_case_identity(self) -> None:
+ dataset = _promoted_dataset()
+ dataset["promotion"]["status"] = "diagnostic"
+ self.assertEqual(
+ {item["disposition"] for item in dataset["coverage"]},
+ {"runnable", "unsupported"},
+ )
+ for item in dataset["coverage"]:
+ self.assertEqual(
+ tuple(item["topology"]), publisher.COVERAGE_TOPOLOGY_FIELDS
+ )
+ publisher.validate_public_dataset(dataset)
+
+ broken = copy.deepcopy(dataset)
+ unsupported = next(
+ item for item in broken["coverage"]
+ if item["disposition"] == "unsupported"
+ )
+ unsupported["topology"]["nodes"] = 2
+ with self.assertRaisesRegex(publisher.PublisherError, "capability registry"):
+ publisher.validate_public_dataset(broken)
+
+ broken = copy.deepcopy(dataset)
+ unsupported = next(
+ item for item in broken["coverage"]
+ if item["disposition"] == "unsupported"
+ )
+ unsupported["sku"] = "mi325x"
+ topology = publisher.capability.topology_for("mi325x", 8)
+ self.assertIsNotNone(topology)
+ unsupported["topology"] = publisher._coverage_topology({
+ "ep_size": 8, **topology,
+ })
+ with self.assertRaisesRegex(publisher.PublisherError, "case identity"):
+ publisher.validate_public_dataset(broken)
+
+ def test_cohort_contract_and_labels_name_mode_explicitly(self) -> None:
+ dataset = _promoted_dataset()
+ dataset["promotion"]["status"] = "diagnostic"
+ publisher.validate_public_dataset(dataset)
+ for cohort in dataset["cohorts"]:
+ self.assertIn("mode", cohort["controlled_factors"])
+ self.assertIn("/ normal /", cohort["label"])
+
+ broken = copy.deepcopy(dataset)
+ cohort = broken["cohorts"][0]
+ cohort["controlled_factors"].remove("mode")
+ cohort["cohort_id"] = publisher._derived_id("cxcohort-v1-", {
+ "kind": cohort["kind"], "series_ids": cohort["series_ids"],
+ "controlled_factors": cohort["controlled_factors"],
+ "varying_factors": cohort["varying_factors"],
+ })
+ with self.assertRaisesRegex(publisher.PublisherError, "cohort factors"):
+ publisher.validate_public_dataset(broken)
+
+ def test_routing_and_eplb_facts_must_match_across_repeats(self) -> None:
+ raw, _ = _native_fixture()
+ descriptor = publisher._eplb_descriptor(raw)
+ facts = publisher._routing_facts(raw["measurement"]["rows"][0])
+ self.assertEqual(
+ publisher._exact_repeat_value([descriptor, copy.deepcopy(descriptor)], "EPLB"),
+ descriptor,
+ )
+ self.assertEqual(
+ publisher._exact_repeat_value([facts, copy.deepcopy(facts)], "routing"),
+ facts,
+ )
+ changed = copy.deepcopy(facts)
+ changed["hotspot_ratio"] += 0.1
+ with self.assertRaisesRegex(publisher.PublisherError, "routing differs"):
+ publisher._exact_repeat_value([facts, changed], "routing")
+
+ dataset = _promoted_dataset()
+ dataset["promotion"]["status"] = "diagnostic"
+ eplb = next(item for item in dataset["series"] if item["eplb"]["enabled"])
+ eplb["points"][0]["routing"]["empty_expert_count"] = 280
+ publisher.validate_public_dataset(dataset)
+ eplb["points"][0]["routing"]["empty_expert_count"] = 288
+ with self.assertRaisesRegex(publisher.PublisherError, "routing/load facts"):
+ publisher.validate_public_dataset(dataset)
+
+ for field, value in (
+ ("mapping_sha256", "0" * 64),
+ ("redundant_experts", 31),
+ ("replicated_experts", 1),
+ ("max_replicas", 2),
+ ("replicated_experts", 257),
+ ("max_replicas", 999),
+ ("imbalance_after", 0.4),
+ ("planner", "different-planner"),
+ ("reference_tokens_per_rank", 1024),
+ ):
+ broken = _promoted_dataset()
+ broken["promotion"]["status"] = "diagnostic"
+ descriptor = next(
+ item["eplb"] for item in broken["series"] if item["eplb"]["enabled"]
+ )
+ descriptor[field] = value
+ with self.subTest(eplb_field=field), self.assertRaisesRegex(
+ publisher.PublisherError, "EPLB descriptor"
+ ):
+ publisher.validate_public_dataset(broken)
+
+ def test_publisher_owns_stable_rankings_and_recommendations(self) -> None:
+ fast, fast_internal = _series("fast", "deepep", decision_grade=True)
+ slow, slow_internal = _series("slow", "uccl", decision_grade=True)
+ reference, reference_internal = _series("reference", "nccl-ep", decision_grade=True)
+ reference_peer, reference_peer_internal = _series(
+ "reference-peer", "nccl-ep", decision_grade=True
+ )
+ reference["backend"]["role"] = "reference"
+ reference_peer["backend"]["role"] = "reference"
+ reference_peer["system"].update({"sku": "h200-dgxc", "label": "NVIDIA H200"})
+ cohorts, rankings, recommendations, _ = publisher.build_decisions(
+ [fast, slow, reference, reference_peer], {
+ fast["series_id"]: fast_internal,
+ slow["series_id"]: slow_internal,
+ reference["series_id"]: reference_internal,
+ reference_peer["series_id"]: reference_peer_internal,
+ }
+ )
+ library = next(item for item in cohorts if item["kind"] == "library")
+ ranking = next(item for item in rankings if item["cohort_id"] == library["cohort_id"]
+ and item["metric"]["measure"] == "latency_us"
+ and item["metric"]["statistic"] == "p99")
+ self.assertTrue(library["eligibility"]["decision_grade"])
+ self.assertEqual(ranking["entries"][0]["series_id"], fast["series_id"])
+ self.assertTrue(any(item["series_id"] == fast["series_id"] for item in recommendations))
+ self.assertFalse(any(
+ entry["series_id"] == reference["series_id"]
+ for item in rankings if item["cohort_id"] == library["cohort_id"]
+ for entry in item["entries"]
+ ))
+ self.assertTrue(any(
+ item["kind"] == "system" and reference["series_id"] in item["series_ids"]
+ for item in cohorts
+ ))
+
+ def test_routing_evidence_is_experimental_and_not_a_configuration_recommendation(self) -> None:
+ dataset = _promoted_dataset()
+ routing = next(item for item in dataset["cohorts"] if item["kind"] == "routing")
+ members = [
+ item for item in dataset["series"]
+ if item["series_id"] in routing["series_ids"]
+ ]
+ self.assertEqual(
+ {(item["workload"]["routing"], item["workload"]["eplb"]) for item in members},
+ {("uniform", False), ("zipf", False), ("zipf", True)},
+ )
+ self.assertIn("implementation-static-build", routing["controlled_factors"])
+ self.assertIn("resource", routing["controlled_factors"])
+ self.assertEqual(
+ routing["varying_factors"],
+ ["workload.routing", "workload.eplb", "implementation-config"],
+ )
+ self.assertEqual(
+ len({item["build"]["routing_control_sha256"] for item in members}),
+ 1,
+ )
+ self.assertGreater(
+ len({item["build"]["implementation_contract_sha256"] for item in members}),
+ 1,
+ )
+ self.assertEqual(len({json.dumps(item["resource"], sort_keys=True) for item in members}), 1)
+ self.assertEqual(routing["publication_tier"], "comparable-experimental")
+ self.assertTrue(any(
+ item["cohort_id"] == routing["cohort_id"] for item in dataset["rankings"]
+ ))
+ self.assertFalse(any(
+ item["cohort_id"] == routing["cohort_id"] for item in dataset["recommendations"]
+ ))
+ self.assertTrue(all(
+ item["publication_tier"] == "official"
+ for item in dataset["recommendations"]
+ ))
+ self.assertFalse(any(
+ dataset_cohort["publication_tier"] == "comparable-experimental"
+ and item["cohort_id"] == dataset_cohort["cohort_id"]
+ for item in dataset["recommendations"]
+ for dataset_cohort in dataset["cohorts"]
+ ))
+ self.assertTrue(all(
+ item["publication_tier"] == "comparable-experimental"
+ for item in dataset["sensitivities"]
+ if item["cohort_id"] == routing["cohort_id"]
+ ))
+
+ def test_routing_implementation_mismatch_blocks_all_decisions(self) -> None:
+ dataset = _promoted_dataset()
+ published = next(item for item in dataset["cohorts"] if item["kind"] == "routing")
+ members = [
+ item for item in dataset["series"]
+ if item["series_id"] in published["series_ids"]
+ ]
+ zipf = next(
+ item for item in members
+ if item["workload"]["routing"] == "zipf" and not item["workload"]["eplb"]
+ )
+ zipf["build"]["implementation_contract_sha256"] = "f" * 64
+ internals = {}
+ for member in members:
+ point = member["points"][0]
+ roundtrip = point["components"]["roundtrip"]
+ metrics = {
+ "latency_us": {
+ name: roundtrip["latency_us"][name] for name in ("p50", "p99")
+ },
+ "logical_payload_rate_gbps_at_latency_percentile": {
+ name: roundtrip[
+ "logical_payload_rate_gbps_at_latency_percentile"
+ ][name]
+ for name in ("p50", "p99")
+ },
+ }
+ internals[member["series_id"]] = {
+ "run_metrics": {
+ str(run): {point["tokens_per_rank"]: metrics}
+ for run in range(3)
+ }
+ }
+ cohorts, rankings, recommendations, sensitivities = publisher.build_decisions(
+ members, internals
+ )
+ routing = next(item for item in cohorts if item["kind"] == "routing")
+ self.assertFalse(routing["eligibility"]["decision_grade"])
+ self.assertIn(
+ "implementation-config-mismatch", routing["eligibility"]["reasons"]
+ )
+ self.assertEqual((rankings, recommendations, sensitivities), ([], [], []))
+
+ def test_promoted_series_fields_are_bound_to_case_and_series_identities(self) -> None:
+ dataset = _promoted_dataset()
+ changed = copy.deepcopy(dataset)
+ series = next(
+ item for item in changed["series"]
+ if item["system"]["sku"] == "h100-dgxc"
+ )
+ series["system"].update({
+ "sku": "h200-dgxc", "label": "NVIDIA H200",
+ "topology_class": "h200-nvlink-island",
+ })
+ for case_id in series["case_ids"]:
+ coverage = next(
+ item for item in changed["coverage"] if item["case_id"] == case_id
+ )
+ coverage["sku"] = "h200-dgxc"
+ coverage["topology"] = publisher._coverage_topology(series["system"])
+ with self.assertRaisesRegex(publisher.PublisherError, "configuration|case identity"):
+ publisher.validate_public_dataset(changed)
+
+ for field, value in (
+ ("source_sha", "b" * 40),
+ ("image_digest", "sha256:" + "4" * 64),
+ ("squash_sha256", "5" * 64),
+ ("runtime_fingerprint_sha256", "6" * 64),
+ ("implementation_contract_sha256", "7" * 64),
+ ("public_config_sha256", "9" * 64),
+ ("routing_control_sha256", "8" * 64),
+ ):
+ changed = copy.deepcopy(dataset)
+ changed["series"][0]["build"][field] = value
+ with self.subTest(build_field=field), self.assertRaisesRegex(
+ publisher.PublisherError, "commit"
+ ):
+ publisher.validate_public_dataset(changed)
+ changed = copy.deepcopy(dataset)
+ changed["series"][0]["workload"]["workload_id"] = identity.workload_id(
+ {"changed": True}
+ )
+ with self.assertRaisesRegex(publisher.PublisherError, "committed factors"):
+ publisher.validate_public_dataset(changed)
+
+ for mutate, message in (
+ (lambda item: item["backend"].update({
+ "generation": "fabricated", "version": "fabricated-999",
+ }), "configuration"),
+ (lambda item: item["resource"].update({
+ "profile": "profile-fabricated", "configured_units": 99,
+ }), "configuration"),
+ (lambda item: item["system"].update({"label": "Fabricated H100"}), "commit"),
+ ):
+ changed = copy.deepcopy(dataset)
+ mutate(changed["series"][0])
+ with self.assertRaisesRegex(publisher.PublisherError, message):
+ publisher.validate_public_dataset(changed)
+
+ diagnostic = _dataset()
+ diagnostic["series"][0]["build"]["source_sha"] = "b" * 40
+ with self.assertRaisesRegex(publisher.PublisherError, "committed factors"):
+ publisher.validate_public_dataset(diagnostic)
+
+ def test_all_decision_metrics_require_stable_repeat_ordering(self) -> None:
+ fast, fast_internal = _series("ordering-fast", "deepep", decision_grade=True)
+ slow, slow_internal = _series("ordering-slow", "uccl", decision_grade=True)
+ internals = {
+ fast["series_id"]: fast_internal,
+ slow["series_id"]: slow_internal,
+ }
+
+ cohorts, rankings, recommendations, _ = publisher.build_decisions(
+ [fast, slow], internals
+ )
+ library = next(item for item in cohorts if item["kind"] == "library")
+ self.assertTrue(library["eligibility"]["decision_grade"])
+ self.assertEqual(
+ len([item for item in rankings if item["cohort_id"] == library["cohort_id"]]),
+ 4,
+ )
+ self.assertEqual(
+ len([
+ item for item in recommendations
+ if item["cohort_id"] == library["cohort_id"]
+ ]),
+ 4,
+ )
+
+ for statistic in ("p50", "p99"):
+ slow_internal["run_metrics"]["1"][8]["logical_payload_rate_gbps_at_latency_percentile"][statistic] = (
+ fast_internal["run_metrics"]["1"][8]["logical_payload_rate_gbps_at_latency_percentile"][statistic] * 2
+ )
+ cohorts, rankings, recommendations, _ = publisher.build_decisions(
+ [fast, slow], internals
+ )
+ library = next(item for item in cohorts if item["kind"] == "library")
+ self.assertFalse(library["eligibility"]["decision_grade"])
+ self.assertIn("unstable-ordering", library["eligibility"]["reasons"])
+ self.assertFalse(any(
+ item["cohort_id"] == library["cohort_id"] for item in rankings
+ ))
+ self.assertFalse(any(
+ item["cohort_id"] == library["cohort_id"] for item in recommendations
+ ))
+
+ def test_extra_eligibility_reason_blocks_decision_grade(self) -> None:
+ allocations = [identity.allocation_id({"run": run}) for run in range(3)]
+ eligibility = publisher._eligibility_record(
+ allocations, complete=True, correct=True, measured=True,
+ stable_ordering=True, p50_ratio=1.01, p99_ratio=1.02,
+ extra_reasons=["incomplete-provenance"],
+ )
+ self.assertFalse(eligibility["decision_grade"])
+ self.assertEqual(eligibility["reasons"], ["incomplete-provenance"])
+ self.assertIs(publisher._eligibility(eligibility, "fixture"), eligibility)
+ broken = {**eligibility, "decision_grade": True}
+ with self.assertRaisesRegex(publisher.PublisherError, "promotion gates"):
+ publisher._eligibility(broken, "fixture")
+
+ def test_schema_is_strict_and_channel_target_must_be_complete(self) -> None:
+ dataset = _dataset()
+ dataset["unexpected"] = True
+ with self.assertRaises(publisher.PublisherError):
+ publisher.validate_public_dataset(dataset)
+ with mock.patch.object(publisher, "MAX_PUBLIC_DATASET_BYTES", 1), self.assertRaisesRegex(
+ publisher.PublisherError, "serving size limit"
+ ):
+ publisher.validate_public_dataset(_dataset())
+ with tempfile.TemporaryDirectory() as temporary:
+ store = publisher.Store(Path(temporary).resolve())
+ dataset = _dataset()
+ digest, size = store.install_dataset(dataset)
+ store.update_channel("latest-attempt", digest, size, dataset["generated_at"])
+ self.assertEqual(store.verify_channel("latest-attempt")["dataset"]["sha256"], digest)
+ channel_path = store.channels / "latest-attempt.json"
+ pointer = publisher.strict_load(channel_path)
+ pointer["generated_at"] = "2099-01-01T00:00:00Z"
+ channel_path.write_bytes(contracts.canonical_json_bytes(pointer))
+ with self.assertRaisesRegex(publisher.PublisherError, "metadata differs"):
+ store.verify_channel("latest-attempt")
+ store.update_channel("latest-attempt", digest, size, dataset["generated_at"])
+ with self.assertRaisesRegex(publisher.PublisherError, "metadata differs"):
+ store.update_channel(
+ "latest-attempt", digest, size + 1, dataset["generated_at"]
+ )
+ with self.assertRaisesRegex(publisher.PublisherError, "metadata differs"):
+ store.update_channel(
+ "latest-attempt", digest, size, "2026-07-05T00:00:00Z"
+ )
+ os.chmod(channel_path, 0o666)
+ with self.assertRaisesRegex(publisher.PublisherError, "regular 644"):
+ store.verify_channel("latest-attempt")
+ os.chmod(channel_path, 0o644)
+ dataset_dir = store.datasets / digest
+ os.chmod(dataset_dir, 0o755)
+ with self.assertRaisesRegex(publisher.PublisherError, "mode differs"):
+ store.verify_channel("latest-attempt")
+ os.chmod(dataset_dir, 0o555)
+ os.chmod(dataset_dir / "dataset.json", 0o644)
+ with self.assertRaisesRegex(publisher.PublisherError, "mode differs"):
+ store.verify_channel("latest-attempt")
+ os.chmod(dataset_dir / "dataset.json", 0o444)
+ os.chmod(dataset_dir, 0o755)
+ (dataset_dir / "COMPLETE").unlink()
+ os.chmod(dataset_dir, 0o555)
+ with self.assertRaisesRegex(publisher.PublisherError, "incomplete"):
+ store.verify_channel("latest-attempt")
+
+ def test_store_modes_do_not_depend_on_process_umask(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ previous = os.umask(0o077)
+ try:
+ store = publisher.Store(Path(temporary).resolve())
+ dataset = _dataset()
+ digest, size = store.install_dataset(dataset)
+ store.update_channel(
+ "latest-attempt", digest, size, dataset["generated_at"]
+ )
+ with store.locked():
+ pass
+ finally:
+ os.umask(previous)
+ self.assertEqual(
+ store.root.stat().st_mode & 0o777,
+ 0o750,
+ )
+ self.assertEqual(
+ (store.channels / "latest-attempt.json").stat().st_mode & 0o777,
+ 0o644,
+ )
+ self.assertEqual(
+ (store.datasets / digest / "dataset.json").stat().st_mode & 0o777,
+ 0o444,
+ )
+ self.assertEqual(
+ (store.locks / "publisher.lock").stat().st_mode & 0o777,
+ 0o600,
+ )
+
+ def test_verify_requires_bootstrap_but_dev_latest_is_optional(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary).resolve()
+ args = types.SimpleNamespace(
+ store_root=str(root / "store"), channel=None, bundle=[]
+ )
+ with self.assertRaises(publisher.PublisherError):
+ publisher.verify_command(args)
+ store = publisher.Store(args.store_root)
+ dataset = publisher._quarantine_dataset(
+ "awaiting-v1-runs", "2026-07-04T00:00:00Z"
+ )
+ digest, size = store.install_dataset(dataset)
+ store.update_channel(
+ "latest-attempt", digest, size, "2026-07-04T00:00:00Z"
+ )
+ result = publisher.verify_command(args)
+ self.assertEqual(set(result["channels"]), {"latest-attempt"})
+ explicit = types.SimpleNamespace(
+ store_root=args.store_root, channel=["dev-latest"], bundle=[]
+ )
+ with self.assertRaises(publisher.PublisherError):
+ publisher.verify_command(explicit)
+ dev_pointer = copy.deepcopy(store.verify_channel("latest-attempt"))
+ dev_pointer["channel"] = "dev-latest"
+ (store.channels / "dev-latest.json").write_bytes(
+ contracts.canonical_json_bytes(dev_pointer)
+ )
+ with self.assertRaisesRegex(publisher.PublisherError, "non-promoted"):
+ publisher.verify_command(args)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
new file mode 100644
index 0000000000..50de2b1ef5
--- /dev/null
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -0,0 +1,3213 @@
+#!/usr/bin/env python3
+"""CPU-only behavioral tests for the CollectiveX v1 execution contract."""
+from __future__ import annotations
+
+import argparse
+import ast
+import copy
+import hashlib
+import io
+import json
+import os
+from pathlib import Path
+import re
+import stat
+import subprocess
+import sys
+import tarfile
+import tempfile
+import types
+import unittest
+from unittest import mock
+
+import numpy as np
+
+HERE = Path(__file__).resolve().parent
+ROOT = HERE.parent
+sys.path[:0] = [str(ROOT), str(HERE)]
+
+import artifact_safety # noqa: E402
+import capability # noqa: E402
+import contracts # noqa: E402
+import eplb # noqa: E402
+import ep_harness # noqa: E402
+import identity # noqa: E402
+import run_ep # noqa: E402
+import source_archive # noqa: E402
+import summarize # noqa: E402
+import sweep_matrix # noqa: E402
+import workload # noqa: E402
+
+
+class SamplingContractTest(unittest.TestCase):
+ def test_identity_and_fixed_sampling_profile(self) -> None:
+ identity.verify_test_vector()
+ self.assertTrue(identity.is_typed_id(identity.IDENTITY_TEST_VECTOR["series_id"], "series"))
+ self.assertEqual(ep_harness.SAMPLING_CONTRACT, "fixed-512-v1")
+ self.assertEqual(
+ (
+ ep_harness.TIMED_ITERS_PER_TRIAL,
+ ep_harness.TRIALS_PER_POINT,
+ ep_harness.TIMED_SAMPLES_PER_POINT,
+ ep_harness.WARMUP_ITERS_PER_TRIAL,
+ ),
+ (8, 64, 512, 32),
+ )
+ self.assertEqual(identity.V1_CASE_PROFILE["activation_profile"], "canonical-counter-source-v3")
+ self.assertEqual(
+ identity.V1_CASE_PROFILE["activation_generator"],
+ "collectivex-activation-counter-v3",
+ )
+ self.assertEqual(identity.V1_CASE_PROFILE["sampling_contract"], "fixed-512-v1")
+ self.assertEqual(identity.V1_CASE_PROFILE["percentile_method"], "nearest-rank")
+ self.assertEqual(
+ identity.V1_CASE_PROFILE["rank_reduction"],
+ "cross-rank-max-per-iteration",
+ )
+ self.assertEqual(
+ identity.V1_CASE_PROFILE["oracle_contract"],
+ "expert-specific-transform-v1",
+ )
+ self.assertEqual(
+ set(identity.V1_CASE_PROFILES), {"normal", "low-latency"}
+ )
+ self.assertEqual(
+ identity.V1_LOW_LATENCY_CASE_PROFILE["payload_unit"], "token-expert"
+ )
+ self.assertNotEqual(
+ identity.digest("case", identity.V1_NORMAL_CASE_PROFILE),
+ identity.digest("case", identity.V1_LOW_LATENCY_CASE_PROFILE),
+ )
+ parser = argparse.ArgumentParser()
+ ep_harness.add_common_args(parser)
+ args = parser.parse_args(
+ [
+ "--runner", "test", "--topology-class", "test",
+ "--scope", "scale-up", "--scale-up-transport", "nvlink",
+ "--out", "result.json",
+ ]
+ )
+ self.assertEqual((args.iters, args.trials, args.warmup), (8, 64, 32))
+ for profile in ((8, 64, 32), (128, 4, 32), (8, 1, 4), (0, 64, 32)):
+ with self.subTest(profile=profile):
+ self.assertEqual(
+ ep_harness.sampling_contract_error(*profile) is None,
+ profile == (8, 64, 32),
+ )
+
+ def test_nearest_rank_percentiles_use_all_512_samples(self) -> None:
+ samples = list(range(1, 513))
+ self.assertEqual(ep_harness.percentile(samples, 50), 256)
+ self.assertEqual(ep_harness.percentile(samples, 99), 507)
+
+ def test_terminal_summary_uses_bound_sku_and_route(self) -> None:
+ terminal = {
+ "format": contracts.TERMINAL_FORMAT,
+ "case": {
+ "backend": "deepep", "phase": "prefill", "ep": 8,
+ "suite": "ep-routing-v1", "routing": "zipf", "eplb": True,
+ "required_publication": "comparable-experimental",
+ },
+ "identity": {"case_factors": {"sku": "h100-dgxc"}},
+ }
+ self.assertEqual(
+ summarize._identity(terminal),
+ (
+ "h100-dgxc", "ep-routing-v1", "zipf", "prefill", True,
+ "comparable-experimental", 8,
+ ),
+ )
+
+ def test_matrix_cases_and_shards_are_identity_bound(self) -> None:
+ matrix = sweep_matrix.validate_matrix_document(
+ sweep_matrix.resolve_matrix(backends="all")
+ )
+ requested = {item["case"]["case_id"]: item for item in matrix["requested_cases"]}
+ assigned = [case_id for shard in matrix["include"] for case_id in shard["case_ids"]]
+ runnable = {
+ case_id for case_id, item in requested.items()
+ if item["disposition"] == "runnable"
+ }
+ runnable_cases = [
+ item for item in matrix["requested_cases"]
+ if item["disposition"] == "runnable"
+ ]
+ unsupported_cases = [
+ item for item in matrix["requested_cases"]
+ if item["disposition"] == "unsupported"
+ ]
+ self.assertEqual(
+ (
+ len(matrix["include"]),
+ len(matrix["requested_cases"]),
+ len(runnable_cases),
+ len(unsupported_cases),
+ sum(
+ len(item["case"]["ladder"].split())
+ for item in matrix["requested_cases"]
+ ),
+ sum(len(item["case"]["ladder"].split()) for item in runnable_cases),
+ sum(len(item["case"]["ladder"].split()) for item in unsupported_cases),
+ ),
+ (58, 608, 364, 244, 1600, 940, 660),
+ )
+ expected_topologies = {}
+ for sku, product in (
+ ("h100-dgxc", "h100"), ("h200-dgxc", "h200"),
+ ("b200-dgxc", "b200"), ("b300", "b300"),
+ ):
+ expected_topologies[sku, 8] = (
+ 1, 8, 8, "scale-up", "nvlink", None, "nvlink",
+ f"{product}-nvlink-island",
+ )
+ expected_topologies[sku, 16] = (
+ 2, 8, 8, "scale-out", "nvlink", "rdma", "nvlink-rdma",
+ f"{product}-nvlink-rdma",
+ )
+ for sku in ("gb200", "gb300"):
+ topology_class = f"{sku}-nvl72-mnnvl"
+ expected_topologies[sku, 8] = (
+ 2, 4, 72, "scale-up", "mnnvl", None, "mnnvl", topology_class,
+ )
+ expected_topologies[sku, 16] = (
+ 4, 4, 72, "scale-up", "mnnvl", None, "mnnvl", topology_class,
+ )
+ for sku in ("mi325x", "mi355x"):
+ expected_topologies[sku, 8] = (
+ 1, 8, 8, "scale-up", "xgmi", None, "xgmi", f"{sku}-xgmi",
+ )
+ expected_topologies[sku, 16] = (
+ 2, 8, 8, "scale-out", "xgmi", "rdma", "xgmi-rdma",
+ f"{sku}-xgmi-rdma",
+ )
+ topology_fields = sweep_matrix.TOPOLOGY_FIELDS
+ observed_topologies: dict[tuple[str, int], set[tuple[object, ...]]] = {}
+ for item in matrix["requested_cases"]:
+ case = item["case"]
+ observed_topologies.setdefault((item["sku"], case["ep"]), set()).add(
+ tuple(case[field] for field in topology_fields)
+ )
+ self.assertEqual(
+ {key: next(iter(values)) for key, values in observed_topologies.items()},
+ expected_topologies,
+ )
+ self.assertTrue(all(len(values) == 1 for values in observed_topologies.values()))
+ self.assertEqual(
+ {
+ (sku, ep): tuple(topology[field] for field in topology_fields)
+ for sku, platform in capability.PLATFORMS.items()
+ for ep, topology in platform["topologies"].items()
+ },
+ expected_topologies,
+ )
+ self.assertEqual(
+ {shard["n"] for shard in matrix["include"]}, {6, 7}
+ )
+ self.assertEqual(
+ sum(shard["n"] == 7 for shard in matrix["include"]), 16
+ )
+ ll_cases = [
+ item for item in matrix["requested_cases"]
+ if item["case"]["mode"] == "low-latency"
+ ]
+ self.assertEqual(len(ll_cases), 32)
+ self.assertTrue(all(
+ item["case"]["suite"] == "ep-low-latency-v1"
+ and item["case"]["backend"] in {"deepep", "uccl"}
+ and item["case"]["phase"] == "decode"
+ and item["case"]["routing"] == "uniform"
+ and not item["case"]["eplb"]
+ and item["case"]["ladder"] == "1 2 4 8 16 32 64 128"
+ for item in ll_cases
+ ))
+ for shard in matrix["include"]:
+ ep = next(
+ requested[case_id]["case"]["ep"] for case_id in shard["case_ids"]
+ )
+ self.assertEqual(
+ tuple(shard[field] for field in topology_fields),
+ expected_topologies[shard["sku"], ep],
+ )
+ routing_points = {
+ phase: {
+ int(point)
+ for item in matrix["requested_cases"]
+ if item["case"]["suite"] == "ep-routing-v1"
+ and item["case"]["phase"] == phase
+ for point in item["case"]["ladder"].split()
+ }
+ for phase in ("decode", "prefill")
+ }
+ self.assertEqual(routing_points, {"decode": {128}, "prefill": {512}})
+ skus = sorted({shard["sku"] for shard in matrix["include"]})
+ self.assertEqual(
+ [shard["sku"] for shard in matrix["include"][:len(skus)]],
+ skus,
+ )
+ self.assertEqual(set(assigned), runnable)
+ self.assertEqual(len(assigned), len(set(assigned)))
+ self.assertEqual({item["case"]["ep"] for item in matrix["requested_cases"]}, {8, 16})
+ self.assertFalse(capability.resolve("gb200", "deepep", ep=8, nodes=1)[0])
+ excluded = {
+ "uccl": {"b200-dgxc", "b300"},
+ }
+ for backend, skus in excluded.items():
+ for sku in skus:
+ with self.subTest(backend=backend, sku=sku):
+ self.assertFalse(capability.resolve(sku, backend)[0])
+ for case_id, item in requested.items():
+ case = {key: value for key, value in item["case"].items() if key != "case_id"}
+ self.assertEqual(
+ case_id,
+ identity.case_id(
+ sku=item["sku"], profile=identity.profile_for_case(case), case=case
+ ),
+ )
+ self.assertEqual(case["timing"], "8:64:32")
+ self.assertEqual(case["samples_per_point"], 512)
+
+ bad_matrix = copy.deepcopy(matrix)
+ bad_matrix["schema_version"] = True
+ with self.assertRaises(sweep_matrix.MatrixError):
+ sweep_matrix.validate_matrix_document(bad_matrix)
+
+ bad_catalog = copy.deepcopy(matrix)
+ wrapper = next(
+ item for item in bad_catalog["requested_cases"]
+ if item["disposition"] == "runnable"
+ )
+ old_id = wrapper["case"]["case_id"]
+ wrapper["case"]["hidden"] = 1
+ factors = {key: value for key, value in wrapper["case"].items() if key != "case_id"}
+ new_id = identity.case_id(
+ sku=wrapper["sku"], profile=identity.V1_CASE_PROFILE, case=factors
+ )
+ wrapper["case"]["case_id"] = new_id
+ for shard in bad_catalog["include"]:
+ shard["case_ids"] = [new_id if value == old_id else value for value in shard["case_ids"]]
+ with self.assertRaisesRegex(sweep_matrix.MatrixError, "frozen v1"):
+ sweep_matrix.validate_matrix_document(bad_catalog)
+
+ bad_topology = copy.deepcopy(matrix)
+ wrapper = next(
+ item for item in bad_topology["requested_cases"]
+ if item["disposition"] == "runnable"
+ )
+ old_id = wrapper["case"]["case_id"]
+ wrapper["case"]["transport"] = "incorrect-transport"
+ factors = {key: value for key, value in wrapper["case"].items() if key != "case_id"}
+ new_id = identity.case_id(
+ sku=wrapper["sku"], profile=identity.V1_CASE_PROFILE, case=factors
+ )
+ wrapper["case"]["case_id"] = new_id
+ for shard in bad_topology["include"]:
+ shard["case_ids"] = [new_id if value == old_id else value for value in shard["case_ids"]]
+ with self.assertRaisesRegex(sweep_matrix.MatrixError, "platform registry"):
+ sweep_matrix.validate_matrix_document(bad_topology)
+
+ shard_meta = matrix["include"][0]
+ requested_cases = {item["case"]["case_id"]: item["case"] for item in matrix["requested_cases"]}
+ shard = {
+ "schema_version": True,
+ "id": shard_meta["id"],
+ "sku": shard_meta["sku"],
+ "backend": shard_meta["backend"],
+ "nodes": shard_meta["nodes"],
+ "n": shard_meta["n"],
+ "cases": [requested_cases[value] for value in shard_meta["case_ids"]],
+ }
+ with self.assertRaises(sweep_matrix.MatrixError):
+ sweep_matrix.validate_shard_control(
+ shard, sku=shard_meta["sku"], backend=shard_meta["backend"],
+ nodes=shard_meta["nodes"],
+ )
+
+ def test_matrix_yaml_and_config_validation_are_strict(self) -> None:
+ suites = sweep_matrix._load("suites.yaml")
+ workloads = sweep_matrix._load("workloads.yaml")
+ self.assertEqual(
+ {tuple(suite["ep_degrees"]) for suite in suites["suites"].values()},
+ {(8, 16)},
+ )
+ invalid = (
+ ("unknown top", lambda s, _w: s.update({"typo": True})),
+ (
+ "unknown suite field",
+ lambda s, _w: s["suites"]["ep-core-v1"].update({"modes": ["normal"]}),
+ ),
+ (
+ "unknown workload field",
+ lambda _s, w: w["model_derived"]["deepseek-v3-v1"].update({"unused": 1}),
+ ),
+ (
+ "string phases",
+ lambda s, _w: s["suites"]["ep-core-v1"].update({"phases": "decode"}),
+ ),
+ (
+ "unknown routing",
+ lambda s, _w: s["suites"]["ep-core-v1"].update({"routings": ["random"]}),
+ ),
+ (
+ "integer EPLB",
+ lambda s, _w: s["suites"]["ep-routing-v1"].update({"eplb": [0, 1]}),
+ ),
+ (
+ "duplicate platform",
+ lambda s, _w: s["suites"]["ep-core-v1"]["platforms"].append("h100-dgxc"),
+ ),
+ (
+ "missing EP degrees",
+ lambda s, _w: s["suites"]["ep-core-v1"].pop("ep_degrees"),
+ ),
+ (
+ "non-v1 EP degrees",
+ lambda s, _w: s["suites"]["ep-core-v1"].update({"ep_degrees": [8]}),
+ ),
+ ("missing top field", lambda s, _w: s.pop("schema_version")),
+ (
+ "string dimension",
+ lambda _s, w: w["model_derived"]["deepseek-v3-v1"].update({"hidden": "7168"}),
+ ),
+ (
+ "unreachable phase ladder",
+ lambda s, _w: s["suites"]["ep-routing-v1"].update({"phases": ["prefill"]}),
+ ),
+ )
+ for label, mutate in invalid:
+ with self.subTest(label=label), self.assertRaises(SystemExit):
+ bad_suites, bad_workloads = copy.deepcopy(suites), copy.deepcopy(workloads)
+ mutate(bad_suites, bad_workloads)
+ sweep_matrix.validate_config_documents(bad_suites, bad_workloads)
+
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary)
+ (root / "configs").mkdir()
+ (root / "configs" / "duplicate.yaml").write_text(
+ "schema_version: 1\nsuites:\n same: 1\n same: 2\n"
+ )
+ with mock.patch.object(sweep_matrix, "HERE", root), self.assertRaisesRegex(
+ SystemExit, "duplicate YAML key"
+ ):
+ sweep_matrix._load("duplicate.yaml")
+
+ def test_semantically_duplicate_suite_points_are_rejected(self) -> None:
+ matrix = sweep_matrix.resolve_matrix()
+ with mock.patch.object(
+ sweep_matrix, "_semantic_points", return_value=["duplicate"]
+ ), self.assertRaisesRegex(
+ sweep_matrix.MatrixError, "duplicates a semantic token point"
+ ):
+ sweep_matrix.validate_matrix_document(matrix)
+
+ def test_only_three_shared_launchers_are_registered(self) -> None:
+ expected = {
+ "launch_single-slurm.sh",
+ "launch_gb-nv.sh",
+ "launch_mi-amds.sh",
+ }
+ self.assertEqual({path.name for path in (ROOT / "launchers").glob("launch_*.sh")}, expected)
+ self.assertEqual(
+ {platform["launcher"] for platform in capability.PLATFORMS.values()},
+ {"single-slurm", "gb-nv", "mi-amds"},
+ )
+ for platform in capability.PLATFORMS.values():
+ launcher = ROOT / "launchers" / f"launch_{platform['launcher']}.sh"
+ self.assertTrue(launcher.is_file())
+ source = launcher.read_text()
+ self.assertNotIn("RUNNER_NAME", source)
+ self.assertIn("cx_preflight_allocation", source)
+ lock_environment = 'cx_lock_canonical_gha_env "$RUNNER"'
+ self.assertIn(lock_environment, source)
+ self.assertLess(
+ source.index("cx_load_operator_config"),
+ source.index(lock_environment),
+ )
+ validate = 'cx_validate_shard_control "$CX_DIR"'
+ stage = 'MOUNT_SRC="$(cx_stage_path '
+ self.assertIn(validate, source)
+ self.assertLess(source.index(validate), source.index(stage))
+ self.assertLess(source.index(stage), source.index('cx_stage_repo "$REPO_ROOT"'))
+ self.assertLess(source.index(validate), source.index("cx_require_vars"))
+ if platform["launcher"] in {"single-slurm", "mi-amds"}:
+ network = "cx_validate_network_profile_on_job"
+ self.assertIn(network, source)
+ self.assertLess(source.index("cx_salloc_jobid"), source.index(network))
+ self.assertLess(source.index(network), source.index("cx_preflight_allocation"))
+ if platform["launcher"] == "single-slurm":
+ self.assertLess(
+ source.index(network),
+ source.index("CX_ENROOT_LOCAL_IMPORT=1 cx_ensure_squash"),
+ )
+
+ common = (ROOT / "runtime" / "common.sh").read_text()
+ workflow = (ROOT.parent.parent / ".github" / "workflows" / "collectivex-sweep.yml").read_text()
+ self.assertNotIn("RUNNER_NAME", common)
+ self.assertNotIn("RUNNER_NAME:", workflow)
+ self.assertNotIn("flashinfer", capability.BACKENDS)
+ self.assertFalse((HERE / "ep_flashinfer.py").exists())
+
+ def test_scaleout_network_profile_is_explicit_and_allowlisted(self) -> None:
+ command = r'''
+ set -euo pipefail
+ source "$1"
+ ! (unset CX_SOCKET_IFNAME CX_RDMA_DEVICES; cx_apply_network_profile 2 nvlink-rdma)
+ ! (export CX_SOCKET_IFNAME=eth0; unset CX_RDMA_DEVICES; cx_apply_network_profile 2 nvlink-rdma)
+ export CX_SOCKET_IFNAME=ib0 CX_RDMA_DEVICES=mlx5_0:1,mlx5_1:1
+ export NCCL_NET=Socket NCCL_IB_HCA=stale NVSHMEM_HCA_LIST=stale
+ cx_apply_network_profile 1 nvlink
+ test -z "${NCCL_NET+x}${NCCL_IB_HCA+x}${NVSHMEM_HCA_LIST+x}"
+ cx_apply_network_profile 4 mnnvl
+ test -z "${NCCL_NET+x}${NCCL_IB_HCA+x}${NVSHMEM_HCA_LIST+x}"
+ export CX_IB_GID_INDEX=3 CX_RDMA_SERVICE_LEVEL=2
+ cx_apply_network_profile 2 nvlink-rdma
+ test "$NCCL_SOCKET_IFNAME:$GLOO_SOCKET_IFNAME:$UCCL_SOCKET_IFNAME" = ib0:ib0:ib0
+ test "$NCCL_NET:$NCCL_IB_HCA" = 'IB:=mlx5_0:1,mlx5_1:1'
+ test "$NVSHMEM_HCA_LIST" = mlx5_0:1,mlx5_1:1
+ test "$MORI_RDMA_DEVICES:$EP_NIC_NAME" = mlx5_0,mlx5_1:mlx5_0
+ test "$NCCL_IB_GID_INDEX:$NCCL_IB_SL" = 3:2
+ test "$NVSHMEM_IB_ENABLE_IBGDA:$NVSHMEM_IBGDA_NIC_HANDLER" = 1:gpu
+ '''
+ subprocess.run(
+ ["bash", "-c", command, "_", str(ROOT / "runtime" / "common.sh")],
+ check=True,
+ env={**os.environ, "COLLECTIVEX_OPERATOR_CONFIG": "/dev/null"},
+ )
+
+ def test_network_profile_validation_is_private_and_all_node(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary).resolve()
+ binary = root / "srun"
+ arguments = root / "arguments"
+ script = root / "script"
+ binary.write_text(
+ "#!/usr/bin/env bash\n"
+ "printf '%s\\n' \"$@\" > \"$CAPTURE_ARGS\"\n"
+ "cat > \"$CAPTURE_SCRIPT\"\n"
+ "exit \"${SRUN_RC:-0}\"\n"
+ )
+ binary.chmod(0o700)
+ command = (
+ 'source "$1"; export COLLECTIVEX_EXECUTION_ID="network-test-$$"; '
+ "trap 'cx_cleanup_private_logs 0' EXIT; "
+ 'cx_validate_network_profile_on_job 42 2 nvlink-rdma'
+ )
+ environment = {
+ **os.environ,
+ "PATH": f"{root}:{os.environ['PATH']}",
+ "COLLECTIVEX_OPERATOR_CONFIG": "/dev/null",
+ "CAPTURE_ARGS": str(arguments),
+ "CAPTURE_SCRIPT": str(script),
+ "CX_SOCKET_IFNAME": "privateif0",
+ "CX_RDMA_DEVICES": "privatehca0:1",
+ "CX_IB_GID_INDEX": "3",
+ }
+ result = subprocess.run(
+ ["bash", "-c", command, "_", str(ROOT / "runtime" / "common.sh")],
+ text=True,
+ capture_output=True,
+ env=environment,
+ )
+ self.assertEqual(result.returncode, 0, result.stderr)
+ invoked = arguments.read_text()
+ self.assertIn("--nodes=2", invoked)
+ self.assertIn("--ntasks=2", invoked)
+ self.assertIn("--input=all", invoked)
+ self.assertIn("CX_SOCKET_IFNAME,CX_RDMA_DEVICES,CX_IB_GID_INDEX", invoked)
+ self.assertIn('/sys/class/infiniband/$device/ports', script.read_text())
+ self.assertNotIn("privateif0", result.stdout + result.stderr)
+ self.assertNotIn("privatehca0", result.stdout + result.stderr)
+
+ failed = subprocess.run(
+ ["bash", "-c", command, "_", str(ROOT / "runtime" / "common.sh")],
+ text=True,
+ capture_output=True,
+ env={**environment, "SRUN_RC": "9"},
+ )
+ self.assertNotEqual(failed.returncode, 0)
+ self.assertNotIn("privateif0", failed.stdout + failed.stderr)
+ self.assertNotIn("privatehca0", failed.stdout + failed.stderr)
+
+ arguments.unlink()
+ subprocess.run(
+ [
+ "bash", "-c",
+ 'source "$1"; cx_validate_network_profile_on_job 42 1 nvlink',
+ "_", str(ROOT / "runtime" / "common.sh"),
+ ],
+ check=True,
+ env=environment,
+ )
+ self.assertFalse(arguments.exists())
+
+ def test_allocation_preflight_proves_shared_write_visibility(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary).resolve()
+ mount = root / "mount"
+ runtime = mount / "experimental" / "CollectiveX" / "runtime"
+ runtime.mkdir(parents=True)
+ (runtime / "run_in_container.sh").write_text("#!/bin/sh\n")
+ squash = root / "image.sqsh"
+ squash.write_bytes(b"squash")
+ binary = root / "bin"
+ binary.mkdir()
+ (binary / "unsquashfs").write_text("#!/bin/sh\nexit 0\n")
+ (binary / "unsquashfs").chmod(0o700)
+ (binary / "srun").write_text(
+ "#!/usr/bin/env bash\n"
+ "set -euo pipefail\n"
+ "case \" $* \" in *' --input=all '*) ;; *) exit 97 ;; esac\n"
+ "worker=\"$FAKE_ROOT/worker.sh\"\n"
+ "cat > \"$worker\"\n"
+ "args=(\"$@\")\n"
+ "start=0\n"
+ "for ((i=0; i<${#args[@]}; i++)); do\n"
+ " [ \"${args[$i]}\" != -- ] || start=$((i + 1))\n"
+ "done\n"
+ "[ \"$start\" -gt 0 ]\n"
+ "worker_args=(\"${args[@]:$start}\")\n"
+ "probe=\"${worker_args[4]}\"\n"
+ "case \"${FAKE_MODE:-success}\" in\n"
+ " missing-source) rm -f -- \"$probe/source\" ;;\n"
+ " readonly) chmod 500 \"$probe\" ;;\n"
+ "esac\n"
+ "for ((node=0; node None:
+ runtime = (ROOT / "runtime" / "run_in_container.sh").read_text()
+ probe = runtime[runtime.index("cx_probe_deepep()"):
+ runtime.index("cx_activate_deepep_v2()")]
+ self.assertIn('expected_version="1.2.1"', probe)
+ self.assertIn('expected_version="1.1.0+814e508"', probe)
+ self.assertNotIn("pip install", probe)
+ self.assertNotIn("cx_fetch_revision", probe)
+ self.assertIn("Path(deep_ep.__file__).resolve() in recorded_files", probe)
+ self.assertIn("Path(buffer_module.__file__).resolve() in recorded_files", probe)
+
+ harness = (HERE / "ep_harness.py").read_text()
+ pass_one = harness[harness.index("# ---- Pass 1"):
+ harness.index("# ---- Pass 2")]
+ self.assertLess(
+ pass_one.index("input_snapshots[T] ="),
+ pass_one.index("oracle = _run_expert_oracle"),
+ )
+ self.assertIn("pre_input_unchanged", pass_one)
+ self.assertIn("hh = prep()\n torch.cuda.synchronize()", harness)
+
+ def test_squash_imports_are_reproducible_and_use_a_fresh_cache_key(self) -> None:
+ common = (ROOT / "runtime" / "common.sh").read_text()
+ amd = (ROOT / "launchers" / "launch_mi-amds.sh").read_text()
+ self.assertIn('CX_SQUASH_FORMAT_VERSION="repro-v1"', common)
+ self.assertIn("SOURCE_DATE_EPOCH=\"$CX_SQUASH_SOURCE_DATE_EPOCH\"", common)
+ self.assertIn("${COLLECTIVEX_IMAGE_DIGEST#sha256:}", common)
+ self.assertIn("cx_ensure_squash_on_job", amd)
+ self.assertIn('"${CX_LOCK_DIR:-}"', amd)
+ self.assertNotIn('"${CX_LOCK_DIR:-/tmp}"', amd)
+ self.assertIn('[ -n "$lock_dir" ] || lock_dir="$squash_dir/.locks"', common)
+ self.assertGreaterEqual(common.count("--chdir=/tmp"), 2)
+ self.assertGreaterEqual(amd.count("--chdir=/tmp"), 2)
+ self.assertIn('ENROOT_CACHE_PATH="$compute_home/enroot-cache"', common)
+ self.assertIn('ENROOT_RUNTIME_PATH="$compute_home/enroot-run"', common)
+ self.assertEqual(common.count('cx_reverify_registry_image "$image"'), 2)
+ result = subprocess.run(
+ [
+ "bash",
+ "-c",
+ f'source "{ROOT / "runtime" / "common.sh"}"; '
+ 'COLLECTIVEX_IMAGE_DIGEST="sha256:$(printf b%.0s {1..64})"; '
+ 'CX_IMAGE_PLATFORM=linux/amd64; cx_squash_path /cache repo/image:tag; '
+ 'printf "\\n"; CX_IMAGE_PLATFORM=linux/arm64; '
+ 'cx_squash_path /cache repo/image:tag',
+ ],
+ text=True,
+ capture_output=True,
+ )
+ self.assertEqual(result.returncode, 0, result.stderr)
+ digest = "b" * 64
+ self.assertEqual(
+ result.stdout.splitlines(),
+ [
+ f"/cache/repro-v1_{digest}_repo_image_tag.sqsh",
+ f"/cache/repro-v1_linux_arm64_{digest}_repo_image_tag.sqsh",
+ ],
+ )
+
+ def test_launchers_preserve_platform_specific_runtime_requirements(self) -> None:
+ single = (ROOT / "launchers" / "launch_single-slurm.sh").read_text()
+ gb = (ROOT / "launchers" / "launch_gb-nv.sh").read_text()
+ amd = (ROOT / "launchers" / "launch_mi-amds.sh").read_text()
+ common = (ROOT / "runtime" / "common.sh").read_text()
+ self.assertIn("ALLOC_EXTRA=(--mem=0)", single)
+ self.assertIn("ALLOC_EXTRA=(-N 1 --mem=0)", single)
+ self.assertIn("SRUN_EXTRA=(--mpi=none --container-remap-root)", single)
+ self.assertIn("CX_ENROOT_LOCAL_IMPORT=1", single)
+ self.assertIn('PRODUCT="${CX_SHARD_SKU:-${CX_GB_PRODUCT:-', gb)
+ self.assertIn("cx_ensure_squash_on_job", gb)
+ self.assertIn("--mem=0 --cpus-per-task=35", gb)
+ self.assertIn("--container-writable", gb)
+ self.assertIn("--container-remap-root", gb)
+ workload_stage = common[
+ common.index("workload_args=("):
+ common.index("workload_log=", common.index("workload_args=("))
+ ]
+ self.assertNotIn("--workload", workload_stage)
+ self.assertIn("mi325x) CPUS_PER_TASK=256", amd)
+ self.assertIn("/dev/kfd:/dev/kfd,/dev/dri:/dev/dri", amd)
+ self.assertIn("--container-writable --container-remap-root", amd)
+ self.assertIn(
+ "CX_DISTRIBUTED_CONTAINER_ARGS=(--container-writable --container-remap-root)",
+ amd,
+ )
+ collect = common[common.index("cx_collect_results()"):
+ common.index("cx_cleanup_stage()")]
+ cleanup = common[common.index("cx_launcher_cleanup()"):
+ common.index("cx_install_launcher_fail_safe()")]
+ self.assertNotIn("cx_cleanup_stage", collect)
+ self.assertLess(cleanup.index("cx_cancel_job"), cleanup.index("cx_cleanup_stage"))
+ runtime = (ROOT / "runtime" / "run_in_container.sh").read_text()
+ self.assertIn('distribution.read_text("direct_url.json")', runtime)
+ self.assertIn("6548e9c504a12b2471af4b7f4d9546321210a57a456b5dc55bd4a8dad0f932ac", runtime)
+ self.assertIn("2671cff7baf8c2c214ff4bac721af875d513130670bec57601998bd1aae82882", runtime)
+
+ def test_deferred_backend_provenance_resolves_before_measurement(self) -> None:
+ harness = (ROOT / "tests" / "ep_harness.py").read_text()
+ conditioning = harness.index("for wt in conditioning_ladder")
+ provenance = harness.index("# Setup may materialize deferred provenance")
+ measurement = harness.index("# ---- Pass 1: build each deterministic problem")
+ self.assertLess(conditioning, provenance)
+ self.assertLess(provenance, measurement)
+
+ def test_backend_specific_routing_contracts_are_explicit(self) -> None:
+ hybrid = (ROOT / "tests" / "ep_deepep_hybrid.py").read_text()
+ self.assertIn("self.domain_rank = int(self.buffer.local_rank)", hybrid)
+ self.assertIn(
+ "probability_columns = self.domain_rank * self.local_experts + local_expert_ids",
+ hybrid,
+ )
+ self.assertIn("h.recv_probs[:count][rows, probability_columns]", hybrid)
+
+ mori = (ROOT / "tests" / "ep_mori.py").read_text()
+ self.assertIn("topk_idx=indices", mori)
+ self.assertIn("indices=indices", mori)
+ self.assertIn(
+ "combine_indices = p.indices if self._async_ll else h.dispatch_indices",
+ mori,
+ )
+ self.assertIn("h.combine_input,\n None,\n combine_indices", mori)
+ self.assertIn('"use_external_inp_buf": self._external_input', mori)
+ self.assertIn("self.block_num = self._block_target = 64", mori)
+ self.assertIn('config_kwargs["block_num"] = self.block_num', mori)
+ self.assertIn(
+ 'config_kwargs["warp_num_per_block"] = self.dispatch_warps', mori
+ )
+ self.assertIn("count > tensor.size(0)", mori)
+ self.assertIn("return combined[:p.T]", mori)
+ self.assertNotIn("return combined\n", mori)
+ self.assertIn(
+ "raw_expert_ids < local_start + experts_per_rank",
+ mori,
+ )
+ self.assertNotIn("MoRI returned a non-local expert", mori)
+ harness = (ROOT / "tests" / "ep_harness.py").read_text()
+ self.assertIn("problem.recv_tokens = backend.recv_tokens(handle)", harness)
+
+ def test_mori_masks_global_topk_metadata_to_the_local_rank(self) -> None:
+ path = HERE / "ep_mori.py"
+ tree = ast.parse(path.read_text(), str(path))
+ helper = next(
+ node
+ for node in tree.body
+ if isinstance(node, ast.FunctionDef) and node.name == "_project_local_metadata"
+ )
+ namespace: dict[str, object] = {}
+ exec(compile(ast.Module(body=[helper], type_ignores=[]), str(path), "exec"), namespace)
+ raw_ids = np.array([[0, 32, 63, -1], [64, 95, 7, 96]], dtype=np.int64)
+ raw_weights = np.arange(8, dtype=np.float32).reshape(2, 4)
+ torch_module = types.SimpleNamespace(
+ where=np.where,
+ full_like=np.full_like,
+ zeros_like=np.zeros_like,
+ )
+ ids, weights, local_ids = namespace["_project_local_metadata"](
+ torch_module, raw_ids, raw_weights, 1, 32
+ )
+ np.testing.assert_array_equal(
+ ids,
+ np.array([[-1, 32, 63, -1], [-1, -1, -1, -1]], dtype=np.int64),
+ )
+ np.testing.assert_array_equal(
+ weights,
+ np.array([[0, 1, 2, 0], [0, 0, 0, 0]], dtype=np.float32),
+ )
+ counts = np.bincount(local_ids, minlength=32)
+ self.assertEqual((counts[0], counts[31], int(counts.sum())), (1, 1, 2))
+ commit_helper = next(
+ node for node in tree.body
+ if isinstance(node, ast.FunctionDef) and node.name == "_mori_source_commit"
+ )
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary)
+ module = root / "python" / "mori" / "__init__.py"
+ module.parent.mkdir(parents=True)
+ module.touch()
+ git = root / ".git"
+ git.mkdir()
+ (git / "HEAD").write_text("a" * 40 + "\n")
+ commit_namespace = {
+ "Path": Path,
+ "re": re,
+ "mori": types.SimpleNamespace(__file__=str(module)),
+ }
+ exec(
+ compile(ast.Module(body=[commit_helper], type_ignores=[]), str(path), "exec"),
+ commit_namespace,
+ )
+ self.assertEqual(commit_namespace["_mori_source_commit"](), "a" * 40)
+ (git / "HEAD").write_text("ref: refs/heads/main\n")
+ with self.assertRaisesRegex(RuntimeError, "detached commit"):
+ commit_namespace["_mori_source_commit"]()
+
+ profile = contracts.project_resource_profile(
+ {
+ "block_num": 64,
+ "device_cus": 304,
+ "kernel_type": "AsyncLL",
+ "tuned_source": "upstream-asyncll-64x8-external-input",
+ }
+ )
+ self.assertIsNone(profile["comm_units_kind"])
+ self.assertIsNone(profile["configured_units"])
+
+ def test_squash_identity_rehashes_instead_of_trusting_a_sidecar(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ image = Path(temporary) / "image.sqsh"
+ image.write_bytes(b"current squash bytes")
+ sidecar = Path(f"{image}.sha256")
+ sidecar.write_text("a" * 64)
+ os.utime(sidecar, (image.stat().st_mtime + 10, image.stat().st_mtime + 10))
+ result = subprocess.run(
+ [
+ "bash", "-c",
+ 'source "$1"; COLLECTIVEX_EXECUTION_ID="squash-hash-$$"; '
+ 'cx_export_squash_identity "$2"; cx_cleanup_private_logs 0; '
+ 'printf "%s" "$COLLECTIVEX_SQUASH_SHA256"',
+ "_", str(ROOT / "runtime" / "common.sh"), str(image),
+ ],
+ text=True,
+ capture_output=True,
+ )
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertEqual(result.stdout, hashlib.sha256(image.read_bytes()).hexdigest())
+
+ def _run_salloc_scenario(
+ self, salloc_body: str, squeue_body: str, *, cleanup: bool
+ ) -> dict[str, object]:
+ prefix = f"inferencex-collectivex-{os.getpid()}-1-"
+ with tempfile.TemporaryDirectory(prefix=prefix, dir="/tmp") as temporary:
+ root = Path(temporary)
+ command_dir = root / "bin"
+ repo = root / "repo"
+ command_dir.mkdir()
+ repo.mkdir()
+ paths = {
+ name: root / name
+ for name in ("arguments", "squeue-calls", "sleep-calls", "scancel-calls")
+ }
+ scripts = {
+ "salloc": (
+ "printf '%s\\n' \"$@\" > \"$CX_TEST_SALLOC_ARGUMENTS\"\n"
+ + salloc_body
+ ),
+ "squeue": (
+ "printf '%s\\n' \"$*\" >> \"$CX_TEST_SQUEUE_CALLS\"\n"
+ + squeue_body
+ ),
+ "sleep": "printf '%s\\n' \"$1\" >> \"$CX_TEST_SLEEP_CALLS\"\n",
+ "scancel": (
+ "printf '%s\\n' \"$*\" >> \"$CX_TEST_SCANCEL_CALLS\"\n"
+ ),
+ }
+ for name, body in scripts.items():
+ path = command_dir / name
+ path.write_text(f"#!/usr/bin/env bash\n{body}\n")
+ path.chmod(0o700)
+ execution_id = f"scheduler-{root.name}"
+ expected_name = "cx-" + hashlib.sha256(
+ execution_id.encode()
+ ).hexdigest()[:24]
+ command = r'''
+ source "$1"
+ JOB_ID=""
+ set +e
+ cx_salloc_jobid --partition=compute
+ run_rc=$?
+ set -e
+ printf '%s:%s:%s\n' \
+ "$run_rc" "$JOB_ID" "$CX_ALLOCATION_UNCERTAIN"
+ cx_cleanup_private_logs 0
+ if [ "$3" = cleanup ]; then
+ export CX_JOB_ROOT="$2" REPO_ROOT="$2/repo" MOUNT_SRC="$2/repo"
+ export COLLECTIVEX_CANONICAL_GHA=1
+ cx_write_cleanup_guard() {
+ rm -f -- "$CX_JOB_ROOT/cleanup-safe" "$CX_JOB_ROOT/cleanup-unsafe"
+ : > "$CX_JOB_ROOT/cleanup-$1"
+ }
+ unset CX_BENCH
+ cx_launcher_cleanup "$run_rc"
+ fi
+ exit "$run_rc"
+ '''
+ result = subprocess.run(
+ [
+ "bash", "-c", command, "_", str(ROOT / "runtime" / "common.sh"),
+ str(root), "cleanup" if cleanup else "no-cleanup",
+ ],
+ text=True,
+ capture_output=True,
+ env={
+ **os.environ,
+ "PATH": f"{command_dir}:{os.environ['PATH']}",
+ "COLLECTIVEX_EXECUTION_ID": execution_id,
+ "CX_TEST_SALLOC_ARGUMENTS": str(paths["arguments"]),
+ "CX_TEST_SQUEUE_CALLS": str(paths["squeue-calls"]),
+ "CX_TEST_SLEEP_CALLS": str(paths["sleep-calls"]),
+ "CX_TEST_SCANCEL_CALLS": str(paths["scancel-calls"]),
+ },
+ )
+ return {
+ "result": result,
+ "job_name": expected_name,
+ "arguments": paths["arguments"].read_text().splitlines(),
+ "squeue_calls": (
+ paths["squeue-calls"].read_text().splitlines()
+ if paths["squeue-calls"].exists() else []
+ ),
+ "sleep_calls": (
+ paths["sleep-calls"].read_text().splitlines()
+ if paths["sleep-calls"].exists() else []
+ ),
+ "scancel_calls": (
+ paths["scancel-calls"].read_text().splitlines()
+ if paths["scancel-calls"].exists() else []
+ ),
+ "cleanup_safe": (root / "cleanup-safe").is_file(),
+ "cleanup_unsafe": (root / "cleanup-unsafe").is_file(),
+ }
+
+ def test_salloc_job_id_parser_uses_the_portable_grant_message(self) -> None:
+ scenario = self._run_salloc_scenario(
+ "printf 'salloc: Granted job allocation 4242\\n' >&2",
+ "exit 2",
+ cleanup=False,
+ )
+ result = scenario["result"]
+ self.assertIsInstance(result, subprocess.CompletedProcess)
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertEqual(
+ result.stdout, "0:4242:0\n"
+ )
+ self.assertEqual(
+ scenario["arguments"],
+ [
+ "--partition=compute",
+ f"--job-name={scenario['job_name']}",
+ "--no-shell",
+ ],
+ )
+ self.assertEqual(scenario["squeue_calls"], [])
+
+ def test_salloc_verified_rejection_is_cleanup_safe(self) -> None:
+ scenario = self._run_salloc_scenario("exit 1", "exit 0", cleanup=True)
+ result = scenario["result"]
+ self.assertEqual(result.returncode, 1)
+ self.assertEqual(result.stdout, "1::0\n")
+ self.assertEqual(len(scenario["squeue_calls"]), 3)
+ scheduler_user = subprocess.check_output(["id", "-un"], text=True).strip()
+ self.assertTrue(all(
+ f"--name={scenario['job_name']}" in call
+ and f"--user={scheduler_user}" in call
+ for call in scenario["squeue_calls"]
+ ))
+ self.assertEqual(scenario["sleep_calls"], ["1", "2"])
+ self.assertTrue(scenario["cleanup_safe"])
+ self.assertFalse(scenario["cleanup_unsafe"])
+
+ def test_salloc_recovers_and_cancels_one_matching_allocation(self) -> None:
+ scenario = self._run_salloc_scenario(
+ "exit 1",
+ r'''
+ case " $* " in
+ *" --name="*) printf '5151\n' ;;
+ *" -j 5151 "*) exit 0 ;;
+ *) exit 2 ;;
+ esac
+ ''',
+ cleanup=True,
+ )
+ result = scenario["result"]
+ self.assertEqual(result.returncode, 1)
+ self.assertEqual(result.stdout, "1:5151:0\n")
+ self.assertEqual(scenario["scancel_calls"], ["5151"])
+ self.assertTrue(scenario["cleanup_safe"])
+ self.assertFalse(scenario["cleanup_unsafe"])
+
+ def test_salloc_ambiguous_lookup_remains_cleanup_unsafe(self) -> None:
+ scenario = self._run_salloc_scenario(
+ "exit 1", "printf '5151\\n5152\\n'", cleanup=True
+ )
+ result = scenario["result"]
+ self.assertEqual(result.returncode, 1)
+ self.assertEqual(result.stdout, "1::1\n")
+ self.assertEqual(scenario["scancel_calls"], [])
+ self.assertFalse(scenario["cleanup_safe"])
+ self.assertTrue(scenario["cleanup_unsafe"])
+
+ def test_salloc_query_failure_and_interruption_remain_cleanup_unsafe(self) -> None:
+ query_failure = self._run_salloc_scenario("exit 1", "exit 2", cleanup=True)
+ self.assertEqual(query_failure["result"].returncode, 1)
+ self.assertEqual(len(query_failure["squeue_calls"]), 1)
+ self.assertFalse(query_failure["cleanup_safe"])
+ self.assertTrue(query_failure["cleanup_unsafe"])
+
+ interrupted = self._run_salloc_scenario("exit 130", "exit 0", cleanup=True)
+ self.assertEqual(interrupted["result"].returncode, 1)
+ self.assertEqual(interrupted["squeue_calls"], [])
+ self.assertFalse(interrupted["cleanup_safe"])
+ self.assertTrue(interrupted["cleanup_unsafe"])
+
+ def test_allocation_cleanup_fails_closed_when_scheduler_queries_fail(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ directory = Path(temporary)
+ for name, body in {
+ "scancel": "exit 0",
+ "squeue": "exit 2",
+ "sleep": "exit 0",
+ }.items():
+ command = directory / name
+ command.write_text(f"#!/usr/bin/env bash\n{body}\n")
+ command.chmod(0o700)
+ result = subprocess.run(
+ [
+ "bash", "-c",
+ 'source "$1"; cx_cancel_job 4242',
+ "_", str(ROOT / "runtime" / "common.sh"),
+ ],
+ text=True,
+ capture_output=True,
+ env={**os.environ, "PATH": f"{directory}:{os.environ['PATH']}"},
+ )
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn("did not terminate", result.stderr)
+
+ workflow = (ROOT.parent.parent / ".github" / "workflows" / "collectivex-sweep.yml").read_text()
+ self.assertIn("cleanup-unsafe", workflow)
+ self.assertIn("cleanup-safe", workflow)
+ self.assertIn("Confirm allocation cleanup", workflow)
+ self.assertIn("Prepare pinned backend source archive", workflow)
+ self.assertIn("Install pinned backend source seed", workflow)
+ self.assertIn("CX_BACKEND_SOURCE_SEED_ROOT", workflow)
+ self.assertIn("steps.gen.outputs.source_backends", workflow)
+ self.assertIn('python3 "$destination/source_archive.py"', workflow)
+ artifact_validation = workflow[workflow.index("- name: Validate shard artifact safety"):]
+ self.assertIn("steps.allocation_cleanup.outcome == 'success'", artifact_validation)
+ sweep_workflow = workflow[workflow.index(" sweep:"):]
+ self.assertNotIn("GITHUB_WORKSPACE", sweep_workflow)
+ self.assertNotIn("RUNNER_WORKSPACE", sweep_workflow)
+ self.assertIn('CX_SOURCE_ROOT: /tmp/inferencex-collectivex-', sweep_workflow)
+ source_step = sweep_workflow[:sweep_workflow.index("- uses: actions/download-artifact")]
+ self.assertNotIn("unsafe_guards=", source_step)
+ self.assertIn("cutoff = time.time() - 86400", source_step)
+ self.assertIn("stat.S_IMODE(metadata.st_mode) != 0o700", source_step)
+ self.assertIn('for marker_name in ("cleanup-safe", "cleanup-unsafe")', source_step)
+ self.assertIn("stat.S_IMODE(marker.st_mode) == 0o600", source_step)
+ self.assertIn("shutil.rmtree(entry.path)", source_step)
+ self.assertLess(
+ source_step.index('rev-parse HEAD'),
+ source_step.index("echo 'prepared=true'"),
+ )
+ upload = workflow[workflow.index("- name: Stage shard artifact"):]
+ self.assertIn("id: stage_artifact", upload)
+ self.assertIn("id: upload_artifact", upload)
+ self.assertIn("steps.stage_artifact.outcome == 'success'", upload)
+ cleanup = workflow[workflow.index("- name: Cleanup isolated workspace"):]
+ for step in (
+ "sweep_shard", "allocation_cleanup", "artifact_safety",
+ "delivery_contracts", "stage_artifact", "upload_artifact",
+ ):
+ self.assertIn(f"steps.{step}.outcome", cleanup)
+ self.assertLess(
+ cleanup.index('cleanup-safe" ]'),
+ cleanup.index('rm -rf -- "$CX_JOB_ROOT"'),
+ )
+
+ def test_v1_publication_requires_explicit_release_markers(self) -> None:
+ workflows = ROOT.parent.parent / ".github" / "workflows"
+ sweep = (workflows / "collectivex-sweep.yml").read_text()
+ publish = (workflows / "collectivex-publish.yml").read_text()
+
+ self.assertIn("release_tag:", sweep)
+ self.assertIn("default: unversioned", sweep)
+ self.assertIn("options: [unversioned, v1]", sweep)
+ self.assertIn("inputs.release_tag == 'v1'", sweep)
+ self.assertIn("collectivex.release-tag.v1", sweep)
+ self.assertIn("V1 release tag requires the locked full matrix", sweep)
+ self.assertIn("EXPECTED_MATRIX_SHA256", sweep)
+ self.assertIn("cxrelease-v1-${{ github.run_id }}-${{ github.run_attempt }}", sweep)
+
+ self.assertIn("run_ids must contain exactly three IDs", publish)
+ self.assertIn("source runs do not share one source SHA", publish)
+ self.assertIn("cxrelease-v1-$run_id-$attempt/release.json", publish)
+ self.assertIn("run $run_id is not tagged for V1 publication", publish)
+ self.assertIn("collectivex.release-tag.v1", publish)
+ self.assertIn("ref: ${{ steps.runs.outputs.source_sha }}", publish)
+ self.assertIn("cxpublication-v1-${{ github.run_id }}-${{ github.run_attempt }}", publish)
+ self.assertIn("retention-days: 90", publish)
+ self.assertNotIn("workflow_run:", publish)
+
+ def test_source_archive_preserves_only_contained_leaf_symlinks(self) -> None:
+ selected = "deepep-hybrid-pinned"
+ other = "deepep-v2-pinned"
+
+ def directory(name: str) -> tarfile.TarInfo:
+ member = tarfile.TarInfo(name)
+ member.type = tarfile.DIRTYPE
+ member.mode = 0o755
+ return member
+
+ def regular(
+ name: str, payload: bytes, mode: int = 0o644
+ ) -> tuple[tarfile.TarInfo, io.BytesIO]:
+ member = tarfile.TarInfo(name)
+ member.size = len(payload)
+ member.mode = mode
+ return member, io.BytesIO(payload)
+
+ def symbolic(name: str, target: str) -> tarfile.TarInfo:
+ member = tarfile.TarInfo(name)
+ member.type = tarfile.SYMTYPE
+ member.linkname = target
+ member.mode = 0o777
+ return member
+
+ def write_archive(path: Path, extras: list[tarfile.TarInfo] | None = None) -> None:
+ root = f".cx_sources/{selected}"
+ with tarfile.open(path, "w") as archive:
+ for name in (
+ ".cx_sources", root, f"{root}/third-party",
+ f"{root}/third-party/nccl", f"{root}/third-party/nccl/pkg",
+ f"{root}/third-party/nccl/pkg/debian",
+ f".cx_sources/{other}",
+ ):
+ archive.addfile(directory(name))
+ member, stream = regular(
+ f"{root}/third-party/nccl/LICENSE.txt", b"license\n"
+ )
+ archive.addfile(member, stream)
+ member, stream = regular(f".cx_sources/{other}/sentinel", b"other\n")
+ archive.addfile(member, stream)
+ member, stream = regular(f"{root}/group-executable", b"exec\n", 0o010)
+ archive.addfile(member, stream)
+ archive.addfile(symbolic(
+ f"{root}/third-party/nccl/pkg/debian/copyright",
+ "../../LICENSE.txt",
+ ))
+ for member in extras or []:
+ archive.addfile(member)
+ path.chmod(0o600)
+
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary).resolve()
+ archive = root / "source.tar"
+ destination = root / "destination"
+ destination.mkdir(mode=0o700)
+ write_archive(archive)
+ source_archive.extract_source_archive(archive, destination, selected)
+ link = (
+ destination / ".cx_sources" / selected / "third-party" / "nccl"
+ / "pkg" / "debian" / "copyright"
+ )
+ self.assertTrue(link.is_symlink())
+ self.assertEqual(os.readlink(link), "../../LICENSE.txt")
+ self.assertEqual(link.read_text(), "license\n")
+ self.assertFalse((destination / ".cx_sources" / other).exists())
+ extracted = destination / ".cx_sources" / selected
+ self.assertEqual(
+ stat.S_IMODE((extracted / "group-executable").stat().st_mode), 0o700
+ )
+ self.assertEqual(
+ stat.S_IMODE(
+ (extracted / "third-party" / "nccl" / "LICENSE.txt").stat().st_mode
+ ),
+ 0o600,
+ )
+
+ invalid: dict[str, list[tarfile.TarInfo]] = {
+ "absolute member": [directory("/outside")],
+ "traversal member": [directory(".cx_sources/../outside")],
+ "duplicate member": [directory(f".cx_sources/{selected}")],
+ "absolute link": [symbolic(f".cx_sources/{selected}/absolute", "/tmp/x")],
+ "escaping link": [symbolic(f".cx_sources/{selected}/escape", "../x")],
+ "cross-root link": [
+ symbolic(f".cx_sources/{selected}/cross", f"../{other}/sentinel")
+ ],
+ "missing target": [symbolic(f".cx_sources/{selected}/missing", "none")],
+ }
+ hardlink = tarfile.TarInfo(f".cx_sources/{selected}/hard")
+ hardlink.type = tarfile.LNKTYPE
+ hardlink.linkname = f".cx_sources/{selected}/third-party/nccl/LICENSE.txt"
+ invalid["hardlink"] = [hardlink]
+ fifo = tarfile.TarInfo(f".cx_sources/{selected}/fifo")
+ fifo.type = tarfile.FIFOTYPE
+ invalid["fifo"] = [fifo]
+ character = tarfile.TarInfo(f".cx_sources/{selected}/character")
+ character.type = tarfile.CHRTYPE
+ invalid["character device"] = [character]
+ block = tarfile.TarInfo(f".cx_sources/{selected}/block")
+ block.type = tarfile.BLKTYPE
+ invalid["block device"] = [block]
+ unknown = tarfile.TarInfo(f".cx_sources/{selected}/unknown")
+ unknown.type = b"Z"
+ invalid["unknown type"] = [unknown]
+ invalid["unsafe unselected root"] = [
+ symbolic(f".cx_sources/{other}/escape", f"../{selected}/group-executable")
+ ]
+ chain_target = symbolic(
+ f".cx_sources/{selected}/chain-target", "third-party/nccl/LICENSE.txt"
+ )
+ invalid["symlink chain"] = [
+ chain_target, symbolic(f".cx_sources/{selected}/chain", "chain-target")
+ ]
+ linked_child = tarfile.TarInfo(f".cx_sources/{selected}/linked-file/child")
+ invalid["symlink parent"] = [
+ symbolic(
+ f".cx_sources/{selected}/linked-file",
+ "third-party/nccl/LICENSE.txt",
+ ),
+ linked_child,
+ ]
+ for label, extras in invalid.items():
+ with self.subTest(label=label), tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary).resolve()
+ archive = root / "source.tar"
+ destination = root / "destination"
+ destination.mkdir(mode=0o700)
+ write_archive(archive, extras)
+ with self.assertRaises(source_archive.SourceArchiveError):
+ source_archive.extract_source_archive(archive, destination, selected)
+ self.assertFalse((destination / ".cx_sources").exists())
+
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary).resolve()
+ archive = root / "source.tar"
+ destination = root / "destination"
+ destination.mkdir(mode=0o700)
+ existing = destination / ".cx_sources"
+ existing.mkdir(mode=0o700)
+ marker = existing / "marker"
+ marker.write_text("existing\n")
+ write_archive(archive)
+ with self.assertRaises(source_archive.SourceArchiveError):
+ source_archive.extract_source_archive(archive, destination, selected)
+ self.assertEqual(marker.read_text(), "existing\n")
+
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary).resolve()
+ archive = root / "source.tar"
+ write_archive(archive)
+ real_destination = root / "real-destination"
+ real_destination.mkdir(mode=0o700)
+ linked_destination = root / "linked-destination"
+ linked_destination.symlink_to(real_destination, target_is_directory=True)
+ with self.assertRaises((OSError, source_archive.SourceArchiveError)):
+ source_archive.extract_source_archive(archive, linked_destination, selected)
+ self.assertFalse((real_destination / ".cx_sources").exists())
+
+ unsafe_destination = root / "unsafe-destination"
+ unsafe_destination.mkdir(mode=0o700)
+ unsafe_destination.chmod(0o755)
+ with self.assertRaises(source_archive.SourceArchiveError):
+ source_archive.extract_source_archive(archive, unsafe_destination, selected)
+ self.assertFalse((unsafe_destination / ".cx_sources").exists())
+
+ for limit, value in (
+ ("MAX_ARCHIVE_MEMBERS", 1),
+ ("MAX_MEMBER_BYTES", 1),
+ ("MAX_EXPANDED_BYTES", 1),
+ ("MAX_ARCHIVE_BYTES", 1),
+ ("MAX_ARCHIVE_HEADERS", 1),
+ ):
+ with self.subTest(limit=limit), tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary).resolve()
+ archive = root / "source.tar"
+ destination = root / "destination"
+ destination.mkdir(mode=0o700)
+ write_archive(archive)
+ with mock.patch.object(source_archive, limit, value):
+ with self.assertRaises(source_archive.SourceArchiveError):
+ source_archive.extract_source_archive(archive, destination, selected)
+ self.assertFalse((destination / ".cx_sources").exists())
+
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary).resolve()
+ archive = root / "source.tar"
+ destination = root / "destination"
+ destination.mkdir(mode=0o700)
+ write_archive(archive)
+ long_name = f".cx_sources/{selected}/long-name-result\0".encode()
+ with tarfile.open(archive, "a") as handle:
+ for _ in range(3):
+ extension = tarfile.TarInfo("././@LongLink")
+ extension.type = tarfile.GNUTYPE_LONGNAME
+ extension.size = len(long_name)
+ handle.addfile(extension, io.BytesIO(long_name))
+ member, stream = regular("placeholder", b"payload\n")
+ handle.addfile(member, stream)
+ archive.chmod(0o600)
+ for limit, value in (
+ ("MAX_EXTENSION_CHAIN", 1),
+ ("MAX_EXTENSION_MEMBER_BYTES", 1),
+ ("MAX_EXTENSION_BYTES", len(long_name) * 2),
+ ):
+ with self.subTest(limit=limit), mock.patch.object(
+ source_archive, limit, value
+ ):
+ with self.assertRaises(source_archive.SourceArchiveError):
+ source_archive.extract_source_archive(
+ archive, destination, selected
+ )
+ self.assertFalse((destination / ".cx_sources").exists())
+
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary).resolve()
+ archive = root / "source.tar"
+ destination = root / "destination"
+ destination.mkdir(mode=0o700)
+ write_archive(archive)
+ with tarfile.open(archive, "a", format=tarfile.PAX_FORMAT) as handle:
+ member, stream = regular(
+ f".cx_sources/{selected}/sparse-v1", b"1\n0\n1\n"
+ )
+ member.pax_headers = {
+ "GNU.sparse.major": "1",
+ "GNU.sparse.minor": "0",
+ "GNU.sparse.realsize": "1",
+ }
+ handle.addfile(member, stream)
+ archive.chmod(0o600)
+ with self.assertRaises(source_archive.SourceArchiveError):
+ source_archive.extract_source_archive(archive, destination, selected)
+ self.assertFalse((destination / ".cx_sources").exists())
+
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary).resolve()
+ archive = root / "source.tar"
+ destination = root / "destination"
+ destination.mkdir(mode=0o700)
+ write_archive(archive)
+ original_next = tarfile.TarFile.next
+
+ def sparse_next(handle: tarfile.TarFile) -> tarfile.TarInfo | None:
+ member = original_next(handle)
+ if member is not None and member.isfile():
+ member.sparse = [(0, 1)]
+ return member
+
+ with mock.patch.object(tarfile.TarFile, "next", sparse_next):
+ with self.assertRaises(source_archive.SourceArchiveError):
+ source_archive.extract_source_archive(archive, destination, selected)
+ self.assertFalse((destination / ".cx_sources").exists())
+
+ def test_runtime_identity_and_realized_placement_are_behavioral(self) -> None:
+ self.assertFalse(capability.runtime_identity_issues(
+ "mi325x", vendor="amd", arch="gfx942", machine="amd64",
+ device_name="AMD Instinct MI325X", device_count=8, world_size=8,
+ ))
+ self.assertTrue(capability.runtime_identity_issues(
+ "mi355x", vendor="amd", arch="gfx942", machine="amd64",
+ device_name="AMD Instinct MI325X", device_count=8, world_size=8,
+ ))
+ records = [("private-a", rank) for rank in range(4)] + [
+ ("private-b", rank) for rank in range(4)
+ ]
+ self.assertEqual(
+ run_ep._summarize_realized_placement(
+ records, expected_nodes=2, expected_gpus_per_node=4, expected_world_size=8
+ ),
+ {
+ "gpus_per_node": 4,
+ "nodes": 2,
+ "ranks_per_node": 4,
+ "unique_local_ranks": True,
+ "valid": True,
+ },
+ )
+ with self.assertRaises(ValueError):
+ run_ep._summarize_realized_placement(
+ records[:-1] + [("private-b", 2)],
+ expected_nodes=2,
+ expected_gpus_per_node=4,
+ expected_world_size=8,
+ )
+
+ def test_collective_version_and_rccl_fingerprint_are_normalized(self) -> None:
+ self.assertEqual(ep_harness.format_collective_version(23004), "2.30.4")
+ self.assertEqual(ep_harness.format_collective_version(21805), "2.18.5")
+ self.assertEqual(ep_harness.format_collective_version((2, 21, 5)), "2.21.5")
+
+ properties = types.SimpleNamespace(
+ multi_processor_count=304, total_memory=1024, warp_size=64
+ )
+ fake = types.SimpleNamespace(
+ __version__="2.9.0",
+ version=types.SimpleNamespace(cuda=None, hip="7.2"),
+ cuda=types.SimpleNamespace(
+ get_device_properties=lambda _device: properties,
+ get_device_name=lambda _device: "AMD Instinct MI325X",
+ nccl=types.SimpleNamespace(version=lambda: 21805),
+ ),
+ )
+ with mock.patch.object(
+ run_ep, "_loaded_collective_version", return_value="2.18.5"
+ ):
+ fingerprint = run_ep._runtime_fingerprint(
+ fake, "device", machine="amd64", vendor="amd", arch="gfx942"
+ )
+ self.assertEqual(fingerprint["collective_library"], {"kind": "rccl", "version": "2.18.5"})
+ self.assertEqual(fingerprint["accelerator_runtime"], {"kind": "hip", "version": "7.2"})
+
+ class FakeCollective:
+ @staticmethod
+ def ncclGetVersion(pointer) -> int:
+ pointer._obj.value = 23004
+ return 0
+
+ maps = "0-1 r-xp 0 00:00 0 /runtime/libnccl.so.2\n"
+ with (
+ mock.patch("builtins.open", return_value=io.StringIO(maps)),
+ mock.patch.object(run_ep.os.path, "isfile", return_value=True),
+ mock.patch.object(
+ run_ep.os.path, "realpath", return_value="/runtime/libnccl.so.2"
+ ),
+ mock.patch.object(run_ep.ctypes, "CDLL", return_value=FakeCollective()),
+ ):
+ self.assertEqual(run_ep._loaded_collective_version(), "2.30.4")
+
+ path = HERE / "ep_nccl.py"
+ tree = ast.parse(path.read_text(), str(path))
+ helper = next(
+ node for node in tree.body
+ if isinstance(node, ast.FunctionDef) and node.name == "_runtime_collective"
+ )
+ namespace = {"re": re}
+ exec(compile(ast.Module(body=[helper], type_ignores=[]), str(path), "exec"), namespace)
+ args = types.SimpleNamespace(
+ runtime_fingerprint={
+ "collective_library": {"kind": "nccl", "version": "2.30.4"}
+ }
+ )
+ cuda = types.SimpleNamespace(version=types.SimpleNamespace(hip=None))
+ self.assertEqual(namespace["_runtime_collective"](args, cuda), ("nccl", "2.30.4"))
+ args.runtime_fingerprint["collective_library"]["version"] = None
+ with self.assertRaisesRegex(RuntimeError, "runtime identity is unavailable"):
+ namespace["_runtime_collective"](args, cuda)
+ self.assertNotIn("torch.cuda.nccl.version", path.read_text())
+
+ def test_workloads_bind_generator_activation_and_trace(self) -> None:
+ args = ("uniform", 7168, 8, 256, 8, 64, 67)
+ first = workload.compute_workload_id(*args)
+ self.assertTrue(identity.is_typed_id(first, "workload"))
+ self.assertEqual(first, workload.compute_workload_id(*args))
+ self.assertNotEqual(first, workload.compute_workload_id(*args[:-1], 68))
+ self.assertNotEqual(
+ first,
+ workload.compute_workload_id(*args, trace_checksum="a" * 64),
+ )
+ _, _, manifest = workload.build_workload(8, 2, 4, "uniform", 4, 67, 2)
+ member, checksums, _, _ = workload.canonical_member(
+ "uniform", 8, 2, 4, 2, 2, 67
+ )
+ self.assertEqual(member, manifest["workload_id"])
+ self.assertEqual(checksums, manifest["checksums"])
+
+ def test_canonical_members_are_bound_to_each_scheduled_row(self) -> None:
+ case = {
+ "routing": "uniform", "hidden": 8, "topk": 2, "experts": 4, "ep": 2,
+ }
+ eplb_record = {
+ "enabled": False, "mapping_hash": None, "num_physical_experts": 4,
+ }
+
+ def expected(
+ *, tokens: int = 1, hidden: int = 8
+ ) -> tuple[str, dict[str, str], str]:
+ member, checksums, row_hash, _, _ = contracts._expected_canonical_trace(
+ "uniform", hidden, 2, 4, 4, 2, tokens, 67, False, 2048
+ )
+ return member, checksums, row_hash
+
+ member, checksums, row_hash = expected()
+ rows = [{"tokens_per_rank": 1, "routing": {"hash": row_hash}}]
+ proof = {
+ "manifest_checksums": {member: checksums},
+ "members": [member],
+ "workload_id": identity.workload_id({
+ "members": [{"checksums": checksums, "workload_id": member}]
+ }),
+ }
+ contracts._validate_canonical_workload(proof, case, rows, eplb_record)
+
+ def replace_member(document: dict, replacement: tuple[str, dict[str, str], str]) -> None:
+ replacement_id, replacement_checksums, _ = replacement
+ document["members"] = [replacement_id]
+ document["manifest_checksums"] = {replacement_id: replacement_checksums}
+ document["workload_id"] = identity.workload_id({
+ "members": [{
+ "checksums": replacement_checksums,
+ "workload_id": replacement_id,
+ }]
+ })
+
+ mutations = {
+ "wrong member token": lambda document, mutated_rows: replace_member(
+ document, expected(tokens=2)
+ ),
+ "wrong member dimensions": lambda document, mutated_rows: replace_member(
+ document, expected(hidden=16)
+ ),
+ "wrong member checksum": lambda document, mutated_rows: replace_member(
+ document,
+ (
+ member,
+ {**checksums, "topk_idx": "0" * 64},
+ row_hash,
+ ),
+ ),
+ "row hash unrelated to member": lambda document, mutated_rows: mutated_rows[0][
+ "routing"
+ ].update({"hash": "f" * 64}),
+ }
+ for label, mutate in mutations.items():
+ with self.subTest(label=label), self.assertRaises(contracts.ContractError):
+ bad_proof, bad_rows = copy.deepcopy(proof), copy.deepcopy(rows)
+ mutate(bad_proof, bad_rows)
+ contracts._validate_canonical_workload(
+ bad_proof, case, bad_rows, eplb_record
+ )
+
+ def test_eplb_row_hash_is_bound_to_the_frozen_remap(self) -> None:
+ case = {"routing": "zipf", "hidden": 8, "topk": 2, "experts": 4, "ep": 2}
+ physical = eplb.physical_count(4, 32, 2)
+ plan = contracts._expected_eplb_plan("zipf", 2, 4, physical, 2, 67, 2048)
+ eplb_record = {
+ "enabled": True,
+ "mapping_hash": eplb.mapping_hash(plan),
+ "num_physical_experts": physical,
+ }
+ member, checksums, row_hash, _, _ = contracts._expected_canonical_trace(
+ "zipf", 8, 2, 4, physical, 2, 1, 67, True, 2048
+ )
+ self.assertNotEqual(row_hash, checksums["trace"])
+ workload_proof = {
+ "manifest_checksums": {member: checksums},
+ "members": [member],
+ "workload_id": identity.workload_id({
+ "members": [{"checksums": checksums, "workload_id": member}]
+ }),
+ }
+ rows = [{"tokens_per_rank": 1, "routing": {"hash": row_hash}}]
+ contracts._validate_canonical_workload(workload_proof, case, rows, eplb_record)
+ with self.assertRaisesRegex(contracts.ContractError, "EPLB mapping"):
+ contracts._validate_canonical_workload(
+ workload_proof, case, rows, {**eplb_record, "mapping_hash": "0" * 64}
+ )
+
+ def test_oracle_pass_cannot_ignore_combined_value_failure(self) -> None:
+ oracle = {
+ "atol": ep_harness.ORACLE_ATOL,
+ "checks": {
+ "combine_values": True,
+ "counts": True,
+ "metadata": True,
+ "multiplicity": True,
+ "payload": True,
+ "source_set": True,
+ "weights": True,
+ },
+ "combine_weight_semantics": "unweighted-rank-sum",
+ "contract": ep_harness.ORACLE_CONTRACT,
+ "dispatch_sha256": "a" * 64,
+ "max_absolute_error": 0.0,
+ "max_elementwise_relative_error": 0.0,
+ "max_relative_error": 0.0,
+ "max_weight_error": 0.0,
+ "order_sha256": "b" * 64,
+ "ordering_contract": "stable-v1",
+ "passed": True,
+ "receive_count": 1,
+ "rtol": ep_harness.ORACLE_RTOL,
+ }
+ contracts._validate_oracle(oracle, "oracle")
+ weighted = copy.deepcopy(oracle)
+ weighted["combine_weight_semantics"] = "native-gate-weighted"
+ with self.assertRaisesRegex(contracts.ContractError, "differs from v1"):
+ contracts._validate_oracle(weighted, "oracle")
+ tampered = copy.deepcopy(oracle)
+ tampered["checks"]["combine_values"] = False
+ with self.assertRaises(contracts.ContractError):
+ contracts._validate_oracle(tampered, "oracle")
+
+ def test_oracle_stability_canonicalizes_native_receive_order(self) -> None:
+ source = (HERE / "ep_harness.py").read_text()
+ begin = source.index("canonical_order = torch.argsort")
+ canonical = source[begin:source.index("problem.recv_tokens = receive_count", begin)]
+ self.assertIn("canonical_sources", canonical)
+ self.assertIn("canonical_ids", canonical)
+ self.assertIn("canonical_weights", canonical)
+ self.assertNotIn("_tensor_sha256(source_ids", canonical)
+ mori = (HERE / "ep_mori.py").read_text()
+ self.assertIn('"inter-node-v1" if self._inter_node', mori)
+ self.assertIn('else "async-ll" if self._async_ll', mori)
+ backend = types.SimpleNamespace(name="mori", kernel_generation="async-ll")
+ self.assertEqual(ep_harness.kernel_generation(backend), "async-ll")
+ backend.kernel_generation = "inter-node-v1"
+ self.assertEqual(ep_harness.kernel_generation(backend), "inter-node-v1")
+
+ def test_terminal_fail_safe_fills_only_missing_shard_cases(self) -> None:
+ matrix = sweep_matrix.resolve_matrix(backends="all", max_cases=128)
+ shard = next(item for item in matrix["include"] if item["n"] >= 2)
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary)
+ matrix_path = root / "matrix.json"
+ control_path = root / "control.json"
+ out_dir = root / "results"
+ matrix_path.write_text(json.dumps(matrix))
+ control = sweep_matrix.extract_shard(
+ matrix_path, shard["id"], control_path,
+ sku=shard["sku"], backend=shard["backend"], nodes=shard["nodes"],
+ )
+ control["cases"] = control["cases"][:2]
+ control["n"] = 2
+ control_path.write_text(json.dumps(control))
+ first = {key: value for key, value in control["cases"][0].items() if key != "case_id"}
+ git_run = {
+ "artifact": "artifact", "job": "job", "ref": "collectivex",
+ "repo": "SemiAnalysisAI/InferenceX", "run_attempt": "1",
+ "run_id": "123", "source_sha": "a" * 40,
+ }
+ allocation = {
+ "artifact": "artifact", "execution_id": "execution", "job": "job",
+ "repo": "SemiAnalysisAI/InferenceX", "run_attempt": "1", "run_id": "123",
+ "runner": shard["sku"], "source_sha": "a" * 40,
+ }
+ out_dir.mkdir()
+ existing = contracts.make_terminal_document(
+ allocation_factors=allocation, attempt_ordinal=1, case=first,
+ case_factors={"case": first, "profile": identity.V1_CASE_PROFILE, "sku": shard["sku"]},
+ control_sha256=hashlib.sha256(control_path.read_bytes()).hexdigest(),
+ failure_mode="setup", generated_at="2026-07-04T00:00:00Z", git_run=git_run,
+ reason="launcher-setup-failed", return_code=7, source="runtime-emitter",
+ status="failed",
+ expected_case_id=control["cases"][0]["case_id"],
+ )
+ (out_dir / "existing.json").write_text(json.dumps(existing))
+ (out_dir / "partial.json").write_text(json.dumps({
+ "format": contracts.RAW_FORMAT,
+ "identity": {"case_id": control["cases"][1]["case_id"]},
+ "sample_artifact": {"path": "partial.samples.json"},
+ }))
+ (out_dir / "partial.samples.json").write_text("{broken")
+ environment = {
+ **os.environ,
+ "COLLECTIVEX_OPERATOR_CONFIG": "/dev/null",
+ "CX_SHARD_FILE": str(control_path),
+ "CX_SHARD_SKU": shard["sku"],
+ "CX_RUNNER": shard["sku"],
+ "CX_BENCH": shard["backend"],
+ "CX_NODES": str(shard["nodes"]),
+ "COLLECTIVEX_EXECUTION_ID": "execution",
+ "COLLECTIVEX_ARTIFACT_NAME": "artifact",
+ "GITHUB_JOB": "job", "GITHUB_REF_NAME": "collectivex",
+ "GITHUB_REPOSITORY": "SemiAnalysisAI/InferenceX",
+ "GITHUB_RUN_ATTEMPT": "1", "GITHUB_RUN_ID": "123",
+ "GITHUB_SHA": "a" * 40,
+ }
+ subprocess.run(
+ [
+ "bash", "-c",
+ 'source "$1"; cx_emit_setup_failures "$2" "$3" "$4" 7',
+ "_", str(ROOT / "runtime" / "common.sh"), str(ROOT),
+ str(out_dir), shard["backend"],
+ ],
+ check=True,
+ env=environment,
+ )
+ attempts = [contracts.strict_load(path) for path in out_dir.glob("*.json")]
+ self.assertEqual(len(attempts), 2)
+ self.assertEqual(
+ contracts.validate_attempt_paths([str(path) for path in out_dir.glob("*.json")]),
+ 2,
+ )
+ delivery = [str(path) for path in out_dir.glob("*.json")]
+ self.assertEqual(contracts.validate_delivery(delivery, str(control_path)), 2)
+ with self.assertRaises(contracts.ContractError):
+ contracts.validate_delivery(delivery[:1], str(control_path))
+ self.assertEqual(
+ {attempt["identity"]["case_id"] for attempt in attempts},
+ {case["case_id"] for case in control["cases"]},
+ )
+ self.assertTrue((out_dir / "partial.json.quarantine").is_file())
+ self.assertTrue((out_dir / "partial.samples.json.quarantine").is_file())
+
+ preallocation = root / "preallocation"
+ preallocation_results = preallocation / "experimental" / "CollectiveX" / "results"
+ preallocation_results.mkdir(parents=True)
+ failed = subprocess.run(
+ [
+ "bash", "-c",
+ 'source "$1"; REPO_ROOT="$2"; export REPO_ROOT; '
+ 'cx_install_launcher_fail_safe; cx_load_operator_config',
+ "_", str(ROOT / "runtime" / "common.sh"), str(preallocation),
+ ],
+ env={**environment, "COLLECTIVEX_OPERATOR_CONFIG_REQUIRED": "1"},
+ )
+ self.assertNotEqual(failed.returncode, 0)
+ preallocation_attempts = [
+ contracts.validate_terminal_document(contracts.strict_load(path))
+ for path in preallocation_results.glob("*.json")
+ ]
+ self.assertEqual(
+ {attempt["identity"]["case_id"] for attempt in preallocation_attempts},
+ {case["case_id"] for case in control["cases"]},
+ )
+
+ def test_runtime_identity_mismatch_is_failed_not_unsupported(self) -> None:
+ wrapper = next(
+ item for item in sweep_matrix.resolve_matrix()["requested_cases"]
+ if item["disposition"] == "runnable"
+ )
+ case = wrapper["case"]
+ environment = {
+ "CX_RUNNER": wrapper["sku"], "CX_CASE_ID": case["case_id"],
+ "CX_SUITE": case["suite"], "CX_WORKLOAD_NAME": case["workload"],
+ "CX_REQUIRED_PUBLICATION": case["required_publication"],
+ "CX_ROUTING": case["routing"], "CX_EPLB": "1" if case["eplb"] else "",
+ "CX_EP": str(case["ep"]), "CX_NGPUS": str(case["ep"]),
+ "CX_HIDDEN": str(case["hidden"]), "CX_TOPK": str(case["topk"]),
+ "CX_EXPERTS": str(case["experts"]), "CX_NODES": str(case["nodes"]),
+ "CX_GPUS_PER_NODE": str(case["gpus_per_node"]),
+ "CX_SCALE_UP_DOMAIN": str(case["scale_up_domain"]),
+ "CX_MODE": case["mode"], "CX_SCOPE": case["scope"],
+ "CX_TOPO": case["topology_class"], "CX_TRANSPORT": case["transport"],
+ "CX_SCALE_UP_TRANSPORT": case["scale_up_transport"],
+ "CX_SCALE_OUT_TRANSPORT": case["scale_out_transport"] or "",
+ "CX_TOKENS_LADDER": case["ladder"], "CX_CANONICAL": "1",
+ "CX_ITERS": "8", "CX_TRIALS": "64", "CX_WARMUP": "32",
+ "CX_SAMPLES_PER_POINT": "512", "GITHUB_RUN_ID": "123",
+ "GITHUB_RUN_ATTEMPT": "1", "GITHUB_REF_NAME": "collectivex",
+ "GITHUB_SHA": "a" * 40, "GITHUB_REPOSITORY": "SemiAnalysisAI/InferenceX",
+ "GITHUB_JOB": "sweep", "COLLECTIVEX_ARTIFACT_NAME": "artifact",
+ "COLLECTIVEX_EXECUTION_ID": "execution",
+ }
+ with mock.patch.dict(os.environ, environment, clear=False):
+ terminal = contracts.make_terminal_from_environment(
+ backend=case["backend"], phase=case["phase"], return_code=5
+ )
+ self.assertEqual(terminal["identity"]["case_id"], case["case_id"])
+ self.assertEqual(
+ terminal["outcome"],
+ {
+ "failure_mode": "runtime-identity",
+ "reason": "runtime-identity-mismatch",
+ "return_code": 5,
+ "status": "failed",
+ },
+ )
+ for mode, reason in contracts.RUNTIME_FAILURE_REASONS.items():
+ with self.subTest(mode=mode), mock.patch.dict(os.environ, environment, clear=False):
+ staged = contracts.make_terminal_from_environment(
+ backend=case["backend"], phase=case["phase"], return_code=1,
+ failure_mode=mode,
+ )
+ self.assertEqual(staged["outcome"]["reason"], reason)
+ mismatched = copy.deepcopy(staged)
+ mismatched["outcome"]["reason"] = "distributed-command-failed"
+ if reason == "distributed-command-failed":
+ mismatched["outcome"]["reason"] = "backend-setup-failed"
+ with self.assertRaisesRegex(
+ contracts.ContractError, "source and outcome are not registered"
+ ):
+ contracts.validate_terminal_document(mismatched)
+ with mock.patch.dict(os.environ, environment, clear=False):
+ with self.assertRaisesRegex(
+ contracts.ContractError, "runtime failure mode is not registered"
+ ) as raised:
+ contracts.make_terminal_from_environment(
+ backend=case["backend"], phase=case["phase"], return_code=1,
+ failure_mode="raw-private-error",
+ )
+ self.assertNotIn("raw-private-error", str(raised.exception))
+ with mock.patch.dict(os.environ, environment, clear=False):
+ generic = contracts.make_terminal_from_environment(
+ backend=case["backend"], phase=case["phase"], return_code=6,
+ )
+ self.assertEqual(
+ generic["outcome"],
+ {
+ "failure_mode": "execution",
+ "reason": "distributed-command-failed",
+ "return_code": 6,
+ "status": "failed",
+ },
+ )
+ manual_environment = {
+ "CX_RUNNER": "manual-runner",
+ "COLLECTIVEX_EXECUTION_ID": "manual-execution",
+ }
+ with mock.patch.dict(os.environ, manual_environment, clear=True):
+ manual = contracts.make_terminal_from_environment(
+ backend="nccl-ep", phase="decode", return_code=6,
+ )
+ self.assertIsNone(manual["provenance"]["git_run"])
+ self.assertEqual(
+ {
+ field: manual["case"][field]
+ for field in ("suite", "workload", "canonical", "required_publication")
+ },
+ {
+ "suite": "manual", "workload": "manual", "canonical": False,
+ "required_publication": "diagnostic",
+ },
+ )
+ self.assertEqual(
+ manual["identity"]["allocation_factors"],
+ {
+ "artifact": None, "execution_id": "manual-execution", "job": None,
+ "repo": None, "run_attempt": None, "run_id": None,
+ "runner": "manual-runner", "source_sha": None,
+ },
+ )
+ broken = copy.deepcopy(manual)
+ broken["identity"]["allocation_factors"]["artifact"] = "forged-artifact"
+ allocation_id = identity.allocation_id(
+ broken["identity"]["allocation_factors"]
+ )
+ broken["identity"]["allocation_id"] = allocation_id
+ broken["identity"]["attempt_id"] = identity.attempt_id(
+ allocation=allocation_id,
+ case=broken["identity"]["case_id"],
+ ordinal=broken["identity"]["attempt_ordinal"],
+ )
+ with self.assertRaisesRegex(
+ contracts.ContractError, "allocation factors differ"
+ ):
+ contracts.validate_terminal_document(broken)
+
+ def test_launchers_use_private_logs_and_allowlisted_failure_stages(self) -> None:
+ expected = {
+ "launch_single-slurm.sh": {
+ "setup", "registry-verification", "container-import", "container-hash",
+ "repository-stage", "scheduler-allocation", "container-launch",
+ "artifact-collection",
+ },
+ "launch_gb-nv.sh": {
+ "setup", "registry-verification", "container-import", "container-hash",
+ "repository-stage", "scheduler-allocation", "container-launch", "backend-setup",
+ "execution", "artifact-collection",
+ },
+ "launch_mi-amds.sh": {
+ "setup", "repository-stage", "registry-verification", "scheduler-allocation",
+ "container-import", "container-hash", "container-launch", "artifact-collection",
+ },
+ }
+ common = (ROOT / "runtime" / "common.sh").read_text()
+ for name, stages in expected.items():
+ launcher = (ROOT / "launchers" / name).read_text()
+ stage_source = launcher + common if name == "launch_gb-nv.sh" else launcher
+ self.assertNotIn("--export=ALL", launcher)
+ if name == "launch_gb-nv.sh":
+ self.assertIn("cx_run_distributed_shard", launcher)
+ else:
+ self.assertIn("cx_container_exports", launcher)
+ self.assertIn("collect_rc=0", launcher)
+ for stage in stages:
+ with self.subTest(launcher=name, stage=stage):
+ self.assertIn(f"cx_set_failure_stage {stage}", stage_source)
+ amd = (ROOT / "launchers" / "launch_mi-amds.sh").read_text()
+ self.assertIn("cx_ensure_squash_on_job", amd)
+ self.assertIn("cx_fail_stage container-hash", amd)
+ self.assertNotIn('cat "$import_log"', amd)
+ self.assertIn('bash -s -- "$sq" "$lock" "$image"', common)
+ self.assertIn("> \"$log\" 2>&1 <<'BASH'", common)
+ self.assertIn("cx_fail_stage container-import", common)
+ runtime = (ROOT / "runtime" / "run_in_container.sh").read_text()
+ export_start = common.index("\ncx_container_exports() {")
+ exports = common[export_start:common.index("\n}", export_start)]
+ export_names = {
+ name
+ for payload in re.findall(r"printf '%s' '([^']*)'", exports)
+ for name in payload.split(",") if name
+ }
+ for private_name in (
+ "COLLECTIVEX_OPERATOR_CONFIG", "GITHUB_TOKEN", "GITHUB_WORKSPACE", "HOME",
+ "CX_PARTITION", "CX_ACCOUNT", "CX_SQUASH_DIR", "CX_STAGE_DIR",
+ ):
+ self.assertNotIn(private_name, export_names)
+ self.assertIn("CX_BACKEND_CACHE_ROOT", export_names)
+ self.assertIn("CX_BACKEND_CACHE_SENTINEL_SHA256", export_names)
+ self.assertNotIn("CX_PREPARED_BACKEND_CACHE", export_names)
+ self.assertIn("MORI_COMMIT", export_names)
+ self.assertIn("cx_write_runtime_stage backend-setup", runtime)
+ self.assertIn("cx_write_runtime_stage execution", runtime)
+ distributed = common[common.index("cx_run_distributed_shard()") :]
+ self.assertIn("cx_private_log_path shard-summary", distributed)
+ self.assertIn("cx_fail_stage execution", distributed)
+ self.assertIn('cx_fail_stage execution "$runtime_log"', distributed)
+
+ def test_case_failure_diagnostic_precedes_normal_srun_footer(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ log = Path(temporary) / "runtime.log"
+ log.write_text(
+ "WARN: deepep decode run failed rc=1 (CX_RUN_TIMEOUT=900s)\n"
+ "SHARD done: 6/6 case(s) failed\n"
+ "srun: error: task exited 1\n"
+ )
+ result = subprocess.run(
+ [
+ "bash", "-c",
+ 'source "$1"; cx_fail_stage execution "$2"',
+ "_", str(ROOT / "runtime" / "common.sh"), str(log),
+ ],
+ text=True,
+ capture_output=True,
+ )
+ self.assertEqual(result.returncode, 1)
+ self.assertIn("diagnostic=benchmark-case-failure", result.stderr)
+
+ def test_non_timeout_failure_warning_is_classified_as_case_failure(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ log = Path(temporary) / "runtime.log"
+ log.write_text("WARN: deepep decode run failed rc=1\nsrun: task exited 1\n")
+ result = subprocess.run(
+ [
+ "bash", "-c",
+ 'source "$1"; cx_fail_stage execution "$2"',
+ "_", str(ROOT / "runtime" / "common.sh"), str(log),
+ ],
+ text=True,
+ capture_output=True,
+ )
+ self.assertEqual(result.returncode, 1)
+ self.assertNotIn("diagnostic=network-or-timeout", result.stderr)
+ self.assertIn("diagnostic=benchmark-case-failure", result.stderr)
+
+ def test_private_runtime_failure_signatures_override_case_footer(self) -> None:
+ signatures = {
+ "DeepEP V2 no-GIN run is outside one realized LSA domain":
+ "accelerator-topology",
+ "NCCL exception (/src/nccl.cu:111): 3": "accelerator-topology",
+ "NCCL exception (/src/nccl.cu:112): 3": "accelerator-topology",
+ "CUDA error: call requires newer driver": "accelerator-driver",
+ "NCCL failure in ncclCommWindowRegister": "nccl-device-api",
+ "Communicator does not support symmetric memory": "nccl-device-api",
+ "NCCL exception (/src/nccl.cu:106): 5": "nccl-device-api",
+ "NCCL exception (/src/nccl.cu:127): 5": "nccl-device-api",
+ "NCCL exception (/src/nccl.cu:128): 5": "nccl-device-api",
+ "NCCL exception (/src/nccl.cu:129): 5": "nccl-device-api",
+ "NCCL exception (/src/nccl.cu:135): 5": "nccl-device-api",
+ "NVCC compilation failed": "jit-toolchain",
+ "CUDA out of memory": "accelerator-memory",
+ "torch rendezvous timed out": "network-or-timeout",
+ }
+ with tempfile.TemporaryDirectory() as temporary:
+ log = Path(temporary) / "runtime.log"
+ for signature, diagnostic in signatures.items():
+ log.write_text(f"{signature}\nSHARD done: 6/6 case(s) failed\n")
+ result = subprocess.run(
+ [
+ "bash", "-c",
+ 'source "$1"; cx_fail_stage execution "$2"',
+ "_", str(ROOT / "runtime" / "common.sh"), str(log),
+ ],
+ text=True,
+ capture_output=True,
+ env={**os.environ, "CX_BENCH": "deepep-v2"},
+ )
+ self.assertEqual(result.returncode, 1)
+ self.assertIn(f"diagnostic={diagnostic}", result.stderr)
+
+ log.write_text(
+ "NCCL exception (/src/nccl.cu:106): 5\n"
+ "SHARD done: 6/6 case(s) failed\n"
+ )
+ result = subprocess.run(
+ [
+ "bash", "-c", 'source "$1"; cx_fail_stage execution "$2"',
+ "_", str(ROOT / "runtime" / "common.sh"), str(log),
+ ],
+ text=True, capture_output=True,
+ env={**os.environ, "CX_BENCH": "deepep"},
+ )
+ self.assertIn("diagnostic=benchmark-case-failure", result.stderr)
+
+ def test_runtime_stage_marker_distinguishes_launch_from_execution(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ mount = Path(temporary)
+ root = mount / "experimental" / "CollectiveX"
+ root.mkdir(parents=True)
+ command = r'''
+ set -euo pipefail
+ source "$1"
+ export COLLECTIVEX_EXECUTION_ID=test_1_shard CX_TS=test
+ cx_set_failure_stage container-launch
+ cx_prepare_runtime_marker "$2"
+ (cd "$2/experimental/CollectiveX"; cx_write_runtime_stage backend-setup)
+ cx_adopt_runtime_stage "$2"
+ test "$CX_FAILSAFE_MODE" = backend-setup
+ (cd "$2/experimental/CollectiveX"; cx_write_runtime_stage execution)
+ cx_adopt_runtime_stage "$2"
+ test "$CX_FAILSAFE_MODE" = execution
+ '''
+ subprocess.run(
+ ["bash", "-c", command, "_", str(ROOT / "runtime" / "common.sh"),
+ str(mount)],
+ check=True,
+ )
+
+ def test_canonical_gha_environment_is_locked_but_manual_overrides_survive(self) -> None:
+ common = ROOT / "runtime" / "common.sh"
+ command = r'''
+ set -euo pipefail
+ source "$1"
+ export COLLECTIVEX_CANONICAL_GHA=1 GITHUB_ACTIONS=true
+ export GITHUB_RUN_ID=123 GITHUB_RUN_ATTEMPT=1
+ export COLLECTIVEX_SOURCE_SHA=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+ export CX_SHARD_FILE=.shards/test.json CX_SHARD_SKU=mi325x
+ export CX_NODES=1 CX_GPUS_PER_NODE=8
+ export CX_IMAGE=untrusted CX_IMAGE_DIGEST=untrusted CX_NGPUS=99
+ export CX_NCCL_HOME=/untrusted CX_LOCK_DIR=/tmp CX_SQUASH_DIR=/shared/containers
+ export CX_STAGE_DIR=/private/stale-stage
+ export CX_MORI_KERNEL_TYPE=intranode MORI_ENABLE_SDMA=0
+ export NCCL_MNNVL_ENABLE=1 MC_FORCE_MNNVL=1 CX_DRYRUN=1
+ export CX_BACKEND_CACHE_ROOT=/untrusted CX_BACKEND_CACHE_SENTINEL_SHA256=bad
+ export CX_PREPARED_BACKEND_CACHE=/untrusted CX_BACKEND_SOURCE_ROOT=/untrusted
+ ! (cx_lock_canonical_gha_env mi325x)
+ export COLLECTIVEX_OPERATOR_CONFIG_LOADED=$$
+ export CX_STAGE_DIR="$GITHUB_WORKSPACE"
+ unset CX_LOCK_DIR
+ cx_lock_canonical_gha_env mi325x
+ test "$CX_IMAGE" = "$CX_IMAGE_AMD_MORI_MI325"
+ test "$CX_IMAGE_DIGEST" = "$CX_IMAGE_AMD_MORI_MI325_DIGEST"
+ test "$CX_NGPUS:$CX_SEED:$CX_RUN_TIMEOUT" = 8:67:1800
+ test "$CX_MORI_KERNEL_TYPE:$MORI_DISABLE_AUTO_XGMI:$MORI_ENABLE_SDMA" = asyncll:0:1
+ test "$MORI_COMMIT" = "$CX_MORI_COMMIT_MI325"
+ test "$MORI_APP_LOG_LEVEL:$MORI_SHMEM_LOG_LEVEL:$MORI_IO_LOG_LEVEL" = info:info:info
+ test "$CX_STAGE_DIR" = "$GITHUB_WORKSPACE"
+ test -z "${CX_NCCL_HOME+x}${CX_LOCK_DIR+x}${NCCL_MNNVL_ENABLE+x}${MC_FORCE_MNNVL+x}"
+ test -z "${CX_BACKEND_CACHE_ROOT+x}${CX_BACKEND_CACHE_SENTINEL_SHA256+x}"
+ test -z "${CX_PREPARED_BACKEND_CACHE+x}${CX_BACKEND_SOURCE_ROOT+x}"
+ test -z "${CX_DRYRUN+x}"
+
+ export CX_STAGE_DIR=/shared/gb-stage
+ export CX_SHARD_SKU=gb300 CX_NODES=2 CX_GPUS_PER_NODE=4
+ export CX_IMAGE=untrusted CX_NGPUS=1 CX_MORI_KERNEL_TYPE=untrusted
+ export MORI_ENABLE_SDMA=0 CX_NCCL_HOME=/untrusted CX_MASTER_PORT=1
+ cx_lock_canonical_gha_env gb300
+ test "$CX_IMAGE" = "$CX_IMAGE_MULTIARCH"
+ test "$CX_IMAGE_DIGEST" = "$CX_IMAGE_MULTIARCH_DIGEST"
+ test "$CX_NGPUS:$CX_SEED:$CX_RUN_TIMEOUT" = 8:67:900
+ test "$CX_NCCL_HOME:$CX_MASTER_PORT" = /usr:29551
+ test "$CX_STAGE_DIR" = /shared/gb-stage
+ test -z "${CX_MORI_KERNEL_TYPE+x}${MORI_ENABLE_SDMA+x}"
+
+ export COLLECTIVEX_OPERATOR_CONFIG_LOADED=$$
+ export CX_SHARD_SKU=mi355x CX_NODES=1 CX_GPUS_PER_NODE=8
+ export CX_LOCK_DIR=/validated/amd-locks CX_STAGE_DIR=/validated/amd-stage
+ cx_lock_canonical_gha_env mi355x
+ test "$CX_LOCK_DIR" = /validated/amd-locks
+ test "$CX_STAGE_DIR" = /validated/amd-stage
+ test "$MORI_COMMIT" = "$CX_MORI_COMMIT_MI355"
+
+ unset COLLECTIVEX_CANONICAL_GHA
+ unset COLLECTIVEX_OPERATOR_CONFIG_LOADED
+ CX_IMAGE=manual CX_IMAGE_DIGEST=manual CX_NGPUS=3
+ CX_MORI_KERNEL_TYPE=manual
+ cx_lock_canonical_gha_env mi355x
+ test "$CX_IMAGE:$CX_IMAGE_DIGEST:$CX_NGPUS:$CX_MORI_KERNEL_TYPE" = manual:manual:3:manual
+ '''
+ with tempfile.TemporaryDirectory(dir=Path.home()) as workspace:
+ Path(workspace).chmod(0o720)
+ subprocess.run(
+ ["bash", "-c", command, "_", str(common)],
+ check=True,
+ env={
+ **os.environ,
+ "COLLECTIVEX_OPERATOR_CONFIG": "/dev/null",
+ "GITHUB_WORKSPACE": workspace,
+ },
+ )
+ self.assertEqual(list(Path(workspace).iterdir()), [])
+
+ def test_canonical_amd_stage_uses_config_not_world_writable_workspace(self) -> None:
+ common = ROOT / "runtime" / "common.sh"
+ command = r'''
+ source "$1"
+ export COLLECTIVEX_CANONICAL_GHA=1 GITHUB_ACTIONS=true
+ export GITHUB_RUN_ID=123 GITHUB_RUN_ATTEMPT=1
+ export COLLECTIVEX_SOURCE_SHA=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+ export CX_SHARD_FILE=.shards/test.json CX_SHARD_SKU=mi325x
+ export CX_NODES=1 CX_GPUS_PER_NODE=8 CX_SQUASH_DIR=/shared/containers
+ export COLLECTIVEX_OPERATOR_CONFIG_LOADED=$$ CX_STAGE_DIR=/shared/amd-stage
+ cx_lock_canonical_gha_env mi325x
+ printf '%s' "$CX_STAGE_DIR"
+ '''
+ with tempfile.TemporaryDirectory(dir=Path.home()) as workspace:
+ Path(workspace).chmod(0o702)
+ result = subprocess.run(
+ ["bash", "-c", command, "_", str(common)],
+ text=True,
+ capture_output=True,
+ env={
+ **os.environ,
+ "COLLECTIVEX_OPERATOR_CONFIG": "/dev/null",
+ "GITHUB_WORKSPACE": workspace,
+ },
+ )
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertEqual(result.stdout, "/shared/amd-stage")
+ self.assertNotIn(workspace, result.stderr)
+
+ def test_canonical_amd_stage_uses_config_not_symlinked_workspace(self) -> None:
+ common = ROOT / "runtime" / "common.sh"
+ command = r'''
+ source "$1"
+ export COLLECTIVEX_CANONICAL_GHA=1 GITHUB_ACTIONS=true
+ export GITHUB_RUN_ID=123 GITHUB_RUN_ATTEMPT=1
+ export COLLECTIVEX_SOURCE_SHA=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+ export CX_SHARD_FILE=.shards/test.json CX_SHARD_SKU=mi325x
+ export CX_NODES=1 CX_GPUS_PER_NODE=8 CX_SQUASH_DIR=/shared/containers
+ export COLLECTIVEX_OPERATOR_CONFIG_LOADED=$$ CX_STAGE_DIR=/shared/amd-stage
+ cx_lock_canonical_gha_env mi325x
+ printf '%s' "$CX_STAGE_DIR"
+ '''
+ with tempfile.TemporaryDirectory(dir=Path.home()) as temporary:
+ root = Path(temporary)
+ real = root / "real"
+ real.mkdir()
+ link = root / "workspace"
+ link.symlink_to(real, target_is_directory=True)
+ result = subprocess.run(
+ ["bash", "-c", command, "_", str(common)],
+ text=True,
+ capture_output=True,
+ env={
+ **os.environ,
+ "COLLECTIVEX_OPERATOR_CONFIG": "/dev/null",
+ "GITHUB_WORKSPACE": str(link),
+ },
+ )
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertEqual(result.stdout, "/shared/amd-stage")
+ self.assertNotIn(str(root), result.stderr)
+
+ def test_image_selection_and_registry_verification_are_fail_closed(self) -> None:
+ common = ROOT / "runtime" / "common.sh"
+ command = r'''
+ source "$1"
+ test "$(cx_default_image mi325x)" = "$CX_IMAGE_AMD_MORI_MI325"
+ test "$(cx_default_image mi355x)" = "$CX_IMAGE_AMD_MORI"
+ pinned="sha256:$(printf 'a%.0s' {1..64})"
+ curl() {
+ case "$*" in
+ *auth.docker.io*) printf '{"token":"test"}' ;;
+ *) printf 'Docker-Content-Digest: %s\r\n' "$pinned" ;;
+ esac
+ }
+ test "$(cx_resolve_registry_digest ubuntu:latest)" = "$pinned"
+ test "$(cx_resolve_registry_digest docker.io/library/ubuntu:latest)" = "$pinned"
+ ! (cx_resolve_registry_digest "ubuntu@$pinned")
+ ! (cx_resolve_registry_digest ghcr.io/example/image:tag)
+ ! (cx_resolve_registry_digest 'ubuntu@sha256:bad')
+ curl() {
+ case "$*" in *auth.docker.io*) printf '{"token":"test"}';; esac
+ }
+ ! (cx_resolve_registry_digest ubuntu:latest)
+ cx_resolve_registry_digest() { printf '%s' "$CX_IMAGE_MULTIARCH_DIGEST"; }
+ cx_verify_registry_image "$CX_IMAGE_MULTIARCH"
+ test "$COLLECTIVEX_IMAGE_DIGEST_VERIFIED" = 1
+ test "$COLLECTIVEX_IMAGE_DIGEST" = "$CX_IMAGE_MULTIARCH_DIGEST"
+ cx_reverify_registry_image "$CX_IMAGE_MULTIARCH"
+ cx_resolve_registry_digest() { printf 'sha256:%064d' 0; }
+ ! (cx_reverify_registry_image "$CX_IMAGE_MULTIARCH")
+ ! (cx_verify_registry_image "$CX_IMAGE_MULTIARCH")
+ '''
+ subprocess.run(
+ ["bash", "-c", command, "_", str(common)],
+ check=True,
+ env={**os.environ, "COLLECTIVEX_OPERATOR_CONFIG": "/dev/null"},
+ )
+
+ def test_canonical_gha_requires_compute_visible_staging(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary).resolve()
+ repo = root / "repo"
+ squash = root / "squash"
+ source = repo / "experimental" / "CollectiveX"
+ source.mkdir(parents=True)
+ squash.mkdir()
+ (source / "public.py").write_text("public\n")
+ (source / "private-infra.md").write_text("private\n")
+ command = r'''
+ set -euo pipefail
+ source "$1"
+ unset CX_SHARD_FILE CX_STAGE_DIR
+ ! (COLLECTIVEX_CANONICAL_GHA=1; cx_stage_path "$2" "")
+ staged="$(COLLECTIVEX_CANONICAL_GHA=0; cx_stage_path "$2" "")"
+ cx_stage_repo "$2" "$staged"
+ test "$staged" != "$2"
+ test -f "$staged/experimental/CollectiveX/public.py"
+ test ! -e "$staged/experimental/CollectiveX/private-infra.md"
+ cx_cleanup_stage "$staged" "$2"
+ test ! -e "$staged"
+ '''
+ subprocess.run(
+ ["bash", "-c", command, "_", str(ROOT / "runtime" / "common.sh"),
+ str(repo)],
+ check=True,
+ env={
+ **os.environ,
+ "COLLECTIVEX_OPERATOR_CONFIG": "/dev/null",
+ "CX_SQUASH_DIR": str(squash),
+ },
+ )
+ self.assertEqual(list(squash.iterdir()), [])
+
+ def test_manual_stage_does_not_write_to_checkout_parent(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ parent = Path(temporary).resolve() / "readonly-parent"
+ repo = parent / "repo"
+ squash = parent / "squash"
+ source = repo / "experimental" / "CollectiveX"
+ source.mkdir(parents=True)
+ squash.mkdir(mode=0o700)
+ (source / "public.py").write_text("public\n")
+ original_mode = parent.stat().st_mode & 0o777
+ parent.chmod(0o555)
+ try:
+ command = r'''
+ set -euo pipefail
+ source "$1"
+ unset CX_STAGE_DIR
+ staged="$(cx_stage_path "$2" "")"
+ cx_stage_repo "$2" "$staged"
+ case "$staged" in "$3"/.collectivex-stage-*) ;; *) exit 1 ;; esac
+ test -f "$staged/experimental/CollectiveX/public.py"
+ test ! -e "$4/.collectivex-stage"
+ cx_cleanup_stage "$staged" "$2"
+ test ! -e "$staged"
+ '''
+ subprocess.run(
+ [
+ "bash", "-c", command, "_",
+ str(ROOT / "runtime" / "common.sh"), str(repo),
+ str(squash), str(parent),
+ ],
+ check=True,
+ env={
+ **os.environ,
+ "COLLECTIVEX_OPERATOR_CONFIG": "/dev/null",
+ "CX_SQUASH_DIR": str(squash),
+ },
+ )
+ finally:
+ parent.chmod(original_mode)
+ self.assertEqual(
+ sorted(path.name for path in parent.iterdir()),
+ ["repo", "squash"],
+ )
+ self.assertEqual(list(squash.iterdir()), [])
+
+ def test_stage_refuses_to_reuse_an_execution_child(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary).resolve()
+ repo = root / "repo"
+ source = repo / "experimental" / "CollectiveX"
+ source.mkdir(parents=True)
+ (source / "public.py").write_text("public\n")
+ base = root / "stage"
+ child = base / "job_collision"
+ child.mkdir(parents=True, mode=0o700)
+ sentinel = child / "keep"
+ sentinel.write_text("keep")
+ command = r'''
+ source "$1"
+ ! (cx_stage_repo "$2" "$3/job_collision")
+ '''
+ subprocess.run(
+ [
+ "bash", "-c", command, "_",
+ str(ROOT / "runtime" / "common.sh"), str(repo), str(base),
+ ],
+ check=True,
+ env={
+ **os.environ,
+ "COLLECTIVEX_OPERATOR_CONFIG": "/dev/null",
+ "COLLECTIVEX_CANONICAL_GHA": "1",
+ "COLLECTIVEX_EXECUTION_ID": "collision",
+ "CX_STAGE_DIR": str(base),
+ },
+ )
+ self.assertEqual(sentinel.read_text(), "keep")
+
+ def test_stage_removes_its_execution_child_when_rsync_fails(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary).resolve()
+ repo = root / "repo"
+ source = repo / "experimental" / "CollectiveX"
+ source.mkdir(parents=True)
+ (source / "public.py").write_text("public\n")
+ base = root / "stage"
+ base.mkdir(mode=0o700)
+ sentinel = root / "rsync-called"
+ command = r'''
+ source "$1"
+ rsync() { : > "$RSYNC_CALLED"; return 1; }
+ staged="$(cx_stage_path "$2" "$3")"
+ ! cx_stage_repo "$2" "$staged"
+ '''
+ subprocess.run(
+ [
+ "bash", "-c", command, "_",
+ str(ROOT / "runtime" / "common.sh"), str(repo), str(base),
+ ],
+ check=True,
+ env={
+ **os.environ,
+ "COLLECTIVEX_OPERATOR_CONFIG": "/dev/null",
+ "COLLECTIVEX_CANONICAL_GHA": "1",
+ "CX_STAGE_DIR": str(base),
+ "RSYNC_CALLED": str(sentinel),
+ },
+ )
+ self.assertTrue(sentinel.is_file())
+ self.assertEqual(list(base.iterdir()), [])
+
+ def test_interrupted_stage_is_cleanup_capable_before_copy(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary).resolve()
+ repo = root / "repo"
+ source = repo / "experimental" / "CollectiveX"
+ source.mkdir(parents=True)
+ (source / "public.py").write_text("public\n")
+ base = root / "stage"
+ base.mkdir(mode=0o700)
+ sibling = base / "keep"
+ sibling.write_text("keep\n")
+ command = r'''
+ set -euo pipefail
+ source "$1"
+ export REPO_ROOT="$2" CX_BENCH=nccl-ep
+ MOUNT_SRC="$(cx_stage_path "$REPO_ROOT" "$3")"
+ cx_install_launcher_fail_safe
+ rsync() { kill -TERM $$; return 143; }
+ cx_stage_repo "$REPO_ROOT" "$MOUNT_SRC"
+ '''
+ result = subprocess.run(
+ [
+ "bash", "-c", command, "_",
+ str(ROOT / "runtime" / "common.sh"), str(repo), str(base),
+ ],
+ text=True,
+ capture_output=True,
+ env={
+ **os.environ,
+ "COLLECTIVEX_OPERATOR_CONFIG": "/dev/null",
+ "COLLECTIVEX_CANONICAL_GHA": "1",
+ "COLLECTIVEX_EXECUTION_ID": "interrupted",
+ "CX_STAGE_DIR": str(base),
+ },
+ )
+ self.assertNotEqual(result.returncode, 0)
+ self.assertFalse((base / "job_interrupted").exists())
+ self.assertEqual(sibling.read_text(), "keep\n")
+
+ def test_stage_base_and_early_cleanup_are_isolated(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary).resolve()
+ repo = root / "repo"
+ source = repo / "experimental" / "CollectiveX"
+ source.mkdir(parents=True)
+ (source / "public.py").write_text("public\n")
+ nested = repo / "stage"
+ nested.mkdir(mode=0o700)
+ group_writable = root / "group-stage"
+ group_writable.mkdir(mode=0o770)
+ group_writable.chmod(0o770)
+ setgid = root / "setgid-stage"
+ setgid.mkdir(mode=0o750)
+ setgid.chmod(0o2750)
+ command = r'''
+ set -euo pipefail
+ source "$1"
+ ! (CX_STAGE_DIR="$3"; cx_stage_path "$2" "$3")
+ ! (CX_STAGE_DIR="$4"; cx_stage_path "$2" "$4")
+ export CX_STAGE_DIR="$5" COLLECTIVEX_EXECUTION_ID="setgid-$$"
+ trap 'cx_cleanup_private_logs 0' EXIT
+ staged="$(cx_stage_path "$2" "$CX_STAGE_DIR")"
+ cx_stage_repo "$2" "$staged"
+ chmod 2700 "$staged"
+ cx_cleanup_stage "$staged" "$2"
+ test ! -e "$staged"
+ '''
+ subprocess.run(
+ [
+ "bash", "-c", command, "_",
+ str(ROOT / "runtime" / "common.sh"), str(repo), str(nested),
+ str(group_writable), str(setgid),
+ ],
+ check=True,
+ env={**os.environ, "COLLECTIVEX_OPERATOR_CONFIG": "/dev/null"},
+ )
+
+ early = r'''
+ set -euo pipefail
+ source "$1"
+ export REPO_ROOT="$2" CX_STAGE_DIR="$3" CX_BENCH=nccl-ep
+ export COLLECTIVEX_EXECUTION_ID="pre-marker-$$"
+ MOUNT_SRC="$(cx_stage_path "$REPO_ROOT" "$CX_STAGE_DIR")"
+ cx_install_launcher_fail_safe
+ mkdir -m 700 "$MOUNT_SRC"
+ exit 17
+ '''
+ result = subprocess.run(
+ [
+ "bash", "-c", early, "_",
+ str(ROOT / "runtime" / "common.sh"), str(repo), str(setgid),
+ ],
+ text=True,
+ capture_output=True,
+ env={**os.environ, "COLLECTIVEX_OPERATOR_CONFIG": "/dev/null"},
+ )
+ self.assertEqual(result.returncode, 17, result.stderr)
+ self.assertEqual(list(setgid.iterdir()), [])
+
+ def test_backend_cache_reuses_v3_and_falls_back_once_without_repair(self) -> None:
+ common = ROOT / "runtime" / "common.sh"
+ with tempfile.TemporaryDirectory() as temporary:
+ parent = Path(temporary) / "stage"
+ parent.mkdir(mode=0o700)
+ concurrent = Path(temporary) / "concurrent"
+ concurrent.mkdir(mode=0o700)
+ command = r'''
+ set -euo pipefail
+ source "$1"
+ for worker in 1 2 3; do
+ (
+ cx_prepare_backend_cache "$2"
+ printf '%s %s\n' "$CX_BACKEND_CACHE_SENTINEL_SHA256" \
+ "$CX_PREPARED_BACKEND_CACHE" > "$3/$worker"
+ ) &
+ done
+ wait
+ cmp "$3/1" "$3/2"
+ cmp "$3/1" "$3/3"
+ cx_prepare_backend_cache "$2"
+ first="$CX_PREPARED_BACKEND_CACHE"
+ first_digest="$CX_BACKEND_CACHE_SENTINEL_SHA256"
+ chmod 2700 "$first"
+ cx_prepare_backend_cache "$2"
+ second="$CX_PREPARED_BACKEND_CACHE"
+ test "$first" = "$second"
+ test "$first_digest" = "$CX_BACKEND_CACHE_SENTINEL_SHA256"
+ test "$first" = "$(cd "$2" && pwd -P)/.collectivex-backend-cache-v3-$(id -u)"
+ export CX_BACKEND_CACHE_ROOT="$first"
+ cx_verify_backend_cache_mount
+ export CX_BACKEND_CACHE_SENTINEL_SHA256="$(printf '0%.0s' {1..64})"
+ ! cx_verify_backend_cache_mount
+ '''
+ subprocess.run(
+ [
+ "bash", "-c", command, "_", str(common), str(parent),
+ str(concurrent),
+ ],
+ check=True,
+ )
+ cache = parent / f".collectivex-backend-cache-v3-{os.getuid()}"
+ self.assertTrue(cache.is_dir())
+ self.assertEqual(cache.stat().st_mode & 0o777, 0o700)
+ self.assertEqual(
+ list(cache.glob(".collectivex-mount-sentinel-v1.tmp.*")), []
+ )
+ alias = Path(temporary) / "stage-alias"
+ alias.symlink_to(parent, target_is_directory=True)
+ canonical = subprocess.run(
+ [
+ "bash", "-c",
+ 'source "$1"; cx_prepare_backend_cache "$2"; '
+ 'printf "%s\\n%s\\n" "$CX_PREPARED_BACKEND_CACHE" '
+ '"$CX_BACKEND_CACHE_SENTINEL_SHA256"',
+ "_", str(common), str(alias),
+ ],
+ text=True,
+ capture_output=True,
+ check=True,
+ )
+ cache_path, digest = canonical.stdout.splitlines()
+ self.assertEqual(cache_path, str(cache.resolve()))
+ self.assertRegex(digest, r"^[0-9a-f]{64}$")
+ saved = parent / "saved-cache"
+ cache.rename(saved)
+ cache.mkdir(mode=0o700)
+ replacement = cache / ".collectivex-mount-sentinel-v1"
+ replacement.write_bytes(b"replacement".ljust(32, b"!"))
+ replacement.chmod(0o600)
+ replaced = subprocess.run(
+ [
+ "bash", "-c",
+ 'source "$1"; export CX_BACKEND_CACHE_ROOT="$2" '
+ 'CX_BACKEND_CACHE_SENTINEL_SHA256="$3"; '
+ 'cx_verify_backend_cache_mount',
+ "_", str(common), str(cache), digest,
+ ]
+ )
+ self.assertNotEqual(replaced.returncode, 0)
+ replacement.unlink()
+ cache.rmdir()
+ saved.rename(cache)
+ (cache / ".collectivex-mount-sentinel-v1").unlink()
+ cache.rmdir()
+ target = Path(temporary) / "target"
+ target.mkdir(mode=0o700)
+ cache.symlink_to(target, target_is_directory=True)
+ fallback = subprocess.run(
+ [
+ "bash", "-c",
+ 'source "$1"; cx_prepare_backend_cache "$2"; '
+ 'printf "%s\\n" "$CX_PREPARED_BACKEND_CACHE"',
+ "_", str(common), str(parent),
+ ],
+ text=True,
+ capture_output=True,
+ check=True,
+ )
+ v4 = parent / f".collectivex-backend-cache-v4-{os.getuid()}"
+ self.assertEqual(fallback.stdout.strip(), str(v4.resolve()))
+ self.assertTrue(cache.is_symlink())
+ self.assertTrue(v4.is_dir())
+ (v4 / ".collectivex-mount-sentinel-v1").unlink()
+ v4.rmdir()
+ v4.symlink_to(target, target_is_directory=True)
+ result = subprocess.run(
+ [
+ "bash", "-c", 'source "$1"; cx_prepare_backend_cache "$2"',
+ "_", str(common), str(parent),
+ ],
+ text=True,
+ capture_output=True,
+ )
+ self.assertNotEqual(result.returncode, 0)
+ self.assertNotIn(str(parent), result.stderr)
+ self.assertTrue(cache.is_symlink())
+ self.assertTrue(v4.is_symlink())
+
+ source = common.read_text().split("cx_prepare_backend_cache() {", 1)[1]
+ program = source.split("<<'PY'\n", 1)[1].split("\nPY\n", 1)[0]
+ with tempfile.TemporaryDirectory() as temporary:
+ parent = Path(temporary) / "stage"
+ parent.mkdir(mode=0o700)
+ fake_os = types.ModuleType("os")
+ fake_os.__dict__.update(os.__dict__)
+ fake_os.fsync = mock.Mock(side_effect=OSError("forced fsync failure"))
+ with (
+ mock.patch.dict(sys.modules, {"os": fake_os}),
+ mock.patch.object(sys, "argv", ["-", str(parent)]),
+ mock.patch.object(sys, "stdout", io.StringIO()),
+ self.assertRaises(SystemExit) as failure,
+ ):
+ exec(compile(program, "", "exec"), {})
+ self.assertEqual(failure.exception.code, 1)
+ self.assertEqual(
+ list(parent.rglob(".collectivex-mount-sentinel-v1.tmp.*")), []
+ )
+
+ def test_nvidia_namespace_package_roots_come_from_distribution_files(self) -> None:
+ runtime = ROOT / "runtime" / "run_in_container.sh"
+ with tempfile.TemporaryDirectory() as temporary:
+ site = Path(temporary) / "site"
+ package = site / "nvidia" / "nccl"
+ (package / "include").mkdir(parents=True)
+ (package / "lib").mkdir()
+ (package / "include" / "nccl.h").write_text("header\n")
+ (package / "lib" / "libnccl.so.2").write_text("library\n")
+ info = site / "nvidia_nccl_cu13-2.30.4.dist-info"
+ info.mkdir()
+ (info / "METADATA").write_text(
+ "Metadata-Version: 2.1\nName: nvidia-nccl-cu13\nVersion: 2.30.4\n"
+ )
+ (info / "RECORD").write_text(
+ "nvidia/nccl/include/nccl.h,,\n"
+ "nvidia/nccl/lib/libnccl.so.2,,\n"
+ "nvidia_nccl_cu13-2.30.4.dist-info/METADATA,,\n"
+ "nvidia_nccl_cu13-2.30.4.dist-info/RECORD,,\n"
+ )
+ command = r'''
+ set -euo pipefail
+ eval "$(sed -n '/^cx_nvidia_package_root()/,/^}/p' "$1")"
+ root="$(cx_nvidia_package_root nvidia-nccl-cu13 nccl)"
+ test "$root" = "$2/nvidia/nccl"
+ ! cx_nvidia_package_root nvidia-nccl-cu13 nvshmem
+ '''
+ subprocess.run(
+ ["bash", "-c", command, "_", str(runtime), str(site.resolve())],
+ check=True,
+ env={**os.environ, "PYTHONPATH": str(site)},
+ )
+
+ def test_cuda_cccl_exports_the_resolved_jit_toolchain_root(self) -> None:
+ runtime = ROOT / "runtime" / "run_in_container.sh"
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary)
+ toolkit = root / "cuda-13.0"
+ (toolkit / "bin").mkdir(parents=True)
+ (toolkit / "include").mkdir()
+ (toolkit / "lib64").mkdir()
+ cccl = toolkit / "targets" / "x86_64-linux" / "include" / "cccl"
+ cccl.mkdir(parents=True)
+ nvcc = toolkit / "bin" / "nvcc"
+ nvcc.write_text("#!/bin/sh\nexit 0\n")
+ nvcc.chmod(0o755)
+ alias = root / "cuda"
+ alias.symlink_to(toolkit, target_is_directory=True)
+ command = r'''
+ set -euo pipefail
+ eval "$(sed -n '/^cx_prepare_cuda_cccl()/,/^}/p' "$1")"
+ cx_prepare_cuda_cccl
+ test "$CUDA_HOME" = "$2"
+ test "$CX_CUDA_CCCL" = "$2/targets/x86_64-linux/include/cccl"
+ test "$CPATH" = "$2/targets/x86_64-linux/include/cccl:"
+ test "$NVCC_PREPEND_FLAGS" = "-I$2/targets/x86_64-linux/include/cccl "
+ '''
+ subprocess.run(
+ ["bash", "-c", command, "_", str(runtime), str(toolkit.resolve())],
+ check=True,
+ env={
+ **os.environ,
+ "PATH": f"{alias / 'bin'}:{os.environ['PATH']}",
+ "CPATH": "",
+ "NVCC_PREPEND_FLAGS": "",
+ },
+ )
+
+ def test_deepep_v2_toolchain_rejects_overlay_lock_failure(self) -> None:
+ runtime = ROOT / "runtime" / "run_in_container.sh"
+ with tempfile.TemporaryDirectory() as temporary:
+ command = r'''
+ set -euo pipefail
+ eval "$(sed -n '/^cx_prepare_deepep_toolchain()/,/^}/p' "$1")"
+ cache_root="$2"
+ cx_nvidia_package_root() { printf '%s' /unused; }
+ cx_deepep_v2_root() { printf '%s' "$cache_root"; }
+ cx_log() { :; }
+ flock() { return 1; }
+ ! cx_prepare_deepep_toolchain
+ '''
+ subprocess.run(
+ ["bash", "-c", command, "_", str(runtime), temporary],
+ check=True,
+ )
+
+ def test_pinned_source_fetch_retries_transient_failures(self) -> None:
+ common = ROOT / "runtime" / "common.sh"
+ with tempfile.TemporaryDirectory() as temporary:
+ command = r'''
+ set -euo pipefail
+ eval "$(sed -n '/^cx_git()/,/^}/p' "$1")"
+ eval "$(sed -n '/^cx_git_in_tree()/,/^}/p' "$1")"
+ eval "$(sed -n '/^cx_fetch_revision()/,/^}/p' "$1")"
+ attempts=0
+ expected_directory="$(cd -P -- "$3" && pwd -P)"
+ sleep() { :; }
+ git() {
+ local argument has_directory=0 has_trust=0
+ if [ "$1" = '-c' ] && [ "$3" = init ]; then
+ mkdir -p "${@: -1}"
+ return 0
+ fi
+ for argument in "$@"; do
+ [ "$argument" != '-C' ] || has_directory=1
+ [ "$argument" != "safe.directory=$expected_directory" ] || has_trust=1
+ [ "$argument" != 'safe.directory=*' ] || return 1
+ done
+ [ "$has_directory" = 0 ] || [ "$has_trust" = 1 ] || return 1
+ case " $* " in
+ *' fetch '*)
+ attempts=$((attempts + 1))
+ [ "$attempts" = 3 ]
+ ;;
+ *' rev-parse HEAD '*) printf '%s\n' "$revision" ;;
+ *) return 0 ;;
+ esac
+ }
+ cx_fetch_revision https://example.invalid/repo "$2" "$3"
+ test "$attempts" = 3
+ '''
+ revision = "a" * 40
+ subprocess.run(
+ ["bash", "-c", command, "_", str(common), revision, temporary],
+ check=True,
+ )
+
+ def test_git_tree_trust_is_exact_and_command_scoped(self) -> None:
+ common = ROOT / "runtime" / "common.sh"
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary)
+ repository = root / "repo"
+ repository.mkdir()
+ alias = root / "alias"
+ alias.symlink_to(repository, target_is_directory=True)
+ wildcard = root / "*"
+ wildcard.mkdir()
+ arguments = root / "arguments"
+ command = r'''
+ set -euo pipefail
+ eval "$(sed -n '/^cx_git()/,/^}/p' "$1")"
+ eval "$(sed -n '/^cx_git_in_tree()/,/^}/p' "$1")"
+ arguments="$4"
+ git() { printf '%s\n' "$@" > "$arguments"; }
+ cx_git_in_tree "$2" status --porcelain
+ ! cx_git_in_tree relative status
+ ! cx_git_in_tree "$3" status
+ ! cx_git_in_tree "$5" status
+ '''
+ subprocess.run(
+ [
+ "bash",
+ "-c",
+ command,
+ "_",
+ str(common),
+ str(repository),
+ str(alias),
+ str(arguments),
+ str(wildcard),
+ ],
+ check=True,
+ )
+ self.assertEqual(
+ arguments.read_text().splitlines(),
+ [
+ "-c",
+ "credential.helper=",
+ "-c",
+ f"safe.directory={repository.resolve()}",
+ "-C",
+ str(repository.resolve()),
+ "status",
+ "--porcelain",
+ ],
+ )
+ self.assertNotIn("safe.directory=*", arguments.read_text())
+
+ def test_runtime_materializes_the_verified_host_source_without_network(self) -> None:
+ common = ROOT / "runtime" / "common.sh"
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary)
+ seed = root / "seed"
+ seed.mkdir()
+ (seed / "pinned").write_text("source\n")
+ destination = root / "build"
+ fetched = root / "network-fetch"
+ command = r'''
+ set -euo pipefail
+ source "$1"
+ export CX_BACKEND_SOURCE_ROOT="$2/source"
+ SEED="$3" FETCHED="$5"
+ copy_mode=
+ cx_backend_source_path() { printf '%s' "$SEED"; }
+ cx_backend_source_is_valid() { test -f "$2/pinned"; }
+ cx_fetch_revision() { : > "$FETCHED"; return 1; }
+ cp() {
+ test "$1" = -R
+ copy_mode=recursive
+ command cp "$@"
+ }
+ cx_materialize_backend_source deepep-hybrid "$4"
+ test -f "$4/pinned"
+ test "$copy_mode" = recursive
+ python3 - "$4" <<'PY'
+import os
+import stat
+import sys
+assert stat.S_IMODE(os.stat(sys.argv[1]).st_mode) == 0o700
+PY
+ test ! -e "$FETCHED"
+ '''
+ subprocess.run(
+ [
+ "bash", "-c", command, "_", str(common), str(root),
+ str(seed), str(destination), str(fetched),
+ ],
+ check=True,
+ )
+
+ def test_backend_source_validation_rejects_status_errors_and_ignored_files(self) -> None:
+ common = ROOT / "runtime" / "common.sh"
+ with tempfile.TemporaryDirectory() as temporary:
+ command = r'''
+ set -euo pipefail
+ source "$1"
+ cx_backend_source_pin() { printf '%s|%s|' revision tree; }
+ git() {
+ case " $* " in
+ *' rev-parse HEAD '*) printf '%s\n' revision ;;
+ *' rev-parse HEAD^{tree} '*) printf '%s\n' tree ;;
+ *' status --porcelain '*) [ "$mode" != status-error ] ;;
+ *' ls-files --others --ignored '*)
+ [ "$mode" != ignored ] || printf '%s\n' ignored.bin
+ ;;
+ *) return 1 ;;
+ esac
+ }
+ mode=status-error
+ ! cx_backend_source_is_valid backend "$2"
+ mode=ignored
+ ! cx_backend_source_is_valid backend "$2"
+ mode=clean
+ cx_backend_source_is_valid backend "$2"
+ '''
+ subprocess.run(
+ ["bash", "-c", command, "_", str(common), temporary],
+ check=True,
+ )
+
+ def test_backend_source_root_normalizes_inherited_special_mode(self) -> None:
+ common = ROOT / "runtime" / "common.sh"
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary)
+ source_root = root / "experimental" / "CollectiveX" / ".cx_sources"
+ source = source_root / "backend-revision"
+ source.mkdir(parents=True)
+ command = r'''
+ set -euo pipefail
+ source "$1"
+ export COLLECTIVEX_EXECUTION_ID="source-mode-$$"
+ trap 'cx_cleanup_private_logs 0' EXIT
+ expected_mount="$2"
+ expected_source="$3"
+ expected_root="${expected_source%/*}"
+ observed_mode=2700
+ mock_stage_owner=4200
+ mock_root_owner=4200
+ chmod_calls=0
+ chmod() {
+ test "$1" = 700 && test "$2" = "$expected_root"
+ chmod_calls=$((chmod_calls + 1))
+ [ "$chmod_calls" = 2 ] || return 1
+ observed_mode=700
+ }
+ stat() {
+ case "$2" in
+ %u)
+ case "$3" in
+ "$expected_mount") printf '%s\n' "$mock_stage_owner" ;;
+ "$expected_root") printf '%s\n' "$mock_root_owner" ;;
+ *) return 1 ;;
+ esac
+ ;;
+ %a)
+ case "$3" in
+ "$expected_mount") printf '2700\n' ;;
+ "$expected_root") printf '%s\n' "$observed_mode" ;;
+ *) return 1 ;;
+ esac
+ ;;
+ *) return 1 ;;
+ esac
+ }
+ cx_backend_source_path() { printf '%s' "$expected_source"; }
+ cx_backend_source_is_valid() {
+ test "$1" = backend && test "$2" = "$expected_source"
+ }
+ cx_prepare_backend_source "$2" backend
+ test "$observed_mode" = 2700
+ test "$chmod_calls" = 0
+ observed_mode=2750
+ ! _cx_prepare_backend_source "$2" backend
+ test "$chmod_calls" = 1
+ _cx_prepare_backend_source "$2" backend
+ test "$observed_mode" = 700
+ mock_root_owner=4300
+ ! _cx_prepare_backend_source "$2" backend
+ '''
+ subprocess.run(
+ ["bash", "-c", command, "_", str(common), str(root), str(source)],
+ check=True,
+ )
+
+ def test_canonical_backend_sources_use_verified_seed_without_network(self) -> None:
+ common = ROOT / "runtime" / "common.sh"
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary)
+ mount = root / "mount"
+ source_root = mount / "experimental" / "CollectiveX" / ".cx_sources"
+ seed_root = root / "seed"
+ seeds = [
+ seed_root / f"{backend}-revision"
+ for backend in ("backend-one", "backend-two")
+ ]
+ mount.mkdir(mode=0o700)
+ source_root.parent.mkdir(parents=True, mode=0o700)
+ for seed in seeds:
+ seed.mkdir(parents=True, mode=0o700)
+ (seed / "pinned").write_text("source\n")
+ network = root / "network"
+ command = r'''
+ set -euo pipefail
+ source "$1"
+ export COLLECTIVEX_CANONICAL_GHA=1
+ export CX_BACKEND_SOURCE_SEED_ROOT="$4"
+ export COLLECTIVEX_EXECUTION_ID="source-seed-$$"
+ trap 'cx_cleanup_private_logs 0' EXIT
+ NETWORK="$5"
+ stat() {
+ case "$2" in
+ %u) printf '4200\n' ;;
+ %a) printf '700\n' ;;
+ *) return 1 ;;
+ esac
+ }
+ cx_backend_source_path() { printf '%s/%s-revision' "$1" "$2"; }
+ cx_backend_source_is_valid() { test -f "$2/pinned"; }
+ cx_fetch_revision() { : > "$NETWORK"; return 1; }
+ cx_prepare_backend_source "$2" backend-one
+ cx_prepare_backend_source "$2" backend-two
+ test -f "$3/backend-one-revision/pinned"
+ test -f "$3/backend-two-revision/pinned"
+ test ! -e "$NETWORK"
+ rm -rf -- "$3/backend-one-revision" "$3/backend-two-revision"
+ unset CX_BACKEND_SOURCE_SEED_ROOT
+ ! _cx_prepare_backend_source "$2" backend-one
+ test ! -e "$NETWORK"
+ '''
+ subprocess.run(
+ [
+ "bash", "-c", command, "_", str(common), str(mount),
+ str(source_root), str(seed_root), str(network),
+ ],
+ check=True,
+ )
+
+ def test_deepep_hybrid_cache_reuse_revalidates_extensions(self) -> None:
+ common = ROOT / "runtime" / "common.sh"
+ runtime = ROOT / "runtime" / "run_in_container.sh"
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary)
+ (root / "deep_ep_cpp.so").write_bytes(b"deep")
+ (root / "hybrid_ep_cpp.so").write_bytes(b"hybrid")
+ command = r'''
+ set -euo pipefail
+ chmod 700 "$3"
+ source "$1"
+ eval "$(sed -n '/^cx_deepep_hybrid_marker_content_sha256()/,/^}/p' "$2")"
+ eval "$(sed -n '/^cx_deepep_hybrid_cache_is_valid()/,/^}/p' "$2")"
+ revision=revision tree=tree
+ cx_git() {
+ case " $* " in
+ *' rev-parse HEAD '*) printf '%s\n' "$revision" ;;
+ *' rev-parse HEAD^{tree} '*) printf '%s\n' "$tree" ;;
+ *' status --porcelain '*|*' ls-files --others '*) return 0 ;;
+ *) return 1 ;;
+ esac
+ }
+ cx_git_in_tree() { shift; cx_git "$@"; }
+ marker="$3/.collectivex-complete"
+ digest="$(cx_extension_pair_sha256 "$3" 'deep_ep_cpp*.so' 'hybrid_ep_cpp*.so')"
+ (umask 077; printf '%s\n%s\n%s\n' "$revision" "$tree" "$digest" > "$marker")
+ cx_deepep_hybrid_cache_is_valid "$3" "$marker" "$revision" "$tree"
+ printf changed > "$3/hybrid_ep_cpp.so"
+ ! cx_deepep_hybrid_cache_is_valid "$3" "$marker" "$revision" "$tree"
+ printf hybrid > "$3/hybrid_ep_cpp.so"
+ cp "$3/deep_ep_cpp.so" "$3/deep_ep_cpp-extra.so"
+ ! cx_deepep_hybrid_cache_is_valid "$3" "$marker" "$revision" "$tree"
+ '''
+ subprocess.run(
+ ["bash", "-c", command, "_", str(common), str(runtime), temporary],
+ check=True,
+ )
+
+ def test_rack_backend_environment_is_shared_per_node_and_required(self) -> None:
+ runtime = ROOT / "runtime" / "run_in_container.sh"
+ launcher = (ROOT / "launchers" / "launch_gb-nv.sh").read_text()
+ assignment = next(
+ line for line in launcher.splitlines()
+ if line.startswith("SOURCE_BACKEND_ENV=")
+ )
+ self.assertNotIn("/tmp/.cx_backend_env", launcher)
+ self.assertIn('[ -f "$env_file" ] && [ -r "$env_file" ]', launcher)
+ self.assertIn('[ ! -L "$env_file" ]', launcher)
+ self.assertIn('$(stat -c "%u" "$env_root"):600', launcher)
+ self.assertIn('case "$(stat -c "%a" "$env_root")" in 700|[1-7]700)', launcher)
+ self.assertIn("node-${SLURM_NODEID}.sh", launcher)
+ self.assertIn("HybridEPBuffer", launcher)
+ self.assertIn('. "$env_file" || exit 66', launcher)
+ with tempfile.TemporaryDirectory() as temporary:
+ consumer = r'''
+ eval "$1"
+ env_root="$2/env"
+ SOURCE_BACKEND_ENV="${SOURCE_BACKEND_ENV//\/ix\/experimental\/CollectiveX\/.cx_backend\/env/$env_root}"
+ mkdir -p "$env_root"
+ env_file="$env_root/node-1.sh"
+ printf 'printf sourced > "$CX_SENTINEL"\n' > "$env_file"
+ chmod 600 "$env_file"
+ export CX_SENTINEL="$2/sentinel"
+ stat() {
+ [ "${STAT_FAIL:-0}" = 0 ] || return 1
+ case "$2" in
+ %a) printf '%s\n' "$ROOT_MODE" ;;
+ %u) printf '1000\n' ;;
+ %u:%a) printf '%s\n' "$FILE_OWNER_MODE" ;;
+ *) return 2 ;;
+ esac
+ }
+ run_case() {
+ rm -f "$CX_SENTINEL"
+ ROOT_MODE="$1" FILE_OWNER_MODE="$2" STAT_FAIL="$3" SLURM_NODEID="$4"
+ ( eval "$SOURCE_BACKEND_ENV" )
+ rc=$?
+ [ "$rc" = "$5" ] || return 1
+ if [ "$5" = 0 ]; then
+ [ -f "$CX_SENTINEL" ]
+ else
+ [ ! -e "$CX_SENTINEL" ]
+ fi
+ }
+ run_case 700 1000:600 0 1 0
+ run_case 2700 1000:600 0 1 0
+ run_case 755 1000:600 0 1 66
+ run_case 700 1000:600 1 1 66
+ run_case 700 2000:600 0 1 66
+ mv "$env_file" "$env_file.real"
+ ln -s "$env_file.real" "$env_file"
+ run_case 700 1000:600 0 1 66
+ rm "$env_file"
+ mv "$env_file.real" "$env_file"
+ run_case 700 1000:600 0 invalid 66
+ '''
+ subprocess.run(
+ ["bash", "-c", consumer, "_", assignment, temporary],
+ check=True,
+ )
+ command = r'''
+ set -euo pipefail
+ cd "$2"
+ eval "$(sed -n '/^cx_persist_backend_env()/,/^}/p' "$1")"
+ export SLURM_NODEID=1 PYTHONPATH=/ix/pinned DEEPEP_COMMIT=abc
+ cx_persist_backend_env
+ env_file="$PWD/.cx_backend/env/node-1.sh"
+ test -f "$env_file"
+ test "$(stat -f %Lp "$env_file" 2>/dev/null || stat -c %a "$env_file")" = 600
+ unset PYTHONPATH DEEPEP_COMMIT
+ . "$env_file"
+ test "$PYTHONPATH" = /ix/pinned
+ test "$DEEPEP_COMMIT" = abc
+ SLURM_NODEID=invalid && ! cx_persist_backend_env
+ '''
+ subprocess.run(
+ ["bash", "-c", command, "_", str(runtime), temporary],
+ check=True,
+ )
+
+ def test_stage_cleanup_failure_fails_job_but_marks_allocation_safe(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary)
+ (root / "repo").mkdir()
+ (root / "stage").mkdir()
+ command = r'''
+ source "$1"
+ cx_write_cleanup_guard() {
+ rm -f -- "$CX_JOB_ROOT/cleanup-safe" "$CX_JOB_ROOT/cleanup-unsafe"
+ : > "$CX_JOB_ROOT/cleanup-$1"
+ }
+ cx_cleanup_stage() { return 1; }
+ cx_cleanup_private_logs() { : > "$CX_JOB_ROOT/logs-deleted"; }
+ export CX_JOB_ROOT="$2" REPO_ROOT="$2/repo" MOUNT_SRC="$2/stage"
+ export COLLECTIVEX_CANONICAL_GHA=1 CX_ALLOCATION_UNCERTAIN=0
+ unset CX_BENCH JOB_ID
+ cx_launcher_cleanup 0
+ '''
+ result = subprocess.run(
+ ["bash", "-c", command, "_", str(ROOT / "runtime" / "common.sh"),
+ str(root)],
+ text=True,
+ capture_output=True,
+ env={**os.environ, "COLLECTIVEX_OPERATOR_CONFIG": "/dev/null"},
+ )
+ self.assertEqual(result.returncode, 1, result.stderr)
+ self.assertTrue((root / "cleanup-safe").is_file())
+ self.assertFalse((root / "cleanup-unsafe").exists())
+ self.assertFalse((root / "logs-deleted").exists())
+
+ def test_generated_stage_cleanup_never_removes_configured_base(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary)
+ base = root / "stage"
+ repo = root / "repo"
+ generated = base / "job_execution"
+ generated.mkdir(parents=True, mode=0o700)
+ repo.mkdir()
+ marker = generated / ".collectivex-stage-v1"
+ marker.write_text("collectivex-stage-v1\nexecution\n")
+ marker.chmod(0o600)
+ (generated / "payload").write_text("temporary")
+ subprocess.run(
+ [
+ "bash", "-c",
+ 'source "$1"; cx_cleanup_stage "$2" "$3"; '
+ '! cx_cleanup_stage "$4" "$3"',
+ "_", str(ROOT / "runtime" / "common.sh"), str(generated),
+ str(repo), str(base),
+ ],
+ check=True,
+ env={
+ **os.environ,
+ "COLLECTIVEX_OPERATOR_CONFIG": "/dev/null",
+ "COLLECTIVEX_EXECUTION_ID": "execution",
+ "CX_STAGE_DIR": str(base),
+ },
+ )
+ self.assertFalse(generated.exists())
+ self.assertTrue(base.is_dir())
+ self.assertTrue(repo.is_dir())
+
+ def test_adapters_do_not_retain_dead_expected_methods(self) -> None:
+ for path in HERE.glob("ep_*.py"):
+ tree = ast.parse(path.read_text(), str(path))
+ methods = {
+ node.name for node in ast.walk(tree)
+ if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))
+ }
+ self.assertNotIn("expected", methods, path.name)
+
+ def test_artifact_safety_rejects_sensitive_material(self) -> None:
+ private_address = ".".join(str(octet) for octet in (10, 0, 0, 1))
+ secret = "github_pat_" + "A" * 24
+ sensitive = {
+ "ipv4": ({"note": private_address}, private_address),
+ "ipv6": ({"note": "[2001:db8::1]:29500"}, "2001:db8::1"),
+ "user-at-host": ({"note": "ssh admin@private-host"}, "admin@private-host"),
+ "hostname": ({"note": "host=compute-17"}, "compute-17"),
+ "private-dns": ({"note": "worker-7.cluster.local"}, "worker-7.cluster.local"),
+ "suffixed-host": ({"worker_hostname": "relative"}, "worker_hostname"),
+ "suffixed-address": ({"control_address": "relative"}, "control_address"),
+ "suffixed-path": ({"scheduler_path": "relative"}, "scheduler_path"),
+ "exact-address": ({"address": "relative"}, "address"),
+ "exact-ip": ({"ip": "relative"}, "ip"),
+ "camel-host": ({"workerHost": "relative"}, "workerHost"),
+ "camel-path": ({"schedulerPath": "relative"}, "schedulerPath"),
+ "acronym-gpu-uuid": ({"gpuUUID": "relative"}, "gpuUUID"),
+ "acronym-device-uuid": ({"deviceUUID": "relative"}, "deviceUUID"),
+ "acronym-pci-bus": ({"pciBusID": "relative"}, "pciBusID"),
+ "mac-address": ({"note": "00:11:22:33:44:55"}, "00:11:22:33:44:55"),
+ "ib-guid": ({"note": "00:11:22:33:44:55:66:77"}, "00:11:22:33:44:55:66:77"),
+ "dgx-host": ({"note": "dgx-b300-001"}, "dgx-b300-001"),
+ "cloud-host": ({"note": "ip-10-20-30-40"}, "ip-10-20-30-40"),
+ "credential-field": ({"service_token": "short"}, "service_token"),
+ "prefixed-token": ({"note": secret}, secret),
+ "hf-token": ({"note": "hf_" + "A" * 24}, "hf_" + "A" * 24),
+ "payment-token": ({"note": "sk_live_" + "A" * 24}, "sk_live_" + "A" * 24),
+ "generic-secret": ({"note": "password=not-a-real-secret"}, "not-a-real-secret"),
+ }
+ for root in ("data", "it-share", "lustre", "raid", "nvme_home", "scratch", "gpfs", "fsx"):
+ value = f"/{root}/collectivex/run"
+ sensitive[f"private-root-{root}"] = ({"note": value}, value)
+ for name, (document, offending) in sensitive.items():
+ with self.subTest(name=name), self.assertRaises(
+ artifact_safety.ArtifactSafetyError
+ ) as caught:
+ artifact_safety.assert_publication_safe([document])
+ self.assertNotIn(offending, str(caught.exception))
+
+ artifact_safety.assert_publication_safe([{
+ "runner": "b300",
+ "redaction": "sanitized-v1",
+ "path": "datasets/" + "a" * 64 + "/dataset.json",
+ "timing": "8:64:32",
+ "image_digest": "sha256:" + "b" * 64,
+ "source": "github.com",
+ }])
+ for ref in ("release@candidate", "worker1-feature", "sk-refactor-long-component-name"):
+ artifact_safety.assert_publication_safe([{"ref": ref}])
+
+ def test_artifact_safety_cli_does_not_echo_sensitive_values(self) -> None:
+ private_value = ".".join(str(octet) for octet in (10, 24, 68, 12))
+ with tempfile.TemporaryDirectory() as temporary:
+ path = Path(temporary) / "artifact.json"
+ path.write_text(json.dumps({"note": private_value}))
+ result = subprocess.run(
+ [sys.executable, str(ROOT / "artifact_safety.py"), str(path)],
+ text=True,
+ capture_output=True,
+ )
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn("forbidden ipv4-address value", result.stderr)
+ self.assertNotIn(private_value, result.stderr)
+
+ def test_artifact_safety_rejects_linked_and_special_inputs(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary)
+ source = root / "source.json"
+ source.write_text("{}")
+ linked = root / "linked.json"
+ linked.symlink_to(source)
+ fifo = root / "fifo.json"
+ os.mkfifo(fifo)
+ for path in (linked, fifo):
+ with self.subTest(path=path.name), self.assertRaises(
+ artifact_safety.ArtifactSafetyError
+ ):
+ artifact_safety.load_documents([str(path)])
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/experimental/CollectiveX/tests/workload.py b/experimental/CollectiveX/tests/workload.py
new file mode 100644
index 0000000000..89a6b46052
--- /dev/null
+++ b/experimental/CollectiveX/tests/workload.py
@@ -0,0 +1,358 @@
+#!/usr/bin/env python3
+"""Canonical, byte-stable CollectiveX routing workloads.
+
+A *canonical workload* is a routing trace generated ONCE, serialized to a platform-independent
+file, and referenced by an immutable `workload_id`. Every promoted benchmark point consumes the
+SAME serialized bytes, so "did NVIDIA and AMD run the identical workload?" is answered by a
+checksum match, not by trusting that two machines re-ran the same seeded generator.
+
+Layout on disk (one workload = two files, basename = workload_id):
+ /.npz topk_idx [gt,topk] int32, topk_weights [gt,topk] float32
+ /.manifest.json dims, routing profile, generator version, seed, SHA-256s
+
+Routing and gate weights come from a stdlib integer counter, not a framework RNG. The same
+parameters therefore produce the same int32/float32 bytes across PyTorch and accelerator images.
+"""
+from __future__ import annotations
+
+from array import array
+import bisect
+import hashlib
+import json
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+import identity # noqa: E402
+
+WORKLOAD_SCHEMA_VERSION = 1
+# Bump when the counter or byte encoding changes. The workload ID binds parameters and trace bytes.
+GENERATOR_VERSION = "collectivex-routing-counter-v3"
+GATE_WEIGHT_FORMAT = "counter-u16-normalized-f32"
+ACTIVATION_GENERATOR = "collectivex-activation-counter-v3"
+_MASK64 = (1 << 64) - 1
+
+
+def _sha256(b: bytes) -> str:
+ return hashlib.sha256(b).hexdigest()
+
+
+def _mix64(value: int) -> int:
+ value = (value + 0x9E3779B97F4A7C15) & _MASK64
+ value = ((value ^ (value >> 30)) * 0xBF58476D1CE4E5B9) & _MASK64
+ value = ((value ^ (value >> 27)) * 0x94D049BB133111EB) & _MASK64
+ return value ^ (value >> 31)
+
+
+def _counter(seed: int, token: int, slot: int, attempt: int, stream: int) -> int:
+ value = (
+ (seed & _MASK64)
+ ^ (((token + 1) * 0xD2B74407B1CE6E93) & _MASK64)
+ ^ (((slot + 1) * 0xCA5A826395121157) & _MASK64)
+ ^ (((attempt + 1) * 0x9E3779B185EBCA87) & _MASK64)
+ ^ (((stream + 1) * 0xA24BAED4963EE407) & _MASK64)
+ )
+ return _mix64(value)
+
+
+def canonical_routing_rows(
+ global_tokens: int, experts: int, topk: int, routing: str, seed: int
+) -> tuple[list[list[int]], list[list[float]]]:
+ """Generate distinct experts and normalized weights using exact integer counters."""
+ if routing not in {"uniform", "zipf"}:
+ raise ValueError(f"unknown routing {routing!r} (uniform|zipf)")
+ if global_tokens <= 0 or experts <= 0 or topk <= 0 or topk > experts:
+ raise ValueError("global_tokens/experts/topk must be positive and topk <= experts")
+
+ cumulative: list[int] | None = None
+ if routing == "zipf":
+ total = 0
+ cumulative = []
+ for expert in range(experts):
+ total += (1 << 32) // (expert + 1)
+ cumulative.append(total)
+
+ indices: list[list[int]] = []
+ weights: list[list[float]] = []
+ for token in range(global_tokens):
+ selected: list[int] = []
+ used: set[int] = set()
+ for slot in range(topk):
+ attempt = 0
+ while True:
+ value = _counter(seed, token, slot, attempt, 0)
+ expert = (
+ value % experts
+ if cumulative is None
+ else bisect.bisect_right(cumulative, value % cumulative[-1])
+ )
+ if expert not in used:
+ used.add(expert)
+ selected.append(expert)
+ break
+ attempt += 1
+ if attempt > experts * 16:
+ raise RuntimeError("counter routing could not select distinct experts")
+ raw = [1 + _counter(seed, token, slot, 0, 1) % 65535 for slot in range(topk)]
+ denominator = float(sum(raw))
+ indices.append(selected)
+ weights.append([value / denominator for value in raw])
+ return indices, weights
+
+
+def _canonical_bytes(
+ indices: list[list[int]], weights: list[list[float]]
+) -> tuple[bytes, bytes]:
+ idx = array("i", (value for row in indices for value in row))
+ gate = array("f", (value for row in weights for value in row))
+ if idx.itemsize != 4 or gate.itemsize != 4:
+ raise RuntimeError("canonical workload requires 32-bit int and float arrays")
+ if sys.byteorder != "little":
+ idx.byteswap()
+ gate.byteswap()
+ return idx.tobytes(), gate.tobytes()
+
+
+def trace_checksums(
+ indices: list[list[int]], weights: list[list[float]]
+) -> dict[str, str]:
+ """Return the manifest hashes for exact logical or remapped routing rows."""
+ idx_bytes, weight_bytes = _canonical_bytes(indices, weights)
+ return {
+ "topk_idx": _sha256(idx_bytes),
+ "topk_weights": _sha256(weight_bytes),
+ "trace": _sha256(idx_bytes + weight_bytes),
+ }
+
+
+def canonical_member(
+ routing: str,
+ hidden: int,
+ topk: int,
+ experts: int,
+ ep_size: int,
+ tokens_per_rank: int,
+ seed: int,
+) -> tuple[str, dict[str, str], list[list[int]], list[list[float]]]:
+ """Derive one canonical manifest member and retain its rows for proof checks."""
+ global_tokens = ep_size * tokens_per_rank
+ indices, weights = canonical_routing_rows(global_tokens, experts, topk, routing, seed)
+ checksums = trace_checksums(indices, weights)
+ member = compute_workload_id(
+ routing,
+ hidden,
+ topk,
+ experts,
+ ep_size,
+ global_tokens,
+ seed,
+ trace_checksum=checksums["trace"],
+ )
+ return member, checksums, indices, weights
+
+
+def compute_workload_id(routing: str, hidden: int, topk: int, experts: int,
+ ep_size: int, global_tokens: int, seed: int,
+ generator: str = GENERATOR_VERSION,
+ trace_checksum: str | None = None) -> str:
+ """Deterministic ID over parameters and canonical trace bytes."""
+ if generator != GENERATOR_VERSION:
+ raise ValueError(f"unsupported workload generator {generator!r}")
+ if trace_checksum is None:
+ indices, weights = canonical_routing_rows(global_tokens, experts, topk, routing, seed)
+ idx_bytes, weight_bytes = _canonical_bytes(indices, weights)
+ trace_checksum = _sha256(idx_bytes + weight_bytes)
+ key = {
+ "generator": generator, "routing": routing, "hidden": hidden, "topk": topk,
+ "experts": experts, "ep_size": ep_size, "global_tokens": global_tokens,
+ "seed": seed, "trace_sha256": trace_checksum,
+ "activation_generator": ACTIVATION_GENERATOR,
+ "activation_identity": compute_activation_identity(seed, hidden),
+ }
+ return identity.workload_id(key)
+
+
+def compute_activation_identity(seed, hidden, generator=ACTIVATION_GENERATOR) -> str:
+ """Identity of the exact counter-derived activation generator."""
+ key = f"counter|seed={seed}|hidden={hidden}|gen={generator}"
+ return _sha256(key.encode())
+
+
+def build_manifest(routing, hidden, topk, experts, global_tokens, seed, experts_per_rank,
+ idx_np, weights_np):
+ """Assemble the manifest dict from the (numpy) trace arrays. Pure numpy/stdlib."""
+ if experts % experts_per_rank:
+ raise ValueError("experts must be divisible by experts_per_rank")
+ idx_bytes = idx_np.astype(" str:
+ import numpy as np
+ os.makedirs(out_dir, exist_ok=True)
+ wid = manifest["workload_id"]
+ np.savez_compressed(os.path.join(out_dir, f"{wid}.npz"),
+ topk_idx=idx_np.astype(np.int32), topk_weights=weights_np.astype(np.float32))
+ with open(os.path.join(out_dir, f"{wid}.manifest.json"), "w") as fh:
+ json.dump(manifest, fh, indent=2, sort_keys=True)
+ return wid
+
+
+def load_workload(npz_path, verify=True):
+ """Load a canonical trace (numpy + stdlib only). Returns (idx_np, weights_np, manifest).
+ Raises ValueError if verify=True and the on-disk bytes don't match the manifest checksums."""
+ import numpy as np
+ base = npz_path[:-4] if npz_path.endswith(".npz") else npz_path
+ with open(base + ".manifest.json") as fh:
+ manifest = json.load(fh)
+ if manifest.get("workload_id") != os.path.basename(base):
+ raise ValueError(f"workload manifest ID does not match filename for {base}")
+ with np.load(base + ".npz", allow_pickle=False) as archive:
+ if set(archive.files) != {"topk_idx", "topk_weights"}:
+ raise ValueError(f"workload archive fields differ for {base}")
+ idx_np = np.ascontiguousarray(archive["topk_idx"])
+ w_np = np.ascontiguousarray(archive["topk_weights"])
+ if verify:
+ ok, reason = verify_workload(manifest, idx_np, w_np)
+ if not ok:
+ raise ValueError(f"workload checksum mismatch for {base}: {reason}")
+ return idx_np, w_np, manifest
+
+
+def verify_workload(manifest, idx_np, weights_np):
+ """Recompute checksums and compare to the manifest. Returns (ok, reason)."""
+ import numpy as np
+ expected_fields = {
+ "schema_version", "workload_id", "generator_version", "gate_weight_format", "dims",
+ "routing_profile", "seed", "checksums", "activation_profile", "activation_generator",
+ "activation_identity",
+ }
+ if not isinstance(manifest, dict) or set(manifest) != expected_fields:
+ return False, "manifest fields differ from the v1 contract"
+ if (manifest["schema_version"] != WORKLOAD_SCHEMA_VERSION
+ or manifest["generator_version"] != GENERATOR_VERSION
+ or manifest["gate_weight_format"] != GATE_WEIGHT_FORMAT
+ or manifest["routing_profile"] not in {"uniform", "zipf"}):
+ return False, "manifest version or generator is unsupported"
+ if (isinstance(manifest["seed"], bool) or not isinstance(manifest["seed"], int)
+ or not identity.is_typed_id(manifest["workload_id"], "workload")):
+ return False, "manifest seed or workload ID is invalid"
+ dims = manifest["dims"]
+ dim_fields = {"hidden", "topk", "experts", "ep_size", "tokens_per_rank",
+ "global_tokens", "experts_per_rank"}
+ if not isinstance(dims, dict) or set(dims) != dim_fields:
+ return False, "manifest dimensions are invalid"
+ if any(isinstance(dims[key], bool) or not isinstance(dims[key], int) or dims[key] <= 0
+ for key in dim_fields):
+ return False, "manifest dimensions must be positive integers"
+ if (dims["experts"] != dims["ep_size"] * dims["experts_per_rank"]
+ or dims["global_tokens"] != dims["ep_size"] * dims["tokens_per_rank"]):
+ return False, "manifest EP dimensions are inconsistent"
+ shape = (dims["global_tokens"], dims["topk"])
+ if (idx_np.dtype != np.int32 or weights_np.dtype != np.float32
+ or idx_np.shape != shape or weights_np.shape != shape
+ or not idx_np.flags.c_contiguous or not weights_np.flags.c_contiguous):
+ return False, "workload array dtype, shape, or layout is invalid"
+ if (np.any(idx_np < 0) or np.any(idx_np >= dims["experts"])
+ or np.any(np.diff(np.sort(idx_np, axis=1), axis=1) == 0)):
+ return False, "expert indices are out of range or repeated"
+ if (not np.isfinite(weights_np).all() or np.any(weights_np < 0)
+ or not np.allclose(weights_np.sum(axis=1), 1.0, rtol=1e-5, atol=1e-6)):
+ return False, "gate weights are invalid"
+ if (manifest["activation_profile"] != "canonical-counter-source-v3"
+ or manifest["activation_generator"] != ACTIVATION_GENERATOR
+ or manifest["activation_identity"]
+ != compute_activation_identity(
+ manifest["seed"], dims["hidden"], manifest["activation_generator"]
+ )):
+ return False, "activation identity is invalid"
+ ib = idx_np.astype(" must fail
+ idx2[0, 0] = (int(idx2[0, 0]) + 1) % 256
+ bad, _ = verify_workload(man2, idx2, w2)
+ assert not bad, "verify must catch tampering"
+ print(f"save/load/verify roundtrip OK (workload_id={wid})")
+ except ImportError:
+ print("(numpy unavailable — skipped serialization roundtrip; id logic passed)")
+ print("workload self-test: PASS")
+ sys.exit(0)
From af511b88dcddec9dae1a4a1fa809366b3b1b8b46 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 15:51:03 +0800
Subject: [PATCH 002/210] feat(collectivex): add V1 contracts and precision
probes
---
.github/workflows/collectivex-publish.yml | 197 ---
.github/workflows/collectivex-sweep.yml | 495 +++++-
AGENTS.md | 2 +
experimental/CollectiveX/README.md | 24 +-
experimental/CollectiveX/capability.py | 345 ++++-
experimental/CollectiveX/configs/suites.yaml | 36 +
experimental/CollectiveX/contracts.py | 341 ++++-
experimental/CollectiveX/docs/methodology.md | 30 +-
experimental/CollectiveX/identity.py | 215 ++-
experimental/CollectiveX/publisher.py | 1322 ++++++++++++++---
experimental/CollectiveX/runtime/common.sh | 125 +-
.../CollectiveX/runtime/run_in_container.sh | 33 +-
.../schemas/channel-v1.schema.json | 2 +-
.../schemas/private-bundle-v1.schema.json | 3 +-
.../schemas/public-dataset-v1.schema.json | 310 +++-
.../schemas/raw-case-v1.schema.json | 372 +++--
.../schemas/samples-v1.schema.json | 53 +-
.../schemas/terminal-outcome-v1.schema.json | 166 ++-
experimental/CollectiveX/sweep_matrix.py | 476 +++++-
experimental/CollectiveX/tests/ep_deepep.py | 158 +-
.../CollectiveX/tests/ep_deepep_hybrid.py | 112 +-
.../CollectiveX/tests/ep_deepep_v2.py | 120 +-
experimental/CollectiveX/tests/ep_harness.py | 533 ++++++-
experimental/CollectiveX/tests/ep_mori.py | 199 ++-
experimental/CollectiveX/tests/ep_nccl.py | 43 +-
.../CollectiveX/tests/ep_precision.py | 371 +++++
experimental/CollectiveX/tests/ep_uccl.py | 158 +-
.../CollectiveX/tests/probe_precision.py | 1162 +++++++++++++++
experimental/CollectiveX/tests/routing.py | 67 +-
experimental/CollectiveX/tests/run_ep.py | 106 +-
.../tests/test_deepep_v2_contract.py | 4 +-
.../tests/test_ep_precision_adapters.py | 130 ++
.../tests/test_precision_scheduling.py | 347 +++++
.../CollectiveX/tests/test_publisher.py | 908 ++++++++---
.../tests/test_qualification_planning.py | 247 +++
.../tests/test_sampling_contract.py | 246 ++-
.../tests/test_schema_v1_contract.py | 436 ++++++
experimental/CollectiveX/tests/workload.py | 72 +-
38 files changed, 8877 insertions(+), 1089 deletions(-)
delete mode 100644 .github/workflows/collectivex-publish.yml
create mode 100644 experimental/CollectiveX/tests/ep_precision.py
create mode 100644 experimental/CollectiveX/tests/probe_precision.py
create mode 100644 experimental/CollectiveX/tests/test_ep_precision_adapters.py
create mode 100644 experimental/CollectiveX/tests/test_precision_scheduling.py
create mode 100644 experimental/CollectiveX/tests/test_qualification_planning.py
create mode 100644 experimental/CollectiveX/tests/test_schema_v1_contract.py
diff --git a/.github/workflows/collectivex-publish.yml b/.github/workflows/collectivex-publish.yml
deleted file mode 100644
index 38143123b9..0000000000
--- a/.github/workflows/collectivex-publish.yml
+++ /dev/null
@@ -1,197 +0,0 @@
-name: CollectiveX Publish V1
-
-on:
- workflow_dispatch:
- inputs:
- run_ids:
- description: Three successful CollectiveX Sweep run IDs, comma-separated
- required: true
- type: string
-
-permissions:
- actions: read
- contents: read
-
-concurrency:
- group: collectivex-publish-v1
- cancel-in-progress: false
-
-jobs:
- publish:
- runs-on: ubuntu-latest
- timeout-minutes: 30
- env:
- GH_TOKEN: ${{ github.token }}
- RUN_IDS: ${{ inputs.run_ids }}
- steps:
- - name: Verify source runs
- id: runs
- env:
- REPOSITORY: ${{ github.repository }}
- run: |
- set -euo pipefail
- IFS=',' read -r -a run_ids <<< "$RUN_IDS"
- [ "${#run_ids[@]}" -eq 3 ] || {
- echo 'run_ids must contain exactly three IDs' >&2
- exit 1
- }
- [ "$(printf '%s\n' "${run_ids[@]}" | sort -u | wc -l)" -eq 3 ] || {
- echo 'run_ids must be unique' >&2
- exit 1
- }
-
- : > "$RUNNER_TEMP/collectivex-runs.tsv"
- source_sha=''
- for run_id in "${run_ids[@]}"; do
- [[ "$run_id" =~ ^[1-9][0-9]*$ ]] || {
- echo 'run_ids contains a non-decimal ID' >&2
- exit 1
- }
- metadata=$(gh api "repos/$REPOSITORY/actions/runs/$run_id")
- name=$(jq -r '.name' <<< "$metadata")
- path=$(jq -r '.path' <<< "$metadata")
- branch=$(jq -r '.head_branch' <<< "$metadata")
- status=$(jq -r '.status' <<< "$metadata")
- conclusion=$(jq -r '.conclusion' <<< "$metadata")
- sha=$(jq -r '.head_sha' <<< "$metadata")
- attempt=$(jq -r '.run_attempt' <<< "$metadata")
- [ "$name" = 'CollectiveX Sweep' ] \
- && [ "$path" = '.github/workflows/collectivex-sweep.yml' ] \
- && [ "$branch" = 'collectivex' ] \
- && [ "$status" = 'completed' ] \
- && [ "$conclusion" = 'success' ] \
- && [[ "$sha" =~ ^[0-9a-f]{40}$ ]] \
- && [[ "$attempt" =~ ^[1-9][0-9]*$ ]] || {
- echo "run $run_id is not an eligible CollectiveX V1 sweep" >&2
- exit 1
- }
- if [ -z "$source_sha" ]; then
- source_sha="$sha"
- else
- [ "$sha" = "$source_sha" ] || {
- echo 'source runs do not share one source SHA' >&2
- exit 1
- }
- fi
- printf '%s\t%s\t%s\n' "$run_id" "$attempt" "$sha" \
- >> "$RUNNER_TEMP/collectivex-runs.tsv"
- done
- echo "source_sha=$source_sha" >> "$GITHUB_OUTPUT"
-
- - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v5.0.0
- with:
- ref: ${{ steps.runs.outputs.source_sha }}
- clean: true
- persist-credentials: false
-
- - name: Install publisher dependencies
- run: python3 -m pip install --quiet -r experimental/CollectiveX/requirements.txt
-
- - name: Build promoted publication
- env:
- REPOSITORY: ${{ github.repository }}
- run: |
- set -euo pipefail
- store="$RUNNER_TEMP/collectivex-publisher"
- downloads="$RUNNER_TEMP/collectivex-downloads"
- output="$RUNNER_TEMP/collectivex-publication"
- umask 027
- mkdir -m 750 "$store"
- mkdir -m 750 "$downloads" "$output"
- : > "$RUNNER_TEMP/collectivex-bundles.txt"
-
- while IFS=$'\t' read -r run_id attempt source_sha; do
- run_dir="$downloads/$run_id"
- mkdir -m 750 "$run_dir"
- gh run download "$run_id" --repo "$REPOSITORY" --dir "$run_dir"
- matrix="$run_dir/cxsweep-matrix-$run_id-$attempt/matrix_full.json"
- marker="$run_dir/cxrelease-v1-$run_id-$attempt/release.json"
- [ -f "$matrix" ] || {
- echo "run $run_id is missing its exact matrix artifact" >&2
- exit 1
- }
- [ -f "$marker" ] || {
- echo "run $run_id is not tagged for V1 publication" >&2
- exit 1
- }
- matrix_sha=$(sha256sum "$matrix" | cut -d' ' -f1)
- jq -e \
- --arg run_id "$run_id" \
- --arg attempt "$attempt" \
- --arg source_sha "$source_sha" \
- --arg matrix_sha "$matrix_sha" \
- 'keys == ["format","matrix_sha256","release_tag","run_attempt","run_id","source_sha"]
- and .format == "collectivex.release-tag.v1"
- and .release_tag == "v1"
- and .run_id == $run_id
- and .run_attempt == $attempt
- and .source_sha == $source_sha
- and .matrix_sha256 == $matrix_sha' \
- "$marker" >/dev/null || {
- echo "run $run_id has an invalid V1 release marker" >&2
- exit 1
- }
-
- mapfile -t artifacts < <(
- find "$run_dir" -mindepth 1 -maxdepth 1 -type d \
- \( -name "cxshard-*-$run_id-$attempt" \
- -o -name "cxunsupported-$run_id-$attempt" \) -print | sort
- )
- [ "${#artifacts[@]}" -gt 0 ] || {
- echo "run $run_id has no result artifacts" >&2
- exit 1
- }
- artifact_args=()
- for artifact in "${artifacts[@]}"; do
- artifact_args+=(--artifact "$artifact")
- done
- result=$(
- python3 experimental/CollectiveX/publisher.py --store-root "$store" ingest \
- --matrix "$matrix" \
- "${artifact_args[@]}" \
- --repository "$REPOSITORY" \
- --run-id "$run_id" \
- --run-attempt "$attempt" \
- --source-sha "$source_sha"
- )
- bundle_id=$(jq -er '.bundle_id' <<< "$result")
- printf '%s\n' "$bundle_id" >> "$RUNNER_TEMP/collectivex-bundles.txt"
- done < "$RUNNER_TEMP/collectivex-runs.tsv"
-
- mapfile -t bundle_ids < "$RUNNER_TEMP/collectivex-bundles.txt"
- promote_args=()
- for bundle_id in "${bundle_ids[@]}"; do
- promote_args+=(--bundle "$bundle_id")
- done
- result=$(
- python3 experimental/CollectiveX/publisher.py --store-root "$store" promote \
- "${promote_args[@]}"
- )
- dataset_id=$(jq -er '.dataset_sha256' <<< "$result")
- dataset="$store/public/datasets/$dataset_id/dataset.json"
- [ -f "$dataset" ] || {
- echo 'publisher did not install the promoted dataset' >&2
- exit 1
- }
- publication="$output/collectivex_public_v1_$dataset_id.ndjson"
- cp -- "$dataset" "$publication"
- python3 experimental/CollectiveX/artifact_safety.py "$publication"
- python3 experimental/CollectiveX/publisher.py --store-root "$store" verify \
- --channel dev-latest "${promote_args[@]}"
- sha256sum "$publication"
- {
- echo '## CollectiveX V1 publication'
- echo
- echo "Dataset: \`$dataset_id\`"
- echo
- echo 'Source runs:'
- sed 's/^/- `/' "$RUNNER_TEMP/collectivex-runs.tsv" | sed 's/$/`/'
- } >> "$GITHUB_STEP_SUMMARY"
-
- - name: Upload JIT publication artifact
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
- with:
- name: cxpublication-v1-${{ github.run_id }}-${{ github.run_attempt }}
- path: ${{ runner.temp }}/collectivex-publication/*.ndjson
- if-no-files-found: error
- retention-days: 90
diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml
index 2837a6da10..47de47a8ce 100644
--- a/.github/workflows/collectivex-sweep.yml
+++ b/.github/workflows/collectivex-sweep.yml
@@ -1,14 +1,20 @@
# CollectiveX Sweep — one structured run instead of thousands of dispatches.
#
-# Shape: setup -> sweep. The matrix unit is a shard: one allocation that sweeps
-# cases sharing (sku, backend, nodes). Each cell uploads its privacy-checked raw
-# result JSONs. The isolated v1 publisher consumes downloaded shards separately.
+# Shape: sweep/probe runs setup -> GPU cells; publish-v1 and refresh-v1 stay on disposable
+# GitHub-hosted storage and skip GPU jobs. This file is registered on the default
+# branch, so its collectivex branch revision can be dispatched with --ref.
name: CollectiveX Sweep
permissions:
+ actions: read
contents: read
on:
workflow_dispatch:
inputs:
+ operation:
+ description: Operation to execute
+ type: choice
+ default: sweep
+ options: [sweep, probe-precision, publish-v1, refresh-v1]
backend:
description: "EP library to sweep — 'all' runs every EP backend in one matrix"
type: choice
@@ -39,17 +45,36 @@ on:
type: choice
default: unversioned
options: [unversioned, v1]
+ qualification_index:
+ description: Deterministic execution-order index; V1 qualification uses 1, then 2, then 3
+ type: choice
+ default: '1'
+ options: ['1', '2', '3']
+ publish_run_ids:
+ description: For publish-v1, exactly three successful V1 sweep run IDs
+ type: string
+ default: ''
+ refresh_run_id:
+ description: For refresh-v1, source publication workflow run ID
+ type: string
+ default: ''
+ refresh_digest:
+ description: For refresh-v1, exact publication dataset SHA-256
+ type: string
+ default: ''
concurrency:
- group: cx-sweep-${{ github.ref }}-${{ inputs.release_tag }}-${{ inputs.backend }}-${{ inputs.only_sku }}
+ group: cx-${{ inputs.operation }}-${{ github.ref }}-${{ inputs.release_tag }}-${{ inputs.backend }}-${{ inputs.only_sku }}
cancel-in-progress: false
jobs:
# ---- setup: resolve the suites into the shard matrix (the "pending jobs" node) ----
setup:
+ if: ${{ inputs.operation == 'sweep' || inputs.operation == 'probe-precision' }}
runs-on: ubuntu-latest
outputs:
matrix: ${{ steps.gen.outputs.matrix }}
n: ${{ steps.gen.outputs.n }}
+ max_parallel: ${{ steps.gen.outputs.max_parallel }}
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v5.0.0
with: { clean: true, persist-credentials: false }
@@ -59,39 +84,74 @@ jobs:
working-directory: experimental/CollectiveX
env:
INPUT_BACKEND: ${{ inputs.backend }}
+ INPUT_OPERATION: ${{ inputs.operation }}
INPUT_SUITES: ${{ inputs.suites }}
INPUT_ONLY_SKU: ${{ inputs.only_sku }}
INPUT_MIN_NODES: ${{ inputs.min_nodes }}
INPUT_MAX_NODES: ${{ inputs.max_nodes }}
INPUT_MAX_CASES: ${{ inputs.max_cases }}
+ INPUT_QUALIFICATION_INDEX: ${{ inputs.qualification_index }}
+ CX_QUALIFICATION_INDEX: ${{ inputs.qualification_index }}
COLLECTIVEX_SOURCE_SHA: ${{ github.sha }}
COLLECTIVEX_ARTIFACT_NAME: cxunsupported-${{ github.run_id }}-${{ github.run_attempt }}
COLLECTIVEX_EXECUTION_ID: ${{ github.run_id }}_${{ github.run_attempt }}_unsupported
run: |
set -euo pipefail
- args=(--suites "$INPUT_SUITES" --max-cases "$INPUT_MAX_CASES")
- case "$INPUT_BACKEND" in
- all) args+=(--backends all) ;;
- *) args+=(--backend "$INPUT_BACKEND") ;;
- esac
- [ -n "$INPUT_ONLY_SKU" ] && args+=(--only-sku "$INPUT_ONLY_SKU")
- [ -n "$INPUT_MIN_NODES" ] && args+=(--min-nodes "$INPUT_MIN_NODES")
- [ -n "$INPUT_MAX_NODES" ] && args+=(--max-nodes "$INPUT_MAX_NODES")
- python3 sweep_matrix.py "${args[@]}" --out matrix_full.json >/dev/null
+ if [ "$INPUT_OPERATION" = probe-precision ] && [ "${{ inputs.release_tag }}" != unversioned ]; then
+ echo 'precision probes cannot carry a V1 release tag' >&2
+ exit 1
+ fi
+ if [ "$INPUT_OPERATION" = sweep ] && [ "${{ inputs.release_tag }}" = v1 ]; then
+ [[ "$INPUT_QUALIFICATION_INDEX" =~ ^[123]$ ]] || {
+ echo 'V1 sweeps require qualification_index 1, 2, or 3' >&2
+ exit 1
+ }
+ [ "$INPUT_BACKEND" = all ] && [ "$INPUT_SUITES" = all ] \
+ && [ -z "$INPUT_ONLY_SKU" ] && [ -z "$INPUT_MIN_NODES" ] \
+ && [ -z "$INPUT_MAX_NODES" ] && [ "$INPUT_MAX_CASES" = 128 ] || {
+ echo 'V1 sweeps require the exact unfiltered full matrix' >&2
+ exit 1
+ }
+ python3 - <<'PY'
+ import capability
+
+ if capability.provisional_precision_targets():
+ raise SystemExit("V1 sweeps require every precision capability cell to be resolved")
+ PY
+ fi
+ if [ "$INPUT_OPERATION" = probe-precision ]; then
+ args=(--workflow-plan --backend "$INPUT_BACKEND" --out matrix_full.json)
+ [ -n "$INPUT_ONLY_SKU" ] && args+=(--only-sku "$INPUT_ONLY_SKU")
+ python3 tests/probe_precision.py "${args[@]}"
+ else
+ args=(--suites "$INPUT_SUITES" --max-cases "$INPUT_MAX_CASES")
+ case "$INPUT_BACKEND" in
+ all) args+=(--backends all) ;;
+ *) args+=(--backend "$INPUT_BACKEND") ;;
+ esac
+ [ -n "$INPUT_ONLY_SKU" ] && args+=(--only-sku "$INPUT_ONLY_SKU")
+ [ -n "$INPUT_MIN_NODES" ] && args+=(--min-nodes "$INPUT_MIN_NODES")
+ [ -n "$INPUT_MAX_NODES" ] && args+=(--max-nodes "$INPUT_MAX_NODES")
+ python3 sweep_matrix.py "${args[@]}" --out matrix_full.json >/dev/null
+ fi
python3 artifact_safety.py matrix_full.json
SLIM=$(python3 -c "import json;m=json.load(open('matrix_full.json'));print(json.dumps({'include':[{k:v for k,v in x.items() if k!='case_ids'} for x in m['include']]}))")
{
echo "matrix=$SLIM"
echo "n=$(python3 -c "import json;print(len(json.load(open('matrix_full.json'))['include']))")"
echo "source_backends=$(python3 -c "import json;m=json.load(open('matrix_full.json'));print(' '.join(sorted({x['backend'] for x in m['include']} & {'deepep-v2','deepep-hybrid'})))")"
+ echo "max_parallel=$(python3 -c "import json;m=json.load(open('matrix_full.json'));w=max(x['execution_weight'] for x in m['include']);b=64 if '$INPUT_OPERATION' == 'probe-precision' else 4096;print(max(1,min(10,b//w)))")"
} >> "$GITHUB_OUTPUT"
- unsupported_n=$(python3 -c "import json;m=json.load(open('matrix_full.json'));print(sum(x['disposition']=='unsupported' for x in m['requested_cases']))")
+ unsupported_n=0
+ if [ "$INPUT_OPERATION" = sweep ]; then
+ unsupported_n=$(python3 -c "import json;m=json.load(open('matrix_full.json'));print(sum(x['disposition']=='unsupported' for x in m['requested_cases']))")
+ fi
echo "unsupported_n=$unsupported_n" >> "$GITHUB_OUTPUT"
if [ "$unsupported_n" -gt 0 ]; then
python3 sweep_matrix.py --emit-unsupported-from matrix_full.json \
--out-dir unsupported
fi
- python3 -c "import json;m=json.load(open('matrix_full.json'));r=m['requested_cases'];print('shard-cells:',len(m['include']),'runnable:',sum(x['disposition']=='runnable' for x in r),'unsupported:',sum(x['disposition']=='unsupported' for x in r))"
+ python3 -c "import json;m=json.load(open('matrix_full.json'));print('execution-cells:',len(m['include']))"
- name: Prepare pinned backend source archive
if: ${{ steps.gen.outputs.source_backends != '' }}
working-directory: experimental/CollectiveX
@@ -123,20 +183,21 @@ jobs:
name: cxbackend-sources-${{ github.run_id }}-${{ github.run_attempt }}
path: ${{ runner.temp }}/collectivex-backend-sources.tar
if-no-files-found: error
- retention-days: 3
+ retention-days: ${{ inputs.release_tag == 'v1' && 90 || 3 }}
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: cxsweep-matrix-${{ github.run_id }}-${{ github.run_attempt }}
path: experimental/CollectiveX/matrix_full.json
if-no-files-found: error
- retention-days: 3
+ retention-days: ${{ inputs.release_tag == 'v1' && 90 || 3 }}
- name: Create V1 release marker
- if: ${{ inputs.release_tag == 'v1' }}
+ if: ${{ inputs.operation == 'sweep' && inputs.release_tag == 'v1' }}
env:
EXPECTED_MATRIX_SHA256: f1ca85f9689922b90edd5767b9ff2a902f6b896f32f68b2ca086dde3fd2157d0
RUN_ID: ${{ github.run_id }}
RUN_ATTEMPT: ${{ github.run_attempt }}
SOURCE_SHA: ${{ github.sha }}
+ QUALIFICATION_INDEX: ${{ inputs.qualification_index }}
run: |
set -euo pipefail
destination="$RUNNER_TEMP/collectivex-release"
@@ -148,13 +209,22 @@ jobs:
import pathlib
import sys
+ sys.path.insert(0, str(pathlib.Path("experimental/CollectiveX").resolve()))
+ import sweep_matrix
+
matrix = pathlib.Path("experimental/CollectiveX/matrix_full.json").read_bytes()
matrix_sha256 = hashlib.sha256(matrix).hexdigest()
if matrix_sha256 != os.environ["EXPECTED_MATRIX_SHA256"]:
raise SystemExit("V1 release tag requires the locked full matrix")
+ qualification_index = int(os.environ["QUALIFICATION_INDEX"])
+ execution_plan_sha256 = sweep_matrix.qualification_execution_plan_sha256(
+ json.loads(matrix), qualification_index
+ )
marker = {
+ "execution_plan_sha256": execution_plan_sha256,
"format": "collectivex.release-tag.v1",
"matrix_sha256": matrix_sha256,
+ "qualification_index": qualification_index,
"release_tag": "v1",
"run_attempt": os.environ["RUN_ATTEMPT"],
"run_id": os.environ["RUN_ID"],
@@ -166,13 +236,13 @@ jobs:
PY
python3 experimental/CollectiveX/artifact_safety.py "$destination/release.json"
- name: Upload V1 release marker
- if: ${{ inputs.release_tag == 'v1' }}
+ if: ${{ inputs.operation == 'sweep' && inputs.release_tag == 'v1' }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: cxrelease-v1-${{ github.run_id }}-${{ github.run_attempt }}
path: ${{ runner.temp }}/collectivex-release/release.json
if-no-files-found: error
- retention-days: 3
+ retention-days: 90
- name: Validate unsupported artifact safety
id: unsupported_safety
if: ${{ always() && fromJSON(steps.gen.outputs.unsupported_n) > 0 }}
@@ -195,15 +265,15 @@ jobs:
name: cxunsupported-${{ github.run_id }}-${{ github.run_attempt }}
path: experimental/CollectiveX/unsupported/*.json
if-no-files-found: error
- retention-days: 3
+ retention-days: ${{ inputs.release_tag == 'v1' && 90 || 3 }}
# ---- sweep: ONE matrix cell per shard (the parent job with child jobs) ----
sweep:
needs: setup
- if: ${{ fromJSON(needs.setup.outputs.n) > 0 }}
+ if: ${{ (inputs.operation == 'sweep' || inputs.operation == 'probe-precision') && fromJSON(needs.setup.outputs.n) > 0 }}
strategy:
fail-fast: false
- max-parallel: 10
+ max-parallel: ${{ fromJSON(needs.setup.outputs.max_parallel) }}
matrix: ${{ fromJSON(needs.setup.outputs.matrix) }}
runs-on: ${{ matrix.sku }}
timeout-minutes: 350
@@ -214,15 +284,17 @@ jobs:
CX_SCALE_UP_DOMAIN: ${{ matrix.scale_up_domain }}
CX_SHARD_FILE: .shards/${{ matrix.id }}.json
CX_SHARD_SKU: ${{ matrix.sku }}
+ CX_PRECISION_PROBE: ${{ inputs.operation == 'probe-precision' && '1' || '0' }}
COLLECTIVEX_CANONICAL_GHA: '1'
COLLECTIVEX_SOURCE_SHA: ${{ github.sha }}
- COLLECTIVEX_ARTIFACT_NAME: cxshard-${{ matrix.id }}-${{ github.run_id }}-${{ github.run_attempt }}
+ COLLECTIVEX_ARTIFACT_NAME: ${{ inputs.operation == 'probe-precision' && format('cxprobe-{0}-{1}-{2}', matrix.id, github.run_id, github.run_attempt) || format('cxshard-{0}-{1}-{2}', matrix.id, github.run_id, github.run_attempt) }}
# Consolidated shards run one bounded build-group in one Slurm allocation, so
# the launcher's default 45-min --time is too short. 300 min covers a cold
# compute-node image import plus the shard. The allocation releases early
# when the shard finishes, so short shards don't waste it.
- CX_TIME: '300'
+ CX_TIME: ${{ inputs.operation == 'probe-precision' && '90' || '300' }}
COLLECTIVEX_EXECUTION_ID: ${{ github.run_id }}_${{ github.run_attempt }}_${{ matrix.id }}
+ CX_QUALIFICATION_INDEX: ${{ inputs.qualification_index }}
CX_JOB_ROOT: /tmp/inferencex-collectivex-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.id }}
CX_SOURCE_ROOT: /tmp/inferencex-collectivex-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.id }}/source
HOME: /tmp/inferencex-collectivex-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.id }}/home
@@ -337,19 +409,29 @@ jobs:
"$archive" "$destination" "$source_basename"
cx_backend_source_is_valid "$EXPECTED_BACKEND" "$source_path"
printf 'CX_BACKEND_SOURCE_SEED_ROOT=%s\n' "$seed_root" >> "$GITHUB_ENV"
- - name: Extract and validate this shard's cases
+ - name: Extract and validate this execution control
run: |
set -euo pipefail
cd "$CX_SOURCE_ROOT/experimental/CollectiveX" 2>/dev/null \
|| { echo "CollectiveX source is unavailable" >&2; exit 1; }
- python3 sweep_matrix.py \
- --extract-from "$CX_JOB_ROOT/control/matrix_full.json" \
- --shard-id '${{ matrix.id }}' \
- --expect-sku '${{ matrix.sku }}' \
- --expect-backend '${{ matrix.backend }}' \
- --expect-nodes '${{ matrix.nodes }}' \
- --out '${{ env.CX_SHARD_FILE }}' >/dev/null
- - name: Sweep shard ${{ matrix.id }} (${{ matrix.n }} cases, one allocation)
+ if [ '${{ inputs.operation }}' = probe-precision ]; then
+ python3 tests/probe_precision.py \
+ --extract-from "$CX_JOB_ROOT/control/matrix_full.json" \
+ --probe-id '${{ matrix.id }}' \
+ --expect-sku '${{ matrix.sku }}' \
+ --expect-backend '${{ matrix.backend }}' \
+ --expect-nodes '${{ matrix.nodes }}' \
+ --out '${{ env.CX_SHARD_FILE }}'
+ else
+ python3 sweep_matrix.py \
+ --extract-from "$CX_JOB_ROOT/control/matrix_full.json" \
+ --shard-id '${{ matrix.id }}' \
+ --expect-sku '${{ matrix.sku }}' \
+ --expect-backend '${{ matrix.backend }}' \
+ --expect-nodes '${{ matrix.nodes }}' \
+ --out '${{ env.CX_SHARD_FILE }}' >/dev/null
+ fi
+ - name: Execute ${{ inputs.operation }} cell ${{ matrix.id }}
id: sweep_shard
env:
COLLECTIVEX_OPERATOR_CONFIG_CONTENT: ${{ secrets.COLLECTIVEX_OPERATOR_CONFIG_V1 }}
@@ -382,11 +464,16 @@ jobs:
run: |
cd "$CX_SOURCE_ROOT" 2>/dev/null \
|| { echo "CollectiveX source is unavailable" >&2; exit 1; }
- python3 experimental/CollectiveX/contracts.py validate-delivery \
- --source "experimental/CollectiveX/${CX_SHARD_FILE}" \
- experimental/CollectiveX/results/*.json
+ if [ '${{ inputs.operation }}' = probe-precision ]; then
+ python3 experimental/CollectiveX/tests/probe_precision.py \
+ --validate-manifest experimental/CollectiveX/results/*.json
+ else
+ python3 experimental/CollectiveX/contracts.py validate-delivery \
+ --source "experimental/CollectiveX/${CX_SHARD_FILE}" \
+ experimental/CollectiveX/results/*.json
+ fi
- name: Shard summary
- if: ${{ always() && steps.artifact_safety.outcome == 'success' && steps.delivery_contracts.outcome == 'success' }}
+ if: ${{ inputs.operation == 'sweep' && always() && steps.artifact_safety.outcome == 'success' && steps.delivery_contracts.outcome == 'success' }}
run: |
cd "$CX_SOURCE_ROOT" 2>/dev/null \
|| { echo "CollectiveX source is unavailable" >&2; exit 1; }
@@ -405,11 +492,11 @@ jobs:
if: always() && steps.stage_artifact.outcome == 'success' && steps.delivery_contracts.outcome == 'success' && steps.artifact_safety.outcome == 'success'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
- name: cxshard-${{ matrix.id }}-${{ github.run_id }}-${{ github.run_attempt }}
+ name: ${{ inputs.operation == 'probe-precision' && format('cxprobe-{0}-{1}-{2}', matrix.id, github.run_id, github.run_attempt) || format('cxshard-{0}-{1}-{2}', matrix.id, github.run_id, github.run_attempt) }}
path: |
${{ env.CX_JOB_ROOT }}/artifact/*.json
if-no-files-found: error
- retention-days: 3
+ retention-days: ${{ inputs.release_tag == 'v1' && 90 || 3 }}
- name: Cleanup isolated workspace
if: ${{ always() && steps.source.outputs.prepared == 'true' }}
run: |
@@ -434,3 +521,331 @@ jobs:
fi
fi
rm -rf -- "$CX_JOB_ROOT"
+
+ probe-summary:
+ needs: [setup, sweep]
+ if: ${{ always() && inputs.operation == 'probe-precision' && needs.sweep.result == 'success' }}
+ runs-on: ubuntu-latest
+ timeout-minutes: 15
+ steps:
+ - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v5.0.0
+ with: { clean: true, persist-credentials: false }
+ - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+ with:
+ name: cxsweep-matrix-${{ github.run_id }}-${{ github.run_attempt }}
+ path: ${{ runner.temp }}/collectivex-probes/control
+ - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+ with:
+ pattern: cxprobe-*-${{ github.run_id }}-${{ github.run_attempt }}
+ merge-multiple: true
+ path: ${{ runner.temp }}/collectivex-probes/results
+ - name: Validate exact precision probe coverage
+ run: |
+ set -euo pipefail
+ plan="$RUNNER_TEMP/collectivex-probes/control/matrix_full.json"
+ shopt -s nullglob
+ manifests=("$RUNNER_TEMP"/collectivex-probes/results/*.json)
+ [ "${#manifests[@]}" -gt 0 ] || {
+ echo 'precision probe artifacts are empty' >&2
+ exit 1
+ }
+ python3 experimental/CollectiveX/artifact_safety.py "$plan" "${manifests[@]}"
+ python3 experimental/CollectiveX/tests/probe_precision.py \
+ --validate-bundle "$plan" --validate-manifest "${manifests[@]}"
+ - name: Upload validated precision probe bundle
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: cxprecision-probes-${{ github.run_id }}-${{ github.run_attempt }}
+ path: ${{ runner.temp }}/collectivex-probes
+ if-no-files-found: error
+ retention-days: 30
+
+ publish:
+ if: ${{ inputs.operation == 'publish-v1' }}
+ runs-on: ubuntu-latest
+ timeout-minutes: 30
+ env:
+ GH_TOKEN: ${{ github.token }}
+ RUN_IDS: ${{ inputs.publish_run_ids }}
+ steps:
+ - name: Verify source runs
+ id: runs
+ env:
+ REPOSITORY: ${{ github.repository }}
+ run: |
+ set -euo pipefail
+ IFS=',' read -r -a run_ids <<< "$RUN_IDS"
+ [ "${#run_ids[@]}" -eq 3 ] || {
+ echo 'publish_run_ids must contain exactly three IDs' >&2
+ exit 1
+ }
+ [ "$(printf '%s\n' "${run_ids[@]}" | sort -u | wc -l)" -eq 3 ] || {
+ echo 'publish_run_ids must be unique' >&2
+ exit 1
+ }
+
+ : > "$RUNNER_TEMP/collectivex-runs.tsv"
+ source_sha=''
+ for run_id in "${run_ids[@]}"; do
+ [[ "$run_id" =~ ^[1-9][0-9]*$ ]] || {
+ echo 'publish_run_ids contains a non-decimal ID' >&2
+ exit 1
+ }
+ metadata=$(gh api "repos/$REPOSITORY/actions/runs/$run_id")
+ name=$(jq -r '.name' <<< "$metadata")
+ path=$(jq -r '.path' <<< "$metadata")
+ branch=$(jq -r '.head_branch' <<< "$metadata")
+ status=$(jq -r '.status' <<< "$metadata")
+ conclusion=$(jq -r '.conclusion' <<< "$metadata")
+ sha=$(jq -r '.head_sha' <<< "$metadata")
+ attempt=$(jq -r '.run_attempt' <<< "$metadata")
+ [ "$name" = 'CollectiveX Sweep' ] \
+ && [ "$path" = '.github/workflows/collectivex-sweep.yml' ] \
+ && [ "$branch" = 'collectivex' ] \
+ && [ "$status" = 'completed' ] \
+ && [ "$conclusion" = 'success' ] \
+ && [ "$attempt" = 1 ] \
+ && [[ "$sha" =~ ^[0-9a-f]{40}$ ]] || {
+ echo "run $run_id is not an eligible first-attempt V1 sweep" >&2
+ exit 1
+ }
+ if [ -z "$source_sha" ]; then
+ source_sha="$sha"
+ else
+ [ "$sha" = "$source_sha" ] || {
+ echo 'source runs do not share one source SHA' >&2
+ exit 1
+ }
+ fi
+ printf '%s\t%s\t%s\n' "$run_id" "$attempt" "$sha" \
+ >> "$RUNNER_TEMP/collectivex-runs.tsv"
+ done
+ echo "source_sha=$source_sha" >> "$GITHUB_OUTPUT"
+
+ - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v5.0.0
+ with:
+ ref: ${{ steps.runs.outputs.source_sha }}
+ clean: true
+ persist-credentials: false
+
+ - name: Install publisher dependencies
+ run: python3 -m pip install --quiet -r experimental/CollectiveX/requirements.txt
+
+ - name: Build promoted publication
+ env:
+ REPOSITORY: ${{ github.repository }}
+ run: |
+ set -euo pipefail
+ store="$RUNNER_TEMP/collectivex-publisher"
+ downloads="$RUNNER_TEMP/collectivex-downloads"
+ output="$RUNNER_TEMP/collectivex-publication"
+ umask 027
+ mkdir -m 750 "$store" "$downloads" "$output"
+ : > "$RUNNER_TEMP/collectivex-bundles.txt"
+ : > "$RUNNER_TEMP/collectivex-qualification-indices.txt"
+
+ while IFS=$'\t' read -r run_id attempt source_sha; do
+ run_dir="$downloads/$run_id"
+ mkdir -m 750 "$run_dir"
+ gh run download "$run_id" --repo "$REPOSITORY" --dir "$run_dir"
+ matrix="$run_dir/cxsweep-matrix-$run_id-$attempt/matrix_full.json"
+ marker="$run_dir/cxrelease-v1-$run_id-$attempt/release.json"
+ [ -f "$matrix" ] || {
+ echo "run $run_id is missing its exact matrix artifact" >&2
+ exit 1
+ }
+ [ -f "$marker" ] || {
+ echo "run $run_id is not tagged for V1 publication" >&2
+ exit 1
+ }
+ matrix_sha=$(sha256sum "$matrix" | cut -d' ' -f1)
+ qualification_index=$(jq -r '.qualification_index' "$marker")
+ execution_plan_sha=$(
+ PYTHONPATH=experimental/CollectiveX python3 -c \
+ 'import json,sys,sweep_matrix as s; print(s.qualification_execution_plan_sha256(json.load(open(sys.argv[1], encoding="utf-8")), int(sys.argv[2])))' \
+ "$matrix" "$qualification_index"
+ )
+ jq -e \
+ --arg run_id "$run_id" \
+ --arg attempt "$attempt" \
+ --arg source_sha "$source_sha" \
+ --arg matrix_sha "$matrix_sha" \
+ --arg execution_plan_sha "$execution_plan_sha" \
+ 'keys == ["execution_plan_sha256","format","matrix_sha256","qualification_index","release_tag","run_attempt","run_id","source_sha"]
+ and .format == "collectivex.release-tag.v1"
+ and .release_tag == "v1"
+ and .run_id == $run_id
+ and .run_attempt == $attempt
+ and .source_sha == $source_sha
+ and (.qualification_index == 1 or .qualification_index == 2 or .qualification_index == 3)
+ and .matrix_sha256 == $matrix_sha
+ and .execution_plan_sha256 == $execution_plan_sha' \
+ "$marker" >/dev/null || {
+ echo "run $run_id has an invalid V1 release marker" >&2
+ exit 1
+ }
+ jq -r '.qualification_index' "$marker" \
+ >> "$RUNNER_TEMP/collectivex-qualification-indices.txt"
+
+ mapfile -t artifacts < <(
+ find "$run_dir" -mindepth 1 -maxdepth 1 -type d \
+ \( -name "cxshard-*-$run_id-$attempt" \
+ -o -name "cxunsupported-$run_id-$attempt" \) -print | sort
+ )
+ [ "${#artifacts[@]}" -gt 0 ] || {
+ echo "run $run_id has no result artifacts" >&2
+ exit 1
+ }
+ artifact_args=()
+ for artifact in "${artifacts[@]}"; do
+ artifact_args+=(--artifact "$artifact")
+ done
+ result=$(
+ python3 experimental/CollectiveX/publisher.py --store-root "$store" ingest \
+ --matrix "$matrix" \
+ "${artifact_args[@]}" \
+ --repository "$REPOSITORY" \
+ --run-id "$run_id" \
+ --run-attempt "$attempt" \
+ --qualification-index "$qualification_index" \
+ --source-sha "$source_sha"
+ )
+ bundle_id=$(jq -er '.bundle_id' <<< "$result")
+ printf '%s\n' "$bundle_id" >> "$RUNNER_TEMP/collectivex-bundles.txt"
+ done < "$RUNNER_TEMP/collectivex-runs.tsv"
+
+ [ "$(sort -n "$RUNNER_TEMP/collectivex-qualification-indices.txt" | tr '\n' ' ')" = '1 2 3 ' ] || {
+ echo 'source runs must contain qualification indices 1, 2, and 3 exactly once' >&2
+ exit 1
+ }
+
+ mapfile -t bundle_ids < "$RUNNER_TEMP/collectivex-bundles.txt"
+ promote_args=()
+ for bundle_id in "${bundle_ids[@]}"; do
+ promote_args+=(--bundle "$bundle_id")
+ done
+ result=$(
+ python3 experimental/CollectiveX/publisher.py --store-root "$store" promote \
+ "${promote_args[@]}"
+ )
+ dataset_id=$(jq -er '.dataset_sha256' <<< "$result")
+ dataset="$store/public/datasets/$dataset_id/dataset.json"
+ [ -f "$dataset" ] || {
+ echo 'publisher did not install the promoted dataset' >&2
+ exit 1
+ }
+ publication="$output/collectivex_public_v1_$dataset_id.ndjson"
+ cp -- "$dataset" "$publication"
+ python3 experimental/CollectiveX/artifact_safety.py "$publication"
+ python3 experimental/CollectiveX/publisher.py --store-root "$store" verify \
+ --channel dev-latest "${promote_args[@]}"
+ [ "$(stat -c %s "$publication")" -le 33554432 ] || {
+ echo 'publication exceeds the 32 MiB frontend limit' >&2
+ exit 1
+ }
+ sha256sum "$publication"
+ {
+ echo '## CollectiveX V1 publication'
+ echo
+ echo "Dataset: \`$dataset_id\`"
+ echo
+ echo 'Source runs:'
+ sed 's/^/- `/' "$RUNNER_TEMP/collectivex-runs.tsv" | sed 's/$/`/'
+ } >> "$GITHUB_STEP_SUMMARY"
+
+ - name: Upload JIT publication artifact
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: cxpublication-v1-${{ github.run_id }}-${{ github.run_attempt }}
+ path: ${{ runner.temp }}/collectivex-publication/*.ndjson
+ if-no-files-found: error
+ retention-days: 90
+
+ refresh:
+ if: ${{ inputs.operation == 'refresh-v1' }}
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+ env:
+ GH_TOKEN: ${{ github.token }}
+ SOURCE_RUN_ID: ${{ inputs.refresh_run_id }}
+ EXPECTED_DIGEST: ${{ inputs.refresh_digest }}
+ steps:
+ - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v5.0.0
+ with: { clean: true, persist-credentials: false }
+
+ - name: Revalidate promoted publication
+ env:
+ REPOSITORY: ${{ github.repository }}
+ run: |
+ set -euo pipefail
+ [[ "$SOURCE_RUN_ID" =~ ^[1-9][0-9]*$ ]] || {
+ echo 'refresh_run_id must be a decimal workflow run ID' >&2
+ exit 1
+ }
+ [[ "$EXPECTED_DIGEST" =~ ^[0-9a-f]{64}$ ]] || {
+ echo 'refresh_digest must be a SHA-256 digest' >&2
+ exit 1
+ }
+ metadata=$(gh api "repos/$REPOSITORY/actions/runs/$SOURCE_RUN_ID")
+ path=$(jq -r '.path' <<< "$metadata")
+ branch=$(jq -r '.head_branch' <<< "$metadata")
+ status=$(jq -r '.status' <<< "$metadata")
+ conclusion=$(jq -r '.conclusion' <<< "$metadata")
+ attempt=$(jq -r '.run_attempt' <<< "$metadata")
+ [ "$path" = '.github/workflows/collectivex-sweep.yml' ] \
+ && [ "$branch" = 'collectivex' ] \
+ && [ "$status" = completed ] \
+ && [ "$conclusion" = success ] \
+ && [[ "$attempt" =~ ^[1-9][0-9]*$ ]] || {
+ echo 'refresh source is not an eligible CollectiveX workflow run' >&2
+ exit 1
+ }
+
+ source="$RUNNER_TEMP/collectivex-refresh-source"
+ output="$RUNNER_TEMP/collectivex-publication"
+ install -d -m 750 "$source" "$output"
+ gh run download "$SOURCE_RUN_ID" --repo "$REPOSITORY" \
+ --name "cxpublication-v1-$SOURCE_RUN_ID-$attempt" --dir "$source"
+ mapfile -t files < <(find "$source" -mindepth 1 -maxdepth 1 -type f -print)
+ [ "${#files[@]}" -eq 1 ] || {
+ echo 'refresh source must contain exactly one root file' >&2
+ exit 1
+ }
+ expected_name="collectivex_public_v1_${EXPECTED_DIGEST}.ndjson"
+ [ "$(basename "${files[0]}")" = "$expected_name" ] || {
+ echo 'refresh source filename differs from its requested digest' >&2
+ exit 1
+ }
+ [ "$(sha256sum "${files[0]}" | cut -d' ' -f1)" = "$EXPECTED_DIGEST" ] || {
+ echo 'refresh source bytes differ from their requested digest' >&2
+ exit 1
+ }
+ [ "$(stat -c %s "${files[0]}")" -le 33554432 ] || {
+ echo 'refresh source exceeds the 32 MiB frontend limit' >&2
+ exit 1
+ }
+ python3 - "${files[0]}" <<'PY'
+ import json
+ import pathlib
+ import sys
+
+ path = pathlib.Path(sys.argv[1])
+ text = path.read_text(encoding="utf-8")
+ if not text.endswith("\n") or text.count("\n") != 1:
+ raise SystemExit("publication must contain exactly one NDJSON record")
+ document = json.loads(text)
+ if document.get("format") != "collectivex.public.v1":
+ raise SystemExit("publication format is invalid")
+ if document.get("promotion", {}).get("status") != "promoted":
+ raise SystemExit("publication is not promoted")
+ PY
+ python3 experimental/CollectiveX/artifact_safety.py "${files[0]}"
+ cp -- "${files[0]}" "$output/$expected_name"
+
+ - name: Upload refreshed JIT publication artifact
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: cxpublication-v1-${{ github.run_id }}-${{ github.run_attempt }}
+ path: ${{ runner.temp }}/collectivex-publication/*.ndjson
+ if-no-files-found: error
+ retention-days: 90
diff --git a/AGENTS.md b/AGENTS.md
index 6a385a5c16..b49f547e2f 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -2,6 +2,8 @@
Guidance for AI agents working with InferenceX.
+> **Temporary language override:** Until the user explicitly removes this override, ignore all other repository instructions that require Simplified Chinese or bilingual output. Work in English only: do not create or update `_zh.md` files, Chinese documentation, translations, or Chinese text in commits, pull requests, issues, reviews, or comments. Leave existing Chinese content untouched.
+
> **Mandatory reading: [`CONTRIBUTING.md`](CONTRIBUTING.md)** — read it before opening or reviewing any PR. It covers the full PR review flow, the CODEOWNER sign-off process, the `/reuse-sweep-run` merge path, post-merge responsibilities, and critical cluster rules (e.g. never leaving root-owned files on AMD runners).
> **PR and GitHub-issue titles & descriptions must be bilingual — include a Simplified Chinese version in addition to English.** Title format: ` / <中文标题>`. In the PR/issue body, follow the English content with its Chinese translation (e.g. a `## 中文说明` section mirroring the summary; don't translate code blocks, logs, or stack traces — summarize around them). **PR comments must include a Chinese translation too** — conversation comments, review summaries, and inline review comments alike: short comments as a single ` / <中文>` line, longer ones with the Chinese translation as a trailing paragraph (`中文：...`). Exception: the CODEOWNER sign-off template stays English-verbatim (the sign-off verifier triggers on its exact phrase); bot-generated comments follow their own workflow templates. This applies to every PR and every issue, matching the bilingual docs rule in Code Conventions.
diff --git a/experimental/CollectiveX/README.md b/experimental/CollectiveX/README.md
index 91469935a0..b58a5af61d 100644
--- a/experimental/CollectiveX/README.md
+++ b/experimental/CollectiveX/README.md
@@ -12,7 +12,11 @@ combine, and paired roundtrip latency across EP libraries and accelerator system
> Publication hold: historical schema 3-5 data is diagnostic. No current dataset is approved for
> rankings, recommendations, or regression baselines.
-## v1 Execution Profile
+> Development status: the sections below document the implemented BF16 pre-V1 baseline, not the
+> final V1 qualification contract. Precision profiles, full point-level publication, and branch-only
+> publication are under active implementation; case counts and digests are not frozen.
+
+## Implemented Pre-V1 Execution Profile
Every scheduled case is BF16 with backend-tuned resources and packed placement. The explicit mode
selects one of two contracts:
@@ -29,7 +33,8 @@ warmups before each measured component at every trial/point. Roundtrip is measur
iteration takes the cross-rank maximum before nearest-rank p50/p90/p95/p99, and roundtrip p99 is the
headline latency. A stdlib integer counter produces byte-identical routing and gate weights.
-The canonical matrix covers H100, H200, B200, B300, GB200, GB300, MI325X, and MI355X. It requests
+The implemented baseline matrix covers H100, H200, B200, B300, GB200, GB300, MI325X, and MI355X.
+It requests
608 cases / 1,600 token points: 364 runnable cases / 940 points, emitted as 58 executable workflow
shards/allocation cells, plus 244 explicit unsupported cases / 660 points. `sweep_matrix.py`
materializes every token ladder and rejects missing, stale, malformed, or altered shard controls.
@@ -70,7 +75,8 @@ reports that its EP8 communicator lacks Device API symmetric-memory support; re-
requires an all-rank CUDA P2P/LSA-capable runtime. Other NVIDIA SKUs remain unvalidated until their
GPU outcomes pass the native correctness and publication gates.
-Removed v1 axes include cached-layout `[cl]`, runtime-visible `[rv]`, FP8, quantized combine,
+Axes not implemented in this baseline include cached-layout `[cl]`, runtime-visible `[rv]`, FP8,
+quantized combine,
extra routing distributions, activation profiles, uneven allocation, placement permutations, model
envelopes, and scaling studies.
@@ -82,11 +88,13 @@ raw GitHub artifacts. Runs default to `release_tag=unversioned` and are diagnost
must explicitly select `release_tag=v1`; setup then requires the locked full-matrix digest and emits
a run/attempt/source-bound `cxrelease-v1-*` marker. Partial and filtered runs cannot receive it.
-`.github/workflows/collectivex-publish.yml` is an explicit V1 gate. It accepts exactly three
-successful tagged sweep run IDs from one source SHA, revalidates their GitHub metadata and release
-markers, and runs `publisher.py` in a disposable runner-local workspace. Only a fully promoted,
-privacy-checked, content-addressed dataset is uploaded as `cxpublication-v1-*`; raw artifacts and
-the private publisher workspace are never exposed to the frontend.
+The main-registered `.github/workflows/collectivex-sweep.yml` provides `sweep`, `publish-v1`, and
+`refresh-v1` operations, so its branch revision can be dispatched with `--ref collectivex` without
+a standalone branch-only workflow. Publication accepts exactly three successful first-attempt tagged
+sweep run IDs from one source SHA, revalidates their metadata and release markers, and runs
+`publisher.py` in a disposable runner-local workspace. Refresh revalidates and reuploads only the
+exact content-addressed sanitized dataset. Raw artifacts and the private publisher workspace are
+never exposed to the frontend.
There is no results server, attached store, Vercel storage, GCP, Neon, managed database, or managed
object store. With the existing server-side `GITHUB_TOKEN`, the frontend discovers the latest
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
index a9431a86b4..346e2ad142 100644
--- a/experimental/CollectiveX/capability.py
+++ b/experimental/CollectiveX/capability.py
@@ -6,6 +6,8 @@
import re
from typing import Any
+import identity
+
DEEPEP_V2_COMMIT = "fa8a9b16898204afd347c663b89e65ef87dc6ce6"
DEEPEP_V2_SKU_CAPABILITIES = {
@@ -158,6 +160,148 @@ def _platform(
}
SWEEP_BACKENDS = tuple(BACKENDS)
+PRECISION_DISPOSITIONS = {
+ "supported", "unsupported", "not-applicable", "provisional",
+}
+_NVIDIA_SKUS = (
+ "h100-dgxc", "h200-dgxc", "b200-dgxc", "b300", "gb200", "gb300",
+)
+_DEEPEP_V2_PRECISION_SKUS = (
+ "h200-dgxc", "b200-dgxc", "b300", "gb200", "gb300",
+)
+_HOPPER_UCCL_SKUS = ("h100-dgxc", "h200-dgxc")
+
+
+def _precision_rule(
+ *,
+ backend: str,
+ skus: tuple[str, ...],
+ ep_degrees: tuple[int, ...],
+ mode: str,
+ basis: str,
+ disposition: str = "provisional",
+) -> dict[str, Any]:
+ return {
+ "backend": backend,
+ "skus": skus,
+ "ep_degrees": ep_degrees,
+ "mode": mode,
+ "disposition": disposition,
+ "basis": basis,
+ }
+
+
+_NORMAL_E4M3FN_PROFILE = "d-fp8-e4m3fn-b128-f32-prequantized.c-bf16"
+_NORMAL_E4M3FNUZ_PROFILE = "d-fp8-e4m3fnuz-b128-f32-prequantized.c-bf16"
+_LL_FP8_PROFILE = "d-fp8-e4m3fn-b128-f32-fused.c-bf16"
+_LL_LOGFMT_PROFILE = "d-bf16.c-logfmt10-dynamic64"
+_LL_FP8_LOGFMT_PROFILE = (
+ "d-fp8-e4m3fn-b128-f32-fused.c-logfmt10-dynamic64"
+)
+_MORI_E4M3FN_DIRECT_PROFILE = "d-bf16.c-fp8-e4m3fn-direct-cast-noscale"
+_MORI_E4M3FN_BOTH_PROFILE = (
+ "d-fp8-e4m3fn-b128-f32-prequantized.c-fp8-e4m3fn-direct-cast-noscale"
+)
+_MORI_E4M3FNUZ_DIRECT_PROFILE = "d-bf16.c-fp8-e4m3fnuz-direct-cast-noscale"
+_MORI_E4M3FNUZ_BOTH_PROFILE = (
+ "d-fp8-e4m3fnuz-b128-f32-prequantized.c-fp8-e4m3fnuz-direct-cast-noscale"
+)
+
+# These are native-path candidates, not executable claims. A cell must be changed
+# from provisional to supported or unsupported after its pinned runtime probe.
+PRECISION_CAPABILITIES: dict[str, tuple[dict[str, Any], ...]] = {
+ _NORMAL_E4M3FN_PROFILE: (
+ _precision_rule(
+ backend="deepep", skus=_NVIDIA_SKUS, ep_degrees=(8, 16), mode="normal",
+ basis="deepep-v1-normal-prequantized-e4m3fn-block128-f32-scale",
+ ),
+ _precision_rule(
+ backend="deepep-v2", skus=_DEEPEP_V2_PRECISION_SKUS,
+ ep_degrees=(8, 16), mode="normal",
+ basis="deepep-v2-normal-prequantized-e4m3fn-block128-f32-scale",
+ ),
+ _precision_rule(
+ backend="deepep-hybrid", skus=_NVIDIA_SKUS,
+ ep_degrees=(8, 16), mode="normal",
+ basis="deepep-hybrid-normal-uint8-e4m3fn-block128-f32-scale",
+ ),
+ _precision_rule(
+ backend="uccl", skus=_HOPPER_UCCL_SKUS, ep_degrees=(8, 16), mode="normal",
+ basis="uccl-deepep-api-normal-prequantized-e4m3fn-block128-f32-scale",
+ ),
+ _precision_rule(
+ backend="mori", skus=("mi355x",), ep_degrees=(8, 16), mode="normal",
+ basis="mori-gfx950-normal-prequantized-ocp-e4m3fn-block128-f32-scale",
+ ),
+ ),
+ _NORMAL_E4M3FNUZ_PROFILE: (
+ _precision_rule(
+ backend="mori", skus=("mi325x",), ep_degrees=(8, 16), mode="normal",
+ basis="mori-gfx942-normal-prequantized-e4m3fnuz-block128-f32-scale",
+ ),
+ ),
+ _LL_FP8_PROFILE: (
+ _precision_rule(
+ backend="deepep", skus=_NVIDIA_SKUS, ep_degrees=(8, 16),
+ mode="low-latency",
+ basis="deepep-v1-low-latency-fused-e4m3fn-block128-f32-scale",
+ ),
+ _precision_rule(
+ backend="uccl", skus=_HOPPER_UCCL_SKUS, ep_degrees=(8, 16),
+ mode="low-latency",
+ basis="uccl-deepep-api-low-latency-fused-e4m3fn-block128-f32-scale",
+ ),
+ ),
+ _LL_LOGFMT_PROFILE: (
+ _precision_rule(
+ backend="deepep", skus=_NVIDIA_SKUS, ep_degrees=(8, 16),
+ mode="low-latency",
+ basis="deepep-v1-low-latency-logfmt10-dynamic-per64-combine",
+ ),
+ _precision_rule(
+ backend="uccl", skus=_HOPPER_UCCL_SKUS, ep_degrees=(8, 16),
+ mode="low-latency",
+ basis="uccl-deepep-api-low-latency-logfmt10-dynamic-per64-combine",
+ ),
+ ),
+ _LL_FP8_LOGFMT_PROFILE: (
+ _precision_rule(
+ backend="deepep", skus=_NVIDIA_SKUS, ep_degrees=(8, 16),
+ mode="low-latency",
+ basis="deepep-v1-low-latency-fused-e4m3fn-dispatch-logfmt10-combine",
+ ),
+ _precision_rule(
+ backend="uccl", skus=_HOPPER_UCCL_SKUS, ep_degrees=(8, 16),
+ mode="low-latency",
+ basis="uccl-deepep-api-low-latency-fused-e4m3fn-dispatch-logfmt10-combine",
+ ),
+ ),
+ _MORI_E4M3FN_DIRECT_PROFILE: (
+ _precision_rule(
+ backend="mori", skus=("mi355x",), ep_degrees=(8,), mode="normal",
+ basis="mori-gfx950-ep8-intranode-e4m3fn-direct-cast-combine",
+ ),
+ ),
+ _MORI_E4M3FN_BOTH_PROFILE: (
+ _precision_rule(
+ backend="mori", skus=("mi355x",), ep_degrees=(8,), mode="normal",
+ basis="mori-gfx950-ep8-intranode-e4m3fn-dispatch-and-direct-cast-combine",
+ ),
+ ),
+ _MORI_E4M3FNUZ_DIRECT_PROFILE: (
+ _precision_rule(
+ backend="mori", skus=("mi325x",), ep_degrees=(8,), mode="normal",
+ basis="mori-gfx942-ep8-asyncll-e4m3fnuz-direct-cast-combine",
+ ),
+ ),
+ _MORI_E4M3FNUZ_BOTH_PROFILE: (
+ _precision_rule(
+ backend="mori", skus=("mi325x",), ep_degrees=(8,), mode="normal",
+ basis="mori-gfx942-ep8-asyncll-e4m3fnuz-dispatch-and-direct-cast-combine",
+ ),
+ ),
+}
+
def runtime_identity_issues(
sku: str, *, vendor: str, arch: str, machine: str, device_name: str,
@@ -191,10 +335,17 @@ def topology_for(sku: str, ep: int) -> dict[str, Any] | None:
return platform["topologies"].get(ep)
-def resolve(sku: str, backend: str, *, ep: int | None = None, nodes: int | None = None,
- routing: str = "uniform", eplb: bool = False,
- mode: str = "normal") -> tuple[bool, str]:
- """Return whether one fixed-v1 case can run on a public GHA runner label."""
+def _resolve_base(
+ sku: str,
+ backend: str,
+ *,
+ ep: int | None = None,
+ nodes: int | None = None,
+ routing: str = "uniform",
+ eplb: bool = False,
+ mode: str = "normal",
+) -> tuple[bool, str]:
+ """Resolve the existing BF16 capability without a precision candidate."""
platform, implementation = PLATFORMS.get(sku), BACKENDS.get(backend)
if platform is None:
return False, f"unknown GHA runner label {sku!r}"
@@ -230,3 +381,189 @@ def resolve(sku: str, backend: str, *, ep: int | None = None, nodes: int | None
if sku in implementation.get("excluded_skus", set()):
return False, f"{backend} is unavailable on {sku}"
return True, "ok"
+
+
+def precision_targets(
+ profile_names: tuple[str, ...] | list[str] | None = None,
+) -> list[dict[str, Any]]:
+ """Expand exact native precision candidates into deterministic target cells."""
+ names = list(PRECISION_CAPABILITIES) if profile_names is None else list(profile_names)
+ unknown = sorted(set(names) - set(PRECISION_CAPABILITIES))
+ if unknown:
+ raise ValueError(f"unknown precision capability profiles {unknown}")
+ targets: list[dict[str, Any]] = []
+ seen: set[tuple[str, str, str, int, str]] = set()
+ for profile_name in names:
+ for rule in PRECISION_CAPABILITIES[profile_name]:
+ for sku in rule["skus"]:
+ for ep in rule["ep_degrees"]:
+ key = (profile_name, rule["backend"], sku, ep, rule["mode"])
+ if key in seen:
+ raise RuntimeError(f"duplicate precision capability target {key}")
+ seen.add(key)
+ targets.append({
+ "precision_profile": profile_name,
+ "backend": rule["backend"],
+ "sku": sku,
+ "ep": ep,
+ "mode": rule["mode"],
+ "disposition": rule["disposition"],
+ "basis": rule["basis"],
+ })
+ return targets
+
+
+def provisional_precision_targets(
+ profile_names: tuple[str, ...] | list[str] | None = None,
+) -> list[dict[str, Any]]:
+ """Return probe-gated targets that must be eliminated before scheduling."""
+ return [
+ target for target in precision_targets(profile_names)
+ if target["disposition"] == "provisional"
+ ]
+
+
+def precision_target_declared(
+ precision_profile: str,
+ *,
+ sku: str,
+ backend: str,
+ ep: int,
+ mode: str,
+) -> bool:
+ """Return whether a profile has an exact native candidate for this cell."""
+ return any(
+ target["precision_profile"] == precision_profile
+ and target["sku"] == sku
+ and target["backend"] == backend
+ and target["ep"] == ep
+ and target["mode"] == mode
+ for target in precision_targets([precision_profile])
+ )
+
+
+def resolve_disposition(
+ sku: str,
+ backend: str,
+ *,
+ ep: int | None = None,
+ nodes: int | None = None,
+ routing: str = "uniform",
+ eplb: bool = False,
+ mode: str = "normal",
+ precision_profile: str | None = None,
+) -> tuple[str, str]:
+ """Resolve a baseline or exact precision cell to its capability disposition."""
+ base_ok, base_detail = _resolve_base(
+ sku,
+ backend,
+ ep=ep,
+ nodes=nodes,
+ routing=routing,
+ eplb=eplb,
+ mode=mode,
+ )
+ if precision_profile is None or precision_profile == identity.V1_CONTROL_PRECISION_PROFILE:
+ return ("supported", "ok") if base_ok else ("unsupported", base_detail)
+ if precision_profile not in identity.V1_PRECISION_PROFILES:
+ return "unsupported", f"unknown precision profile {precision_profile!r}"
+ profile = identity.V1_PRECISION_PROFILES[precision_profile]
+ if mode not in profile["modes"]:
+ return (
+ "not-applicable",
+ f"precision profile {precision_profile} is not defined for {mode} mode",
+ )
+ if ep is None:
+ platform = PLATFORMS.get(sku)
+ if platform is None:
+ return "unsupported", base_detail
+ if nodes is None:
+ ep = platform["ep_degrees"][0]
+ else:
+ matches = [
+ degree for degree, topology in platform["topologies"].items()
+ if topology["nodes"] == nodes
+ ]
+ if len(matches) != 1:
+ return "unsupported", base_detail
+ ep = matches[0]
+ matches = [
+ target for target in precision_targets([precision_profile])
+ if target["sku"] == sku
+ and target["backend"] == backend
+ and target["ep"] == ep
+ and target["mode"] == mode
+ ]
+ if not matches:
+ return (
+ "not-applicable",
+ f"{precision_profile} has no native {backend} target on {sku} EP{ep}",
+ )
+ if not base_ok:
+ return "unsupported", base_detail
+ target = matches[0]
+ return target["disposition"], target["basis"]
+
+
+def resolve(
+ sku: str,
+ backend: str,
+ *,
+ ep: int | None = None,
+ nodes: int | None = None,
+ routing: str = "uniform",
+ eplb: bool = False,
+ mode: str = "normal",
+ precision_profile: str | None = None,
+) -> tuple[bool, str]:
+ """Return whether one fixed-v1 case can run on a public GHA runner label."""
+ disposition, detail = resolve_disposition(
+ sku,
+ backend,
+ ep=ep,
+ nodes=nodes,
+ routing=routing,
+ eplb=eplb,
+ mode=mode,
+ precision_profile=precision_profile,
+ )
+ return disposition == "supported", detail
+
+
+def _validate_precision_capabilities() -> None:
+ expected = set(identity.V1_PRECISION_PROFILES) - {
+ identity.V1_CONTROL_PRECISION_PROFILE
+ }
+ if set(PRECISION_CAPABILITIES) != expected:
+ raise RuntimeError("precision capability profiles differ from the identity registry")
+ empty = sorted(
+ profile for profile, rules in PRECISION_CAPABILITIES.items() if not rules
+ )
+ if empty:
+ raise RuntimeError(f"precision profiles have no native targets: {empty}")
+ for target in precision_targets():
+ if target["backend"] not in BACKENDS or target["sku"] not in PLATFORMS:
+ raise RuntimeError(f"unknown precision target: {target}")
+ if target["ep"] not in PLATFORMS[target["sku"]]["ep_degrees"]:
+ raise RuntimeError(f"invalid precision target EP degree: {target}")
+ if target["disposition"] not in PRECISION_DISPOSITIONS - {"not-applicable"}:
+ raise RuntimeError(f"invalid declared precision disposition: {target}")
+ if target["mode"] not in identity.V1_PRECISION_PROFILES[
+ target["precision_profile"]
+ ]["modes"]:
+ raise RuntimeError(f"precision target mode differs from its profile: {target}")
+ topology = topology_for(target["sku"], target["ep"])
+ base_ok, base_detail = _resolve_base(
+ target["sku"],
+ target["backend"],
+ ep=target["ep"],
+ nodes=topology["nodes"] if topology is not None else None,
+ mode=target["mode"],
+ )
+ if target["disposition"] in {"supported", "provisional"} and not base_ok:
+ raise RuntimeError(
+ f"precision target exceeds its backend capability: {target}: {base_detail}"
+ )
+
+
+_validate_precision_capabilities()
diff --git a/experimental/CollectiveX/configs/suites.yaml b/experimental/CollectiveX/configs/suites.yaml
index 1508695637..2f52147248 100644
--- a/experimental/CollectiveX/configs/suites.yaml
+++ b/experimental/CollectiveX/configs/suites.yaml
@@ -34,3 +34,39 @@ suites:
phases: [decode]
token_points_decode: [1, 2, 4, 8, 16, 32, 64, 128]
required_publication: official
+
+ ep-precision-normal-v1:
+ mode: normal
+ backends: [deepep, deepep-v2, uccl, deepep-hybrid, mori]
+ workloads: [deepseek-v3-v1]
+ platforms: [h100-dgxc, h200-dgxc, b300, b200-dgxc, gb300, gb200, mi355x, mi325x]
+ ep_degrees: [8, 16]
+ routings: [uniform]
+ phases: [decode, prefill]
+ token_points_decode: [128]
+ token_points_prefill: [512]
+ precision_profiles:
+ - d-fp8-e4m3fn-b128-f32-prequantized.c-bf16
+ - d-fp8-e4m3fnuz-b128-f32-prequantized.c-bf16
+ - d-bf16.c-fp8-e4m3fn-direct-cast-noscale
+ - d-fp8-e4m3fn-b128-f32-prequantized.c-fp8-e4m3fn-direct-cast-noscale
+ - d-bf16.c-fp8-e4m3fnuz-direct-cast-noscale
+ - d-fp8-e4m3fnuz-b128-f32-prequantized.c-fp8-e4m3fnuz-direct-cast-noscale
+ provisional: true
+ required_publication: comparable-experimental
+
+ ep-precision-low-latency-v1:
+ mode: low-latency
+ backends: [deepep, uccl]
+ workloads: [deepseek-v3-v1]
+ platforms: [h100-dgxc, h200-dgxc, b300, b200-dgxc, gb300, gb200, mi355x, mi325x]
+ ep_degrees: [8, 16]
+ routings: [uniform]
+ phases: [decode]
+ token_points_decode: [128]
+ precision_profiles:
+ - d-fp8-e4m3fn-b128-f32-fused.c-bf16
+ - d-bf16.c-logfmt10-dynamic64
+ - d-fp8-e4m3fn-b128-f32-fused.c-logfmt10-dynamic64
+ provisional: true
+ required_publication: comparable-experimental
diff --git a/experimental/CollectiveX/contracts.py b/experimental/CollectiveX/contracts.py
index fbebe070c8..04357da78c 100644
--- a/experimental/CollectiveX/contracts.py
+++ b/experimental/CollectiveX/contracts.py
@@ -35,9 +35,12 @@
}
ALLOCATION_FACTOR_FIELDS = {
"artifact", "execution_id", "job", "repo", "run_attempt", "run_id", "runner",
+ "source_sha", "qualification_index",
+}
+GIT_RUN_FIELDS = {
+ "artifact", "job", "qualification_index", "ref", "repo", "run_attempt", "run_id",
"source_sha",
}
-GIT_RUN_FIELDS = {"artifact", "job", "ref", "repo", "run_attempt", "run_id", "source_sha"}
PRE_EXECUTION_FAILURE_REASONS = {
"setup": "launcher-setup-failed",
"repository-stage": "repository-staging-failed",
@@ -166,6 +169,14 @@ def scheduled_case_profile(case: dict[str, Any], path: str = "case") -> dict[str
raise ContractError(f"{path}: {exc}") from exc
+def _scheduled_case(value: Any, path: str) -> dict[str, Any]:
+ """Validate baseline or explicit-precision scheduled case fields."""
+ fields = set(TERMINAL_CASE_FIELDS)
+ if isinstance(value, dict) and "precision_profile" in value:
+ fields.add("precision_profile")
+ return _keys(value, fields, path)
+
+
def resolve_deepep_mnnvl(
*, requested: bool, signature_parameters: Iterable[str], deepep_commit: str | None
) -> tuple[dict[str, bool], str]:
@@ -213,8 +224,8 @@ def project_resource_profile(provenance: dict[str, Any]) -> dict[str, Any]:
"comm_units_kind": kind,
"configured_units": configured,
"conformance_class": (
- "not-applicable" if fixed else "best-known" if "default" not in source
- else "backend-default"
+ "not-applicable" if fixed else "backend-default" if "default" in source
+ else "pinned-upstream"
),
"device_units": device_units,
"fixed_kernel": fixed,
@@ -223,7 +234,7 @@ def project_resource_profile(provenance: dict[str, Any]) -> dict[str, Any]:
"persistent_bytes": persistent_bytes,
"qps_per_rank": provenance.get("num_qps_per_rank"),
"requested_fraction": None,
- "resource_class": "fixed-kernel" if fixed else "backend-tuned",
+ "resource_class": "fixed-kernel" if fixed else "fixed-profile",
"target_achieved_within_tol": None,
"tolerance": 0.10,
"tuned_source": provenance.get("tuned_source"),
@@ -649,12 +660,15 @@ def backend_provenance_issues(backend: str, provenance: dict[str, Any]) -> list[
def provenance_complete(
provenance: dict[str, Any], backend: str, git_run: dict[str, Any] | None,
- *, image_digest: Any, image_verified: Any, squash_sha256: Any,
+ *, allocation_stratum_sha256: Any, image_digest: Any, image_verified: Any,
+ squash_sha256: Any,
) -> bool:
image = str(image_digest or "")
squash = str(squash_sha256 or "")
+ allocation_stratum = str(allocation_stratum_sha256 or "")
return (
not backend_provenance_issues(backend, provenance)
+ and bool(re.fullmatch(r"[0-9a-f]{64}", allocation_stratum))
and image_verified is True
and bool(re.fullmatch(r"sha256:[0-9a-f]{64}", image))
and bool(re.fullmatch(r"[0-9a-f]{64}", squash))
@@ -822,28 +836,89 @@ def _sha256_json(value: Any) -> str:
return hashlib.sha256(payload).hexdigest()
+def _precision_byte_provenance(
+ axis: dict[str, Any], logical_copies: int, hidden: int
+) -> dict[str, Any]:
+ bits_per_value = {
+ "bf16": 16,
+ "fp8-e4m3fn": 8,
+ "fp8-e4m3fnuz": 8,
+ "logfmt10": 10,
+ }.get(axis["communication_format"])
+ if bits_per_value is None:
+ raise ContractError("unknown communication precision format")
+ scale_size = {None: 0, "f32": 4, "implicit-logfmt10": 0}.get(axis["scale_dtype"])
+ if scale_size is None:
+ raise ContractError("unknown communication scale dtype")
+ group_size = axis["scale_group_size"]
+ groups = math.ceil(hidden / group_size) if group_size is not None else 0
+ activation = logical_copies * math.ceil(hidden * bits_per_value / 8)
+ scales = logical_copies * groups * scale_size
+ return {
+ "accounting_contract": "activation-data-plus-scales-v1",
+ "activation_data_bytes": activation,
+ "scale_bytes": scales,
+ "total_logical_bytes": activation + scales,
+ }
+
+
@lru_cache(maxsize=None)
-def _expected_eplb_plan(
+def _expected_eplb_calibration(
routing: str,
+ hidden: int,
topk: int,
logical_experts: int,
physical_experts: int,
ep_size: int,
seed: int,
reference_tokens_per_rank: int,
-) -> dict[str, Any]:
- indices, _ = workload_contract.canonical_routing_rows(
- reference_tokens_per_rank * ep_size,
- logical_experts,
- topk,
+) -> tuple[dict[str, Any], dict[str, Any]]:
+ member, checksums, indices, _ = workload_contract.canonical_eplb_calibration_member(
routing,
+ hidden,
+ topk,
+ logical_experts,
+ ep_size,
+ reference_tokens_per_rank,
seed,
)
load = [0] * logical_experts
for row in indices:
for expert in row:
load[expert] += 1
- return eplb_contract.build_plan(load, physical_experts, ep_size)
+ plan = eplb_contract.build_plan(load, physical_experts, ep_size)
+ descriptor = {
+ "calibration_token_offset": workload_contract.EPLB_CALIBRATION_TOKEN_OFFSET,
+ "calibration_trace_sha256": checksums["trace"],
+ "calibration_window": workload_contract.EPLB_CALIBRATION_WINDOW,
+ "calibration_workload_id": member,
+ }
+ return plan, descriptor
+
+
+@lru_cache(maxsize=None)
+def _expected_eplb_plan(
+ routing: str,
+ topk: int,
+ logical_experts: int,
+ physical_experts: int,
+ ep_size: int,
+ seed: int,
+ reference_tokens_per_rank: int,
+ hidden: int = 7168,
+) -> dict[str, Any]:
+ """Compatibility wrapper returning the disjoint calibration plan."""
+ plan, _ = _expected_eplb_calibration(
+ routing,
+ hidden,
+ topk,
+ logical_experts,
+ physical_experts,
+ ep_size,
+ seed,
+ reference_tokens_per_rank,
+ )
+ return plan
@lru_cache(maxsize=None)
@@ -877,6 +952,7 @@ def _expected_canonical_trace(
ep_size,
seed,
reference_tokens_per_rank,
+ hidden,
)
indices = eplb_contract.remap_rows(indices, plan)
routing_hash = workload_contract.trace_checksums(indices, weights)["trace"]
@@ -986,6 +1062,7 @@ def _expected_anomalies(
tokens: int, components: dict[str, Any]
) -> list[dict[str, Any]]:
dispatch = components["dispatch"]["percentiles_us"]
+ stage = components["stage"]["percentiles_us"]
combine = components["combine"]["percentiles_us"]
roundtrip = components["roundtrip"]["percentiles_us"]
isolated = components["isolated_sum"]["percentiles_us"]
@@ -999,7 +1076,10 @@ def _expected_anomalies(
"ratio": round(roundtrip["p99"] / isolated["p99"], 2),
"threshold": 3.0,
})
- floor = max(dispatch["p50"], combine["p50"]) if dispatch and combine else None
+ floor = (
+ max(dispatch["p50"], combine["p50"], stage["p50"] if stage is not None else 0.0)
+ if dispatch and combine else None
+ )
if floor and roundtrip["p50"] < 0.95 * floor:
anomalies.append({
"type": "roundtrip_lt_component_floor",
@@ -1017,7 +1097,7 @@ def _validate_canonical_workload(
eplb: dict[str, Any],
) -> None:
"""Bind every canonical member and measured routing hash to its scheduled token row."""
- profile = identity.V1_CASE_PROFILE
+ profile = identity.profile_for_case(scheduled_case)
if eplb["enabled"]:
plan = _expected_eplb_plan(
scheduled_case["routing"],
@@ -1027,6 +1107,7 @@ def _validate_canonical_workload(
scheduled_case["ep"],
profile["seed"],
profile["eplb_reference_tokens_per_rank"],
+ scheduled_case["hidden"],
)
if eplb["mapping_hash"] != eplb_contract.mapping_hash(plan):
raise ContractError("raw EPLB mapping differs from the frozen canonical plan")
@@ -1259,8 +1340,8 @@ def validate_samples_document(document: Any) -> dict[str, Any]:
_validate_native_schema("samples-v1.schema.json", document)
doc = _keys(
document,
- {"allocation_id", "attempt_id", "case_id", "format", "points", "sampling",
- "schema_version", "series_id"},
+ {"allocation_id", "attempt_id", "case_id", "format", "points",
+ "qualification_index", "sampling", "schema_version", "series_id"},
"samples",
)
if doc["format"] != SAMPLES_FORMAT or doc["schema_version"] != 1:
@@ -1270,6 +1351,11 @@ def validate_samples_document(document: Any) -> dict[str, Any]:
("case_id", "case"), ("series_id", "series"),
):
_typed(doc[field], kind, f"samples.{field}")
+ qualification_index = _integer(
+ doc["qualification_index"], "samples.qualification_index", minimum=1
+ )
+ if qualification_index > 3:
+ raise ContractError("samples.qualification_index must be in 1..3")
sampling = _keys(
doc["sampling"], {"iterations_per_trial", "reduction", "trials"}, "samples.sampling"
)
@@ -1296,7 +1382,10 @@ def validate_samples_document(document: Any) -> dict[str, Any]:
seen.add(tokens)
_typed(point["point_id"], "point", f"{path}.point_id")
_typed(point["evidence_id"], "evidence", f"{path}.evidence_id")
- components = _keys(point["components"], {"combine", "dispatch", "roundtrip"}, f"{path}.components")
+ components = _keys(
+ point["components"], {"combine", "dispatch", "roundtrip", "stage"},
+ f"{path}.components",
+ )
for name, component_value in components.items():
component = _keys(
component_value, {"availability", "sample_count", "trials"},
@@ -1421,6 +1510,55 @@ def _validate_oracle(
return oracle
+def _validate_precision_evidence(
+ value: Any, profile_id: str, communication_precision: dict[str, Any], path: str
+) -> dict[str, Any]:
+ precision = _keys(value, {"combine", "dispatch", "passed", "profile_id"}, path)
+ if precision["profile_id"] != profile_id or type(precision["passed"]) is not bool:
+ raise ContractError(f"{path} profile/outcome differs")
+ for direction in ("dispatch", "combine"):
+ axis_path = f"{path}.{direction}"
+ axis = _keys(
+ precision[direction],
+ {"dequantized_semantics", "encoded_payload_valid", "max_abs_error",
+ "max_rel_error", "passed", "saturation_count", "saturation_rate",
+ "scales_finite", "scales_positive"},
+ axis_path,
+ )
+ for field in ("dequantized_semantics", "encoded_payload_valid", "passed"):
+ if type(axis[field]) is not bool:
+ raise ContractError(f"{axis_path}.{field} must be boolean")
+ expects_scales = communication_precision[direction]["scale_dtype"] is not None
+ for field in ("scales_finite", "scales_positive"):
+ if expects_scales:
+ if type(axis[field]) is not bool:
+ raise ContractError(f"{axis_path}.{field} must be boolean")
+ elif axis[field] is not None:
+ raise ContractError(f"{axis_path}.{field} must be null without scales")
+ saturation_count = _integer(
+ axis["saturation_count"], f"{axis_path}.saturation_count"
+ )
+ saturation_rate = _number(
+ axis["saturation_rate"], f"{axis_path}.saturation_rate", minimum=0.0
+ )
+ if saturation_rate > 1.0:
+ raise ContractError(f"{axis_path}.saturation_rate must be <= 1")
+ _number(axis["max_abs_error"], f"{axis_path}.max_abs_error", minimum=0.0)
+ _number(axis["max_rel_error"], f"{axis_path}.max_rel_error", minimum=0.0)
+ expected_pass = (
+ axis["encoded_payload_valid"]
+ and axis["dequantized_semantics"]
+ and (not expects_scales or (axis["scales_finite"] and axis["scales_positive"]))
+ and saturation_count >= 0
+ )
+ if axis["passed"] != bool(expected_pass):
+ raise ContractError(f"{axis_path}.passed differs from its evidence")
+ expected_pass = precision["dispatch"]["passed"] and precision["combine"]["passed"]
+ if precision["passed"] != expected_pass:
+ raise ContractError(f"{path}.passed differs from direction evidence")
+ return precision
+
+
def validate_raw_document(document: Any, samples_document: Any) -> dict[str, Any]:
"""Validate identities, exact samples, formulas, privacy, and the native raw shape."""
_validate_native_schema("raw-case-v1.schema.json", document)
@@ -1451,12 +1589,19 @@ def validate_raw_document(document: Any, samples_document: Any) -> dict[str, Any
identifiers["allocation_factors"], ALLOCATION_FACTOR_FIELDS,
"raw.identity.allocation_factors",
)
+ qualification_index = _integer(
+ allocation_factors["qualification_index"],
+ "raw.identity.allocation_factors.qualification_index",
+ minimum=1,
+ )
+ if qualification_index > 3:
+ raise ContractError("raw qualification index must be in 1..3")
case_factors = _keys(
identifiers["case_factors"], {"case", "profile", "sku"},
"raw.identity.case_factors",
)
- scheduled_case = _keys(
- case_factors["case"], TERMINAL_CASE_FIELDS, "raw.identity.case_factors.case"
+ scheduled_case = _scheduled_case(
+ case_factors["case"], "raw.identity.case_factors.case"
)
profile = scheduled_case_profile(scheduled_case, "raw.identity.case_factors.case")
if case_factors["profile"] != profile:
@@ -1484,6 +1629,8 @@ def validate_raw_document(document: Any, samples_document: Any) -> dict[str, Any
for field in ("allocation_id", "attempt_id", "case_id", "series_id"):
if samples[field] != identifiers[field]:
raise ContractError(f"samples.{field} differs from raw identity")
+ if samples["qualification_index"] != qualification_index:
+ raise ContractError("samples qualification index differs from raw allocation")
sample_by_token = {point["tokens_per_rank"]: point for point in samples["points"]}
case = _keys(
@@ -1500,8 +1647,9 @@ def validate_raw_document(document: Any, samples_document: Any) -> dict[str, Any
_text(case[field], f"raw.case.{field}")
shape = _keys(
case["shape"],
- {"activation_profile", "dispatch_dtype", "eplb", "experts", "experts_per_rank",
- "hidden", "kernel_gen", "num_logical_experts", "quant", "routing", "topk"},
+ {"activation_profile", "combine_precision", "dispatch_precision", "eplb", "experts",
+ "experts_per_rank", "hidden", "kernel_gen", "num_logical_experts",
+ "precision_profile", "routing", "topk"},
"raw.case.shape",
)
hidden = _integer(shape["hidden"], "raw.case.shape.hidden", minimum=1)
@@ -1517,17 +1665,22 @@ def validate_raw_document(document: Any, samples_document: Any) -> dict[str, Any
experts_per_rank = _integer(
shape["experts_per_rank"], "raw.case.shape.experts_per_rank", minimum=1
)
- quant = _keys(
- shape["quant"],
- {"combine_accum_dtype", "combine_input_dtype", "combine_output_dtype",
- "combine_quant_mode", "scale_layout"},
- "raw.case.shape.quant",
+ precision_profile_id = scheduled_case.get(
+ "precision_profile", identity.V1_CONTROL_PRECISION_PROFILE
)
+ communication_precision = identity.precision_profile(precision_profile_id)
+ if (
+ shape["precision_profile"] != precision_profile_id
+ or shape["dispatch_precision"] != communication_precision["dispatch"]
+ or shape["combine_precision"] != communication_precision["combine"]
+ ):
+ raise ContractError("raw communication precision differs from scheduled case")
eplb = _keys(
case["eplb"],
- {"enabled", "imbalance_after", "imbalance_before", "mapping_hash", "max_replicas",
- "num_logical_experts", "num_physical_experts", "num_redundant", "planner",
- "reference_tokens_per_rank", "replicated_experts"},
+ {"calibration_token_offset", "calibration_trace_sha256", "calibration_window",
+ "calibration_workload_id", "enabled", "imbalance_after", "imbalance_before",
+ "mapping_hash", "max_replicas", "num_logical_experts", "num_physical_experts",
+ "num_redundant", "planner", "reference_tokens_per_rank", "replicated_experts"},
"raw.case.eplb",
)
if not isinstance(eplb["enabled"], bool):
@@ -1549,8 +1702,9 @@ def validate_raw_document(document: Any, samples_document: Any) -> dict[str, Any
):
raise ContractError("raw EPLB/shape dimensions differ from the frozen profile")
if eplb["enabled"]:
- expected_plan = _expected_eplb_plan(
+ expected_plan, calibration_descriptor = _expected_eplb_calibration(
scheduled_case["routing"],
+ hidden,
topk,
logical_experts,
physical_experts,
@@ -1559,6 +1713,7 @@ def validate_raw_document(document: Any, samples_document: Any) -> dict[str, Any
profile["eplb_reference_tokens_per_rank"],
)
expected_eplb = {
+ **calibration_descriptor,
"enabled": True,
"imbalance_after": expected_plan["imbalance_after"],
"imbalance_before": expected_plan["imbalance_before"],
@@ -1575,6 +1730,10 @@ def validate_raw_document(document: Any, samples_document: Any) -> dict[str, Any
}
else:
expected_eplb = {
+ "calibration_token_offset": None,
+ "calibration_trace_sha256": None,
+ "calibration_window": None,
+ "calibration_workload_id": None,
"enabled": False,
"imbalance_after": None,
"imbalance_before": None,
@@ -1635,10 +1794,16 @@ def validate_raw_document(document: Any, samples_document: Any) -> dict[str, Any
measurement = _keys(
doc["measurement"],
- {"component_order_contract", "conditioning", "contract", "rows",
- "sampling", "source_allocation"},
+ {"component_order_contract", "conditioning", "contract", "execution_order_sha256",
+ "qualification_index", "rows", "sampling", "source_allocation"},
"raw.measurement",
)
+ if measurement["qualification_index"] != qualification_index:
+ raise ContractError("raw measurement qualification index differs from allocation")
+ if not isinstance(measurement["execution_order_sha256"], str) or not re.fullmatch(
+ r"[0-9a-f]{64}", measurement["execution_order_sha256"]
+ ):
+ raise ContractError("raw measurement execution order digest is invalid")
validate_conditioning_contract(measurement["conditioning"], case["phase"])
sampling = _keys(
measurement["sampling"],
@@ -1662,11 +1827,6 @@ def validate_raw_document(document: Any, samples_document: Any) -> dict[str, Any
or measurement["component_order_contract"] != profile["component_order_contract"]
or measurement["source_allocation"] != "even"
or shape["activation_profile"] != profile["activation_profile"]
- or shape["dispatch_dtype"] != profile["dtype"]
- or quant["combine_input_dtype"] != profile["combine_dtype"]
- or quant["combine_output_dtype"] != profile["combine_dtype"]
- or quant["combine_quant_mode"] != profile["combine_quant_mode"]
- or quant["scale_layout"] is not None
or workload["activation_generator"] != profile["activation_generator"]
or workload["activation_profile"] != profile["activation_profile"]
or workload["routing_generator"] != profile["routing_generator"]
@@ -1690,8 +1850,8 @@ def validate_raw_document(document: Any, samples_document: Any) -> dict[str, Any
path = f"raw.measurement.rows[{index}]"
row = _keys(
row_value,
- {"anomalies", "components", "correctness", "evidence_id", "global_tokens",
- "logical_bytes", "point_id", "receive", "routing",
+ {"anomalies", "byte_provenance", "components", "correctness", "evidence_id",
+ "global_tokens", "point_id", "receive", "routing",
"sample_histograms", "sample_sha256", "token_rate_at_latency_percentile",
"tokens_per_rank"},
path,
@@ -1716,16 +1876,24 @@ def validate_raw_document(document: Any, samples_document: Any) -> dict[str, Any
if row["sample_sha256"] != sample_point["sample_sha256"]:
raise ContractError(f"{path}.sample_sha256 differs")
components = _keys(
- row["components"], {"combine", "dispatch", "isolated_sum", "roundtrip"},
+ row["components"], {"combine", "dispatch", "isolated_sum", "roundtrip", "stage"},
f"{path}.components",
)
- for name in ("combine", "dispatch", "roundtrip"):
+ for name in ("combine", "dispatch", "roundtrip", "stage"):
_validate_component(
components[name], sample_point["components"][name], f"{path}.components.{name}"
)
_validate_component(
components["isolated_sum"], None, f"{path}.components.isolated_sum", derived=True
)
+ expected_stage_availability = (
+ "measured"
+ if communication_precision["dispatch"]["communication_format"] != "bf16"
+ or (case["backend"] == "mori" and shape["kernel_gen"] == "intranode")
+ else "unavailable"
+ )
+ if components["stage"]["availability"] != expected_stage_availability:
+ raise ContractError(f"{path}.components.stage differs from adapter device work")
_, _, _, expected_indices, expected_weights = _expected_canonical_trace(
scheduled_case["routing"],
hidden,
@@ -1769,7 +1937,7 @@ def validate_raw_document(document: Any, samples_document: Any) -> dict[str, Any
)
correctness = _keys(
row["correctness"],
- {"contract", "max_relative_error", "passed", "rank_evidence", "scope"},
+ {"contract", "max_relative_error", "passed", "precision", "rank_evidence", "scope"},
f"{path}.correctness",
)
if (
@@ -1778,6 +1946,10 @@ def validate_raw_document(document: Any, samples_document: Any) -> dict[str, Any
or type(correctness["passed"]) is not bool
):
raise ContractError(f"{path}.correctness contract differs")
+ precision_evidence = _validate_precision_evidence(
+ correctness["precision"], precision_profile_id, communication_precision,
+ f"{path}.correctness.precision",
+ )
_number(
correctness["max_relative_error"],
f"{path}.correctness.max_relative_error",
@@ -1830,6 +2002,7 @@ def validate_raw_document(document: Any, samples_document: Any) -> dict[str, Any
evidence_passed = evidence_passed and all(
(evidence["input_unchanged"], evidence["order_stable"], pre["passed"], post["passed"])
)
+ evidence_passed = evidence_passed and precision_evidence["passed"]
if ranks != set(range(ep_size)) or correctness["passed"] != evidence_passed:
raise ContractError(f"{path}.correctness rank coverage or outcome differs")
_close(
@@ -1840,6 +2013,11 @@ def validate_raw_document(document: Any, samples_document: Any) -> dict[str, Any
for percentile in PERCENTILES:
expected = (
components["dispatch"]["percentiles_us"][percentile]
+ + (
+ components["stage"]["percentiles_us"][percentile]
+ if components["stage"]["availability"] == "measured"
+ else 0.0
+ )
+ components["combine"]["percentiles_us"][percentile]
)
_close(
@@ -1851,13 +2029,36 @@ def validate_raw_document(document: Any, samples_document: Any) -> dict[str, Any
if profile["payload_unit"] == "token-expert"
else expected_routing["routed_copies"]
)
- expected_bytes = logical_copies * hidden * 2
- expected_logical = {
- "combine": expected_bytes,
- "dispatch": expected_bytes,
- "roundtrip": expected_bytes * 2,
+ dispatch_bytes = _precision_byte_provenance(
+ communication_precision["dispatch"], logical_copies, hidden
+ )
+ combine_bytes = _precision_byte_provenance(
+ communication_precision["combine"], logical_copies, hidden
+ )
+ stage_bytes = {
+ "accounting_contract": "activation-data-plus-scales-v1",
+ "activation_data_bytes": 0,
+ "scale_bytes": 0,
+ "total_logical_bytes": 0,
+ }
+ roundtrip_bytes = {
+ "accounting_contract": "activation-data-plus-scales-v1",
+ **{
+ field: dispatch_bytes[field] + combine_bytes[field]
+ for field in (
+ "activation_data_bytes", "scale_bytes", "total_logical_bytes"
+ )
+ },
+ }
+ expected_byte_provenance = {
+ "combine": combine_bytes,
+ "dispatch": dispatch_bytes,
+ "roundtrip": roundtrip_bytes,
+ "stage": stage_bytes,
}
- _equivalent(row["logical_bytes"], expected_logical, f"{path}.logical_bytes")
+ _equivalent(
+ row["byte_provenance"], expected_byte_provenance, f"{path}.byte_provenance"
+ )
max_receive = max(expected_payload_counts)
expected_receive = {
@@ -1877,7 +2078,7 @@ def validate_raw_document(document: Any, samples_document: Any) -> dict[str, Any
if sample_point["components"][name]["availability"] == "measured"
else None
)
- for name in ("dispatch", "combine", "roundtrip")
+ for name in ("dispatch", "stage", "combine", "roundtrip")
}
_equivalent(
row["sample_histograms"], expected_histograms, f"{path}.sample_histograms"
@@ -2085,9 +2286,17 @@ def validate_raw_document(document: Any, samples_document: Any) -> dict[str, Any
+ "; ".join(identity_issues)
)
raw_provenance = _keys(
- doc["provenance"], {"command", "distributed_launcher", "git_run", "image", "redaction"},
+ doc["provenance"],
+ {"allocation_stratum_sha256", "command", "distributed_launcher", "git_run",
+ "image", "redaction"},
"raw.provenance",
)
+ allocation_stratum = raw_provenance["allocation_stratum_sha256"]
+ if workload["source"] == "canonical-serialized" and not (
+ isinstance(allocation_stratum, str)
+ and re.fullmatch(r"[0-9a-f]{64}", allocation_stratum)
+ ):
+ raise ContractError("canonical raw evidence is missing its private allocation stratum")
image = _keys(
raw_provenance["image"],
{"arch", "digest", "digest_verified", "reference", "squash_sha256"},
@@ -2104,10 +2313,13 @@ def validate_raw_document(document: Any, samples_document: Any) -> dict[str, Any
git_run = raw_provenance["git_run"]
if git_run is not None:
git_run = _keys(git_run, GIT_RUN_FIELDS, "raw.provenance.git_run")
+ if git_run["qualification_index"] != qualification_index:
+ raise ContractError("raw git run qualification index differs from allocation")
expected_provenance_complete = provenance_complete(
provenance_fields,
case["backend"],
git_run,
+ allocation_stratum_sha256=allocation_stratum,
image_digest=image["digest"],
image_verified=image["digest_verified"],
squash_sha256=image["squash_sha256"],
@@ -2143,6 +2355,8 @@ def validate_raw_document(document: Any, samples_document: Any) -> dict[str, Any
"warmup_semantics": sampling["warmup_semantics"],
"workload": case["workload_name"],
}
+ if "precision_profile" in scheduled_case:
+ actual_scheduled_case["precision_profile"] = shape["precision_profile"]
if scheduled_case != actual_scheduled_case:
mismatches = sorted(
field for field in scheduled_case
@@ -2182,6 +2396,7 @@ def validate_raw_document(document: Any, samples_document: Any) -> dict[str, Any
"artifact": git_run["artifact"] if git_run is not None else None,
"execution_id": allocation_factors["execution_id"],
"job": git_run["job"] if git_run is not None else None,
+ "qualification_index": qualification_index,
"repo": git_run["repo"] if git_run is not None else None,
"run_attempt": git_run["run_attempt"] if git_run is not None else None,
"run_id": git_run["run_id"] if git_run is not None else None,
@@ -2306,7 +2521,7 @@ def validate_terminal_document(document: Any) -> dict[str, Any]:
for field, kind in (("allocation_id", "allocation"), ("attempt_id", "attempt"), ("case_id", "case")):
_typed(ids[field], kind, f"terminal.identity.{field}")
ordinal = _integer(ids["attempt_ordinal"], "terminal.identity.attempt_ordinal", minimum=1)
- case = _keys(doc["case"], TERMINAL_CASE_FIELDS, "terminal.case")
+ case = _scheduled_case(doc["case"], "terminal.case")
factors = _keys(ids["case_factors"], {"case", "profile", "sku"}, "terminal.identity.case_factors")
if factors["case"] != case or factors["profile"] != scheduled_case_profile(
case, "terminal.case"
@@ -2317,6 +2532,13 @@ def validate_terminal_document(document: Any) -> dict[str, Any]:
ids["allocation_factors"], ALLOCATION_FACTOR_FIELDS,
"terminal.identity.allocation_factors",
)
+ qualification_index = _integer(
+ allocation["qualification_index"],
+ "terminal.identity.allocation_factors.qualification_index",
+ minimum=1,
+ )
+ if qualification_index > 3:
+ raise ContractError("terminal qualification index must be in 1..3")
expected_case = identity.digest("case", factors)
expected_allocation = identity.allocation_id(allocation)
expected_attempt = identity.attempt_id(
@@ -2333,6 +2555,10 @@ def validate_terminal_document(document: Any) -> dict[str, Any]:
git_run = provenance["git_run"]
if git_run is not None:
git_run = _keys(git_run, GIT_RUN_FIELDS, "terminal.provenance.git_run")
+ if git_run["qualification_index"] != qualification_index:
+ raise ContractError(
+ "terminal git run qualification index differs from allocation"
+ )
control = provenance["control_sha256"]
if control is not None and (
not isinstance(control, str) or len(control) != 64
@@ -2373,6 +2599,7 @@ def validate_terminal_document(document: Any) -> dict[str, Any]:
"artifact": git_run["artifact"] if git_run is not None else None,
"execution_id": allocation["execution_id"],
"job": git_run["job"] if git_run is not None else None,
+ "qualification_index": qualification_index,
"repo": git_run["repo"] if git_run is not None else None,
"run_attempt": git_run["run_attempt"] if git_run is not None else None,
"run_id": git_run["run_id"] if git_run is not None else None,
@@ -2480,7 +2707,7 @@ def _terminal_case_from_environment(backend: str, phase: str) -> dict[str, Any]:
if phase == "decode"
else "128 256 512 1024 2048 4096"
)
- return {
+ case = {
"suite": os.environ.get("CX_SUITE") or "manual",
"workload": os.environ.get("CX_WORKLOAD_NAME") or "manual",
"required_publication": os.environ.get("CX_REQUIRED_PUBLICATION") or "diagnostic",
@@ -2513,6 +2740,10 @@ def _terminal_case_from_environment(backend: str, phase: str) -> dict[str, Any]:
"scale_up_transport": os.environ.get("CX_SCALE_UP_TRANSPORT", "unknown"),
"scale_out_transport": os.environ.get("CX_SCALE_OUT_TRANSPORT") or None,
}
+ precision_profile = os.environ.get("CX_PRECISION_PROFILE") or None
+ if precision_profile is not None:
+ case["precision_profile"] = precision_profile
+ return case
def _git_run_from_environment() -> dict[str, Any] | None:
@@ -2528,7 +2759,10 @@ def value(name: str) -> str | None:
"job": value("GITHUB_JOB"),
"artifact": value("COLLECTIVEX_ARTIFACT_NAME"),
}
- return git_run if any(value is not None for value in git_run.values()) else None
+ if not any(item is not None for item in git_run.values()):
+ return None
+ git_run["qualification_index"] = _env_integer("CX_QUALIFICATION_INDEX", 1)
+ return git_run
def _allocation_factors_from_environment(
@@ -2538,6 +2772,7 @@ def _allocation_factors_from_environment(
"artifact": git_run["artifact"] if git_run is not None else None,
"execution_id": os.environ.get("COLLECTIVEX_EXECUTION_ID") or None,
"job": git_run["job"] if git_run is not None else None,
+ "qualification_index": _env_integer("CX_QUALIFICATION_INDEX", 1),
"repo": git_run["repo"] if git_run is not None else None,
"run_attempt": git_run["run_attempt"] if git_run is not None else None,
"run_id": git_run["run_id"] if git_run is not None else None,
diff --git a/experimental/CollectiveX/docs/methodology.md b/experimental/CollectiveX/docs/methodology.md
index 606e7c8995..396a705b6e 100644
--- a/experimental/CollectiveX/docs/methodology.md
+++ b/experimental/CollectiveX/docs/methodology.md
@@ -1,4 +1,4 @@
-# CollectiveX EP v1 Contract
+# CollectiveX EP Pre-V1 Baseline
@@ -6,7 +6,10 @@
-This document defines new CollectiveX results. Historical run notes are evidence, not contract.
+This document describes the implemented BF16 baseline. It is not yet the V1 qualification contract.
+Before any V1-tagged run, this English document must match the implemented precision, measurement,
+publication, and frontend contracts; counts and digests remain unfrozen. Chinese documentation
+synchronization is explicitly deferred for the V1 implementation phase.
## Product Boundary
@@ -18,9 +21,9 @@ CollectiveX is a communication microbenchmark for:
It does not predict serving throughput without a separate correlation study.
-## Matrix
+## Implemented Matrix
-The promoted workload is `deepseek-v3-v1`: hidden 7168, top-k 8, 256 routed experts, BF16 dispatch
+The implemented workload is `deepseek-v3-v1`: hidden 7168, top-k 8, 256 routed experts, BF16 dispatch
and combine, packed placement, and backend-tuned resources. Each case explicitly selects normal
`layout-and-dispatch-v1` or low-latency `expert-packed-weighted-combine-v1` semantics.
@@ -28,7 +31,8 @@ and combine, packed placement, and backend-tuned resources. Each case explicitly
- `ep-routing-v1`: Zipf with EPLB off/on; decode T=128; prefill T=512.
- `ep-low-latency-v1`: DeepEP V1/UCCL native low-latency APIs; uniform decode T=1..128 powers of
two; the capability contract rejects every other backend instead of fabricating a low-latency path.
-- Canonical surface: 608 requested cases / 1,600 token points; 364 runnable cases / 940 points in
+- Implemented baseline surface: 608 requested cases / 1,600 token points; 364 runnable cases / 940
+ points in
58 executable workflow shards/allocation cells; 244 unsupported cases / 660 points.
| Systems | EP8 | EP16 |
@@ -49,7 +53,7 @@ remains MNNVL scale-up and uses LSA. NVIDIA capabilities declared in source rema
GPU outcomes pass the native oracle and publisher gates. H100 V2 on the current runner pool is a
declared unsupported combination in v1 because NCCL 2.30.4 reports no Device API symmetric-memory
support for its EP8 communicator; that pool can return only after all-rank CUDA P2P/LSA support is
-restored. Removed axes include `[cl]`, `[rv]`, quantization, alternate activation/routing profiles,
+restored. This baseline omits `[cl]`, `[rv]`, quantization, alternate activation/routing profiles,
uneven allocation, placement permutations, model envelopes, and scaling.
FlashInfer is excluded from v1 after repeatable intermittent execution failures; those failures are
not converted into planned-unsupported coverage.
@@ -285,12 +289,14 @@ artifact. `promote` accepts explicit immutable bundle IDs. Default `verify` requ
one-record `collectivex_public_v1_.ndjson` artifact. Raw artifacts and private workspace
content are never bundled into the application.
-Sweeps default to `release_tag=unversioned`. Selecting `v1` requires the locked full-matrix digest
-and emits a marker bound to the run ID, attempt, source SHA, and matrix SHA-256. The manual
-publication workflow accepts exactly three unique successful `CollectiveX Sweep` run IDs from one
-source SHA, revalidates their metadata and exact markers, downloads their immutable artifacts, and
-passes the same provenance assertions to `publisher.py ingest`. Partial, filtered, untagged,
-cross-source, failed, or expired inputs fail closed.
+Sweeps default to `release_tag=unversioned`. The main-registered `collectivex-sweep.yml` owns
+`sweep`, `publish-v1`, and `refresh-v1`, so its branch revision remains dispatchable. V1 emits a
+marker bound to the run ID, first attempt, qualification index, source SHA, and locked matrix digest.
+Publication accepts exactly three unique successful run IDs from one source SHA with qualification
+indices 1, 2, and 3, downloads their immutable artifacts, and passes the same provenance assertions
+to `publisher.py ingest`. Refresh requires an exact source run and dataset digest and reuploads the
+same validated sanitized bytes. Partial, filtered, untagged, cross-source, rerun, failed, expired,
+or digest-mismatched inputs fail closed.
Using a server-side GitHub read token, the frontend discovers the latest successful version-scoped
publication run and downloads the publication artifact just in time. It requires exactly one root
diff --git a/experimental/CollectiveX/identity.py b/experimental/CollectiveX/identity.py
index ff5b116996..3b263cc9c3 100644
--- a/experimental/CollectiveX/identity.py
+++ b/experimental/CollectiveX/identity.py
@@ -5,6 +5,7 @@
import hashlib
import json
import re
+from copy import deepcopy
from typing import Any
IDENTITY_VERSION = 1
@@ -19,12 +20,12 @@
"attempt": "cxattempt-v1-",
}
V1_NORMAL_CASE_PROFILE = {
- "activation_generator": "collectivex-activation-counter-v3",
- "activation_profile": "canonical-counter-source-v3",
+ "activation_generator": "collectivex-activation-counter-v4",
+ "activation_profile": "canonical-counter-source-v4",
"combine_dtype": "bf16",
"combine_quant_mode": "none",
"combine_semantics": "activation-only",
- "component_order_contract": "roundtrip-dispatch-activation-only-combine-v2",
+ "component_order_contract": "qualification-hash-rotated-components-v1",
"conditioning_contract": "fixed-phase-ramp-8-roundtrips-v1",
"contract": "layout-and-dispatch-v1",
"correctness_scope": "dispatch-metadata-and-transformed-combine",
@@ -39,15 +40,16 @@
"placement": "packed",
"percentile_method": "nearest-rank",
"rank_reduction": "cross-rank-max-per-iteration",
- "resource_mode": "tuned",
+ "resource_mode": "fixed-profile",
"routing_generator": "collectivex-routing-counter-v3",
"sampling_contract": "fixed-512-v1",
"seed": 67,
+ "source_identity_contract": "bounded-sign-bit-source-v1",
}
V1_LOW_LATENCY_CASE_PROFILE = {
**V1_NORMAL_CASE_PROFILE,
- "component_order_contract": "roundtrip-dispatch-gate-weighted-combine-v1",
+ "component_order_contract": "qualification-hash-rotated-components-v1",
"combine_semantics": "gate-weighted",
"contract": "expert-packed-weighted-combine-v1",
"correctness_scope": "expert-assignment-and-weighted-combine",
@@ -64,6 +66,187 @@
"low-latency": V1_LOW_LATENCY_CASE_PROFILE,
}
+V1_CONTROL_PRECISION_PROFILE = "d-bf16.c-bf16"
+V1_NORMAL_PRECISION_PROFILE_IDS = (
+ "d-fp8-e4m3fn-b128-f32-prequantized.c-bf16",
+ "d-fp8-e4m3fnuz-b128-f32-prequantized.c-bf16",
+ "d-bf16.c-fp8-e4m3fn-direct-cast-noscale",
+ "d-fp8-e4m3fn-b128-f32-prequantized.c-fp8-e4m3fn-direct-cast-noscale",
+ "d-bf16.c-fp8-e4m3fnuz-direct-cast-noscale",
+ "d-fp8-e4m3fnuz-b128-f32-prequantized.c-fp8-e4m3fnuz-direct-cast-noscale",
+)
+V1_LOW_LATENCY_PRECISION_PROFILE_IDS = (
+ "d-fp8-e4m3fn-b128-f32-fused.c-bf16",
+ "d-bf16.c-logfmt10-dynamic64",
+ "d-fp8-e4m3fn-b128-f32-fused.c-logfmt10-dynamic64",
+)
+
+
+def _communication_axis(
+ *,
+ api_input_dtype: str,
+ api_output_dtype: str,
+ communication_format: str,
+ scale_dtype: str | None,
+ scale_layout: str,
+ scale_group_size: int | None,
+ padding_contract: str,
+ alignment_contract: str,
+ quantization_origin: str,
+ conversion_boundary: str,
+) -> dict[str, Any]:
+ return {
+ "api_input_dtype": api_input_dtype,
+ "api_output_dtype": api_output_dtype,
+ "communication_format": communication_format,
+ "scale_dtype": scale_dtype,
+ "scale_layout": scale_layout,
+ "scale_group_size": scale_group_size,
+ "padding_contract": padding_contract,
+ "alignment_contract": alignment_contract,
+ "quantization_origin": quantization_origin,
+ "conversion_boundary": conversion_boundary,
+ }
+
+
+_BF16_AXIS = _communication_axis(
+ api_input_dtype="bf16",
+ api_output_dtype="bf16",
+ communication_format="bf16",
+ scale_dtype=None,
+ scale_layout="none",
+ scale_group_size=None,
+ padding_contract="none",
+ alignment_contract="native-bf16-vector-alignment",
+ quantization_origin="none",
+ conversion_boundary="none",
+)
+_FP8_E4M3FN_PREQUANTIZED_DISPATCH = _communication_axis(
+ api_input_dtype="fp8-e4m3fn-with-f32-scale",
+ api_output_dtype="fp8-e4m3fn-with-f32-scale",
+ communication_format="fp8-e4m3fn",
+ scale_dtype="f32",
+ scale_layout="per-token-hidden-block",
+ scale_group_size=128,
+ padding_contract="right-zero-pad-hidden-to-128",
+ alignment_contract="hidden-block-128",
+ quantization_origin="caller-prequantized",
+ conversion_boundary="before-dispatch-timing",
+)
+_FP8_E4M3FNUZ_PREQUANTIZED_DISPATCH = _communication_axis(
+ api_input_dtype="fp8-e4m3fnuz-with-f32-scale",
+ api_output_dtype="fp8-e4m3fnuz-with-f32-scale",
+ communication_format="fp8-e4m3fnuz",
+ scale_dtype="f32",
+ scale_layout="per-token-hidden-block",
+ scale_group_size=128,
+ padding_contract="right-zero-pad-hidden-to-128",
+ alignment_contract="hidden-block-128",
+ quantization_origin="caller-prequantized",
+ conversion_boundary="before-dispatch-timing",
+)
+_FP8_E4M3FN_FUSED_DISPATCH = _communication_axis(
+ api_input_dtype="bf16",
+ api_output_dtype="fp8-e4m3fn-with-f32-scale",
+ communication_format="fp8-e4m3fn",
+ scale_dtype="f32",
+ scale_layout="per-token-hidden-block",
+ scale_group_size=128,
+ padding_contract="right-zero-pad-hidden-to-128",
+ alignment_contract="hidden-block-128",
+ quantization_origin="backend-fused",
+ conversion_boundary="inside-dispatch-timing",
+)
+_LOGFMT10_DYNAMIC64_COMBINE = _communication_axis(
+ api_input_dtype="bf16",
+ api_output_dtype="bf16",
+ communication_format="logfmt10",
+ scale_dtype="implicit-logfmt10",
+ scale_layout="dynamic-per-64-values",
+ scale_group_size=64,
+ padding_contract="right-zero-pad-values-to-64",
+ alignment_contract="value-block-64",
+ quantization_origin="backend-internal",
+ conversion_boundary="inside-combine-timing",
+)
+_FP8_E4M3FN_DIRECT_CAST_COMBINE = _communication_axis(
+ api_input_dtype="bf16",
+ api_output_dtype="bf16",
+ communication_format="fp8-e4m3fn",
+ scale_dtype=None,
+ scale_layout="none",
+ scale_group_size=None,
+ padding_contract="none",
+ alignment_contract="native-fp8-vector-alignment",
+ quantization_origin="backend-internal-direct-cast",
+ conversion_boundary="inside-combine-timing",
+)
+_FP8_E4M3FNUZ_DIRECT_CAST_COMBINE = _communication_axis(
+ api_input_dtype="bf16",
+ api_output_dtype="bf16",
+ communication_format="fp8-e4m3fnuz",
+ scale_dtype=None,
+ scale_layout="none",
+ scale_group_size=None,
+ padding_contract="none",
+ alignment_contract="native-fp8-vector-alignment",
+ quantization_origin="backend-internal-direct-cast",
+ conversion_boundary="inside-combine-timing",
+)
+
+V1_PRECISION_PROFILES: dict[str, dict[str, Any]] = {
+ V1_CONTROL_PRECISION_PROFILE: {
+ "modes": ["normal", "low-latency"],
+ "dispatch": _BF16_AXIS,
+ "combine": _BF16_AXIS,
+ },
+ "d-fp8-e4m3fn-b128-f32-prequantized.c-bf16": {
+ "modes": ["normal"],
+ "dispatch": _FP8_E4M3FN_PREQUANTIZED_DISPATCH,
+ "combine": _BF16_AXIS,
+ },
+ "d-fp8-e4m3fnuz-b128-f32-prequantized.c-bf16": {
+ "modes": ["normal"],
+ "dispatch": _FP8_E4M3FNUZ_PREQUANTIZED_DISPATCH,
+ "combine": _BF16_AXIS,
+ },
+ "d-fp8-e4m3fn-b128-f32-fused.c-bf16": {
+ "modes": ["low-latency"],
+ "dispatch": _FP8_E4M3FN_FUSED_DISPATCH,
+ "combine": _BF16_AXIS,
+ },
+ "d-bf16.c-logfmt10-dynamic64": {
+ "modes": ["low-latency"],
+ "dispatch": _BF16_AXIS,
+ "combine": _LOGFMT10_DYNAMIC64_COMBINE,
+ },
+ "d-fp8-e4m3fn-b128-f32-fused.c-logfmt10-dynamic64": {
+ "modes": ["low-latency"],
+ "dispatch": _FP8_E4M3FN_FUSED_DISPATCH,
+ "combine": _LOGFMT10_DYNAMIC64_COMBINE,
+ },
+ "d-bf16.c-fp8-e4m3fn-direct-cast-noscale": {
+ "modes": ["normal"],
+ "dispatch": _BF16_AXIS,
+ "combine": _FP8_E4M3FN_DIRECT_CAST_COMBINE,
+ },
+ "d-fp8-e4m3fn-b128-f32-prequantized.c-fp8-e4m3fn-direct-cast-noscale": {
+ "modes": ["normal"],
+ "dispatch": _FP8_E4M3FN_PREQUANTIZED_DISPATCH,
+ "combine": _FP8_E4M3FN_DIRECT_CAST_COMBINE,
+ },
+ "d-bf16.c-fp8-e4m3fnuz-direct-cast-noscale": {
+ "modes": ["normal"],
+ "dispatch": _BF16_AXIS,
+ "combine": _FP8_E4M3FNUZ_DIRECT_CAST_COMBINE,
+ },
+ "d-fp8-e4m3fnuz-b128-f32-prequantized.c-fp8-e4m3fnuz-direct-cast-noscale": {
+ "modes": ["normal"],
+ "dispatch": _FP8_E4M3FNUZ_PREQUANTIZED_DISPATCH,
+ "combine": _FP8_E4M3FNUZ_DIRECT_CAST_COMBINE,
+ },
+}
+
def case_profile(mode: str) -> dict[str, Any]:
"""Return the immutable measurement profile for one scheduled mode."""
@@ -73,12 +256,32 @@ def case_profile(mode: str) -> dict[str, Any]:
raise IdentityError(f"unknown CollectiveX case mode {mode!r}") from exc
+def precision_profile(name: str) -> dict[str, Any]:
+ """Return one exact dispatch/combine communication-format profile."""
+ try:
+ profile = V1_PRECISION_PROFILES[name]
+ except KeyError as exc:
+ raise IdentityError(f"unknown CollectiveX precision profile {name!r}") from exc
+ return {"profile_id": name, **deepcopy(profile)}
+
+
def profile_for_case(case: dict[str, Any]) -> dict[str, Any]:
"""Resolve a scheduled case's explicit mode to its identity profile."""
mode = case.get("mode")
if not isinstance(mode, str):
raise IdentityError("scheduled case mode is missing")
- return case_profile(mode)
+ base = case_profile(mode)
+ precision_name = case.get("precision_profile")
+ if precision_name is None:
+ return base
+ if not isinstance(precision_name, str):
+ raise IdentityError("scheduled case precision_profile must be a string")
+ precision = precision_profile(precision_name)
+ if mode not in precision["modes"]:
+ raise IdentityError(
+ f"precision profile {precision_name!r} is not valid in mode {mode!r}"
+ )
+ return {**base, "communication_precision": precision}
class IdentityError(ValueError):
diff --git a/experimental/CollectiveX/publisher.py b/experimental/CollectiveX/publisher.py
index e04ba1f514..e64f52c124 100644
--- a/experimental/CollectiveX/publisher.py
+++ b/experimental/CollectiveX/publisher.py
@@ -22,6 +22,7 @@
import zipfile
import jsonschema
+import numpy as np
HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE))
@@ -40,6 +41,9 @@
OUTCOMES = ("success", "unsupported", "failed", "invalid", "diagnostic")
REQUIRED_ALLOCATIONS = 3
REQUIRED_COHORT_KINDS = ("library", "chip", "system", "routing")
+PRECISION_COHORT_KINDS = (
+ "dispatch-precision", "combine-precision", "precision-pair",
+)
REQUIRED_PROMOTION_COHORT_COUNTS = {"library": 76, "system": 12, "routing": 116}
CANONICAL_FULL_V1_MATRIX_SHA256 = (
"f1ca85f9689922b90edd5767b9ff2a902f6b896f32f68b2ca086dde3fd2157d0"
@@ -49,6 +53,14 @@
)
P50_STABILITY_LIMIT = 1.10
P99_STABILITY_LIMIT = 1.25
+TRIAL_DRIFT_RATIO_LIMIT = 1.10
+TRIAL_OUTLIER_FRACTION_LIMIT = 0.05
+TRIAL_OUTLIER_MAD_MULTIPLIER = 6.0
+BOOTSTRAP_RESAMPLES = 10_000
+BOOTSTRAP_CONFIDENCE = 0.95
+BOOTSTRAP_EQUIVALENCE_BAND = 0.05
+BOOTSTRAP_POLICY = "hierarchical-run-trial-p99-ratio-v1"
+BOOTSTRAP_CHUNK_SIZE = 250
MAX_ARCHIVE_MEMBERS = 20_000
MAX_ARCHIVE_MEMBER_BYTES = 2 * 1024**3
MAX_ARCHIVE_TOTAL_BYTES = 16 * 1024**3
@@ -66,6 +78,7 @@
CHANNEL_PATH = re.compile(r"datasets/([0-9a-f]{64})/dataset\.json")
SCHEMA_DIR = HERE / "schemas"
_SCHEMAS: dict[str, jsonschema.protocols.Validator] = {}
+_BOOTSTRAP_CACHE: dict[tuple[Any, ...], dict[str, Any]] = {}
class PublisherError(ValueError):
@@ -188,9 +201,9 @@ def _metric_value(series: dict[str, Any], metric: dict[str, Any]) -> tuple[str,
value = component["latency_us"][metric["statistic"]]
unit = "us"
else:
- rates = component["logical_payload_rate_gbps_at_latency_percentile"]
+ rates = component[metric["measure"]]
if rates is None:
- raise PublisherError("logical bandwidth decision has no logical byte contract")
+ raise PublisherError("data-rate decision has no byte accounting contract")
value = rates[metric["statistic"]]
unit = "GB/s"
return point["point_id"], value, unit
@@ -203,11 +216,14 @@ def _validate_metric(metric: dict[str, Any]) -> None:
def _metric_label(measure: str, statistic: str) -> str:
- return (
- f"{statistic} latency"
- if measure == "latency_us"
- else f"payload rate at {statistic} latency"
+ if measure == "latency_us":
+ return f"{statistic} latency"
+ label = (
+ "activation data rate"
+ if measure == "activation_data_rate_gbps_at_latency_percentile"
+ else "total logical data rate"
)
+ return f"{label} at {statistic} latency"
def _routing_build_control(build: dict[str, Any]) -> dict[str, Any]:
@@ -262,6 +278,8 @@ def _public_case_factors(series: dict[str, Any]) -> dict[str, Any]:
"warmup_semantics": sweep_matrix.ep_harness.WARMUP_SEMANTICS,
"workload": series["model"],
}
+ if workload["precision_profile"] != identity.V1_CONTROL_PRECISION_PROFILE:
+ case["precision_profile"] = workload["precision_profile"]
return {
"case": case,
"profile": identity.profile_for_case(case),
@@ -289,9 +307,12 @@ def _coverage_coordinates(case: dict[str, Any]) -> dict[str, Any]:
def _canonical_coverage_cases() -> dict[str, dict[str, Any]]:
matrix = sweep_matrix.resolve_matrix(suites="all", max_cases=128, backends="all")
return {
- item["case"]["case_id"]: _coverage_coordinates({
- "sku": item["sku"], **item["case"],
- })
+ item["case"]["case_id"]: {
+ "sku": item["sku"],
+ **item["case"],
+ "disposition": item["disposition"],
+ "reason": item["reason"],
+ }
for item in matrix["requested_cases"]
}
@@ -313,8 +334,8 @@ def _public_cohort_factors(kind: str, item: dict[str, Any]) -> tuple[Any, Any]:
shape = {
key: workload[key]
for key in (
- "hidden", "top_k", "experts", "dispatch_dtype", "combine_dtype",
- "activation_profile",
+ "hidden", "top_k", "experts", "precision_profile", "dispatch_precision",
+ "combine_precision", "activation_profile",
)
}
common = {
@@ -345,6 +366,47 @@ def _public_cohort_factors(kind: str, item: dict[str, Any]) -> tuple[Any, Any]:
[workload["routing"], workload["eplb"],
build["implementation_contract_sha256"]],
)
+ if kind in PRECISION_COHORT_KINDS:
+ static_shape = {
+ key: workload[key]
+ for key in ("hidden", "top_k", "experts", "activation_profile")
+ }
+ control = {
+ "backend": item["backend"],
+ "build": {
+ key: build[key]
+ for key in (
+ "image_digest", "runtime_fingerprint_sha256", "source_sha",
+ "squash_sha256",
+ )
+ },
+ "measurement": item["measurement"],
+ "mode": item["mode"],
+ "model": item["model"],
+ "phase": item["phase"],
+ "resource": item["resource"],
+ "shape": static_shape,
+ "system": item["system"],
+ "workload": {
+ "eplb": workload["eplb"],
+ "routing": workload["routing"],
+ },
+ }
+ if kind == "dispatch-precision":
+ control["combine_precision"] = workload["combine_precision"]
+ variant = workload["dispatch_precision"]
+ elif kind == "combine-precision":
+ control["dispatch_precision"] = workload["dispatch_precision"]
+ variant = workload["combine_precision"]
+ else:
+ control.pop("resource")
+ variant = {
+ "combine_precision": workload["combine_precision"],
+ "dispatch_precision": workload["dispatch_precision"],
+ "precision_profile": workload["precision_profile"],
+ "resource": item["resource"],
+ }
+ return control, variant
raise PublisherError(f"unknown cohort kind {kind}")
@@ -412,15 +474,40 @@ def validate_public_dataset(doc: Any) -> dict[str, Any]:
)
):
raise PublisherError("coverage topology differs from the capability registry")
- coordinates = {
- "sku": item["sku"], "backend": item["backend"],
- "mode": item["mode"], "phase": item["phase"], "topology": topology,
- }
canonical = canonical_cases.get(item["case_id"])
- if canonical is not None and coordinates != canonical:
- raise PublisherError("coverage coordinates differ from its case identity")
+ if canonical is not None:
+ precision_profile = canonical.get(
+ "precision_profile", identity.V1_CONTROL_PRECISION_PROFILE
+ )
+ precision = identity.precision_profile(precision_profile)
+ expected_projection = {
+ "sku": canonical["sku"],
+ "suite": canonical["suite"],
+ "workload": canonical["workload"],
+ "publication_tier": canonical["required_publication"],
+ "backend": canonical["backend"],
+ "mode": canonical["mode"],
+ "phase": canonical["phase"],
+ "routing": canonical["routing"],
+ "eplb": canonical["eplb"],
+ "precision_profile": precision_profile,
+ "dispatch_precision": precision["dispatch"],
+ "combine_precision": precision["combine"],
+ "topology": _coverage_topology(canonical),
+ "disposition": canonical["disposition"],
+ }
+ if any(item[field] != value for field, value in expected_projection.items()):
+ raise PublisherError("coverage dimensions differ from its case identity")
+ expected_tokens = [int(value) for value in canonical["ladder"].split()]
+ if [point["tokens_per_rank"] for point in item["points"]] != expected_tokens:
+ raise PublisherError("coverage points differ from the requested token ladder")
if canonical is None and item["case_id"] not in series_case_ids:
raise PublisherError("coverage case identity is outside the v1 catalog")
+ for point in item["points"]:
+ if point["global_tokens"] != point["tokens_per_rank"] * topology["ep_size"]:
+ raise PublisherError("coverage point global token count differs")
+ if (point["terminal_status"] == "measured") != (point["reason"] is None):
+ raise PublisherError("coverage point terminal reason differs from status")
for item in doc["attempts"]:
if item["case_id"] not in case_ids or item["allocation_id"] not in allocation_ids:
raise PublisherError("attempt references undeclared coverage or allocation")
@@ -483,7 +570,54 @@ def validate_public_dataset(doc: Any) -> dict[str, Any]:
)
if selected != latest["attempt_id"]:
raise PublisherError("coverage does not select the latest canonical allocation")
- if promotion["requested_cases"] != len(doc["coverage"]) or promotion["terminal_cases"] != terminal:
+ expected_status = (
+ "measured" if attempts[selected]["outcome"] == "success"
+ else attempts[selected]["outcome"]
+ )
+ if any(point["terminal_status"] != expected_status for point in item["points"]):
+ raise PublisherError("coverage point status differs from selected attempt")
+ if expected_status == "measured":
+ selected_series = series.get(attempts[selected]["series_id"])
+ if selected_series is None:
+ raise PublisherError("measured coverage points lack a public series")
+ public_points = {
+ point["tokens_per_rank"]: point for point in selected_series["points"]
+ }
+ if any(
+ point["series_id"] != selected_series["series_id"]
+ or point["point_id"]
+ != public_points.get(point["tokens_per_rank"], {}).get("point_id")
+ for point in item["points"]
+ ):
+ raise PublisherError("coverage point identities differ from series")
+ measured_cases = sum(
+ all(point["terminal_status"] == "measured" for point in item["points"])
+ for item in doc["coverage"]
+ )
+ unsupported_cases = sum(
+ all(point["terminal_status"] == "unsupported" for point in item["points"])
+ for item in doc["coverage"]
+ )
+ requested_points = sum(len(item["points"]) for item in doc["coverage"])
+ measured_points = sum(
+ point["terminal_status"] == "measured"
+ for item in doc["coverage"] for point in item["points"]
+ )
+ unsupported_points = sum(
+ point["terminal_status"] == "unsupported"
+ for item in doc["coverage"] for point in item["points"]
+ )
+ expected_counts = {
+ "requested_cases": len(doc["coverage"]),
+ "terminal_cases": terminal,
+ "measured_cases": measured_cases,
+ "unsupported_cases": unsupported_cases,
+ "requested_points": requested_points,
+ "terminal_points": requested_points,
+ "measured_points": measured_points,
+ "unsupported_points": unsupported_points,
+ }
+ if any(promotion[field] != value for field, value in expected_counts.items()):
raise PublisherError("promotion coverage counts differ")
selected_evidence: dict[tuple[str, str], set[str]] = {}
for attempt in doc["attempts"]:
@@ -500,7 +634,11 @@ def validate_public_dataset(doc: Any) -> dict[str, Any]:
coordinate = (
item["mode"], item["phase"], workload["routing"], workload["eplb"]
)
- profile = identity.case_profile(item["mode"])
+ profile_case = {"mode": item["mode"]}
+ if workload["precision_profile"] != identity.V1_CONTROL_PRECISION_PROFILE:
+ profile_case["precision_profile"] = workload["precision_profile"]
+ profile = identity.profile_for_case(profile_case)
+ communication_precision = identity.precision_profile(workload["precision_profile"])
if (
item["model"] != model
or (workload["hidden"], workload["top_k"], workload["experts"])
@@ -517,6 +655,12 @@ def validate_public_dataset(doc: Any) -> dict[str, Any]:
!= profile["component_order_contract"]
or item["measurement"]["combine_semantics"] != profile["combine_semantics"]
or item["measurement"]["payload_unit"] != profile["payload_unit"]
+ or workload["dispatch_precision"] != communication_precision["dispatch"]
+ or workload["combine_precision"] != communication_precision["combine"]
+ or item["measurement"]["qualification_indices"]
+ != sorted(item["measurement"]["qualification_indices"])
+ or len(set(item["measurement"]["qualification_indices"]))
+ != len(item["measurement"]["qualification_indices"])
):
raise PublisherError("series differs from the frozen v1 workload/suite profile")
backend_id = item["backend"]["id"]
@@ -534,13 +678,18 @@ def validate_public_dataset(doc: Any) -> dict[str, Any]:
registered_topology = capability.topology_for(sku, ep_size)
if platform is None or registered_topology is None:
raise PublisherError("series system projection differs from v1")
- supported, _ = capability.resolve(
+ disposition, _ = capability.resolve_disposition(
sku, backend_id, ep=ep_size, nodes=item["system"]["nodes"],
routing=workload["routing"], eplb=workload["eplb"],
mode=item["mode"],
+ precision_profile=(
+ workload["precision_profile"]
+ if workload["precision_profile"] != identity.V1_CONTROL_PRECISION_PROFILE
+ else None
+ ),
)
if (
- not supported
+ disposition != "supported"
or item["system"]["vendor"] != platform["vendor"]
or any(
item["system"][field] != registered_topology[field]
@@ -585,7 +734,8 @@ def validate_public_dataset(doc: Any) -> dict[str, Any]:
expected_physical = eplb["logical_experts"] + eplb["redundant_experts"]
nullable_eplb = (
"planner", "mapping_sha256", "reference_tokens_per_rank", "max_replicas",
- "imbalance_before", "imbalance_after",
+ "imbalance_before", "imbalance_after", "calibration_workload_id",
+ "calibration_trace_sha256", "calibration_window", "calibration_token_offset",
)
if eplb["enabled"]:
if (
@@ -605,13 +755,14 @@ def validate_public_dataset(doc: Any) -> dict[str, Any]:
or not 1 <= eplb["imbalance_after"] <= eplb["imbalance_before"] <= ep_size
):
raise PublisherError("enabled EPLB descriptor is incomplete")
- expected_plan = contracts._expected_eplb_plan(
- workload["routing"], workload["top_k"],
+ expected_plan, calibration = contracts._expected_eplb_calibration(
+ workload["routing"], workload["hidden"], workload["top_k"],
eplb["logical_experts"], eplb["physical_experts"], ep_size,
identity.V1_CASE_PROFILE["seed"],
identity.V1_CASE_PROFILE["eplb_reference_tokens_per_rank"],
)
expected_eplb = {
+ **calibration,
"enabled": True,
"planner": identity.V1_CASE_PROFILE["eplb_planner"],
"mapping_sha256": contracts.eplb_contract.mapping_hash(expected_plan),
@@ -645,7 +796,11 @@ def validate_public_dataset(doc: Any) -> dict[str, Any]:
raise PublisherError("series status differs from eligibility")
if (
set(eligibility["allocation_ids"]) != set(item["allocation_ids"])
- or eligibility["correct"] != all(point["correct"] for point in item["points"])
+ or eligibility["correct"] != all(
+ point["correctness"]["semantic_pass"]
+ and point["correctness"]["precision"]["passed"]
+ for point in item["points"]
+ )
):
raise PublisherError("series eligibility differs from its evidence")
selected_attempts = selected_by_series.get(item["series_id"], [])
@@ -653,6 +808,8 @@ def validate_public_dataset(doc: Any) -> dict[str, Any]:
set(item["case_ids"]) != {attempt["case_id"] for attempt in selected_attempts}
or set(item["allocation_ids"])
!= {attempt["allocation_id"] for attempt in selected_attempts}
+ or item["measurement"]["qualification_indices"]
+ != sorted({attempt["qualification_index"] for attempt in selected_attempts})
):
raise PublisherError("series case/allocation catalog differs from selected attempts")
if item["eligibility"]["decision_grade"] and len(
@@ -712,6 +869,44 @@ def validate_public_dataset(doc: Any) -> dict[str, Any]:
)
if set(point["evidence_ids"]) != expected_evidence:
raise PublisherError("point evidence differs from selected series attempts")
+ point_correctness = point["correctness"]
+ if (
+ point_correctness["precision"]["profile_id"]
+ != workload["precision_profile"]
+ or (
+ point_correctness["semantic_pass"]
+ and not point_correctness["precision"]["passed"]
+ )
+ or point["stability"]["qualification_indices"]
+ != item["measurement"]["qualification_indices"]
+ ):
+ raise PublisherError("point correctness/stability differs from series evidence")
+ diagnostics = point["trial_diagnostics"]
+ diagnostic_reasons = set(diagnostics["reasons"])
+ component_reasons: set[str] = set()
+ for name, summary in diagnostics["components"].items():
+ if summary is None:
+ if point["components"][name] is not None:
+ raise PublisherError("trial diagnostics omit a measured component")
+ continue
+ if point["components"][name] is None:
+ raise PublisherError("trial diagnostics describe an unavailable component")
+ if summary["drift_flagged"] != (
+ summary["first_last_median_ratio"] > TRIAL_DRIFT_RATIO_LIMIT
+ ) or summary["outlier_flagged"] != (
+ summary["robust_outlier_fraction"] > TRIAL_OUTLIER_FRACTION_LIMIT
+ ):
+ raise PublisherError("trial diagnostic flags differ from their thresholds")
+ if summary["drift_flagged"]:
+ component_reasons.add("trial-drift")
+ if summary["outlier_flagged"]:
+ component_reasons.add("trial-outliers")
+ if (
+ diagnostic_reasons != component_reasons
+ or diagnostics["flagged"] != bool(diagnostic_reasons)
+ or not diagnostic_reasons.issubset(point["anomalies"])
+ ):
+ raise PublisherError("trial diagnostic summary is inconsistent")
components = point["components"]
if (components["dispatch"] is None) != (components["combine"] is None):
raise PublisherError("dispatch/combine availability differs")
@@ -722,24 +917,31 @@ def validate_public_dataset(doc: Any) -> dict[str, Any]:
expected_samples = None if name == "isolated_sum" else 512
if component["origin"] != expected_origin or component["sample_count"] != expected_samples:
raise PublisherError(f"{name} origin or sample count differs")
- if name == "isolated_sum" and (
- component["logical_bytes"] is not None
- or component["logical_payload_rate_gbps_at_latency_percentile"] is not None
- ):
- raise PublisherError("isolated_sum cannot publish logical bandwidth")
- if name != "isolated_sum" and (
- component["logical_bytes"] is None
- or component["logical_payload_rate_gbps_at_latency_percentile"] is None
- ):
- raise PublisherError(f"{name} measured logical bandwidth is missing")
+ rate_fields = (
+ "activation_data_rate_gbps_at_latency_percentile",
+ "total_logical_data_rate_gbps_at_latency_percentile",
+ )
+ if name == "isolated_sum" and any(component[field] is not None for field in rate_fields):
+ raise PublisherError("isolated_sum cannot publish a derived data rate")
+ if name != "isolated_sum" and any(component[field] is None for field in rate_fields):
+ raise PublisherError(f"{name} measured data rates are missing")
latency = component["latency_us"]
if list(latency.values()) != sorted(latency.values()):
raise PublisherError("latency percentiles are not ordered")
- if component["logical_payload_rate_gbps_at_latency_percentile"] is not None:
- for statistic, rate in component["logical_payload_rate_gbps_at_latency_percentile"].items():
- expected = component["logical_bytes"] / (latency[statistic] * 1000.0)
- if not math.isclose(rate, expected, rel_tol=1e-9, abs_tol=1e-12):
- raise PublisherError("logical GB/s formula differs")
+ byte_provenance = component["byte_provenance"]
+ if byte_provenance["total_logical_bytes"] != (
+ byte_provenance["activation_data_bytes"] + byte_provenance["scale_bytes"]
+ ):
+ raise PublisherError("component byte accounting does not reconcile")
+ for field, byte_field in (
+ ("activation_data_rate_gbps_at_latency_percentile", "activation_data_bytes"),
+ ("total_logical_data_rate_gbps_at_latency_percentile", "total_logical_bytes"),
+ ):
+ if component[field] is not None:
+ for statistic, rate in component[field].items():
+ expected = byte_provenance[byte_field] / (latency[statistic] * 1000.0)
+ if not math.isclose(rate, expected, rel_tol=1e-9, abs_tol=1e-12):
+ raise PublisherError("component GB/s formula differs")
if components["roundtrip"] is None or components["roundtrip"]["origin"] != "measured":
raise PublisherError("roundtrip must be measured")
for statistic, throughput in point["roundtrip_token_rate_at_latency_percentile"].items():
@@ -753,11 +955,19 @@ def validate_public_dataset(doc: Any) -> dict[str, Any]:
if derived is None or any(not math.isclose(
derived["latency_us"][statistic],
components["dispatch"]["latency_us"][statistic]
+ + (
+ components["stage"]["latency_us"][statistic]
+ if components["stage"] is not None else 0.0
+ )
+ components["combine"]["latency_us"][statistic], rel_tol=1e-12
) for statistic in ("p50", "p90", "p95", "p99")):
raise PublisherError("isolated_sum is not the component percentile sum")
elif components["isolated_sum"] is not None:
raise PublisherError("isolated_sum requires measured dispatch/combine components")
+ if any(point["trial_diagnostics"]["flagged"] for point in item["points"]) != (
+ "unresolved-trial-diagnostic" in item["eligibility"]["reasons"]
+ ):
+ raise PublisherError("series trial diagnostic eligibility is inconsistent")
cohorts = {item["cohort_id"]: item for item in doc["cohorts"]}
if len(cohorts) != len(doc["cohorts"]):
raise PublisherError("dataset has duplicate cohort IDs")
@@ -779,7 +989,7 @@ def validate_public_dataset(doc: Any) -> dict[str, Any]:
raise PublisherError("library cohort contains non-library evidence")
if item["kind"] == "system" and roles != {"reference"}:
raise PublisherError("system cohort is not a portable reference comparison")
- if item["kind"] in {"chip", "routing"} and len(
+ if item["kind"] in {"chip", "routing", *PRECISION_COHORT_KINDS} and len(
{_canonical(member["backend"]) for member in members}
) != 1:
raise PublisherError(f"{item['kind']} cohort mixes backend implementations")
@@ -803,6 +1013,31 @@ def validate_public_dataset(doc: Any) -> dict[str, Any]:
mismatch_reason = "implementation-config-mismatch" in item["eligibility"]["reasons"]
if mismatch != mismatch_reason:
raise PublisherError("routing implementation control and eligibility disagree")
+ if item["kind"] in PRECISION_COHORT_KINDS:
+ if item["publication_tier"] != "comparable-experimental":
+ raise PublisherError("precision cohorts must be experimental")
+ if item["kind"] in {"dispatch-precision", "combine-precision"}:
+ axis = (
+ "dispatch"
+ if item["kind"] == "dispatch-precision"
+ else "combine"
+ )
+ field = f"{axis}_precision"
+ bf16 = identity.precision_profile(
+ identity.V1_CONTROL_PRECISION_PROFILE
+ )[axis]
+ has_baseline = sum(
+ _canonical(member["workload"][field]) == _canonical(bf16)
+ for member in members
+ ) == 1
+ missing_reason = (
+ "missing-bf16-precision-baseline"
+ in item["eligibility"]["reasons"]
+ )
+ if has_baseline == missing_reason:
+ raise PublisherError(
+ "precision baseline and eligibility reason disagree"
+ )
expected_id = _derived_id("cxcohort-v1-", {
"kind": item["kind"], "series_ids": item["series_ids"],
"controlled_factors": item["controlled_factors"],
@@ -827,6 +1062,33 @@ def validate_public_dataset(doc: Any) -> dict[str, Any]:
["backend", "implementation-static-build", "system", "model-shape", "mode", "phase", "measurement", "resource"],
["workload.routing", "workload.eplb", "implementation-config"],
),
+ "dispatch-precision": (
+ [
+ "backend", "implementation-static-build", "system", "model-shape",
+ "mode", "phase", "workload.routing", "workload.eplb",
+ "measurement", "resource", "combine-precision",
+ ],
+ ["dispatch-precision"],
+ ),
+ "combine-precision": (
+ [
+ "backend", "implementation-static-build", "system", "model-shape",
+ "mode", "phase", "workload.routing", "workload.eplb",
+ "measurement", "resource", "dispatch-precision",
+ ],
+ ["combine-precision"],
+ ),
+ "precision-pair": (
+ [
+ "backend", "implementation-static-build", "system", "model-shape",
+ "mode", "phase", "workload.routing", "workload.eplb",
+ "measurement",
+ ],
+ [
+ "dispatch-precision", "combine-precision", "precision-profile",
+ "resource",
+ ],
+ ),
}[item["kind"]]
member_allocations = {
allocation for series_id in item["series_ids"]
@@ -850,10 +1112,15 @@ def validate_public_dataset(doc: Any) -> dict[str, Any]:
expected_ranking_keys.update(
(cohort["cohort_id"], measure, statistic, token)
for token in tokens
- for measure in ("latency_us", "logical_payload_rate_gbps_at_latency_percentile")
+ for measure in (
+ "latency_us", "activation_data_rate_gbps_at_latency_percentile",
+ "total_logical_data_rate_gbps_at_latency_percentile",
+ )
for statistic in ("p50", "p99")
)
- ranking_top: dict[tuple[str, str, str, int], dict[str, Any]] = {}
+ ranking_top: dict[
+ tuple[str, str, str, int], dict[str, Any] | None
+ ] = {}
ranking_ids: set[str] = set()
for ranking in doc["rankings"]:
cohort = cohorts.get(ranking["cohort_id"])
@@ -879,32 +1146,51 @@ def validate_public_dataset(doc: Any) -> dict[str, Any]:
raise PublisherError("ranking entry differs from series data")
reverse = ranking["metric"]["objective"] == "max"
expected = sorted(entries, key=lambda entry: (entry["value"], entry["series_id"]), reverse=reverse)
- if entries != expected or [entry["rank"] for entry in entries] != list(range(1, len(entries) + 1)):
- raise PublisherError("ranking order differs")
metric = ranking["metric"]
+ ranks = [entry["rank"] for entry in entries]
+ if metric["measure"] == "latency_us" and metric["statistic"] == "p99":
+ tied_first = sum(rank == 1 for rank in ranks)
+ expected_ranks = [1] * tied_first + list(
+ range(tied_first + 1, len(entries) + 1)
+ )
+ else:
+ expected_ranks = list(range(1, len(entries) + 1))
+ if entries != expected or not ranks or ranks != expected_ranks:
+ raise PublisherError("ranking order differs")
expected_id = _derived_id("cxranking-v1-", {
"cohort_id": ranking["cohort_id"], "metric": metric,
})
if ranking["ranking_id"] != expected_id or expected_id in ranking_ids:
raise PublisherError("ranking ID is duplicate or differs")
ranking_ids.add(expected_id)
- ranking_top[(ranking["cohort_id"], metric["measure"], metric["statistic"], metric["tokens_per_rank"])] = entries[0]
+ ranking_top[(ranking["cohort_id"], metric["measure"], metric["statistic"], metric["tokens_per_rank"])] = (
+ entries[0] if ranks.count(1) == 1 else None
+ )
if set(ranking_top) != expected_ranking_keys:
raise PublisherError("rankings do not cover every eligible cohort metric")
objective = {
"min-p50-latency": ("latency_us", "p50"), "min-p99-latency": ("latency_us", "p99"),
- "max-payload-rate-at-p50-latency": (
- "logical_payload_rate_gbps_at_latency_percentile", "p50"
+ "max-activation-data-rate-at-p50-latency": (
+ "activation_data_rate_gbps_at_latency_percentile", "p50"
),
- "max-payload-rate-at-p99-latency": (
- "logical_payload_rate_gbps_at_latency_percentile", "p99"
+ "max-activation-data-rate-at-p99-latency": (
+ "activation_data_rate_gbps_at_latency_percentile", "p99"
+ ),
+ "max-total-logical-data-rate-at-p50-latency": (
+ "total_logical_data_rate_gbps_at_latency_percentile", "p50"
+ ),
+ "max-total-logical-data-rate-at-p99-latency": (
+ "total_logical_data_rate_gbps_at_latency_percentile", "p99"
),
}
recommendation_ids: set[str] = set()
for item in doc["recommendations"]:
+ if item["objective"] != "min-p99-latency":
+ raise PublisherError("recommendation is not a unique p99 latency winner")
measure, statistic = objective[item["objective"]]
candidates = [top for key, top in ranking_top.items()
- if key[:3] == (item["cohort_id"], measure, statistic) and top["point_id"] == item["point_id"]]
+ if key[:3] == (item["cohort_id"], measure, statistic)
+ and top is not None and top["point_id"] == item["point_id"]]
if len(candidates) != 1 or any(item[field] != candidates[0][field] for field in ("series_id", "point_id", "value", "unit")):
raise PublisherError("recommendation is not a ranking winner")
matching_ranking = next(
@@ -926,6 +1212,9 @@ def validate_public_dataset(doc: Any) -> dict[str, Any]:
recommendation_ids.add(expected_id)
expected_recommendations = sum(
cohorts[ranking["cohort_id"]]["publication_tier"] == "official"
+ and ranking["metric"]["measure"] == "latency_us"
+ and ranking["metric"]["statistic"] == "p99"
+ and sum(entry["rank"] == 1 for entry in ranking["entries"]) == 1
for ranking in doc["rankings"]
)
if len(doc["recommendations"]) != expected_recommendations:
@@ -936,24 +1225,39 @@ def validate_public_dataset(doc: Any) -> dict[str, Any]:
cohort = cohorts.get(item["cohort_id"])
if (
cohort is None
- or cohort["kind"] != "routing"
+ or cohort["kind"] not in {
+ "routing", "dispatch-precision", "combine-precision",
+ }
or not cohort["eligibility"]["decision_grade"]
or item["publication_tier"] != cohort["publication_tier"]
or item["eligibility"] != cohort["eligibility"]
):
- raise PublisherError("sensitivity references a non-routing cohort")
+ raise PublisherError("sensitivity references an ineligible contrast cohort")
if (
item["baseline_series_id"] == item["candidate_series_id"]
or not {item["baseline_series_id"], item["candidate_series_id"]}.issubset(cohort["series_ids"])
):
- raise PublisherError("sensitivity series differ from its routing cohort")
+ raise PublisherError("sensitivity series differ from its cohort")
_validate_metric(item["metric"])
baseline_series = series[item["baseline_series_id"]]
- if (
- baseline_series["workload"]["routing"] != "uniform"
- or baseline_series["workload"]["eplb"]
- ):
- raise PublisherError("sensitivity baseline is not uniform without EPLB")
+ if cohort["kind"] == "routing":
+ if (
+ baseline_series["workload"]["routing"] != "uniform"
+ or baseline_series["workload"]["eplb"]
+ ):
+ raise PublisherError("sensitivity baseline is not uniform without EPLB")
+ else:
+ axis = (
+ "dispatch"
+ if cohort["kind"] == "dispatch-precision"
+ else "combine"
+ )
+ field = f"{axis}_precision"
+ bf16 = identity.precision_profile(
+ identity.V1_CONTROL_PRECISION_PROFILE
+ )[axis]
+ if _canonical(baseline_series["workload"][field]) != _canonical(bf16):
+ raise PublisherError("precision sensitivity baseline is not BF16")
_, baseline, _ = _metric_value(series[item["baseline_series_id"]], item["metric"])
_, candidate, _ = _metric_value(series[item["candidate_series_id"]], item["metric"])
if not math.isclose(item["signed_change_ratio"], (candidate - baseline) / baseline, rel_tol=1e-12):
@@ -973,13 +1277,34 @@ def validate_public_dataset(doc: Any) -> dict[str, Any]:
))
expected_sensitivity_keys: set[tuple[str, str, str, str, str, int]] = set()
for cohort in doc["cohorts"]:
- if cohort["kind"] != "routing" or not cohort["eligibility"]["decision_grade"]:
+ if (
+ cohort["kind"] not in {
+ "routing", "dispatch-precision", "combine-precision",
+ }
+ or not cohort["eligibility"]["decision_grade"]
+ ):
continue
members = [series[series_id] for series_id in cohort["series_ids"]]
- baseline = next((
- member for member in members
- if member["workload"]["routing"] == "uniform" and not member["workload"]["eplb"]
- ), None)
+ if cohort["kind"] == "routing":
+ baseline = next((
+ member for member in members
+ if member["workload"]["routing"] == "uniform"
+ and not member["workload"]["eplb"]
+ ), None)
+ else:
+ axis = (
+ "dispatch"
+ if cohort["kind"] == "dispatch-precision"
+ else "combine"
+ )
+ field = f"{axis}_precision"
+ bf16 = identity.precision_profile(
+ identity.V1_CONTROL_PRECISION_PROFILE
+ )[axis]
+ baseline = next((
+ member for member in members
+ if _canonical(member["workload"][field]) == _canonical(bf16)
+ ), None)
if baseline is None:
continue
tokens = set.intersection(*(
@@ -991,19 +1316,27 @@ def validate_public_dataset(doc: Any) -> dict[str, Any]:
measure, statistic, token)
for candidate in members if candidate is not baseline
for token in tokens
- for measure in ("latency_us", "logical_payload_rate_gbps_at_latency_percentile")
+ for measure in (
+ "latency_us", "activation_data_rate_gbps_at_latency_percentile",
+ "total_logical_data_rate_gbps_at_latency_percentile",
+ )
for statistic in ("p50", "p99")
)
if sensitivity_keys != expected_sensitivity_keys:
- raise PublisherError("sensitivities do not cover every routing contrast metric")
+ raise PublisherError("sensitivities do not cover every declared contrast metric")
+ observed_qualification_indices = sorted({
+ item["qualification_index"] for item in doc["attempts"] if item["selected"]
+ })
+ if promotion["qualification_indices"] != observed_qualification_indices:
+ raise PublisherError("promotion qualification index catalog differs from attempts")
if promotion["status"] == "promoted":
run_ids = {item["run_id"] for item in doc["attempts"] if item["selected"]}
repeated_cases = all(
- len({
- attempts[attempt_id]["run_id"]
+ {
+ attempts[attempt_id]["qualification_index"]
for attempt_id in coverage["attempt_ids"]
if attempts[attempt_id]["selected"]
- }) == REQUIRED_ALLOCATIONS
+ } == {1, 2, 3}
for coverage in doc["coverage"]
)
if promotion["matrix_id"] != CANONICAL_FULL_V1_MATRIX_SHA256:
@@ -1015,6 +1348,12 @@ def validate_public_dataset(doc: Any) -> dict[str, Any]:
raise PublisherError("promotion requires the canonical case/disposition catalog")
if (
terminal != len(doc["coverage"])
+ or promotion["qualification_indices"] != [1, 2, 3]
+ or promotion["measured_cases"] + promotion["unsupported_cases"]
+ != promotion["requested_cases"]
+ or promotion["measured_points"] + promotion["unsupported_points"]
+ != promotion["requested_points"]
+ or promotion["terminal_points"] != promotion["requested_points"]
or len(doc["source_bundle_ids"]) != REQUIRED_ALLOCATIONS
or len(run_ids) != REQUIRED_ALLOCATIONS
or not repeated_cases
@@ -1045,8 +1384,8 @@ def validate_public_dataset(doc: Any) -> dict[str, Any]:
)
_require_promotion_series(doc["series"])
_require_promotion_cohorts(doc["cohorts"], doc["series"])
- if not doc["rankings"] or not doc["recommendations"]:
- raise PublisherError("promoted dataset lacks eligible decisions")
+ if not doc["rankings"]:
+ raise PublisherError("promoted dataset lacks eligible rankings")
if promotion["status"] == "quarantined" and any((
doc["source_bundle_ids"], promotion["allocation_ids"], doc["coverage"],
doc["attempts"], doc["series"], doc["cohorts"], doc["rankings"],
@@ -1653,6 +1992,7 @@ def _run_matches(document: dict[str, Any], run: dict[str, Any]) -> bool:
return (
str(git_run.get("run_id")) == run["run_id"]
and str(git_run.get("run_attempt")) == str(run["run_attempt"])
+ and git_run.get("qualification_index") == run["qualification_index"]
and git_run.get("source_sha") == run["source_sha"]
and (git_run.get("repo") or git_run.get("repository")) == run["repository"]
)
@@ -1665,7 +2005,7 @@ def _case_matches(document: dict[str, Any], expected: dict[str, Any]) -> bool:
}
return document.get("identity", {}).get("case_factors") == {
"case": scheduled,
- "profile": identity.V1_CASE_PROFILE,
+ "profile": identity.profile_for_case(scheduled),
"sku": expected["sku"],
}
@@ -2025,6 +2365,7 @@ def _public_attempt(document: dict[str, Any], *, selected: bool = False) -> dict
"allocation_id": normalized["allocation_id"],
"run_id": str(run["run_id"]),
"run_attempt": int(run["run_attempt"]),
+ "qualification_index": int(run["qualification_index"]),
"attempt_index": document["identity"]["attempt_ordinal"],
"selected": selected,
"outcome": status,
@@ -2039,6 +2380,238 @@ def _ratio(values: Sequence[float]) -> float | None:
return max(values) / min(values) if len(values) >= REQUIRED_ALLOCATIONS and min(values) > 0 else None
+def _private_trial_components(sample_document: dict[str, Any]) -> dict[int, dict[str, Any]]:
+ """Copy validated trial blocks into publisher-private memory without fixing component names."""
+ points: dict[int, dict[str, Any]] = {}
+ for point in sample_document["points"]:
+ token = point["tokens_per_rank"]
+ components: dict[str, Any] = {}
+ for name, component in point["components"].items():
+ availability = component["availability"]
+ if availability in {"unavailable", "not-applicable"}:
+ components[name] = None
+ continue
+ if availability != "measured":
+ raise PublisherError(f"private sample component {name} has invalid availability")
+ trials = component["trials"]
+ if (
+ not isinstance(trials, list)
+ or len(trials) != 64
+ or any(not isinstance(trial, list) or len(trial) != 8 for trial in trials)
+ ):
+ raise PublisherError(f"private sample component {name} is not 64x8")
+ copied = tuple(
+ tuple(float(sample) for sample in trial)
+ for trial in trials
+ )
+ if any(
+ not math.isfinite(sample) or sample < 0
+ for trial in copied for sample in trial
+ ):
+ raise PublisherError(f"private sample component {name} is not finite")
+ components[name] = copied
+ points[token] = components
+ return points
+
+
+def _trial_diagnostics(
+ trial_blocks: dict[str, dict[int, dict[str, Any]]], token: int,
+) -> dict[str, Any]:
+ components: dict[str, Any] = {}
+ reasons: set[str] = set()
+ for name in ("dispatch", "stage", "combine", "roundtrip"):
+ values = [trial_blocks[run_id][token][name] for run_id in sorted(trial_blocks)]
+ if all(value is None for value in values):
+ components[name] = None
+ continue
+ if any(value is None for value in values):
+ raise PublisherError(f"{name} trial availability differs across qualification runs")
+ array = np.asarray(values, dtype=np.float64)
+ if array.shape != (REQUIRED_ALLOCATIONS, 64, 8) or not np.isfinite(array).all():
+ raise PublisherError(f"{name} trial diagnostics require three finite 64x8 runs")
+ medians = np.median(array, axis=2)
+ first = np.median(medians[:, :8], axis=1)
+ last = np.median(medians[:, -8:], axis=1)
+ if np.any(first <= 0) or np.any(last <= 0):
+ raise PublisherError(f"{name} trial diagnostics require positive latency")
+ drift_ratio = float(np.max(np.maximum(first / last, last / first)))
+ center = float(np.median(medians))
+ mad = float(np.median(np.abs(medians - center)))
+ if mad == 0:
+ outliers = np.abs(medians - center) > 0
+ else:
+ outliers = np.abs(medians - center) > (
+ TRIAL_OUTLIER_MAD_MULTIPLIER * 1.4826 * mad
+ )
+ outlier_fraction = float(np.count_nonzero(outliers) / medians.size)
+ drift_flagged = drift_ratio > TRIAL_DRIFT_RATIO_LIMIT
+ outlier_flagged = outlier_fraction > TRIAL_OUTLIER_FRACTION_LIMIT
+ if drift_flagged:
+ reasons.add("trial-drift")
+ if outlier_flagged:
+ reasons.add("trial-outliers")
+ components[name] = {
+ "drift_flagged": drift_flagged,
+ "first_last_median_ratio": drift_ratio,
+ "outlier_flagged": outlier_flagged,
+ "robust_outlier_fraction": outlier_fraction,
+ "trial_count": int(medians.size),
+ }
+ return {
+ "flagged": bool(reasons),
+ "reasons": sorted(reasons),
+ "components": components,
+ }
+
+
+def _nearest_rank_p99(blocks: Sequence[Sequence[float]]) -> float:
+ samples = sorted(float(sample) for block in blocks for sample in block)
+ if len(samples) != 512 or samples[0] < 0 or not all(map(math.isfinite, samples)):
+ raise PublisherError("p99 bootstrap input must contain 512 finite samples")
+ return samples[math.ceil(0.99 * len(samples)) - 1]
+
+
+def _roundtrip_trial_array(
+ internal: dict[str, Any], token: int
+) -> tuple[tuple[str, ...], np.ndarray]:
+ trial_blocks = internal.get("trial_blocks")
+ if not isinstance(trial_blocks, dict):
+ raise PublisherError("series is missing private trial blocks")
+ run_ids = tuple(sorted(trial_blocks, key=lambda value: (int(value), value)))
+ if len(run_ids) != REQUIRED_ALLOCATIONS:
+ raise PublisherError("p99 bootstrap requires exactly three run blocks")
+ values = []
+ for run_id in run_ids:
+ point = trial_blocks[run_id].get(token)
+ blocks = point.get("roundtrip") if isinstance(point, dict) else None
+ if blocks is None:
+ raise PublisherError("p99 bootstrap requires measured roundtrip blocks")
+ if len(blocks) != 64 or any(len(block) != 8 for block in blocks):
+ raise PublisherError("p99 bootstrap roundtrip blocks must be 64x8")
+ values.append(blocks)
+ array = np.asarray(values, dtype=np.float64)
+ if array.shape != (REQUIRED_ALLOCATIONS, 64, 8):
+ raise PublisherError("p99 bootstrap trial array shape differs")
+ if not np.isfinite(array).all() or np.any(array <= 0):
+ raise PublisherError("p99 bootstrap latencies must be finite and positive")
+ return run_ids, array
+
+
+def _bootstrap_seed(
+ dataset_binding: str, baseline_series_id: str, candidate_series_id: str, token: int
+) -> tuple[str, int]:
+ payload = _canonical({
+ "policy": BOOTSTRAP_POLICY,
+ "resamples": BOOTSTRAP_RESAMPLES,
+ "confidence": BOOTSTRAP_CONFIDENCE,
+ "equivalence_band": BOOTSTRAP_EQUIVALENCE_BAND,
+ "dataset_binding": dataset_binding,
+ "baseline_series_id": baseline_series_id,
+ "candidate_series_id": candidate_series_id,
+ "tokens_per_rank": token,
+ })
+ digest = hashlib.sha256(payload).digest()
+ return digest.hex(), int.from_bytes(digest[:16], "big")
+
+
+def _hierarchical_p99_ratio(
+ baseline_series_id: str,
+ candidate_series_id: str,
+ token: int,
+ internals: dict[str, dict[str, Any]],
+ dataset_binding: str,
+) -> dict[str, Any]:
+ """Bootstrap candidate/baseline p99 across runs, then 64 trial blocks."""
+ baseline_runs, baseline = _roundtrip_trial_array(
+ internals[baseline_series_id], token
+ )
+ candidate_runs, candidate = _roundtrip_trial_array(
+ internals[candidate_series_id], token
+ )
+ if baseline_runs != candidate_runs:
+ raise PublisherError("p99 bootstrap run blocks are not aligned")
+ seed_sha256, seed = _bootstrap_seed(
+ dataset_binding, baseline_series_id, candidate_series_id, token
+ )
+ cache_key = (
+ seed_sha256,
+ _sha_bytes(baseline.tobytes()),
+ _sha_bytes(candidate.tobytes()),
+ )
+ cached = _BOOTSTRAP_CACHE.get(cache_key)
+ if cached is not None:
+ return dict(cached)
+
+ baseline_run_p99 = np.asarray(
+ [_nearest_rank_p99(run) for run in baseline], dtype=np.float64
+ )
+ candidate_run_p99 = np.asarray(
+ [_nearest_rank_p99(run) for run in candidate], dtype=np.float64
+ )
+ run_ratios = candidate_run_p99 / baseline_run_p99
+ point_ratio = float(np.median(candidate_run_p99) / np.median(baseline_run_p99))
+
+ rng = np.random.Generator(np.random.PCG64(seed))
+ ratios = np.empty(BOOTSTRAP_RESAMPLES, dtype=np.float64)
+ p99_index = math.ceil(0.99 * 512) - 1
+ for start in range(0, BOOTSTRAP_RESAMPLES, BOOTSTRAP_CHUNK_SIZE):
+ size = min(BOOTSTRAP_CHUNK_SIZE, BOOTSTRAP_RESAMPLES - start)
+ sampled_runs = rng.integers(0, REQUIRED_ALLOCATIONS, size=(size, 3))
+ sampled_blocks = rng.integers(0, 64, size=(size, 3, 64))
+ run_index = sampled_runs[:, :, None]
+ baseline_sample = baseline[run_index, sampled_blocks].reshape(size, 3, 512)
+ candidate_sample = candidate[run_index, sampled_blocks].reshape(size, 3, 512)
+ baseline_p99 = np.partition(baseline_sample, p99_index, axis=2)[:, :, p99_index]
+ candidate_p99 = np.partition(candidate_sample, p99_index, axis=2)[:, :, p99_index]
+ ratios[start:start + size] = (
+ np.median(candidate_p99, axis=1) / np.median(baseline_p99, axis=1)
+ )
+ ratios.sort()
+ tail = (1.0 - BOOTSTRAP_CONFIDENCE) / 2.0
+ lower_index = max(0, math.ceil(tail * BOOTSTRAP_RESAMPLES) - 1)
+ upper_index = min(
+ BOOTSTRAP_RESAMPLES - 1,
+ math.ceil((1.0 - tail) * BOOTSTRAP_RESAMPLES) - 1,
+ )
+ ci = [float(ratios[lower_index]), float(ratios[upper_index])]
+ threshold = 1.0 + BOOTSTRAP_EQUIVALENCE_BAND
+ baseline_wins = ci[0] > threshold and bool(np.all(run_ratios > threshold))
+ result = {
+ "policy": BOOTSTRAP_POLICY,
+ "resamples": BOOTSTRAP_RESAMPLES,
+ "confidence": BOOTSTRAP_CONFIDENCE,
+ "equivalence_band": BOOTSTRAP_EQUIVALENCE_BAND,
+ "seed_sha256": seed_sha256,
+ "point_ratio": point_ratio,
+ "ci95": ci,
+ "run_ratios": [float(value) for value in run_ratios],
+ "all_runs_agree": bool(np.all(run_ratios > threshold)),
+ "baseline_wins": baseline_wins,
+ "tie": not baseline_wins,
+ }
+ _BOOTSTRAP_CACHE[cache_key] = result
+ return dict(result)
+
+
+def _bootstrap_inputs_ready(
+ members: Sequence[dict[str, Any]],
+ internals: dict[str, dict[str, Any]],
+ tokens: Sequence[int],
+) -> bool:
+ try:
+ expected_runs: tuple[str, ...] | None = None
+ for member in members:
+ for token in tokens:
+ run_ids, _ = _roundtrip_trial_array(internals[member["series_id"]], token)
+ if expected_runs is None:
+ expected_runs = run_ids
+ elif run_ids != expected_runs:
+ return False
+ return expected_runs is not None
+ except (KeyError, PublisherError, TypeError, ValueError):
+ return False
+
+
def _eligibility_record(
allocations: Sequence[str],
*,
@@ -2099,23 +2672,38 @@ def _aggregate_component(
raise PublisherError("component availability differs across repeat allocations")
latency = _aggregate_percentiles([component["percentiles_us"] for component in components])
if name == "isolated_sum":
+ byte_provenance = {
+ "accounting_contract": "activation-data-plus-scales-v1",
+ "activation_data_bytes": 0,
+ "scale_bytes": 0,
+ "total_logical_bytes": 0,
+ }
return {
"origin": "derived",
"latency_us": latency,
- "logical_bytes": None,
- "logical_payload_rate_gbps_at_latency_percentile": None,
+ "byte_provenance": byte_provenance,
+ "activation_data_rate_gbps_at_latency_percentile": None,
+ "total_logical_data_rate_gbps_at_latency_percentile": None,
"sample_count": None,
}
- byte_values = {row["logical_bytes"][name] for row in rows}
- if len(byte_values) != 1:
- raise PublisherError("logical byte accounting differs across repeat allocations")
- logical_bytes = byte_values.pop()
- rates = {statistic: logical_bytes / (latency[statistic] * 1000.0) for statistic in latency}
+ byte_provenance = _exact_repeat_value(
+ [row["byte_provenance"][name] for row in rows],
+ f"{name} byte accounting",
+ )
+ activation_rates = {
+ statistic: byte_provenance["activation_data_bytes"] / (latency[statistic] * 1000.0)
+ for statistic in latency
+ }
+ total_rates = {
+ statistic: byte_provenance["total_logical_bytes"] / (latency[statistic] * 1000.0)
+ for statistic in latency
+ }
return {
"origin": "measured",
"latency_us": latency,
- "logical_bytes": logical_bytes,
- "logical_payload_rate_gbps_at_latency_percentile": rates,
+ "byte_provenance": byte_provenance,
+ "activation_data_rate_gbps_at_latency_percentile": activation_rates,
+ "total_logical_data_rate_gbps_at_latency_percentile": total_rates,
"sample_count": 512,
}
@@ -2130,6 +2718,10 @@ def _eplb_descriptor(document: dict[str, Any]) -> dict[str, Any]:
value = document["case"]["eplb"]
return {
"enabled": value["enabled"],
+ "calibration_workload_id": value["calibration_workload_id"],
+ "calibration_trace_sha256": value["calibration_trace_sha256"],
+ "calibration_window": value["calibration_window"],
+ "calibration_token_offset": value["calibration_token_offset"],
"planner": value["planner"],
"mapping_sha256": value["mapping_hash"],
"logical_experts": value["num_logical_experts"],
@@ -2157,6 +2749,35 @@ def _routing_facts(row: dict[str, Any]) -> dict[str, Any]:
}
+def _aggregate_precision_evidence(rows: Sequence[dict[str, Any]]) -> dict[str, Any]:
+ values = [row["correctness"]["precision"] for row in rows]
+ profile_ids = {value["profile_id"] for value in values}
+ if len(profile_ids) != 1:
+ raise PublisherError("precision evidence profile differs across qualification runs")
+ result: dict[str, Any] = {"profile_id": profile_ids.pop()}
+ for direction in ("dispatch", "combine"):
+ axes = [value[direction] for value in values]
+ finite = [axis["scales_finite"] for axis in axes]
+ positive = [axis["scales_positive"] for axis in axes]
+ result[direction] = {
+ "encoded_payload_valid": all(axis["encoded_payload_valid"] for axis in axes),
+ "scales_finite": None if all(value is None for value in finite) else all(
+ value is True for value in finite
+ ),
+ "scales_positive": None if all(value is None for value in positive) else all(
+ value is True for value in positive
+ ),
+ "dequantized_semantics": all(axis["dequantized_semantics"] for axis in axes),
+ "saturation_count": max(axis["saturation_count"] for axis in axes),
+ "saturation_rate": max(axis["saturation_rate"] for axis in axes),
+ "max_abs_error": max(axis["max_abs_error"] for axis in axes),
+ "max_rel_error": max(axis["max_rel_error"] for axis in axes),
+ "passed": all(axis["passed"] for axis in axes),
+ }
+ result["passed"] = result["dispatch"]["passed"] and result["combine"]["passed"]
+ return result
+
+
def _series_extra_reasons(documents: Sequence[dict[str, Any]]) -> list[str]:
reasons: set[str] = set()
for document in documents:
@@ -2192,6 +2813,7 @@ def _series_extra_reasons(documents: Sequence[dict[str, Any]]) -> list[str]:
def _build_series(
series_id: str,
documents: Sequence[dict[str, Any]],
+ sample_documents: Sequence[dict[str, Any]],
expected_repeats: int,
) -> tuple[dict[str, Any], dict[str, Any]]:
if not documents:
@@ -2199,6 +2821,8 @@ def _build_series(
first = documents[0]
if any(document["identity"]["series_id"] != series_id for document in documents):
raise PublisherError("series aggregation mixed identities")
+ if len(sample_documents) != len(documents):
+ raise PublisherError("series aggregation lost private sample documents")
allocations = [document["identity"]["allocation_id"] for document in documents]
if len(allocations) != len(set(allocations)):
raise PublisherError("series repeats reuse an allocation identity")
@@ -2210,6 +2834,9 @@ def _build_series(
if len(token_sets) != 1:
raise PublisherError("series token coverage differs across allocations")
tokens = list(next(iter(token_sets)))
+ qualification_indices = sorted(
+ document["measurement"]["qualification_index"] for document in documents
+ )
p50_ratios = [
_ratio([rows[token]["components"]["roundtrip"]["percentiles_us"]["p50"] for rows in row_maps])
for token in tokens
@@ -2229,17 +2856,7 @@ def _build_series(
and row["components"]["roundtrip"]["percentiles_us"].get("p99") is not None
for document in documents for row in document["measurement"]["rows"]
)
- eligibility = _eligibility_record(
- allocations,
- complete=len(documents) == expected_repeats,
- correct=correct,
- measured=measured,
- # Ordering is defined only across alternatives in a controlled cohort.
- stable_ordering=True,
- p50_ratio=p50_ratio,
- p99_ratio=p99_ratio,
- extra_reasons=_series_extra_reasons(documents),
- )
+ extra_reasons = _series_extra_reasons(documents)
case = first["case"]
shape = case["shape"]
topology = first["topology"]
@@ -2269,47 +2886,101 @@ def _build_series(
)
points: list[dict[str, Any]] = []
run_metrics: dict[str, dict[int, dict[str, float]]] = {}
- for document, rows in zip(documents, row_maps, strict=True):
+ trial_blocks: dict[str, dict[int, dict[str, Any]]] = {}
+ for document, sample_document, rows in zip(
+ documents, sample_documents, row_maps, strict=True
+ ):
+ if any(
+ sample_document[field] != document["identity"][field]
+ for field in ("allocation_id", "attempt_id", "case_id", "series_id")
+ ):
+ raise PublisherError("private samples differ from their selected raw attempt")
+ if sample_document["qualification_index"] != document["measurement"]["qualification_index"]:
+ raise PublisherError("private sample qualification index differs from raw attempt")
run_id = str(_git_run(document)["run_id"])
if run_id in run_metrics:
raise PublisherError("series has two allocations from one workflow run")
+ trial_blocks[run_id] = _private_trial_components(sample_document)
run_metrics[run_id] = {}
for token in tokens:
latency = rows[token]["components"]["roundtrip"]["percentiles_us"]
- logical_bytes = rows[token]["logical_bytes"]["roundtrip"]
+ byte_provenance = rows[token]["byte_provenance"]["roundtrip"]
run_metrics[run_id][token] = {
"latency_us": {statistic: latency[statistic] for statistic in ("p50", "p99")},
- "logical_payload_rate_gbps_at_latency_percentile": {
- statistic: logical_bytes / (latency[statistic] * 1000.0)
+ "activation_data_rate_gbps_at_latency_percentile": {
+ statistic: byte_provenance["activation_data_bytes"]
+ / (latency[statistic] * 1000.0)
+ for statistic in ("p50", "p99")
+ },
+ "total_logical_data_rate_gbps_at_latency_percentile": {
+ statistic: byte_provenance["total_logical_bytes"]
+ / (latency[statistic] * 1000.0)
for statistic in ("p50", "p99")
},
}
for token in tokens:
rows = [row_map[token] for row_map in row_maps]
+ diagnostics = _trial_diagnostics(trial_blocks, token)
+ if diagnostics["flagged"]:
+ extra_reasons.append("unresolved-trial-diagnostic")
routing = _exact_repeat_value(
[_routing_facts(row) for row in rows], "routing/load facts"
)
components = {
name: _aggregate_component(rows, name)
- for name in ("dispatch", "combine", "roundtrip")
+ for name in ("dispatch", "stage", "combine", "roundtrip")
}
if components["dispatch"] is None:
components["isolated_sum"] = None
else:
latency = {
statistic: components["dispatch"]["latency_us"][statistic]
+ + (
+ components["stage"]["latency_us"][statistic]
+ if components["stage"] is not None else 0.0
+ )
+ components["combine"]["latency_us"][statistic]
for statistic in ("p50", "p90", "p95", "p99")
}
components["isolated_sum"] = {
- "origin": "derived", "latency_us": latency, "logical_bytes": None,
- "logical_payload_rate_gbps_at_latency_percentile": None, "sample_count": None,
+ "origin": "derived",
+ "latency_us": latency,
+ "byte_provenance": components["roundtrip"]["byte_provenance"],
+ "activation_data_rate_gbps_at_latency_percentile": None,
+ "total_logical_data_rate_gbps_at_latency_percentile": None,
+ "sample_count": None,
}
points.append({
"point_id": rows[0]["point_id"],
"tokens_per_rank": token,
"global_tokens": token * case["ep_size"],
- "correct": all(row["correctness"]["passed"] for row in rows),
+ "correctness": {
+ "semantic_pass": all(row["correctness"]["passed"] for row in rows),
+ "precision": _aggregate_precision_evidence(rows),
+ },
+ "anomalies": sorted({
+ anomaly["type"].replace("_", "-")
+ for row in rows for anomaly in row["anomalies"]
+ } | set(diagnostics["reasons"])),
+ "stability": {
+ "complete": qualification_indices == [1, 2, 3],
+ "qualification_indices": qualification_indices,
+ "p50_max_min_ratio": p50_ratios[tokens.index(token)]
+ if qualification_indices == [1, 2, 3] else None,
+ "p99_max_min_ratio": p99_ratios[tokens.index(token)]
+ if qualification_indices == [1, 2, 3] else None,
+ "stable_p50": bool(
+ qualification_indices == [1, 2, 3]
+ and p50_ratios[tokens.index(token)] is not None
+ and p50_ratios[tokens.index(token)] <= P50_STABILITY_LIMIT
+ ),
+ "stable_p99": bool(
+ qualification_indices == [1, 2, 3]
+ and p99_ratios[tokens.index(token)] is not None
+ and p99_ratios[tokens.index(token)] <= P99_STABILITY_LIMIT
+ ),
+ },
+ "trial_diagnostics": diagnostics,
"routing": routing,
"components": components,
"roundtrip_token_rate_at_latency_percentile": {
@@ -2319,6 +2990,17 @@ def _build_series(
},
"evidence_ids": [row["evidence_id"] for row in rows],
})
+ eligibility = _eligibility_record(
+ allocations,
+ complete=len(documents) == expected_repeats,
+ correct=correct,
+ measured=measured,
+ # Ordering is defined only across alternatives in a controlled cohort.
+ stable_ordering=True,
+ p50_ratio=p50_ratio,
+ p99_ratio=p99_ratio,
+ extra_reasons=sorted(set(extra_reasons)),
+ )
series = {
"series_id": series_id,
"label": (
@@ -2386,8 +3068,9 @@ def _build_series(
"experts": case["eplb"]["num_logical_experts"],
"routing": shape["routing"],
"eplb": case["eplb"]["enabled"],
- "dispatch_dtype": shape["dispatch_dtype"],
- "combine_dtype": shape["quant"]["combine_output_dtype"],
+ "precision_profile": shape["precision_profile"],
+ "dispatch_precision": shape["dispatch_precision"],
+ "combine_precision": shape["combine_precision"],
"activation_profile": shape["activation_profile"],
},
"eplb": eplb,
@@ -2402,6 +3085,7 @@ def _build_series(
"trials": first["measurement"]["sampling"]["trials"],
"warmups": first["measurement"]["sampling"]["warmup_iterations"],
"samples_per_component": first["measurement"]["sampling"]["samples_per_component"],
+ "qualification_indices": qualification_indices,
"headline_component": "roundtrip",
"headline_percentile": "p99",
},
@@ -2411,6 +3095,7 @@ def _build_series(
internal = {
"documents": list(documents),
"run_metrics": run_metrics,
+ "trial_blocks": trial_blocks,
"series_factors": first["identity"]["series_factors"],
}
return series, internal
@@ -2477,6 +3162,7 @@ def load_bundle(store: Store, bundle_id: str) -> dict[str, Any]:
if {item["case_id"] for item in manifest["coverage"]["selections"]} != set(expected_by_id):
raise PublisherError("bundle selected coverage differs from requested matrix")
documents: dict[str, dict[str, Any]] = {}
+ sample_documents: dict[str, dict[str, Any]] = {}
runtime_fingerprints: set[str] = set()
for attempt in manifest["attempts"]:
document_path = _resolve_bundle_file(root, attempt["document"])
@@ -2493,7 +3179,9 @@ def load_bundle(store: Store, bundle_id: str) -> dict[str, Any]:
sample_document = contracts.strict_load(sample_path)
artifact_safety.assert_publication_safe([sample_document])
_schema("samples-v1.schema.json", sample_document)
+ sample_document = contracts.validate_samples_document(sample_document)
document = contracts.load_raw_attempt(document_path)
+ sample_documents[attempt["attempt_id"]] = sample_document
else:
if attempt["samples"] is not None:
raise PublisherError("terminal attempt unexpectedly names a sample artifact")
@@ -2523,6 +3211,7 @@ def load_bundle(store: Store, bundle_id: str) -> dict[str, Any]:
"manifest": manifest,
"cases": cases,
"documents": documents,
+ "sample_documents": sample_documents,
"selected": selected,
}
@@ -2535,7 +3224,10 @@ def _cohort_control(
workload = series["workload"]
shape = {
key: workload[key]
- for key in ("hidden", "top_k", "experts", "dispatch_dtype", "combine_dtype", "activation_profile")
+ for key in (
+ "hidden", "top_k", "experts", "precision_profile", "dispatch_precision",
+ "combine_precision", "activation_profile",
+ )
}
common = {
"model": series["model"], "mode": series["mode"],
@@ -2572,6 +3264,31 @@ def _cohort_control(
["workload.routing", "workload.eplb", "implementation-config"],
varying,
)
+ if kind in PRECISION_COHORT_KINDS:
+ control, variant = _public_cohort_factors(kind, series)
+ if kind == "dispatch-precision":
+ controlled = [
+ "backend", "implementation-static-build", "system", "model-shape",
+ "mode", "phase", "workload.routing", "workload.eplb", "measurement",
+ "resource", "combine-precision",
+ ]
+ varying = ["dispatch-precision"]
+ elif kind == "combine-precision":
+ controlled = [
+ "backend", "implementation-static-build", "system", "model-shape",
+ "mode", "phase", "workload.routing", "workload.eplb", "measurement",
+ "resource", "dispatch-precision",
+ ]
+ varying = ["combine-precision"]
+ else:
+ controlled = [
+ "backend", "implementation-static-build", "system", "model-shape",
+ "mode", "phase", "workload.routing", "workload.eplb", "measurement",
+ ]
+ varying = [
+ "dispatch-precision", "combine-precision", "precision-profile", "resource",
+ ]
+ return control, controlled, varying, variant
raise PublisherError(f"unknown cohort kind {kind}")
@@ -2586,7 +3303,10 @@ def _cohort_ordering(
orders: list[tuple[str, str, int, str, tuple[str, ...]]] = []
for run_id in sorted(run_ids):
for token in tokens:
- for measure in ("latency_us", "logical_payload_rate_gbps_at_latency_percentile"):
+ for measure in (
+ "latency_us", "activation_data_rate_gbps_at_latency_percentile",
+ "total_logical_data_rate_gbps_at_latency_percentile",
+ ):
for statistic in ("p50", "p99"):
ordered = tuple(
member["series_id"]
@@ -2596,12 +3316,15 @@ def _cohort_ordering(
internals[item["series_id"]]["run_metrics"][run_id][token][measure][statistic],
item["series_id"],
),
- reverse=measure == "logical_payload_rate_gbps_at_latency_percentile",
+ reverse=measure != "latency_us",
)
)
orders.append((measure, statistic, token, run_id, ordered))
for token in tokens:
- for measure in ("latency_us", "logical_payload_rate_gbps_at_latency_percentile"):
+ for measure in (
+ "latency_us", "activation_data_rate_gbps_at_latency_percentile",
+ "total_logical_data_rate_gbps_at_latency_percentile",
+ ):
for statistic in ("p50", "p99"):
observed = {
entry[4]
@@ -2613,11 +3336,61 @@ def _cohort_ordering(
return True, len(run_ids)
+def _p99_top_tie_ids(
+ members: Sequence[dict[str, Any]],
+ internals: dict[str, dict[str, Any]],
+ token: int,
+ dataset_binding: str,
+ cohort_id: str,
+) -> set[str]:
+ metric = {
+ "operation": "roundtrip",
+ "statistic": "p99",
+ "measure": "latency_us",
+ "objective": "min",
+ "tokens_per_rank": token,
+ "phase": members[0]["phase"],
+ }
+ ordered = sorted(
+ members,
+ key=lambda member: (
+ _metric_value(member, metric)[1], member["series_id"]
+ ),
+ )
+ baseline_id = ordered[0]["series_id"]
+ comparisons: dict[str, dict[str, Any]] = {}
+ tie_end = 0
+ for index, candidate in enumerate(ordered[1:], 1):
+ candidate_id = candidate["series_id"]
+ result = _hierarchical_p99_ratio(
+ baseline_id, candidate_id, token, internals, dataset_binding
+ )
+ comparisons[candidate_id] = result
+ if not result["baseline_wins"]:
+ tie_end = index
+ tie_ids = {member["series_id"] for member in ordered[:tie_end + 1]}
+ internals[baseline_id].setdefault("decision_statistics", {})[
+ f"{cohort_id}:p99:{token}"
+ ] = {
+ "baseline_series_id": baseline_id,
+ "comparisons": comparisons,
+ "tie_series_ids": sorted(tie_ids),
+ }
+ return tie_ids
+
+
def build_decisions(
- series: Sequence[dict[str, Any]], internals: dict[str, dict[str, Any]]
+ series: Sequence[dict[str, Any]],
+ internals: dict[str, dict[str, Any]],
+ *,
+ dataset_binding: str | None = None,
) -> tuple[list[dict[str, Any]], list[dict[str, Any]], list[dict[str, Any]], list[dict[str, Any]]]:
+ if dataset_binding is None:
+ dataset_binding = _sha_bytes(_canonical({
+ "series_ids": sorted(item["series_id"] for item in series),
+ }))
cohorts: list[dict[str, Any]] = []
- for kind in ("library", "chip", "system", "routing"):
+ for kind in (*REQUIRED_COHORT_KINDS, *PRECISION_COHORT_KINDS):
groups: dict[bytes, list[tuple[dict[str, Any], Any, list[str], list[str]]]] = {}
for item in series:
if kind == "library" and item["backend"]["role"] != "library":
@@ -2650,6 +3423,8 @@ def build_decisions(
}
if aligned_runs < REQUIRED_ALLOCATIONS:
extra.add("incomplete-aligned-repeats")
+ if tokens and not _bootstrap_inputs_ready(members, internals, tokens):
+ extra.add("missing-trial-blocks")
if kind == "routing" and sum(
member["workload"]["routing"] == "uniform"
and not member["workload"]["eplb"]
@@ -2663,12 +3438,24 @@ def build_decisions(
extra.add("incomplete-routing-anchors")
if kind == "routing" and _routing_implementation_mismatch(members):
extra.add("implementation-config-mismatch")
- if not tokens or (kind != "routing" and not same_points):
+ endpoint_contrast = kind == "routing" or kind in PRECISION_COHORT_KINDS
+ if not tokens or (not endpoint_contrast and not same_points):
extra.add("unmatched-token-coverage")
+ if kind in {"dispatch-precision", "combine-precision"}:
+ axis = "dispatch" if kind == "dispatch-precision" else "combine"
+ field = f"{axis}_precision"
+ bf16 = identity.precision_profile(
+ identity.V1_CONTROL_PRECISION_PROFILE
+ )[axis]
+ if sum(
+ _canonical(member["workload"][field]) == _canonical(bf16)
+ for member in members
+ ) != 1:
+ extra.add("missing-bf16-precision-baseline")
eligibility = _eligibility_record(
allocations,
complete=all(member["eligibility"]["complete"] for member in members)
- and bool(tokens) and (kind == "routing" or same_points),
+ and bool(tokens) and (endpoint_contrast or same_points),
correct=all(member["eligibility"]["correct"] for member in members),
measured=all(member["eligibility"]["measured_roundtrip_p99"] for member in members),
stable_ordering=ordering,
@@ -2687,7 +3474,12 @@ def build_decisions(
"kind": kind, "series_ids": member_ids,
"controlled_factors": controlled, "varying_factors": varying,
})
- kind_label = "Platform" if kind == "chip" else kind.title()
+ kind_label = {
+ "chip": "Platform",
+ "dispatch-precision": "Dispatch precision",
+ "combine-precision": "Combine precision",
+ "precision-pair": "Precision profile",
+ }.get(kind, kind.title())
first = members[0]
routing_label = first["workload"]["routing"] + (
"+EPLB" if first["workload"]["eplb"] else ""
@@ -2709,6 +3501,18 @@ def build_decisions(
f"{first['system']['sku'].upper()} / {first['backend']['label']} / "
f"EP{first['system']['ep_size']} / {first['mode']} / {first['phase']}"
),
+ "dispatch-precision": (
+ f"{first['system']['sku'].upper()} / {first['backend']['label']} / "
+ f"EP{first['system']['ep_size']} / {first['mode']} / {first['phase']}"
+ ),
+ "combine-precision": (
+ f"{first['system']['sku'].upper()} / {first['backend']['label']} / "
+ f"EP{first['system']['ep_size']} / {first['mode']} / {first['phase']}"
+ ),
+ "precision-pair": (
+ f"{first['system']['sku'].upper()} / {first['backend']['label']} / "
+ f"EP{first['system']['ep_size']} / {first['mode']} / {first['phase']}"
+ ),
}[kind]
cohorts.append({
"cohort_id": cohort_id,
@@ -2717,7 +3521,11 @@ def build_decisions(
"description": (
"Publisher-controlled NCCL/RCCL system comparison"
if kind == "system"
- else f"Publisher-controlled {kind_label.lower()} comparison"
+ else (
+ "Descriptive configured-stack precision comparison; no isolated axis claim"
+ if kind == "precision-pair"
+ else f"Publisher-controlled {kind_label.lower()} comparison"
+ )
),
"series_ids": member_ids,
"controlled_factors": controlled,
@@ -2738,8 +3546,13 @@ def build_decisions(
{point["tokens_per_rank"] for point in member["points"]} for member in members
)))
for token in tokens:
+ p99_tie_ids = _p99_top_tie_ids(
+ members, internals, token, dataset_binding, cohort["cohort_id"]
+ )
for measure, objective, unit in (
- ("latency_us", "min", "us"), ("logical_payload_rate_gbps_at_latency_percentile", "max", "GB/s")
+ ("latency_us", "min", "us"),
+ ("activation_data_rate_gbps_at_latency_percentile", "max", "GB/s"),
+ ("total_logical_data_rate_gbps_at_latency_percentile", "max", "GB/s"),
):
for statistic in ("p50", "p99"):
metric = {
@@ -2758,7 +3571,13 @@ def build_decisions(
})
entries.sort(key=lambda item: (item["value"], item["series_id"]), reverse=objective == "max")
for rank, entry in enumerate(entries, 1):
- entry["rank"] = rank
+ entry["rank"] = (
+ 1
+ if measure == "latency_us"
+ and statistic == "p99"
+ and entry["series_id"] in p99_tie_ids
+ else rank
+ )
ranking_id = _derived_id("cxranking-v1-", {
"cohort_id": cohort["cohort_id"], "metric": metric,
})
@@ -2770,13 +3589,14 @@ def build_decisions(
"publication_tier": cohort["publication_tier"],
"eligibility": cohort["eligibility"],
})
- if cohort["publication_tier"] != "official":
+ if (
+ cohort["publication_tier"] != "official"
+ or measure != "latency_us"
+ or statistic != "p99"
+ or sum(entry["rank"] == 1 for entry in entries) != 1
+ ):
continue
- objective_name = (
- f"min-{statistic}-latency"
- if measure == "latency_us"
- else f"max-payload-rate-at-{statistic}-latency"
- )
+ objective_name = "min-p99-latency"
top = entries[0]
recommendation_id = _derived_id("cxrecommendation-v1-", {
"objective": objective_name, "ranking_id": ranking_id,
@@ -2788,7 +3608,10 @@ def build_decisions(
"objective": objective_name,
"series_id": top["series_id"], "point_id": top["point_id"],
"value": top["value"], "unit": top["unit"],
- "rationale": "Top stable measured roundtrip result in a controlled cohort",
+ "rationale": (
+ "Unique p99 winner after deterministic hierarchical bootstrap "
+ "and all-run agreement"
+ ),
"publication_tier": cohort["publication_tier"],
"eligibility": cohort["eligibility"],
})
@@ -2803,7 +3626,11 @@ def build_decisions(
if candidate is baseline:
continue
for token in tokens:
- for measure, objective in (("latency_us", "min"), ("logical_payload_rate_gbps_at_latency_percentile", "max")):
+ for measure, objective in (
+ ("latency_us", "min"),
+ ("activation_data_rate_gbps_at_latency_percentile", "max"),
+ ("total_logical_data_rate_gbps_at_latency_percentile", "max"),
+ ):
for statistic in ("p50", "p99"):
metric = {
"operation": "roundtrip", "statistic": statistic,
@@ -2830,6 +3657,62 @@ def build_decisions(
"publication_tier": cohort["publication_tier"],
"eligibility": cohort["eligibility"],
})
+ if cohort["kind"] in {"dispatch-precision", "combine-precision"}:
+ axis = (
+ "dispatch"
+ if cohort["kind"] == "dispatch-precision"
+ else "combine"
+ )
+ field = f"{axis}_precision"
+ bf16 = identity.precision_profile(
+ identity.V1_CONTROL_PRECISION_PROFILE
+ )[axis]
+ baseline = next(
+ member for member in members
+ if _canonical(member["workload"][field]) == _canonical(bf16)
+ )
+ for candidate in members:
+ if candidate is baseline:
+ continue
+ for token in tokens:
+ for measure, objective in (
+ ("latency_us", "min"),
+ ("activation_data_rate_gbps_at_latency_percentile", "max"),
+ ("total_logical_data_rate_gbps_at_latency_percentile", "max"),
+ ):
+ for statistic in ("p50", "p99"):
+ metric = {
+ "operation": "roundtrip",
+ "statistic": statistic,
+ "measure": measure,
+ "objective": objective,
+ "tokens_per_rank": token,
+ "phase": baseline["phase"],
+ }
+ _, base_value, _ = _metric_value(baseline, metric)
+ _, candidate_value, _ = _metric_value(candidate, metric)
+ sensitivity_id = _derived_id("cxsensitivity-v1-", {
+ "baseline": baseline["series_id"],
+ "candidate": candidate["series_id"],
+ "cohort": cohort["cohort_id"],
+ "metric": metric,
+ })
+ sensitivities.append({
+ "sensitivity_id": sensitivity_id,
+ "cohort_id": cohort["cohort_id"],
+ "label": (
+ f"{axis.title()} precision sensitivity: "
+ f"{_metric_label(measure, statistic)} T={token}"
+ ),
+ "baseline_series_id": baseline["series_id"],
+ "candidate_series_id": candidate["series_id"],
+ "metric": metric,
+ "signed_change_ratio": (
+ candidate_value - base_value
+ ) / base_value,
+ "publication_tier": cohort["publication_tier"],
+ "eligibility": cohort["eligibility"],
+ })
rankings.sort(key=lambda item: item["ranking_id"])
recommendations.sort(key=lambda item: item["recommendation_id"])
sensitivities.sort(key=lambda item: item["sensitivity_id"])
@@ -2876,7 +3759,16 @@ def _require_promotion_cohorts(
for cohort in cohorts
if cohort["eligibility"]["decision_grade"]
}
- missing = [kind for kind in REQUIRED_COHORT_KINDS if kind not in eligible_kinds]
+ required_kinds = list(REQUIRED_COHORT_KINDS)
+ if any(
+ item["workload"].get(
+ "precision_profile", identity.V1_CONTROL_PRECISION_PROFILE
+ )
+ != identity.V1_CONTROL_PRECISION_PROFILE
+ for item in series
+ ):
+ required_kinds.extend(PRECISION_COHORT_KINDS)
+ missing = [kind for kind in required_kinds if kind not in eligible_kinds]
if missing:
raise PublisherError(
"promotion lacks decision-grade cohort kinds: " + ", ".join(missing)
@@ -2947,11 +3839,18 @@ def build_dataset(
if len(matrix_ids) != 1 or len({tuple(sorted(values)) for values in case_sets}) != 1:
raise PublisherError("dataset bundles do not share one exact requested matrix")
run_ids = [bundle["manifest"]["run"]["run_id"] for bundle in loaded]
+ qualification_indices = sorted(
+ bundle["manifest"]["run"]["qualification_index"] for bundle in loaded
+ )
if promote and (
len(loaded) != REQUIRED_ALLOCATIONS
or len(run_ids) != len(set(run_ids))
+ or qualification_indices != [1, 2, 3]
+ or any(bundle["manifest"]["run"]["run_attempt"] != 1 for bundle in loaded)
):
- raise PublisherError("promotion requires three independent complete workflow runs")
+ raise PublisherError(
+ "promotion requires qualification indices 1, 2, and 3 from first-attempt runs"
+ )
if promote and matrix_ids != {CANONICAL_FULL_V1_MATRIX_SHA256}:
raise PublisherError("promotion requires the canonical full-v1 matrix")
cases = {case["case_id"]: case for case in loaded[0]["cases"]}
@@ -2975,6 +3874,11 @@ def build_dataset(
case_id: [bundle["selected"][case_id] for bundle in loaded]
for case_id in sorted(cases)
}
+ samples_by_attempt = {
+ attempt_id: sample_document
+ for bundle in loaded
+ for attempt_id, sample_document in bundle["sample_documents"].items()
+ }
coverage: list[dict[str, Any]] = []
for case_id, case in sorted(cases.items()):
attempts = sorted(
@@ -2984,7 +3888,64 @@ def build_dataset(
attempt["attempt_index"], attempt["attempt_id"],
),
)
- selected = _public_attempt(selected_by_case[case_id][-1], selected=True)
+ selected_document = selected_by_case[case_id][-1]
+ selected = _public_attempt(selected_document, selected=True)
+ precision_profile = case.get(
+ "precision_profile", identity.V1_CONTROL_PRECISION_PROFILE
+ )
+ precision = identity.precision_profile(precision_profile)
+ selected_raw = (
+ selected_document
+ if selected_document["format"] == contracts.RAW_FORMAT
+ and selected_document["outcome"]["status"] == "success"
+ else None
+ )
+ if selected_raw is not None:
+ backend_generation = selected_raw["implementation"]["kernel_generation"]
+ projected = contracts.public_series_config(
+ kernel_generation=backend_generation,
+ provenance=selected_raw["implementation"]["provenance"],
+ resource_profile=selected_raw["implementation"]["resource_profile"],
+ resource_mode=selected_raw["case"]["resource_mode"],
+ device_product=selected_raw["topology"]["device_product"],
+ )
+ resource = projected["resource"]
+ rows_by_token = {
+ row["tokens_per_rank"]: row for row in selected_raw["measurement"]["rows"]
+ }
+ series_id = selected_raw["identity"]["series_id"]
+ else:
+ backend_generation = None
+ resource = {
+ "mode": "fixed-profile",
+ "profile": None,
+ "comm_units_kind": None,
+ "configured_units": None,
+ }
+ rows_by_token = {}
+ series_id = None
+ point_status = (
+ "measured" if selected["outcome"] == "success" else selected["outcome"]
+ )
+ point_reason = (
+ None
+ if point_status == "measured"
+ else case["_reason"]
+ if point_status == "unsupported"
+ else selected["reason"]
+ )
+ token_ladder = [int(value) for value in case["ladder"].split()]
+ coverage_points = []
+ for token in token_ladder:
+ row = rows_by_token.get(token)
+ coverage_points.append({
+ "point_id": row["point_id"] if row is not None else None,
+ "series_id": series_id if row is not None else None,
+ "tokens_per_rank": token,
+ "global_tokens": token * case["ep"],
+ "terminal_status": point_status,
+ "reason": point_reason,
+ })
coverage.append({
"case_id": case_id,
"label": (
@@ -2993,10 +3954,21 @@ def build_dataset(
),
"required": True,
"sku": _slug(case["sku"]),
+ "suite": _slug(case["suite"]),
+ "workload": _slug(case["workload"]),
+ "publication_tier": case["required_publication"],
"backend": _slug(case["backend"]),
+ "backend_generation": backend_generation,
"mode": case["mode"],
"phase": case["phase"],
+ "routing": case["routing"],
+ "eplb": case["eplb"],
+ "precision_profile": precision_profile,
+ "dispatch_precision": precision["dispatch"],
+ "combine_precision": precision["combine"],
+ "resource": resource,
"topology": _coverage_topology(case),
+ "points": coverage_points,
"disposition": case["_disposition"],
"selected_attempt_id": selected["attempt_id"],
"outcome": selected["outcome"],
@@ -3015,11 +3987,46 @@ def build_dataset(
series: list[dict[str, Any]] = []
internals: dict[str, dict[str, Any]] = {}
for series_id, documents in sorted(by_series.items()):
- item, internal = _build_series(series_id, documents, len(loaded))
+ try:
+ sample_documents = [
+ samples_by_attempt[document["identity"]["attempt_id"]]
+ for document in documents
+ ]
+ except KeyError as exc:
+ raise PublisherError(
+ "selected raw evidence is missing its private sample document"
+ ) from exc
+ item, internal = _build_series(
+ series_id, documents, sample_documents, len(loaded)
+ )
series.append(item)
internals[series_id] = internal
- cohorts, rankings, recommendations, sensitivities = build_decisions(series, internals)
+ dataset_binding = _sha_bytes(_canonical({
+ "matrix_id": next(iter(matrix_ids)),
+ "source_bundle_ids": sorted(bundle_ids),
+ }))
+ cohorts, rankings, recommendations, sensitivities = build_decisions(
+ series, internals, dataset_binding=dataset_binding
+ )
allocation_ids = sorted({attempt["allocation_id"] for attempt in public_attempts})
+ qualification_indices = sorted({int(value) for value in qualification_indices})
+ measured_cases = sum(
+ all(point["terminal_status"] == "measured" for point in item["points"])
+ for item in coverage
+ )
+ unsupported_cases = sum(
+ all(point["terminal_status"] == "unsupported" for point in item["points"])
+ for item in coverage
+ )
+ requested_points = sum(len(item["points"]) for item in coverage)
+ measured_points = sum(
+ point["terminal_status"] == "measured"
+ for item in coverage for point in item["points"]
+ )
+ unsupported_points = sum(
+ point["terminal_status"] == "unsupported"
+ for item in coverage for point in item["points"]
+ )
status = "promoted" if promote else "diagnostic"
dataset = {
"format": FORMAT_PUBLIC,
@@ -3034,8 +4041,15 @@ def build_dataset(
"matrix_id": next(iter(matrix_ids)),
"allocation_ids": allocation_ids,
"required_allocations": REQUIRED_ALLOCATIONS,
+ "qualification_indices": qualification_indices,
"requested_cases": len(coverage),
"terminal_cases": len(coverage),
+ "measured_cases": measured_cases,
+ "unsupported_cases": unsupported_cases,
+ "requested_points": requested_points,
+ "terminal_points": requested_points,
+ "measured_points": measured_points,
+ "unsupported_points": unsupported_points,
"policy": POLICY,
},
"coverage": coverage,
@@ -3053,34 +4067,6 @@ def build_dataset(
return dataset
-def _quarantine_dataset(reason: str, generated_at: str) -> dict[str, Any]:
- dataset = {
- "format": FORMAT_PUBLIC,
- "schema_version": 1,
- "generated_at": generated_at,
- "source_bundle_ids": [],
- "promotion": {
- "status": "quarantined",
- "reason": reason,
- "matrix_id": None,
- "allocation_ids": [],
- "required_allocations": REQUIRED_ALLOCATIONS,
- "requested_cases": 0,
- "terminal_cases": 0,
- "policy": POLICY,
- },
- "coverage": [],
- "attempts": [],
- "series": [],
- "cohorts": [],
- "rankings": [],
- "recommendations": [],
- "sensitivities": [],
- }
- validate_public_dataset(dataset)
- return dataset
-
-
def quarantine_incoming(
store: Store, ingest_id: str, reason: str, generated_at: str
) -> str:
@@ -3103,11 +4089,6 @@ def quarantine_incoming(
store.install(stage, store.quarantine / digest, private=True)
if _sha_bytes(_canonical(strict_load(store.quarantine / digest / "quarantine.json"))) != digest:
raise PublisherError("existing quarantine object differs")
- # The incoming digest distinguishes separate rejected deliveries while preserving
- # byte-identical output when the operator retries the same immutable input.
- dataset = _quarantine_dataset(public_reason, generated_at)
- dataset_digest, size = store.install_dataset(dataset)
- store.update_channel("latest-attempt", dataset_digest, size, generated_at)
return digest
@@ -3131,6 +4112,7 @@ def _run_metadata(args: argparse.Namespace) -> dict[str, Any]:
"repository": args.repository,
"run_id": args.run_id,
"run_attempt": args.run_attempt,
+ "qualification_index": args.qualification_index,
"source_sha": args.source_sha,
}
# Reuse the authoritative private schema constraints before any filesystem mutation.
@@ -3140,6 +4122,8 @@ def _run_metadata(args: argparse.Namespace) -> dict[str, Any]:
raise PublisherError("--run-id must be a positive decimal string")
if type(run["run_attempt"]) is not int or run["run_attempt"] < 1:
raise PublisherError("--run-attempt must be positive")
+ if type(run["qualification_index"]) is not int or run["qualification_index"] not in range(1, 4):
+ raise PublisherError("--qualification-index must be 1, 2, or 3")
if not re.fullmatch(r"[0-9a-f]{40}", run["source_sha"] or ""):
raise PublisherError("--source-sha must be a 40-character lowercase Git SHA")
return run
@@ -3186,16 +4170,9 @@ def ingest_command(args: argparse.Namespace) -> dict[str, Any]:
)
try:
bundle_id, _, _ = build_bundle(store, ingest_id, incoming, run)
- dataset = build_dataset(store, [bundle_id], promote=False)
- dataset_id, size = store.install_dataset(dataset)
- store.update_channel(
- "latest-attempt", dataset_id, size, dataset["generated_at"]
- )
- store.verify_channel("latest-attempt")
return {
"status": "accepted", "incoming_id": ingest_id,
- "bundle_id": bundle_id, "dataset_sha256": dataset_id,
- "channel": "latest-attempt",
+ "bundle_id": bundle_id,
}
except (
PublisherError, contracts.ContractError, artifact_safety.ArtifactSafetyError,
@@ -3228,12 +4205,10 @@ def promote_command(args: argparse.Namespace) -> dict[str, Any]:
def verify_command(args: argparse.Namespace) -> dict[str, Any]:
bundle_ids = _bundle_ids(args.bundle, promote=False) if args.bundle else []
- channels = args.channel or ["latest-attempt"]
- if any(channel not in {"latest-attempt", "dev-latest"} for channel in channels):
+ channels = args.channel or ["dev-latest"]
+ if any(channel != "dev-latest" for channel in channels):
raise PublisherError("unknown channel")
store = _store_from_args(args)
- if args.channel is None and (store.channels / "dev-latest.json").is_file():
- channels.append("dev-latest")
with store.locked():
pointers = {channel: store.verify_channel(channel) for channel in channels}
bundles = [load_bundle(store, bundle_id)["id"] for bundle_id in bundle_ids]
@@ -3250,11 +4225,12 @@ def _parser() -> argparse.ArgumentParser:
ingest.add_argument("--repository", required=True)
ingest.add_argument("--run-id", required=True)
ingest.add_argument("--run-attempt", required=True, type=int)
+ ingest.add_argument("--qualification-index", required=True, type=int)
ingest.add_argument("--source-sha", required=True)
promote = subparsers.add_parser("promote", help="publish explicit independent bundles")
promote.add_argument("--bundle", action="append", required=True)
verify = subparsers.add_parser("verify", help="verify immutable targets and pointers")
- verify.add_argument("--channel", action="append", choices=["latest-attempt", "dev-latest"])
+ verify.add_argument("--channel", action="append", choices=["dev-latest"])
verify.add_argument("--bundle", action="append", default=[])
return parser
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 1f6a32c2da..772ac8b0a1 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -94,6 +94,7 @@ cx_load_operator_config() {
unset ENROOT_CACHE_PATH
unset CX_EXCLUDE_NODES CX_NODELIST CX_LOCK_DIR CX_MASTER_PORT
unset CX_SOCKET_IFNAME CX_RDMA_DEVICES CX_IB_GID_INDEX CX_RDMA_SERVICE_LEVEL
+ unset CX_AUDIT_SALT
unset MASTER_ADDR MASTER_PORT RANK WORLD_SIZE LOCAL_RANK LOCAL_WORLD_SIZE
config_path="${COLLECTIVEX_OPERATOR_CONFIG:-${XDG_CONFIG_HOME:-${HOME}/.config}/inferencex/collectivex.json}"
if [ -n "${COLLECTIVEX_OPERATOR_CONFIG_CONTENT:-}" ]; then
@@ -122,6 +123,8 @@ cx_load_operator_config() {
fi
unset COLLECTIVEX_OPERATOR_CONFIG_CONTENT COLLECTIVEX_OPERATOR_CONFIG_REQUIRED
if [ ! -e "$config_path" ]; then
+ [ "${COLLECTIVEX_CANONICAL_GHA:-0}" != 1 ] \
+ || cx_die "runner configuration is unavailable"
COLLECTIVEX_OPERATOR_CONFIG_LOADED="$$"
return 0
fi
@@ -132,6 +135,7 @@ cx_load_operator_config() {
}
config_log="$(cx_private_log_path operator-config)"
if ! python3 - "$config_path" "${CX_RUNNER:-${CX_SHARD_SKU:-${CX_PUBLIC_RUNNER:-}}}" \
+ "${COLLECTIVEX_CANONICAL_GHA:-0}" \
> "$parsed_path" 2> "$config_log" <<'PY'
import json
import os
@@ -188,6 +192,7 @@ PATH = re.compile(r"^/[A-Za-z0-9._/+\-]+$")
IPV4 = re.compile(r"(?/dev/null 2>&1 \
- || cx_die "invalid shard control"
+ if [ "${CX_PRECISION_PROBE:-0}" = 1 ]; then
+ python3 "${cx_root%/}/tests/probe_precision.py" \
+ --validate-control "$path" --expect-sku "$expected_sku" \
+ --expect-backend "$CX_BENCH" --expect-nodes "$CX_NODES" >/dev/null 2>&1 \
+ || cx_die "invalid precision probe control"
+ else
+ python3 "${cx_root%/}/sweep_matrix.py" \
+ --validate-control "$path" --expect-sku "$expected_sku" \
+ --expect-backend "$CX_BENCH" --expect-nodes "$CX_NODES" >/dev/null 2>&1 \
+ || cx_die "invalid shard control"
+ fi
control_sha256="$(sha256sum "$path" | awk '{print $1}')"
[[ "$control_sha256" =~ ^[0-9a-f]{64}$ ]] \
|| cx_die "cannot hash shard control"
export COLLECTIVEX_CONTROL_SHA256="$control_sha256"
}
+cx_precision_probe_control_fields() {
+ local cx_root="$1" shard="${CX_SHARD_FILE:-}" path
+ [ "${CX_PRECISION_PROBE:-0}" = 1 ] || return 1
+ path="$shard"
+ [ -f "$path" ] || path="${cx_root%/}/$shard"
+ python3 - "$path" <<'PY'
+import json
+import pathlib
+import sys
+
+path = pathlib.Path(sys.argv[1])
+document = json.loads(path.read_text())
+target = document["target"]
+values = (
+ document["id"], target["backend"], target["sku"], target["ep"],
+ target["mode"], target["precision_profile"],
+)
+if any("|" in str(value) or "\n" in str(value) for value in values):
+ raise SystemExit("unsafe precision probe control field")
+print("|".join(map(str, values)))
+PY
+}
+
cx_apply_timing_profile() {
[ -n "${CX_TIMING:-}" ] || return 0
local iters trials warmup extra
@@ -1364,6 +1416,7 @@ cx_lock_canonical_gha_env() {
local trusted_stage_dir=""
local trusted_socket_ifname="" trusted_rdma_devices=""
local trusted_ib_gid_index="" trusted_rdma_service_level=""
+ local trusted_audit_salt=""
[ "${COLLECTIVEX_CANONICAL_GHA:-0}" = 1 ] || return 0
[ "${GITHUB_ACTIONS:-}" = true ] \
|| cx_die "canonical CollectiveX execution requires GitHub Actions"
@@ -1383,10 +1436,12 @@ cx_lock_canonical_gha_env() {
trusted_rdma_devices="${CX_RDMA_DEVICES:-}"
trusted_ib_gid_index="${CX_IB_GID_INDEX:-}"
trusted_rdma_service_level="${CX_RDMA_SERVICE_LEVEL:-}"
+ trusted_audit_salt="${CX_AUDIT_SALT:-}"
fi
unset CX_NCCL_HOME CX_MASTER_PORT CX_MORI_KERNEL_TYPE CX_LOCK_DIR CX_STAGE_DIR
unset MASTER_ADDR MASTER_PORT RANK WORLD_SIZE LOCAL_RANK LOCAL_WORLD_SIZE
unset CX_SOCKET_IFNAME CX_RDMA_DEVICES CX_IB_GID_INDEX CX_RDMA_SERVICE_LEVEL
+ unset CX_AUDIT_SALT
unset NCCL_NET NCCL_SOCKET_IFNAME GLOO_SOCKET_IFNAME NCCL_IB_HCA
unset NCCL_IB_GID_INDEX NCCL_IB_SL
unset NVSHMEM_HCA_LIST NVSHMEM_IB_GID_INDEX NVSHMEM_IB_SL
@@ -1404,6 +1459,8 @@ cx_lock_canonical_gha_env() {
|| cx_die "canonical CollectiveX execution requires shared container storage"
[ -n "$trusted_stage_dir" ] \
|| cx_die "canonical CollectiveX execution requires a configured shared stage directory"
+ [[ "$trusted_audit_salt" =~ ^[0-9a-f]{64}$ ]] \
+ || cx_die "canonical CollectiveX execution requires a private audit salt"
case "$runner" in
h100-dgxc|h200-dgxc|b200-dgxc|b300)
@@ -1468,7 +1525,8 @@ cx_lock_canonical_gha_env() {
|| export CX_IB_GID_INDEX="$trusted_ib_gid_index"
[ -z "$trusted_rdma_service_level" ] \
|| export CX_RDMA_SERVICE_LEVEL="$trusted_rdma_service_level"
- export CX_STAGE_DIR
+ CX_AUDIT_SALT="$trusted_audit_salt"
+ export CX_STAGE_DIR CX_AUDIT_SALT
[ "${CX_NODES:-}" = "$expected_nodes" ] \
&& [ "${CX_GPUS_PER_NODE:-}" = "$expected_gpn" ] \
|| cx_die "canonical CollectiveX placement differs from the shard"
@@ -1987,7 +2045,7 @@ cx_emit_setup_failures() {
local root="$1" out_dir="$2" backend="$3" rc="$4" shard="${CX_SHARD_FILE:-}" path
local phase case_id suite workload required routing eplb ep hidden topk experts nodes
local gpn domain ladder canonical timing mode scope scale_up_transport scale_out_transport
- local warmup_semantics
+ local warmup_semantics precision_profile
local transport topology_class
local cases_file expected emitted=0 covered=0
mkdir -p "$out_dir" || return 1
@@ -2035,6 +2093,7 @@ for case in cases:
case["transport"], case["topology_class"], case["ladder"],
case["warmup_semantics"],
"1" if case["canonical"] else "", case["timing"],
+ case.get("precision_profile") or "",
)
print("|".join(map(str, fields)))
PY
@@ -2047,7 +2106,7 @@ PY
[ "$expected" -gt 0 ] || { rm -f "$cases_file"; unset CX_FAILURE_MODE; return 1; }
while IFS='|' read -r phase mode case_id suite workload required routing eplb ep hidden topk \
experts nodes gpn domain scope scale_up_transport scale_out_transport transport \
- topology_class ladder warmup_semantics canonical timing; do
+ topology_class ladder warmup_semantics canonical timing precision_profile; do
export CX_CASE_ID="$case_id" CX_SUITE="$suite" CX_WORKLOAD_NAME="$workload"
export CX_REQUIRED_PUBLICATION="$required" CX_ROUTING="$routing" CX_EPLB="$eplb"
export CX_EP="$ep" CX_NGPUS="$ep" CX_HIDDEN="$hidden" CX_TOPK="$topk" CX_EXPERTS="$experts"
@@ -2057,6 +2116,7 @@ PY
export CX_SCALE_OUT_TRANSPORT="$scale_out_transport"
export CX_TRANSPORT="$transport" CX_TOPO="$topology_class"
export CX_TOKENS_LADDER="$ladder" CX_CANONICAL="$canonical"
+ export CX_PRECISION_PROFILE="$precision_profile"
export CX_WARMUP_SEMANTICS="$warmup_semantics"
IFS=: read -r CX_ITERS CX_TRIALS CX_WARMUP <<< "$timing"
export CX_ITERS CX_TRIALS CX_WARMUP CX_SAMPLES_PER_POINT="$((CX_ITERS * CX_TRIALS))"
@@ -2083,6 +2143,7 @@ cx_run_distributed_shard() {
local ph mode routing eplb hidden topk experts ladder suite workload required_pub
local canonical case_id ep timing case_iters case_trials case_warmup case_stem
local scope scale_up_transport scale_out_transport transport topology_class nodes gpn domain
+ local precision_profile
local workload_dir workload_ladder workload_log stage_rc attempt_tag out failure_out
local runtime_log run_rc expected_out case_ok summary_log
local -a container_args workload_args ep_args
@@ -2122,12 +2183,41 @@ cx_run_distributed_shard() {
set -e
if [ "$build_rc" != 0 ]; then
cx_fail_stage backend-setup "$build_log" || true
+ [ "${CX_PRECISION_PROBE:-0}" != 1 ] || return "$build_rc"
cx_emit_setup_failures "$CX_DIR" "$MOUNT_SRC/experimental/CollectiveX/results" \
"$CX_BENCH" "$build_rc"
return "$build_rc"
fi
cx_set_failure_stage execution
+ if [ "${CX_PRECISION_PROBE:-0}" = 1 ]; then
+ local fields probe_id backend sku ep mode profile
+ fields="$(cx_precision_probe_control_fields "$CX_DIR")" || return 1
+ IFS='|' read -r probe_id backend sku ep mode profile <<< "$fields"
+ [ "$backend" = "$CX_BENCH" ] && [ "$sku" = "$RUNNER" ] && [ "$ep" = "$NGPUS" ] \
+ || cx_die "precision probe control differs from runtime placement"
+ out="results/${probe_id}.json"
+ expected_out="$MOUNT_SRC/experimental/CollectiveX/$out"
+ runtime_log="$(cx_private_log_path precision-probe)"
+ set +e
+ timeout -k 30 "${CX_RUN_TIMEOUT:-900}" srun --jobid="$JOB_ID" --nodes="$NODES" \
+ --ntasks="$NGPUS" --ntasks-per-node="$GPN" --chdir=/tmp \
+ --container-name="$container_name" "${container_args[@]}" \
+ --export="$(cx_container_exports)" \
+ bash -c "$WRAP" _ --backend "$backend" --sku "$sku" --ep "$ep" \
+ --mode "$mode" --precision-profile "$profile" --out "$out" \
+ "$runtime_log" 2>&1
+ run_rc=$?
+ set -e
+ if [ "$run_rc" != 0 ] || ! python3 "$CX_DIR/tests/probe_precision.py" \
+ --validate-manifest "$expected_out" >/dev/null 2>&1; then
+ [ "$run_rc" != 0 ] || run_rc=1
+ cx_fail_stage execution "$runtime_log" || true
+ return "$run_rc"
+ fi
+ return 0
+ fi
+
cases_file="$(mktemp)" || return 1
local shard="${CX_SHARD_FILE:-}"
[ -z "$shard" ] || [ -f "$shard" ] || shard="$CX_DIR/$shard"
@@ -2149,6 +2239,7 @@ for case in cases:
get("timing", "8:64:32"), get("nodes"), get("gpus_per_node"),
get("scale_up_domain"), get("scope"), get("scale_up_transport"),
get("scale_out_transport"), get("transport"), get("topology_class"),
+ get("precision_profile"),
)
print("|".join(fields))
PY
@@ -2163,12 +2254,13 @@ PY
"${CX_EPLB:-}" "${CX_HIDDEN:-7168}" "${CX_TOPK:-8}" "${CX_EXPERTS:-256}" \
"${CX_TOKENS_LADDER:-}" "${CX_SUITE:-}" "${CX_WORKLOAD_NAME:-}" \
"${CX_REQUIRED_PUBLICATION:-}" "${CX_CANONICAL:-}" "${CX_CASE_ID:-}" \
+ "${CX_PRECISION_PROFILE:-}" \
"${CX_ITERS:-8}" "${CX_TRIALS:-64}" "${CX_WARMUP:-32}" \
"${CX_SCOPE:-scale-up}" \
"${CX_SCALE_UP_TRANSPORT:-unknown}" "${CX_SCALE_OUT_TRANSPORT:-}" \
"${CX_TRANSPORT:-unknown}" "${CX_TOPO:-manual}"
for phase in $phases; do
- (IFS='|'; printf '%s\n' "$phase|${CX_MODE:-normal}|${CX_ROUTING:-uniform}|${CX_EPLB:-}|${CX_HIDDEN:-7168}|${CX_TOPK:-8}|${CX_EXPERTS:-256}|${CX_TOKENS_LADDER:-}|${CX_SUITE:-}|${CX_WORKLOAD_NAME:-}|${CX_REQUIRED_PUBLICATION:-}|${CX_CANONICAL:-}|${CX_CASE_ID:-}|$NGPUS|${CX_ITERS:-8}:${CX_TRIALS:-64}:${CX_WARMUP:-32}|$NODES|$GPN|$SCALE_UP_DOMAIN|${CX_SCOPE:-scale-up}|${CX_SCALE_UP_TRANSPORT:-unknown}|${CX_SCALE_OUT_TRANSPORT:-}|${CX_TRANSPORT:-unknown}|${CX_TOPO:-manual}")
+ (IFS='|'; printf '%s\n' "$phase|${CX_MODE:-normal}|${CX_ROUTING:-uniform}|${CX_EPLB:-}|${CX_HIDDEN:-7168}|${CX_TOPK:-8}|${CX_EXPERTS:-256}|${CX_TOKENS_LADDER:-}|${CX_SUITE:-}|${CX_WORKLOAD_NAME:-}|${CX_REQUIRED_PUBLICATION:-}|${CX_CANONICAL:-}|${CX_CASE_ID:-}|$NGPUS|${CX_ITERS:-8}:${CX_TRIALS:-64}:${CX_WARMUP:-32}|$NODES|$GPN|$SCALE_UP_DOMAIN|${CX_SCOPE:-scale-up}|${CX_SCALE_UP_TRANSPORT:-unknown}|${CX_SCALE_OUT_TRANSPORT:-}|${CX_TRANSPORT:-unknown}|${CX_TOPO:-manual}|${CX_PRECISION_PROFILE:-}")
done > "$cases_file"
fi
expected_cases="$(wc -l < "$cases_file" | tr -d ' ')"
@@ -2177,7 +2269,7 @@ PY
while IFS='|' read -r ph mode routing eplb hidden topk experts ladder suite workload \
required_pub canonical case_id ep timing nodes gpn domain scope scale_up_transport \
- scale_out_transport transport topology_class; do
+ scale_out_transport transport topology_class precision_profile; do
[ -n "$ph" ] || continue
ci=$((ci + 1))
case_stem="${RUNNER}_${CX_BENCH}_${ph}_${TS}-c$(printf '%03d' "$ci")"
@@ -2188,6 +2280,7 @@ PY
ep="${ep:-$NGPUS}"
export CX_MODE="$mode" CX_CASE_ID="$case_id" CX_SUITE="$suite" CX_WORKLOAD_NAME="$workload"
export CX_REQUIRED_PUBLICATION="$required_pub" CX_CANONICAL="$canonical" CX_EP="$ep"
+ export CX_PRECISION_PROFILE="$precision_profile"
export CX_ROUTING="$routing" CX_EPLB="$eplb" CX_TOKENS_LADDER="$ladder"
export CX_HIDDEN="$hidden" CX_TOPK="$topk" CX_EXPERTS="$experts"
export CX_NODES="$nodes" CX_GPUS_PER_NODE="$gpn" CX_SCALE_UP_DOMAIN="$domain"
@@ -2234,13 +2327,15 @@ PY
fi
ep_args=(--backend "$CX_BENCH" --mode "$mode" --phase "$ph" --routing "$routing"
+ --precision-profile "$precision_profile"
--gpus-per-node "$gpn" --scale-up-domain "$domain" --scope "$scope"
--scale-up-transport "$scale_up_transport" --scale-out-transport "$scale_out_transport"
--tokens-ladder "$ladder" --hidden "$hidden" --topk "$topk" --experts "$experts"
--warmup "$case_warmup" --iters "$case_iters" --trials "$case_trials"
--seed "${CX_SEED:-67}" --runner "$RUNNER" --topology-class "$topology_class"
--transport "$transport" --case-id "$case_id" --suite "$suite"
- --workload-name "$workload" --required-publication "$required_pub")
+ --workload-name "$workload" --required-publication "$required_pub"
+ --qualification-index "${CX_QUALIFICATION_INDEX:-1}")
cx_bool_enabled "$eplb" && ep_args+=(--eplb)
[ -z "$workload_dir" ] || ep_args+=(--workload-dir "$workload_dir")
export CX_ATTEMPT_ID=1
diff --git a/experimental/CollectiveX/runtime/run_in_container.sh b/experimental/CollectiveX/runtime/run_in_container.sh
index 63396c48a0..eeb8f632f9 100644
--- a/experimental/CollectiveX/runtime/run_in_container.sh
+++ b/experimental/CollectiveX/runtime/run_in_container.sh
@@ -80,6 +80,7 @@ run_ep_suite() {
cx_log "ep backend=$backend phase=$phase ngpus=$CX_NGPUS ladder='${ladder:-}'"
local out="results/${CX_RUNNER}_${backend}_${phase}_${CX_TS}.json"
local -a EPARGS=(--backend "$backend" --mode "${CX_MODE:-normal}" --phase "$phase"
+ --precision-profile "${CX_PRECISION_PROFILE:-}"
--tokens-ladder "$ladder"
--hidden "${CX_HIDDEN:-7168}" --topk "${CX_TOPK:-8}" --experts "${CX_EXPERTS:-256}"
--routing "${CX_ROUTING:-uniform}" --seed "${CX_SEED:-67}" --iters "${CX_ITERS:-8}"
@@ -89,6 +90,7 @@ run_ep_suite() {
--scale-out-transport "${CX_SCALE_OUT_TRANSPORT:-}"
--case-id "${CX_CASE_ID:-}" --suite "${CX_SUITE:-}" --workload-name "${CX_WORKLOAD_NAME:-}"
--required-publication "${CX_REQUIRED_PUBLICATION:-}"
+ --qualification-index "${CX_QUALIFICATION_INDEX:-1}"
--runner "$CX_RUNNER" --topology-class "$CX_TOPO" --transport "$CX_TRANSPORT"
--out "$out")
cx_bool_enabled "${CX_EPLB:-0}" && EPARGS+=(--eplb)
@@ -987,6 +989,25 @@ dispatch_bench() {
esac
}
+run_precision_probe() {
+ local fields probe_id backend sku ep mode profile out rc_run
+ fields="$(cx_precision_probe_control_fields "$PWD")" || return 1
+ IFS='|' read -r probe_id backend sku ep mode profile <<< "$fields"
+ [ "$backend" = "$CX_BENCH" ] && [ "$sku" = "$CX_RUNNER" ] && [ "$ep" = "$CX_NGPUS" ] \
+ || { cx_log "ERROR: precision probe control differs from runtime placement"; return 1; }
+ out="results/${probe_id}.json"
+ cx_write_runtime_stage execution || return 1
+ if timeout -k 30 "${CX_RUN_TIMEOUT:-900}" torchrun --nproc_per_node="$CX_NGPUS" \
+ tests/probe_precision.py --backend "$backend" --sku "$sku" --ep "$ep" \
+ --mode "$mode" --precision-profile "$profile" --out "$out"; then
+ rc_run=0
+ else
+ rc_run=$?
+ fi
+ [ "$rc_run" = 0 ] || return "$rc_run"
+ python3 tests/probe_precision.py --validate-manifest "$out"
+}
+
rc=0
cx_validate_shard_control "$PWD"
# Build-only mode: rack launchers run the shared backend preparation hook once per
@@ -1000,7 +1021,13 @@ if [ -n "${CX_BUILD_ONLY:-}" ]; then
cx_log "backend preparation: bench=${CX_BENCH:-unknown} rc=$rc"
exit "$rc"
fi
-if [ -n "${CX_SHARD_FILE:-}" ]; then
+if [ "${CX_PRECISION_PROBE:-0}" = 1 ]; then
+ if cx_probe_scaleout_network && cx_prepare_backend "${CX_BENCH:-}"; then
+ run_precision_probe || rc=$?
+ else
+ rc=1
+ fi
+elif [ -n "${CX_SHARD_FILE:-}" ]; then
# SHARD/SWEEP mode (collectivex-sweep.yml): run EVERY case of this shard in THIS one allocation.
# All cases share (sku, backend, nodes), so backend preparation is paid once and cached.
ncases="$(python3 -c "import json;print(len(json.load(open('$CX_SHARD_FILE'))['cases']))")"
@@ -1083,5 +1110,7 @@ else
fi
# Summary table for the log; also fails the job if no valid results were produced.
-python3 summarize.py --results-dir results --runner "$CX_RUNNER" --ts "$CX_TS" || rc=1
+if [ "${CX_PRECISION_PROBE:-0}" != 1 ]; then
+ python3 summarize.py --results-dir results --runner "$CX_RUNNER" --ts "$CX_TS" || rc=1
+fi
exit "$rc"
diff --git a/experimental/CollectiveX/schemas/channel-v1.schema.json b/experimental/CollectiveX/schemas/channel-v1.schema.json
index 663e22914b..87ffa86b03 100644
--- a/experimental/CollectiveX/schemas/channel-v1.schema.json
+++ b/experimental/CollectiveX/schemas/channel-v1.schema.json
@@ -7,7 +7,7 @@
"required": ["format","channel","dataset","generated_at"],
"properties": {
"format": {"const": "collectivex.channel.v1"},
- "channel": {"enum": ["latest-attempt","dev-latest"]},
+ "channel": {"const": "dev-latest"},
"dataset": {
"type": "object",
"additionalProperties": false,
diff --git a/experimental/CollectiveX/schemas/private-bundle-v1.schema.json b/experimental/CollectiveX/schemas/private-bundle-v1.schema.json
index 166c808930..7891196927 100644
--- a/experimental/CollectiveX/schemas/private-bundle-v1.schema.json
+++ b/experimental/CollectiveX/schemas/private-bundle-v1.schema.json
@@ -26,11 +26,12 @@
"run": {
"type": "object",
"additionalProperties": false,
- "required": ["repository","run_id","run_attempt","source_sha"],
+ "required": ["repository","run_id","run_attempt","qualification_index","source_sha"],
"properties": {
"repository": {"type": "string","pattern": "^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$"},
"run_id": {"type": "string","pattern": "^[1-9][0-9]*$"},
"run_attempt": {"type": "integer","minimum": 1},
+ "qualification_index": {"type":"integer","minimum":1,"maximum":3},
"source_sha": {"type": "string","pattern": "^[0-9a-f]{40}$"}
}
},
diff --git a/experimental/CollectiveX/schemas/public-dataset-v1.schema.json b/experimental/CollectiveX/schemas/public-dataset-v1.schema.json
index 1b51a8c579..cf5a5eed42 100644
--- a/experimental/CollectiveX/schemas/public-dataset-v1.schema.json
+++ b/experimental/CollectiveX/schemas/public-dataset-v1.schema.json
@@ -32,8 +32,15 @@
"matrix_id",
"allocation_ids",
"required_allocations",
+ "qualification_indices",
"requested_cases",
"terminal_cases",
+ "measured_cases",
+ "unsupported_cases",
+ "requested_points",
+ "terminal_points",
+ "measured_points",
+ "unsupported_points",
"policy"
],
"properties": {
@@ -42,12 +49,19 @@
"matrix_id": {"oneOf": [{"type": "null"},{"$ref": "#/$defs/sha256"}]},
"allocation_ids": {"type": "array","uniqueItems": true,"items": {"$ref": "#/$defs/allocationId"}},
"required_allocations": {"const": 3},
+ "qualification_indices": {"type":"array","minItems":0,"maxItems":3,"uniqueItems":true,"items":{"enum":[1,2,3]}},
"requested_cases": {"type": "integer","minimum": 0},
"terminal_cases": {"type": "integer","minimum": 0},
+ "measured_cases": {"type": "integer","minimum": 0},
+ "unsupported_cases": {"type": "integer","minimum": 0},
+ "requested_points": {"type": "integer","minimum": 0},
+ "terminal_points": {"type": "integer","minimum": 0},
+ "measured_points": {"type": "integer","minimum": 0},
+ "unsupported_points": {"type": "integer","minimum": 0},
"policy": {"const": "collectivex-decision-grade-v1"}
}
},
- "coverage": {"type": "array","items": {"$ref": "#/$defs/coverage"}},
+ "coverage": {"type": "array","uniqueItems":true,"items": {"$ref": "#/$defs/coverage"}},
"attempts": {"type": "array","items": {"$ref": "#/$defs/attempt"}},
"series": {"type": "array","items": {"$ref": "#/$defs/series"}},
"cohorts": {"type": "array","items": {"$ref": "#/$defs/cohort"}},
@@ -65,11 +79,129 @@
"allocationId": {"type": "string","pattern": "^cxallocation-v1-[0-9a-f]{64}$"},
"attemptId": {"type": "string","pattern": "^cxattempt-v1-[0-9a-f]{64}$"},
"safeId": {"type": "string","pattern": "^[a-z0-9][a-z0-9_.-]*$","maxLength": 128},
+ "communicationAxis": {
+ "type":"object",
+ "additionalProperties":false,
+ "required": [
+ "alignment_contract","api_input_dtype","api_output_dtype","communication_format",
+ "conversion_boundary","padding_contract","quantization_origin","scale_dtype",
+ "scale_group_size","scale_layout"
+ ],
+ "properties": {
+ "alignment_contract":{"enum":["native-bf16-vector-alignment","hidden-block-128","native-fp8-vector-alignment","value-block-64"]},
+ "api_input_dtype":{"enum":["bf16","fp8-e4m3fn-with-f32-scale","fp8-e4m3fnuz-with-f32-scale"]},
+ "api_output_dtype":{"enum":["bf16","fp8-e4m3fn-with-f32-scale","fp8-e4m3fnuz-with-f32-scale"]},
+ "communication_format":{"enum":["bf16","fp8-e4m3fn","fp8-e4m3fnuz","logfmt10"]},
+ "conversion_boundary":{"enum":["none","before-dispatch-timing","inside-dispatch-timing","inside-combine-timing"]},
+ "padding_contract":{"enum":["none","right-zero-pad-hidden-to-128","right-zero-pad-values-to-64"]},
+ "quantization_origin":{"enum":["none","caller-prequantized","backend-fused","backend-internal","backend-internal-direct-cast"]},
+ "scale_dtype":{"oneOf":[{"type":"null"},{"enum":["f32","implicit-logfmt10"]}]},
+ "scale_group_size":{"oneOf":[{"type":"null"},{"enum":[64,128]}]},
+ "scale_layout":{"enum":["none","per-token-hidden-block","dynamic-per-64-values"]}
+ }
+ },
+ "precisionProfile": {
+ "enum": [
+ "d-bf16.c-bf16",
+ "d-fp8-e4m3fn-b128-f32-prequantized.c-bf16",
+ "d-fp8-e4m3fnuz-b128-f32-prequantized.c-bf16",
+ "d-fp8-e4m3fn-b128-f32-fused.c-bf16",
+ "d-bf16.c-logfmt10-dynamic64",
+ "d-fp8-e4m3fn-b128-f32-fused.c-logfmt10-dynamic64",
+ "d-bf16.c-fp8-e4m3fn-direct-cast-noscale",
+ "d-fp8-e4m3fn-b128-f32-prequantized.c-fp8-e4m3fn-direct-cast-noscale",
+ "d-bf16.c-fp8-e4m3fnuz-direct-cast-noscale",
+ "d-fp8-e4m3fnuz-b128-f32-prequantized.c-fp8-e4m3fnuz-direct-cast-noscale"
+ ]
+ },
+ "byteAccounting": {
+ "type":"object",
+ "additionalProperties":false,
+ "required":["accounting_contract","activation_data_bytes","scale_bytes","total_logical_bytes"],
+ "properties": {
+ "accounting_contract":{"const":"activation-data-plus-scales-v1"},
+ "activation_data_bytes":{"type":"integer","minimum":0},
+ "scale_bytes":{"type":"integer","minimum":0},
+ "total_logical_bytes":{"type":"integer","minimum":0}
+ }
+ },
"publicationTier": {"enum": ["official","comparable-experimental"]},
"label": {"type": "string","minLength": 1,"maxLength": 160},
"nullableLabel": {"oneOf": [{"type": "null"},{"$ref": "#/$defs/label"}]},
"reason": {"oneOf": [{"type": "null"},{"type": "string","pattern": "^[a-z0-9][a-z0-9.-]*$","maxLength": 96}]},
+ "reasonId": {"type":"string","pattern":"^[a-z0-9][a-z0-9.-]*$","maxLength":96},
"outcome": {"enum": ["success","unsupported","failed","invalid","diagnostic"]},
+ "precisionAxisEvidence": {
+ "type":"object",
+ "additionalProperties":false,
+ "required":[
+ "dequantized_semantics","encoded_payload_valid","max_abs_error","max_rel_error",
+ "passed","saturation_count","saturation_rate","scales_finite","scales_positive"
+ ],
+ "properties": {
+ "dequantized_semantics":{"type":"boolean"},
+ "encoded_payload_valid":{"type":"boolean"},
+ "max_abs_error":{"type":"number","minimum":0},
+ "max_rel_error":{"type":"number","minimum":0},
+ "passed":{"type":"boolean"},
+ "saturation_count":{"type":"integer","minimum":0},
+ "saturation_rate":{"type":"number","minimum":0,"maximum":1},
+ "scales_finite":{"oneOf":[{"type":"null"},{"type":"boolean"}]},
+ "scales_positive":{"oneOf":[{"type":"null"},{"type":"boolean"}]}
+ }
+ },
+ "precisionEvidence": {
+ "type":"object",
+ "additionalProperties":false,
+ "required":["combine","dispatch","passed","profile_id"],
+ "properties": {
+ "combine":{"$ref":"#/$defs/precisionAxisEvidence"},
+ "dispatch":{"$ref":"#/$defs/precisionAxisEvidence"},
+ "passed":{"type":"boolean"},
+ "profile_id":{"$ref":"#/$defs/precisionProfile"}
+ }
+ },
+ "pointCorrectness": {
+ "type":"object",
+ "additionalProperties":false,
+ "required":["semantic_pass","precision"],
+ "properties": {
+ "semantic_pass":{"type":"boolean"},
+ "precision":{"$ref":"#/$defs/precisionEvidence"}
+ }
+ },
+ "pointStability": {
+ "type":"object",
+ "additionalProperties":false,
+ "allOf":[
+ {
+ "if":{"properties":{"complete":{"const":true}},"required":["complete"]},
+ "then":{"properties":{
+ "qualification_indices":{"const":[1,2,3]},
+ "p50_max_min_ratio":{"type":"number","minimum":1},
+ "p99_max_min_ratio":{"type":"number","minimum":1}
+ }},
+ "else":{"properties":{
+ "p50_max_min_ratio":{"type":"null"},
+ "p99_max_min_ratio":{"type":"null"},
+ "stable_p50":{"const":false},
+ "stable_p99":{"const":false}
+ }}
+ }
+ ],
+ "required":[
+ "complete","qualification_indices","p50_max_min_ratio","p99_max_min_ratio",
+ "stable_p50","stable_p99"
+ ],
+ "properties": {
+ "complete":{"type":"boolean"},
+ "qualification_indices":{"type":"array","minItems":1,"maxItems":3,"uniqueItems":true,"items":{"enum":[1,2,3]}},
+ "p50_max_min_ratio":{"oneOf":[{"type":"null"},{"type":"number","minimum":1}]},
+ "p99_max_min_ratio":{"oneOf":[{"type":"null"},{"type":"number","minimum":1}]},
+ "stable_p50":{"type":"boolean"},
+ "stable_p99":{"type":"boolean"}
+ }
+ },
"coverageTopology": {
"type": "object",
"additionalProperties": false,
@@ -96,6 +228,44 @@
"topology_class": {"$ref": "#/$defs/safeId"}
}
},
+ "coverageResource": {
+ "type":"object",
+ "additionalProperties":false,
+ "required":["mode","profile","comm_units_kind","configured_units"],
+ "properties": {
+ "mode":{"const":"fixed-profile"},
+ "profile":{"oneOf":[{"type":"null"},{"$ref":"#/$defs/safeId"}]},
+ "comm_units_kind":{"$ref":"#/$defs/nullableLabel"},
+ "configured_units":{"oneOf":[{"type":"null"},{"type":"integer","minimum":1}]}
+ }
+ },
+ "coveragePoint": {
+ "type":"object",
+ "additionalProperties":false,
+ "allOf":[
+ {
+ "if":{"properties":{"terminal_status":{"const":"measured"}},"required":["terminal_status"]},
+ "then":{"properties":{"point_id":{"$ref":"#/$defs/pointId"},"series_id":{"$ref":"#/$defs/seriesId"},"reason":{"type":"null"}}}
+ },
+ {
+ "if":{"properties":{"terminal_status":{"const":"unsupported"}},"required":["terminal_status"]},
+ "then":{"properties":{"point_id":{"type":"null"},"series_id":{"type":"null"},"reason":{"$ref":"#/$defs/reasonId"}}}
+ },
+ {
+ "if":{"properties":{"terminal_status":{"enum":["failed","invalid"]}},"required":["terminal_status"]},
+ "then":{"properties":{"reason":{"$ref":"#/$defs/reasonId"}}}
+ }
+ ],
+ "required":["point_id","series_id","tokens_per_rank","global_tokens","terminal_status","reason"],
+ "properties": {
+ "point_id":{"oneOf":[{"type":"null"},{"$ref":"#/$defs/pointId"}]},
+ "series_id":{"oneOf":[{"type":"null"},{"$ref":"#/$defs/seriesId"}]},
+ "tokens_per_rank":{"type":"integer","minimum":1},
+ "global_tokens":{"type":"integer","minimum":1},
+ "terminal_status":{"enum":["measured","unsupported","failed","invalid","diagnostic"]},
+ "reason":{"$ref":"#/$defs/reason"}
+ }
+ },
"coverage": {
"type": "object",
"additionalProperties": false,
@@ -104,10 +274,21 @@
"label",
"required",
"sku",
+ "suite",
+ "workload",
+ "publication_tier",
"backend",
+ "backend_generation",
"mode",
"phase",
+ "routing",
+ "eplb",
+ "precision_profile",
+ "dispatch_precision",
+ "combine_precision",
+ "resource",
"topology",
+ "points",
"disposition",
"selected_attempt_id",
"outcome",
@@ -120,10 +301,21 @@
"label": {"$ref": "#/$defs/label"},
"required": {"type": "boolean"},
"sku": {"$ref": "#/$defs/safeId"},
+ "suite": {"$ref":"#/$defs/safeId"},
+ "workload": {"$ref":"#/$defs/safeId"},
+ "publication_tier": {"$ref":"#/$defs/publicationTier"},
"backend": {"$ref": "#/$defs/safeId"},
+ "backend_generation": {"$ref":"#/$defs/nullableLabel"},
"mode": {"enum": ["normal","low-latency"]},
"phase": {"enum": ["decode","prefill"]},
+ "routing": {"enum":["uniform","zipf"]},
+ "eplb": {"type":"boolean"},
+ "precision_profile": {"$ref":"#/$defs/precisionProfile"},
+ "dispatch_precision": {"$ref":"#/$defs/communicationAxis"},
+ "combine_precision": {"$ref":"#/$defs/communicationAxis"},
+ "resource": {"$ref":"#/$defs/coverageResource"},
"topology": {"$ref": "#/$defs/coverageTopology"},
+ "points": {"type":"array","minItems":1,"uniqueItems":true,"items":{"$ref":"#/$defs/coveragePoint"}},
"disposition": {"enum": ["runnable","unsupported"]},
"selected_attempt_id": {"oneOf": [{"type": "null"},{"$ref": "#/$defs/attemptId"}]},
"outcome": {"$ref": "#/$defs/outcome"},
@@ -142,6 +334,7 @@
"allocation_id",
"run_id",
"run_attempt",
+ "qualification_index",
"attempt_index",
"selected",
"outcome",
@@ -166,6 +359,7 @@
"allocation_id": {"$ref": "#/$defs/allocationId"},
"run_id": {"type": "string","pattern": "^[1-9][0-9]*$"},
"run_attempt": {"type": "integer","minimum": 1},
+ "qualification_index": {"type":"integer","minimum":1,"maximum":3},
"attempt_index": {"type": "integer","minimum": 1},
"selected": {"type": "boolean"},
"outcome": {"$ref": "#/$defs/outcome"},
@@ -228,16 +422,60 @@
"component": {
"type": "object",
"additionalProperties": false,
- "required": ["origin","latency_us","logical_bytes","logical_payload_rate_gbps_at_latency_percentile","sample_count"],
+ "allOf":[
+ {
+ "if":{"properties":{"origin":{"const":"measured"}},"required":["origin"]},
+ "then":{"properties":{"sample_count":{"const":512}}}
+ },
+ {
+ "if":{"properties":{"origin":{"const":"derived"}},"required":["origin"]},
+ "then":{"properties":{"sample_count":{"type":"null"}}}
+ }
+ ],
+ "required": ["origin","latency_us","byte_provenance","activation_data_rate_gbps_at_latency_percentile","total_logical_data_rate_gbps_at_latency_percentile","sample_count"],
"properties": {
"origin": {"enum": ["measured","derived"]},
"latency_us": {"$ref": "#/$defs/percentiles"},
- "logical_bytes": {"oneOf": [{"type": "null"},{"type": "integer","minimum": 1}]},
- "logical_payload_rate_gbps_at_latency_percentile": {"oneOf": [{"type": "null"},{"$ref": "#/$defs/percentiles"}]},
+ "byte_provenance": {"$ref":"#/$defs/byteAccounting"},
+ "activation_data_rate_gbps_at_latency_percentile": {"oneOf":[{"type":"null"},{"$ref":"#/$defs/percentiles"}]},
+ "total_logical_data_rate_gbps_at_latency_percentile": {"oneOf":[{"type":"null"},{"$ref":"#/$defs/percentiles"}]},
"sample_count": {"oneOf": [{"type": "null"},{"type": "integer","minimum": 1}]}
}
},
"nullableComponent": {"oneOf": [{"type": "null"},{"$ref": "#/$defs/component"}]},
+ "trialDiagnosticComponent": {
+ "type":"object",
+ "additionalProperties":false,
+ "required":["drift_flagged","first_last_median_ratio","outlier_flagged","robust_outlier_fraction","trial_count"],
+ "properties": {
+ "drift_flagged":{"type":"boolean"},
+ "first_last_median_ratio":{"type":"number","minimum":1},
+ "outlier_flagged":{"type":"boolean"},
+ "robust_outlier_fraction":{"type":"number","minimum":0,"maximum":1},
+ "trial_count":{"const":192}
+ }
+ },
+ "nullableTrialDiagnosticComponent": {"oneOf":[{"type":"null"},{"$ref":"#/$defs/trialDiagnosticComponent"}]},
+ "trialDiagnostics": {
+ "type":"object",
+ "additionalProperties":false,
+ "required":["components","flagged","reasons"],
+ "properties": {
+ "components": {
+ "type":"object",
+ "additionalProperties":false,
+ "required":["dispatch","stage","combine","roundtrip"],
+ "properties": {
+ "dispatch":{"$ref":"#/$defs/nullableTrialDiagnosticComponent"},
+ "stage":{"$ref":"#/$defs/nullableTrialDiagnosticComponent"},
+ "combine":{"$ref":"#/$defs/nullableTrialDiagnosticComponent"},
+ "roundtrip":{"$ref":"#/$defs/nullableTrialDiagnosticComponent"}
+ }
+ },
+ "flagged":{"type":"boolean"},
+ "reasons":{"type":"array","maxItems":2,"uniqueItems":true,"items":{"enum":["trial-drift","trial-outliers"]}}
+ }
+ },
"point": {
"type": "object",
"additionalProperties": false,
@@ -245,7 +483,10 @@
"point_id",
"tokens_per_rank",
"global_tokens",
- "correct",
+ "anomalies",
+ "correctness",
+ "stability",
+ "trial_diagnostics",
"routing",
"components",
"roundtrip_token_rate_at_latency_percentile",
@@ -255,7 +496,10 @@
"point_id": {"$ref": "#/$defs/pointId"},
"tokens_per_rank": {"type": "integer","minimum": 1},
"global_tokens": {"type": "integer","minimum": 1},
- "correct": {"type": "boolean"},
+ "anomalies": {"type":"array","maxItems":16,"uniqueItems":true,"items":{"$ref":"#/$defs/reasonId"}},
+ "correctness": {"$ref":"#/$defs/pointCorrectness"},
+ "stability": {"$ref":"#/$defs/pointStability"},
+ "trial_diagnostics":{"$ref":"#/$defs/trialDiagnostics"},
"routing": {
"type": "object",
"additionalProperties": false,
@@ -283,16 +527,17 @@
"components": {
"type": "object",
"additionalProperties": false,
- "required": ["dispatch","combine","roundtrip","isolated_sum"],
+ "required": ["dispatch","stage","combine","roundtrip","isolated_sum"],
"properties": {
"dispatch": {"$ref": "#/$defs/nullableComponent"},
+ "stage": {"$ref":"#/$defs/nullableComponent"},
"combine": {"$ref": "#/$defs/nullableComponent"},
"roundtrip": {"$ref": "#/$defs/nullableComponent"},
"isolated_sum": {"$ref": "#/$defs/nullableComponent"}
}
},
"roundtrip_token_rate_at_latency_percentile": {"$ref": "#/$defs/percentiles"},
- "evidence_ids": {"type": "array","uniqueItems": true,"items": {"$ref": "#/$defs/evidenceId"}}
+ "evidence_ids": {"type": "array","minItems":1,"maxItems":3,"uniqueItems": true,"items": {"$ref": "#/$defs/evidenceId"}}
}
},
"series": {
@@ -387,8 +632,9 @@
"experts",
"routing",
"eplb",
- "dispatch_dtype",
- "combine_dtype",
+ "precision_profile",
+ "dispatch_precision",
+ "combine_precision",
"activation_profile"
],
"properties": {
@@ -398,16 +644,38 @@
"experts": {"type": "integer","minimum": 1},
"routing": {"enum": ["uniform","zipf"]},
"eplb": {"type": "boolean"},
- "dispatch_dtype": {"const": "bf16"},
- "combine_dtype": {"const": "bf16"},
- "activation_profile": {"const": "canonical-counter-source-v3"}
+ "precision_profile": {"$ref":"#/$defs/precisionProfile"},
+ "dispatch_precision": {"$ref":"#/$defs/communicationAxis"},
+ "combine_precision": {"$ref":"#/$defs/communicationAxis"},
+ "activation_profile": {"const": "canonical-counter-source-v4"}
}
},
"eplb": {
"type": "object",
"additionalProperties": false,
+ "allOf":[
+ {
+ "if":{"properties":{"enabled":{"const":true}},"required":["enabled"]},
+ "then":{"properties":{
+ "calibration_workload_id":{"$ref":"#/$defs/workloadId"},
+ "calibration_trace_sha256":{"$ref":"#/$defs/sha256"},
+ "calibration_window":{"const":"collectivex-eplb-calibration-window-v1"},
+ "calibration_token_offset":{"type":"integer","minimum":0}
+ }},
+ "else":{"properties":{
+ "calibration_workload_id":{"type":"null"},
+ "calibration_trace_sha256":{"type":"null"},
+ "calibration_window":{"type":"null"},
+ "calibration_token_offset":{"type":"null"}
+ }}
+ }
+ ],
"required": [
"enabled",
+ "calibration_workload_id",
+ "calibration_trace_sha256",
+ "calibration_window",
+ "calibration_token_offset",
"planner",
"mapping_sha256",
"logical_experts",
@@ -421,6 +689,10 @@
],
"properties": {
"enabled": {"type": "boolean"},
+ "calibration_workload_id": {"oneOf":[{"type":"null"},{"$ref":"#/$defs/workloadId"}]},
+ "calibration_trace_sha256": {"oneOf":[{"type":"null"},{"$ref":"#/$defs/sha256"}]},
+ "calibration_window": {"oneOf":[{"type":"null"},{"const":"collectivex-eplb-calibration-window-v1"}]},
+ "calibration_token_offset": {"oneOf":[{"type":"null"},{"type":"integer","minimum":0}]},
"planner": {"$ref": "#/$defs/nullableLabel"},
"mapping_sha256": {"oneOf": [{"type": "null"},{"$ref": "#/$defs/sha256"}]},
"logical_experts": {"type": "integer","minimum": 1},
@@ -438,7 +710,7 @@
"additionalProperties": false,
"required": ["mode","profile","comm_units_kind","configured_units"],
"properties": {
- "mode": {"const": "tuned"},
+ "mode": {"const": "fixed-profile"},
"profile": {"$ref": "#/$defs/safeId"},
"comm_units_kind": {"$ref": "#/$defs/nullableLabel"},
"configured_units": {"oneOf": [{"type": "null"},{"type": "integer","minimum": 1}]}
@@ -457,12 +729,13 @@
"trials",
"warmups",
"samples_per_component",
+ "qualification_indices",
"headline_component",
"headline_percentile"
],
"properties": {
"contract": {"enum": ["layout-and-dispatch-v1","expert-packed-weighted-combine-v1"]},
- "component_order_contract": {"enum": ["roundtrip-dispatch-activation-only-combine-v2","roundtrip-dispatch-gate-weighted-combine-v1"]},
+ "component_order_contract": {"const":"qualification-hash-rotated-components-v1"},
"combine_semantics": {"enum": ["activation-only","gate-weighted"]},
"payload_unit": {"enum": ["token-rank","token-expert"]},
"sampling_contract": {"const": "fixed-512-v1"},
@@ -470,6 +743,7 @@
"trials": {"const": 64},
"warmups": {"const": 32},
"samples_per_component": {"const": 512},
+ "qualification_indices": {"type":"array","minItems":1,"maxItems":3,"uniqueItems":true,"items":{"enum":[1,2,3]}},
"headline_component": {"const": "roundtrip"},
"headline_percentile": {"const": "p99"}
}
@@ -494,7 +768,7 @@
],
"properties": {
"cohort_id": {"type": "string","pattern": "^cxcohort-v1-[0-9a-f]{64}$"},
- "kind": {"enum": ["library","chip","system","routing"]},
+ "kind": {"enum": ["library","chip","system","routing","dispatch-precision","combine-precision","precision-pair"]},
"label": {"$ref": "#/$defs/label"},
"description": {"$ref": "#/$defs/label"},
"series_ids": {"type": "array","minItems": 2,"uniqueItems": true,"items": {"$ref": "#/$defs/seriesId"}},
@@ -511,7 +785,7 @@
"properties": {
"operation": {"const": "roundtrip"},
"statistic": {"enum": ["p50","p99"]},
- "measure": {"enum": ["latency_us","logical_payload_rate_gbps_at_latency_percentile"]},
+ "measure": {"enum": ["latency_us","activation_data_rate_gbps_at_latency_percentile","total_logical_data_rate_gbps_at_latency_percentile"]},
"objective": {"enum": ["min","max"]},
"tokens_per_rank": {"type": "integer","minimum": 1},
"phase": {"enum": ["decode","prefill"]}
@@ -566,7 +840,7 @@
"recommendation_id": {"type": "string","pattern": "^cxrecommendation-v1-[0-9a-f]{64}$"},
"cohort_id": {"type": "string","pattern": "^cxcohort-v1-[0-9a-f]{64}$"},
"label": {"$ref": "#/$defs/label"},
- "objective": {"enum": ["min-p50-latency","min-p99-latency","max-payload-rate-at-p50-latency","max-payload-rate-at-p99-latency"]},
+ "objective": {"enum": ["min-p50-latency","min-p99-latency","max-activation-data-rate-at-p50-latency","max-activation-data-rate-at-p99-latency","max-total-logical-data-rate-at-p50-latency","max-total-logical-data-rate-at-p99-latency"]},
"series_id": {"$ref": "#/$defs/seriesId"},
"point_id": {"$ref": "#/$defs/pointId"},
"value": {"type": "number","exclusiveMinimum": 0},
diff --git a/experimental/CollectiveX/schemas/raw-case-v1.schema.json b/experimental/CollectiveX/schemas/raw-case-v1.schema.json
index be0d98d76c..d5c8a73a5f 100644
--- a/experimental/CollectiveX/schemas/raw-case-v1.schema.json
+++ b/experimental/CollectiveX/schemas/raw-case-v1.schema.json
@@ -77,6 +77,164 @@
"type": "object"
},
"nullable_sha256": {"oneOf":[{"type":"null"},{"pattern":"^[0-9a-f]{64}$","type":"string"}]},
+ "precision_profile": {
+ "enum": [
+ "d-bf16.c-bf16",
+ "d-fp8-e4m3fn-b128-f32-prequantized.c-bf16",
+ "d-fp8-e4m3fnuz-b128-f32-prequantized.c-bf16",
+ "d-fp8-e4m3fn-b128-f32-fused.c-bf16",
+ "d-bf16.c-logfmt10-dynamic64",
+ "d-fp8-e4m3fn-b128-f32-fused.c-logfmt10-dynamic64",
+ "d-bf16.c-fp8-e4m3fn-direct-cast-noscale",
+ "d-fp8-e4m3fn-b128-f32-prequantized.c-fp8-e4m3fn-direct-cast-noscale",
+ "d-bf16.c-fp8-e4m3fnuz-direct-cast-noscale",
+ "d-fp8-e4m3fnuz-b128-f32-prequantized.c-fp8-e4m3fnuz-direct-cast-noscale"
+ ]
+ },
+ "communication_axis": {
+ "additionalProperties": false,
+ "properties": {
+ "alignment_contract": {"enum":["native-bf16-vector-alignment","hidden-block-128","native-fp8-vector-alignment","value-block-64"]},
+ "api_input_dtype": {"enum":["bf16","fp8-e4m3fn-with-f32-scale","fp8-e4m3fnuz-with-f32-scale"]},
+ "api_output_dtype": {"enum":["bf16","fp8-e4m3fn-with-f32-scale","fp8-e4m3fnuz-with-f32-scale"]},
+ "communication_format": {"enum":["bf16","fp8-e4m3fn","fp8-e4m3fnuz","logfmt10"]},
+ "conversion_boundary": {"enum":["none","before-dispatch-timing","inside-dispatch-timing","inside-combine-timing"]},
+ "padding_contract": {"enum":["none","right-zero-pad-hidden-to-128","right-zero-pad-values-to-64"]},
+ "quantization_origin": {"enum":["none","caller-prequantized","backend-fused","backend-internal","backend-internal-direct-cast"]},
+ "scale_dtype": {"oneOf":[{"type":"null"},{"enum":["f32","implicit-logfmt10"]}]},
+ "scale_group_size": {"oneOf":[{"type":"null"},{"enum":[64,128]}]},
+ "scale_layout": {"enum":["none","per-token-hidden-block","dynamic-per-64-values"]}
+ },
+ "required": [
+ "alignment_contract","api_input_dtype","api_output_dtype","communication_format",
+ "conversion_boundary","padding_contract","quantization_origin","scale_dtype",
+ "scale_group_size","scale_layout"
+ ],
+ "type": "object"
+ },
+ "communication_precision": {
+ "additionalProperties": false,
+ "properties": {
+ "combine": {"$ref":"#/$defs/communication_axis"},
+ "dispatch": {"$ref":"#/$defs/communication_axis"},
+ "modes": {
+ "items":{"enum":["normal","low-latency"]},
+ "minItems":1,
+ "type":"array",
+ "uniqueItems":true
+ },
+ "profile_id": {"$ref":"#/$defs/precision_profile"}
+ },
+ "required": ["combine","dispatch","modes","profile_id"],
+ "type": "object"
+ },
+ "byte_accounting": {
+ "additionalProperties": false,
+ "properties": {
+ "accounting_contract": {"const":"activation-data-plus-scales-v1"},
+ "activation_data_bytes": {"minimum":0,"type":"integer"},
+ "scale_bytes": {"minimum":0,"type":"integer"},
+ "total_logical_bytes": {"minimum":0,"type":"integer"}
+ },
+ "required": [
+ "accounting_contract","activation_data_bytes","scale_bytes","total_logical_bytes"
+ ],
+ "type": "object"
+ },
+ "precision_axis_evidence": {
+ "additionalProperties": false,
+ "properties": {
+ "dequantized_semantics": {"type":"boolean"},
+ "encoded_payload_valid": {"type":"boolean"},
+ "max_abs_error": {"minimum":0,"type":"number"},
+ "max_rel_error": {"minimum":0,"type":"number"},
+ "passed": {"type":"boolean"},
+ "saturation_count": {"minimum":0,"type":"integer"},
+ "saturation_rate": {"maximum":1,"minimum":0,"type":"number"},
+ "scales_finite": {"oneOf":[{"type":"null"},{"type":"boolean"}]},
+ "scales_positive": {"oneOf":[{"type":"null"},{"type":"boolean"}]}
+ },
+ "required": [
+ "dequantized_semantics","encoded_payload_valid","max_abs_error","max_rel_error",
+ "passed","saturation_count","saturation_rate","scales_finite","scales_positive"
+ ],
+ "type":"object"
+ },
+ "precision_evidence": {
+ "additionalProperties": false,
+ "properties": {
+ "combine": {"$ref":"#/$defs/precision_axis_evidence"},
+ "dispatch": {"$ref":"#/$defs/precision_axis_evidence"},
+ "passed": {"type":"boolean"},
+ "profile_id": {"$ref":"#/$defs/precision_profile"}
+ },
+ "required": ["combine","dispatch","passed","profile_id"],
+ "type":"object"
+ },
+ "case_profile": {
+ "additionalProperties": false,
+ "allOf": [
+ {
+ "if": {"properties":{"mode":{"const":"normal"}},"required":["mode"]},
+ "then": {"properties": {
+ "combine_semantics":{"const":"activation-only"},
+ "component_order_contract":{"const":"qualification-hash-rotated-components-v1"},
+ "contract":{"const":"layout-and-dispatch-v1"},
+ "correctness_scope":{"const":"dispatch-metadata-and-transformed-combine"},
+ "oracle_contract":{"const":"expert-specific-transform-v1"},
+ "payload_unit":{"const":"token-rank"}
+ }}
+ },
+ {
+ "if": {"properties":{"mode":{"const":"low-latency"}},"required":["mode"]},
+ "then": {"properties": {
+ "combine_semantics":{"const":"gate-weighted"},
+ "component_order_contract":{"const":"qualification-hash-rotated-components-v1"},
+ "contract":{"const":"expert-packed-weighted-combine-v1"},
+ "correctness_scope":{"const":"expert-assignment-and-weighted-combine"},
+ "oracle_contract":{"const":"expert-assignment-transform-v1"},
+ "payload_unit":{"const":"token-expert"}
+ }}
+ }
+ ],
+ "properties": {
+ "activation_generator": {"const":"collectivex-activation-counter-v4"},
+ "activation_profile": {"const":"canonical-counter-source-v4"},
+ "combine_dtype": {"const":"bf16"},
+ "combine_quant_mode": {"const":"none"},
+ "combine_semantics": {"enum":["activation-only","gate-weighted"]},
+ "communication_precision": {"$ref":"#/$defs/communication_precision"},
+ "component_order_contract": {"const":"qualification-hash-rotated-components-v1"},
+ "conditioning_contract": {"const":"fixed-phase-ramp-8-roundtrips-v1"},
+ "contract": {"enum":["layout-and-dispatch-v1","expert-packed-weighted-combine-v1"]},
+ "correctness_scope": {"enum":["dispatch-metadata-and-transformed-combine","expert-assignment-and-weighted-combine"]},
+ "dtype": {"const":"bf16"},
+ "eplb_planner": {"const":"greedy-rank-major-v1"},
+ "eplb_redundant_experts": {"const":32},
+ "eplb_reference_tokens_per_rank": {"const":2048},
+ "mode": {"enum":["normal","low-latency"]},
+ "oracle_contract": {"enum":["expert-specific-transform-v1","expert-assignment-transform-v1"]},
+ "oracle_tolerances": {"const":"rtol=0.05,atol=0.02"},
+ "payload_unit": {"enum":["token-rank","token-expert"]},
+ "placement": {"const":"packed"},
+ "percentile_method": {"const":"nearest-rank"},
+ "rank_reduction": {"const":"cross-rank-max-per-iteration"},
+ "resource_mode": {"const":"fixed-profile"},
+ "routing_generator": {"const":"collectivex-routing-counter-v3"},
+ "sampling_contract": {"const":"fixed-512-v1"},
+ "seed": {"const":67},
+ "source_identity_contract": {"const":"bounded-sign-bit-source-v1"}
+ },
+ "required": [
+ "activation_generator","activation_profile","combine_dtype","combine_quant_mode",
+ "combine_semantics","component_order_contract","conditioning_contract","contract",
+ "correctness_scope","dtype","eplb_planner","eplb_redundant_experts",
+ "eplb_reference_tokens_per_rank","mode","oracle_contract","oracle_tolerances",
+ "payload_unit","placement","percentile_method","rank_reduction","resource_mode",
+ "routing_generator","sampling_contract","seed","source_identity_contract"
+ ],
+ "type": "object"
+ },
"oracle": {
"additionalProperties": false,
"properties": {
@@ -139,6 +297,27 @@
},
"component": {
"additionalProperties": false,
+ "allOf": [
+ {
+ "if": {"properties":{"availability":{"const":"measured"}},"required":["availability"]},
+ "then": {
+ "properties": {
+ "origin": {"const":"measured"},
+ "percentiles_us": {"$ref":"#/$defs/percentiles"},
+ "sample_count": {"const":512}
+ }
+ }
+ },
+ {
+ "if": {"properties":{"availability":{"const":"unavailable"}},"required":["availability"]},
+ "then": {
+ "properties": {
+ "percentiles_us": {"type":"null"},
+ "sample_count": {"const":0}
+ }
+ }
+ }
+ ],
"properties": {
"availability": {"enum":["measured","derived","unavailable"]},
"origin": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]},
@@ -174,6 +353,7 @@
"mode": {"enum":["normal","low-latency"]},
"nodes": {"minimum":1,"type":"integer"},
"phase": {"enum":["decode","prefill"]},
+ "precision_profile": {"$ref":"#/$defs/precision_profile"},
"required_publication": {"enum":["official","comparable-experimental"]},
"routing": {"enum":["uniform","zipf"]},
"samples_per_point": {"const":512},
@@ -227,13 +407,40 @@
"repo": {"pattern":"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$","type":"string"},
"run_attempt": {"pattern":"^[1-9][0-9]*$","type":"string"},
"run_id": {"pattern":"^[1-9][0-9]*$","type":"string"},
+ "qualification_index": {"maximum":3,"minimum":1,"type":"integer"},
"source_sha": {"pattern":"^[0-9a-f]{40}$","type":"string"}
},
- "required": ["artifact","job","ref","repo","run_attempt","run_id","source_sha"],
+ "required": ["artifact","job","qualification_index","ref","repo","run_attempt","run_id","source_sha"],
"type": "object"
}
},
"additionalProperties": false,
+ "allOf": [
+ {
+ "if": {
+ "properties": {
+ "workload": {
+ "properties": {"source": {"const":"canonical-serialized"}},
+ "required": ["source"]
+ }
+ },
+ "required": ["workload"]
+ },
+ "then": {
+ "properties": {
+ "provenance": {
+ "properties": {
+ "allocation_stratum_sha256": {
+ "pattern":"^[0-9a-f]{64}$",
+ "type":"string"
+ }
+ },
+ "required": ["allocation_stratum_sha256"]
+ }
+ }
+ }
+ }
+ ],
"properties": {
"case": {
"additionalProperties": false,
@@ -243,7 +450,32 @@
"ep_size": {"minimum":1,"type":"integer"},
"eplb": {
"additionalProperties": false,
+ "allOf": [
+ {
+ "if": {"properties":{"enabled":{"const":true}},"required":["enabled"]},
+ "then": {
+ "properties": {
+ "calibration_token_offset": {"minimum":0,"type":"integer"},
+ "calibration_trace_sha256": {"pattern":"^[0-9a-f]{64}$","type":"string"},
+ "calibration_window": {"const":"collectivex-eplb-calibration-window-v1"},
+ "calibration_workload_id": {"pattern":"^cxwork-v1-[0-9a-f]{64}$","type":"string"}
+ }
+ },
+ "else": {
+ "properties": {
+ "calibration_token_offset": {"type":"null"},
+ "calibration_trace_sha256": {"type":"null"},
+ "calibration_window": {"type":"null"},
+ "calibration_workload_id": {"type":"null"}
+ }
+ }
+ }
+ ],
"properties": {
+ "calibration_token_offset": {"oneOf":[{"type":"null"},{"minimum":0,"type":"integer"}]},
+ "calibration_trace_sha256": {"$ref":"#/$defs/nullable_sha256"},
+ "calibration_window": {"oneOf":[{"type":"null"},{"const":"collectivex-eplb-calibration-window-v1"}]},
+ "calibration_workload_id": {"oneOf":[{"type":"null"},{"pattern":"^cxwork-v1-[0-9a-f]{64}$","type":"string"}]},
"enabled": {"type":"boolean"},
"imbalance_after": {"oneOf":[{"type":"null"},{"minimum":0,"type":"number"}]},
"imbalance_before": {"oneOf":[{"type":"null"},{"minimum":0,"type":"number"}]},
@@ -257,6 +489,10 @@
"replicated_experts": {"minimum":0,"type":"integer"}
},
"required": [
+ "calibration_token_offset",
+ "calibration_trace_sha256",
+ "calibration_window",
+ "calibration_workload_id",
"enabled",
"imbalance_after",
"imbalance_before",
@@ -274,50 +510,35 @@
"mode": {"enum":["normal","low-latency"]},
"phase": {"enum":["decode","prefill"]},
"required_publication": {"enum":["official","comparable-experimental"]},
- "resource_mode": {"const":"tuned"},
+ "resource_mode": {"const":"fixed-profile"},
"runner": {"maxLength":128,"pattern":"^[a-z0-9][a-z0-9_.-]*$","type":"string"},
"shape": {
"additionalProperties": false,
"properties": {
- "activation_profile": {"const":"canonical-counter-source-v3"},
- "dispatch_dtype": {"const":"bf16"},
+ "activation_profile": {"const":"canonical-counter-source-v4"},
+ "combine_precision": {"$ref":"#/$defs/communication_axis"},
+ "dispatch_precision": {"$ref":"#/$defs/communication_axis"},
"eplb": {"type":"boolean"},
"experts": {"minimum":1,"type":"integer"},
"experts_per_rank": {"minimum":1,"type":"integer"},
"hidden": {"minimum":1,"type":"integer"},
"kernel_gen": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]},
"num_logical_experts": {"minimum":1,"type":"integer"},
- "quant": {
- "additionalProperties": false,
- "properties": {
- "combine_accum_dtype": {"minLength":1,"type":"string"},
- "combine_input_dtype": {"const":"bf16"},
- "combine_output_dtype": {"const":"bf16"},
- "combine_quant_mode": {"const":"none"},
- "scale_layout": {"type":"null"}
- },
- "required": [
- "combine_accum_dtype",
- "combine_input_dtype",
- "combine_output_dtype",
- "combine_quant_mode",
- "scale_layout"
- ],
- "type": "object"
- },
+ "precision_profile": {"$ref":"#/$defs/precision_profile"},
"routing": {"enum":["uniform","zipf"]},
"topk": {"minimum":1,"type":"integer"}
},
"required": [
"activation_profile",
- "dispatch_dtype",
+ "combine_precision",
+ "dispatch_precision",
"eplb",
"experts",
"experts_per_rank",
"hidden",
"kernel_gen",
"num_logical_experts",
- "quant",
+ "precision_profile",
"routing",
"topk"
],
@@ -356,10 +577,11 @@
"repo": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]},
"run_attempt": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]},
"run_id": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]},
+ "qualification_index": {"maximum":3,"minimum":1,"type":"integer"},
"runner": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]},
"source_sha": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]}
},
- "required": ["artifact","execution_id","job","repo","run_attempt","run_id","runner","source_sha"],
+ "required": ["artifact","execution_id","job","qualification_index","repo","run_attempt","run_id","runner","source_sha"],
"type": "object"
},
"allocation_id": {"pattern":"^cxallocation-v1-[0-9a-f]{64}$","type":"string"},
@@ -369,61 +591,7 @@
"additionalProperties": false,
"properties": {
"case": {"$ref":"#/$defs/scheduled_case"},
- "profile": {
- "oneOf": [
- {"const": {
- "activation_generator": "collectivex-activation-counter-v3",
- "activation_profile": "canonical-counter-source-v3",
- "combine_dtype": "bf16",
- "combine_quant_mode": "none",
- "combine_semantics": "activation-only",
- "component_order_contract": "roundtrip-dispatch-activation-only-combine-v2",
- "conditioning_contract": "fixed-phase-ramp-8-roundtrips-v1",
- "contract": "layout-and-dispatch-v1",
- "correctness_scope": "dispatch-metadata-and-transformed-combine",
- "dtype": "bf16",
- "eplb_planner": "greedy-rank-major-v1",
- "eplb_redundant_experts": 32,
- "eplb_reference_tokens_per_rank": 2048,
- "mode": "normal",
- "oracle_contract": "expert-specific-transform-v1",
- "oracle_tolerances": "rtol=0.05,atol=0.02",
- "payload_unit": "token-rank",
- "placement": "packed",
- "percentile_method": "nearest-rank",
- "rank_reduction": "cross-rank-max-per-iteration",
- "resource_mode": "tuned",
- "routing_generator": "collectivex-routing-counter-v3",
- "sampling_contract": "fixed-512-v1",
- "seed": 67
- }},
- {"const": {
- "activation_generator": "collectivex-activation-counter-v3",
- "activation_profile": "canonical-counter-source-v3",
- "combine_dtype": "bf16",
- "combine_quant_mode": "none",
- "combine_semantics": "gate-weighted",
- "component_order_contract": "roundtrip-dispatch-gate-weighted-combine-v1",
- "conditioning_contract": "fixed-phase-ramp-8-roundtrips-v1",
- "contract": "expert-packed-weighted-combine-v1",
- "correctness_scope": "expert-assignment-and-weighted-combine",
- "dtype": "bf16",
- "eplb_planner": "greedy-rank-major-v1",
- "eplb_redundant_experts": 32,
- "eplb_reference_tokens_per_rank": 2048,
- "mode": "low-latency",
- "oracle_contract": "expert-assignment-transform-v1",
- "oracle_tolerances": "rtol=0.05,atol=0.02",
- "payload_unit": "token-expert",
- "placement": "packed",
- "percentile_method": "nearest-rank",
- "rank_reduction": "cross-rank-max-per-iteration",
- "resource_mode": "tuned",
- "routing_generator": "collectivex-routing-counter-v3",
- "sampling_contract": "fixed-512-v1",
- "seed": 67
- }}]
- },
+ "profile": {"$ref":"#/$defs/case_profile"},
"sku": {"maxLength":128,"pattern":"^[a-z0-9][a-z0-9_.-]*$","type":"string"}
},
"required": ["case","profile","sku"],
@@ -679,7 +847,7 @@
"measurement": {
"additionalProperties": false,
"properties": {
- "component_order_contract": {"enum":["roundtrip-dispatch-activation-only-combine-v2","roundtrip-dispatch-gate-weighted-combine-v1"]},
+ "component_order_contract": {"const":"qualification-hash-rotated-components-v1"},
"conditioning": {
"additionalProperties": false,
"properties": {
@@ -691,6 +859,8 @@
"type": "object"
},
"contract": {"enum":["layout-and-dispatch-v1","expert-packed-weighted-combine-v1"]},
+ "execution_order_sha256": {"pattern":"^[0-9a-f]{64}$","type":"string"},
+ "qualification_index": {"maximum":3,"minimum":1,"type":"integer"},
"rows": {
"items": {
"additionalProperties": false,
@@ -719,9 +889,10 @@
"combine": {"$ref":"#/$defs/component"},
"dispatch": {"$ref":"#/$defs/component"},
"isolated_sum": {"$ref":"#/$defs/component"},
- "roundtrip": {"$ref":"#/$defs/component"}
+ "roundtrip": {"$ref":"#/$defs/component"},
+ "stage": {"$ref":"#/$defs/component"}
},
- "required": ["combine","dispatch","isolated_sum","roundtrip"],
+ "required": ["combine","dispatch","isolated_sum","roundtrip","stage"],
"type": "object"
},
"correctness": {
@@ -730,6 +901,7 @@
"contract": {"enum":["expert-specific-transform-v1","expert-assignment-transform-v1"]},
"max_relative_error": {"minimum":0,"type":"number"},
"passed": {"type":"boolean"},
+ "precision": {"$ref":"#/$defs/precision_evidence"},
"rank_evidence": {
"items": {
"additionalProperties": false,
@@ -748,19 +920,20 @@
},
"scope": {"enum":["dispatch-metadata-and-transformed-combine","expert-assignment-and-weighted-combine"]}
},
- "required": ["contract","max_relative_error","passed","rank_evidence","scope"],
+ "required": ["contract","max_relative_error","passed","precision","rank_evidence","scope"],
"type": "object"
},
"evidence_id": {"pattern":"^cxevidence-v1-[0-9a-f]{64}$","type":"string"},
"global_tokens": {"minimum":1,"type":"integer"},
- "logical_bytes": {
+ "byte_provenance": {
"additionalProperties": false,
"properties": {
- "combine": {"minimum":1,"type":"integer"},
- "dispatch": {"minimum":1,"type":"integer"},
- "roundtrip": {"minimum":1,"type":"integer"}
+ "combine": {"$ref":"#/$defs/byte_accounting"},
+ "dispatch": {"$ref":"#/$defs/byte_accounting"},
+ "roundtrip": {"$ref":"#/$defs/byte_accounting"},
+ "stage": {"$ref":"#/$defs/byte_accounting"}
},
- "required": ["combine","dispatch","roundtrip"],
+ "required": ["combine","dispatch","roundtrip","stage"],
"type": "object"
},
"point_id": {"pattern":"^cxpoint-v1-[0-9a-f]{64}$","type":"string"},
@@ -874,9 +1047,10 @@
"properties": {
"combine": {"oneOf":[{"type":"null"},{"$ref":"#/$defs/histogram"}]},
"dispatch": {"oneOf":[{"type":"null"},{"$ref":"#/$defs/histogram"}]},
- "roundtrip": {"$ref":"#/$defs/histogram"}
+ "roundtrip": {"$ref":"#/$defs/histogram"},
+ "stage": {"oneOf":[{"type":"null"},{"$ref":"#/$defs/histogram"}]}
},
- "required": ["dispatch","combine","roundtrip"],
+ "required": ["dispatch","combine","roundtrip","stage"],
"type": "object"
},
"sample_sha256": {"pattern":"^[0-9a-f]{64}$","type":"string"},
@@ -887,9 +1061,9 @@
"anomalies",
"components",
"correctness",
+ "byte_provenance",
"evidence_id",
"global_tokens",
- "logical_bytes",
"point_id",
"receive",
"routing",
@@ -933,6 +1107,8 @@
"component_order_contract",
"conditioning",
"contract",
+ "execution_order_sha256",
+ "qualification_index",
"rows",
"sampling",
"source_allocation"
@@ -978,6 +1154,12 @@
"provenance": {
"additionalProperties": false,
"properties": {
+ "allocation_stratum_sha256": {
+ "oneOf": [
+ {"type":"null"},
+ {"pattern":"^[0-9a-f]{64}$","type":"string"}
+ ]
+ },
"command": {"minLength":1,"type":"string"},
"distributed_launcher": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]},
"git_run": {"oneOf":[{"type":"null"},{"$ref":"#/$defs/git_run"}]},
@@ -997,7 +1179,7 @@
},
"redaction": {"const":"sanitized-v1"}
},
- "required": ["command","distributed_launcher","git_run","image","redaction"],
+ "required": ["allocation_stratum_sha256","command","distributed_launcher","git_run","image","redaction"],
"type": "object"
},
"record_type": {"const":"case-attempt"},
@@ -1117,9 +1299,9 @@
"workload": {
"additionalProperties": false,
"properties": {
- "activation_generator": {"const":"collectivex-activation-counter-v3"},
+ "activation_generator": {"const":"collectivex-activation-counter-v4"},
"activation_identity": {"pattern":"^[0-9a-f]{64}$","type":"string"},
- "activation_profile": {"const":"canonical-counter-source-v3"},
+ "activation_profile": {"const":"canonical-counter-source-v4"},
"cross_rank_consistent": {"const":true},
"manifest_checksums": {
"oneOf": [
diff --git a/experimental/CollectiveX/schemas/samples-v1.schema.json b/experimental/CollectiveX/schemas/samples-v1.schema.json
index b9a1df0541..f216860a80 100644
--- a/experimental/CollectiveX/schemas/samples-v1.schema.json
+++ b/experimental/CollectiveX/schemas/samples-v1.schema.json
@@ -4,7 +4,7 @@
"title": "CollectiveX exact private samples v1",
"type": "object",
"additionalProperties": false,
- "required": ["allocation_id","attempt_id","case_id","format","points","sampling","schema_version","series_id"],
+ "required": ["allocation_id","attempt_id","case_id","format","points","qualification_index","sampling","schema_version","series_id"],
"properties": {
"allocation_id": {"$ref": "#/$defs/allocationId"},
"attempt_id": {"$ref": "#/$defs/attemptId"},
@@ -21,11 +21,12 @@
"components": {
"type": "object",
"additionalProperties": false,
- "required": ["combine","dispatch","roundtrip"],
+ "required": ["combine","dispatch","roundtrip","stage"],
"properties": {
"combine": {"$ref": "#/$defs/component"},
"dispatch": {"$ref": "#/$defs/component"},
- "roundtrip": {"$ref": "#/$defs/component"}
+ "roundtrip": {"$ref": "#/$defs/component"},
+ "stage": {"$ref":"#/$defs/component"}
}
},
"evidence_id": {"$ref": "#/$defs/evidenceId"},
@@ -35,6 +36,7 @@
}
}
},
+ "qualification_index": {"type":"integer","minimum":1,"maximum":3},
"sampling": {
"type": "object",
"additionalProperties": false,
@@ -56,25 +58,36 @@
"evidenceId": {"type": "string","pattern": "^cxevidence-v1-[0-9a-f]{64}$"},
"allocationId": {"type": "string","pattern": "^cxallocation-v1-[0-9a-f]{64}$"},
"attemptId": {"type": "string","pattern": "^cxattempt-v1-[0-9a-f]{64}$"},
- "component": {
- "type": "object",
- "additionalProperties": false,
- "required": ["availability","sample_count","trials"],
+ "trials": {
+ "type":"array",
+ "minItems":64,
+ "maxItems":64,
+ "items":{"type":"array","minItems":8,"maxItems":8,"items":{"type":"number","minimum":0}}
+ },
+ "measuredComponent": {
+ "type":"object",
+ "additionalProperties":false,
+ "required":["availability","sample_count","trials"],
"properties": {
- "availability": {"enum": ["measured","unavailable"]},
- "sample_count": {"type": "integer","minimum": 0,"maximum": 512},
- "trials": {
- "oneOf": [
- {"type": "null"},
- {
- "type": "array",
- "minItems": 64,
- "maxItems": 64,
- "items": {"type": "array","minItems": 8,"maxItems": 8,"items": {"type": "number","minimum": 0}}
- }
- ]
- }
+ "availability":{"const":"measured"},
+ "sample_count":{"const":512},
+ "trials":{"$ref":"#/$defs/trials"}
}
+ },
+ "component": {
+ "oneOf": [
+ {"$ref":"#/$defs/measuredComponent"},
+ {
+ "type":"object",
+ "additionalProperties":false,
+ "required":["availability","sample_count","trials"],
+ "properties": {
+ "availability":{"const":"unavailable"},
+ "sample_count":{"const":0},
+ "trials":{"type":"null"}
+ }
+ }
+ ]
}
}
}
diff --git a/experimental/CollectiveX/schemas/terminal-outcome-v1.schema.json b/experimental/CollectiveX/schemas/terminal-outcome-v1.schema.json
index dede78a43c..f4a9d99ca2 100644
--- a/experimental/CollectiveX/schemas/terminal-outcome-v1.schema.json
+++ b/experimental/CollectiveX/schemas/terminal-outcome-v1.schema.json
@@ -24,62 +24,7 @@
"required": ["case","profile","sku"],
"properties": {
"case": {"$ref": "#/$defs/case"},
- "profile": {
- "oneOf": [
- {"const": {
- "activation_generator": "collectivex-activation-counter-v3",
- "activation_profile": "canonical-counter-source-v3",
- "combine_dtype": "bf16",
- "combine_quant_mode": "none",
- "combine_semantics": "activation-only",
- "component_order_contract": "roundtrip-dispatch-activation-only-combine-v2",
- "conditioning_contract": "fixed-phase-ramp-8-roundtrips-v1",
- "contract": "layout-and-dispatch-v1",
- "correctness_scope": "dispatch-metadata-and-transformed-combine",
- "dtype": "bf16",
- "eplb_planner": "greedy-rank-major-v1",
- "eplb_redundant_experts": 32,
- "eplb_reference_tokens_per_rank": 2048,
- "mode": "normal",
- "oracle_contract": "expert-specific-transform-v1",
- "oracle_tolerances": "rtol=0.05,atol=0.02",
- "payload_unit": "token-rank",
- "placement": "packed",
- "percentile_method": "nearest-rank",
- "rank_reduction": "cross-rank-max-per-iteration",
- "resource_mode": "tuned",
- "routing_generator": "collectivex-routing-counter-v3",
- "sampling_contract": "fixed-512-v1",
- "seed": 67
- }},
- {"const": {
- "activation_generator": "collectivex-activation-counter-v3",
- "activation_profile": "canonical-counter-source-v3",
- "combine_dtype": "bf16",
- "combine_quant_mode": "none",
- "combine_semantics": "gate-weighted",
- "component_order_contract": "roundtrip-dispatch-gate-weighted-combine-v1",
- "conditioning_contract": "fixed-phase-ramp-8-roundtrips-v1",
- "contract": "expert-packed-weighted-combine-v1",
- "correctness_scope": "expert-assignment-and-weighted-combine",
- "dtype": "bf16",
- "eplb_planner": "greedy-rank-major-v1",
- "eplb_redundant_experts": 32,
- "eplb_reference_tokens_per_rank": 2048,
- "mode": "low-latency",
- "oracle_contract": "expert-assignment-transform-v1",
- "oracle_tolerances": "rtol=0.05,atol=0.02",
- "payload_unit": "token-expert",
- "placement": "packed",
- "percentile_method": "nearest-rank",
- "rank_reduction": "cross-rank-max-per-iteration",
- "resource_mode": "tuned",
- "routing_generator": "collectivex-routing-counter-v3",
- "sampling_contract": "fixed-512-v1",
- "seed": 67
- }}
- ]
- },
+ "profile": {"$ref":"#/$defs/caseProfile"},
"sku": {"$ref": "#/$defs/safeId"}
}
},
@@ -187,25 +132,126 @@
"reason": {
"enum": [
"backend-platform-unsupported",
- "backend-token-capacity"
+ "backend-token-capacity",
+ "precision-profile-unsupported"
]
}
}
},
"sha256": {"type": "string","pattern": "^[0-9a-f]{64}$"},
"safeId": {"type": "string","pattern": "^[a-z0-9][a-z0-9_.-]*$","maxLength": 128},
+ "precisionProfile": {
+ "enum":[
+ "d-bf16.c-bf16",
+ "d-fp8-e4m3fn-b128-f32-prequantized.c-bf16",
+ "d-fp8-e4m3fnuz-b128-f32-prequantized.c-bf16",
+ "d-fp8-e4m3fn-b128-f32-fused.c-bf16",
+ "d-bf16.c-logfmt10-dynamic64",
+ "d-fp8-e4m3fn-b128-f32-fused.c-logfmt10-dynamic64",
+ "d-bf16.c-fp8-e4m3fn-direct-cast-noscale",
+ "d-fp8-e4m3fn-b128-f32-prequantized.c-fp8-e4m3fn-direct-cast-noscale",
+ "d-bf16.c-fp8-e4m3fnuz-direct-cast-noscale",
+ "d-fp8-e4m3fnuz-b128-f32-prequantized.c-fp8-e4m3fnuz-direct-cast-noscale"
+ ]
+ },
"caseId": {"type": "string","pattern": "^cxcase-v1-[0-9a-f]{64}$"},
"allocationId": {"type": "string","pattern": "^cxallocation-v1-[0-9a-f]{64}$"},
"attemptId": {"type": "string","pattern": "^cxattempt-v1-[0-9a-f]{64}$"},
"nullableText": {"oneOf": [{"type": "null"},{"type": "string","minLength": 1}]},
+ "communicationAxis": {
+ "type":"object",
+ "additionalProperties":false,
+ "required":["alignment_contract","api_input_dtype","api_output_dtype","communication_format","conversion_boundary","padding_contract","quantization_origin","scale_dtype","scale_group_size","scale_layout"],
+ "properties": {
+ "alignment_contract":{"enum":["native-bf16-vector-alignment","hidden-block-128","native-fp8-vector-alignment","value-block-64"]},
+ "api_input_dtype":{"enum":["bf16","fp8-e4m3fn-with-f32-scale","fp8-e4m3fnuz-with-f32-scale"]},
+ "api_output_dtype":{"enum":["bf16","fp8-e4m3fn-with-f32-scale","fp8-e4m3fnuz-with-f32-scale"]},
+ "communication_format":{"enum":["bf16","fp8-e4m3fn","fp8-e4m3fnuz","logfmt10"]},
+ "conversion_boundary":{"enum":["none","before-dispatch-timing","inside-dispatch-timing","inside-combine-timing"]},
+ "padding_contract":{"enum":["none","right-zero-pad-hidden-to-128","right-zero-pad-values-to-64"]},
+ "quantization_origin":{"enum":["none","caller-prequantized","backend-fused","backend-internal","backend-internal-direct-cast"]},
+ "scale_dtype":{"oneOf":[{"type":"null"},{"enum":["f32","implicit-logfmt10"]}]},
+ "scale_group_size":{"oneOf":[{"type":"null"},{"enum":[64,128]}]},
+ "scale_layout":{"enum":["none","per-token-hidden-block","dynamic-per-64-values"]}
+ }
+ },
+ "communicationPrecision": {
+ "type":"object",
+ "additionalProperties":false,
+ "required":["combine","dispatch","modes","profile_id"],
+ "properties": {
+ "combine":{"$ref":"#/$defs/communicationAxis"},
+ "dispatch":{"$ref":"#/$defs/communicationAxis"},
+ "modes":{"type":"array","minItems":1,"uniqueItems":true,"items":{"enum":["normal","low-latency"]}},
+ "profile_id":{"$ref":"#/$defs/precisionProfile"}
+ }
+ },
+ "caseProfile": {
+ "type":"object",
+ "additionalProperties":false,
+ "allOf":[
+ {
+ "if":{"properties":{"mode":{"const":"normal"}},"required":["mode"]},
+ "then":{"properties":{
+ "combine_semantics":{"const":"activation-only"},
+ "component_order_contract":{"const":"qualification-hash-rotated-components-v1"},
+ "contract":{"const":"layout-and-dispatch-v1"},
+ "correctness_scope":{"const":"dispatch-metadata-and-transformed-combine"},
+ "oracle_contract":{"const":"expert-specific-transform-v1"},
+ "payload_unit":{"const":"token-rank"}
+ }}
+ },
+ {
+ "if":{"properties":{"mode":{"const":"low-latency"}},"required":["mode"]},
+ "then":{"properties":{
+ "combine_semantics":{"const":"gate-weighted"},
+ "component_order_contract":{"const":"qualification-hash-rotated-components-v1"},
+ "contract":{"const":"expert-packed-weighted-combine-v1"},
+ "correctness_scope":{"const":"expert-assignment-and-weighted-combine"},
+ "oracle_contract":{"const":"expert-assignment-transform-v1"},
+ "payload_unit":{"const":"token-expert"}
+ }}
+ }
+ ],
+ "required":["activation_generator","activation_profile","combine_dtype","combine_quant_mode","combine_semantics","component_order_contract","conditioning_contract","contract","correctness_scope","dtype","eplb_planner","eplb_redundant_experts","eplb_reference_tokens_per_rank","mode","oracle_contract","oracle_tolerances","payload_unit","placement","percentile_method","rank_reduction","resource_mode","routing_generator","sampling_contract","seed","source_identity_contract"],
+ "properties": {
+ "activation_generator":{"const":"collectivex-activation-counter-v4"},
+ "activation_profile":{"const":"canonical-counter-source-v4"},
+ "combine_dtype":{"const":"bf16"},
+ "combine_quant_mode":{"const":"none"},
+ "combine_semantics":{"enum":["activation-only","gate-weighted"]},
+ "communication_precision":{"$ref":"#/$defs/communicationPrecision"},
+ "component_order_contract":{"const":"qualification-hash-rotated-components-v1"},
+ "conditioning_contract":{"const":"fixed-phase-ramp-8-roundtrips-v1"},
+ "contract":{"enum":["layout-and-dispatch-v1","expert-packed-weighted-combine-v1"]},
+ "correctness_scope":{"enum":["dispatch-metadata-and-transformed-combine","expert-assignment-and-weighted-combine"]},
+ "dtype":{"const":"bf16"},
+ "eplb_planner":{"const":"greedy-rank-major-v1"},
+ "eplb_redundant_experts":{"const":32},
+ "eplb_reference_tokens_per_rank":{"const":2048},
+ "mode":{"enum":["normal","low-latency"]},
+ "oracle_contract":{"enum":["expert-specific-transform-v1","expert-assignment-transform-v1"]},
+ "oracle_tolerances":{"const":"rtol=0.05,atol=0.02"},
+ "payload_unit":{"enum":["token-rank","token-expert"]},
+ "placement":{"const":"packed"},
+ "percentile_method":{"const":"nearest-rank"},
+ "rank_reduction":{"const":"cross-rank-max-per-iteration"},
+ "resource_mode":{"const":"fixed-profile"},
+ "routing_generator":{"const":"collectivex-routing-counter-v3"},
+ "sampling_contract":{"const":"fixed-512-v1"},
+ "seed":{"const":67},
+ "source_identity_contract":{"const":"bounded-sign-bit-source-v1"}
+ }
+ },
"allocationFactors": {
"type": "object",
"additionalProperties": false,
- "required": ["artifact","execution_id","job","repo","run_attempt","run_id","runner","source_sha"],
+ "required": ["artifact","execution_id","job","qualification_index","repo","run_attempt","run_id","runner","source_sha"],
"properties": {
"artifact": {"$ref": "#/$defs/nullableText"},
"execution_id": {"$ref": "#/$defs/nullableText"},
"job": {"$ref": "#/$defs/nullableText"},
+ "qualification_index": {"type":"integer","minimum":1,"maximum":3},
"repo": {"$ref": "#/$defs/nullableText"},
"run_attempt": {"$ref": "#/$defs/nullableText"},
"run_id": {"$ref": "#/$defs/nullableText"},
@@ -216,10 +262,11 @@
"gitRun": {
"type": "object",
"additionalProperties": false,
- "required": ["artifact","job","ref","repo","run_attempt","run_id","source_sha"],
+ "required": ["artifact","job","qualification_index","ref","repo","run_attempt","run_id","source_sha"],
"properties": {
"artifact": {"type": "string","minLength": 1},
"job": {"type": "string","minLength": 1},
+ "qualification_index": {"type":"integer","minimum":1,"maximum":3},
"ref": {"type": "string","minLength": 1},
"repo": {"type": "string","pattern": "^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$"},
"run_attempt": {"type": "string","pattern": "^[1-9][0-9]*$"},
@@ -269,6 +316,7 @@
"mode": {"enum": ["normal","low-latency"]},
"nodes": {"type": "integer","minimum": 1},
"phase": {"enum": ["decode","prefill"]},
+ "precision_profile": {"$ref":"#/$defs/precisionProfile"},
"required_publication": {"enum": ["official","comparable-experimental","diagnostic"]},
"routing": {"enum": ["uniform","zipf"]},
"samples_per_point": {"const": 512},
diff --git a/experimental/CollectiveX/sweep_matrix.py b/experimental/CollectiveX/sweep_matrix.py
index fcf1ffc233..309199fe2a 100644
--- a/experimental/CollectiveX/sweep_matrix.py
+++ b/experimental/CollectiveX/sweep_matrix.py
@@ -1,9 +1,9 @@
#!/usr/bin/env python3
"""Resolve CollectiveX v1 suites and extract validated execution shards.
-The promoted v1 profile contains normal and explicit low-latency BF16 contracts.
-Mode changes measurement semantics and therefore participates in case identity;
-resource mode and quantization remain fixed rather than becoming matrix axes.
+Mode changes measurement semantics and therefore participates in case identity.
+Precision sensitivity uses allowlisted communication profiles; provisional native
+paths remain outside the executable matrix until their probes are resolved.
"""
from __future__ import annotations
@@ -69,12 +69,31 @@
"coordinates": {("low-latency", "decode", "uniform", False)},
"ladders": {"decode": tuple(ep_harness.DECODE_LADDER)},
},
+ "ep-precision-normal-v1": {
+ "mode": "normal",
+ "publication": "comparable-experimental",
+ "backends": {"deepep", "deepep-v2", "uccl", "deepep-hybrid", "mori"},
+ "precision_profiles": identity.V1_NORMAL_PRECISION_PROFILE_IDS,
+ "coordinates": {
+ ("normal", "decode", "uniform", False),
+ ("normal", "prefill", "uniform", False),
+ },
+ "ladders": {"decode": (128,), "prefill": (512,)},
+ },
+ "ep-precision-low-latency-v1": {
+ "mode": "low-latency",
+ "publication": "comparable-experimental",
+ "backends": {"deepep", "uccl"},
+ "precision_profiles": identity.V1_LOW_LATENCY_PRECISION_PROFILE_IDS,
+ "coordinates": {("low-latency", "decode", "uniform", False)},
+ "ladders": {"decode": (128,)},
+ },
}
IDENTIFIER = re.compile(r"[a-z0-9][a-z0-9.-]*")
SUITE_FIELDS = {
"backends", "ep_degrees", "eplb", "mode", "phases", "platforms",
- "required_publication", "routings", "token_points", "token_points_decode",
- "token_points_prefill", "workloads",
+ "precision_profiles", "provisional", "required_publication", "routings",
+ "token_points", "token_points_decode", "token_points_prefill", "workloads",
}
SUITE_REQUIRED = {
"ep_degrees", "mode", "phases", "platforms", "required_publication", "routings",
@@ -84,6 +103,7 @@
"nodes", "gpus_per_node", "scale_up_domain", "scope", "scale_up_transport",
"scale_out_transport", "transport", "topology_class",
)
+QUALIFICATION_INDICES = range(1, 4)
class MatrixError(ValueError):
@@ -215,6 +235,52 @@ def validate_config_documents(
raise SystemExit(f"suite {name}.backends differs from the frozen v1 catalog")
if expected_backends is None and "backends" in suite:
raise SystemExit(f"suite {name}.backends must be omitted")
+ expected_profiles = contract.get("precision_profiles")
+ if expected_profiles is None:
+ if "precision_profiles" in suite or "provisional" in suite:
+ raise SystemExit(
+ f"suite {name} cannot add precision fields to a baseline suite"
+ )
+ precision_profiles: list[str] = []
+ else:
+ precision_profiles = _list(
+ suite.get("precision_profiles"),
+ f"suite {name}.precision_profiles",
+ str,
+ set(identity.V1_PRECISION_PROFILES),
+ )
+ if tuple(precision_profiles) != expected_profiles:
+ raise SystemExit(
+ f"suite {name}.precision_profiles differs from the frozen v1 catalog"
+ )
+ if identity.V1_CONTROL_PRECISION_PROFILE in precision_profiles:
+ raise SystemExit(
+ f"suite {name} must reference existing BF16 evidence, not duplicate it"
+ )
+ if any(
+ mode not in identity.V1_PRECISION_PROFILES[profile]["modes"]
+ for profile in precision_profiles
+ ):
+ raise SystemExit(f"suite {name} contains a precision profile for another mode")
+ if type(suite.get("provisional")) is not bool:
+ raise SystemExit(f"suite {name}.provisional must be a boolean")
+ unresolved = cap.provisional_precision_targets(precision_profiles)
+ if suite["provisional"] != bool(unresolved):
+ raise SystemExit(
+ f"suite {name}.provisional must track unresolved capability targets"
+ )
+ candidates = cap.precision_targets(precision_profiles)
+ covered_candidates = [
+ target for target in candidates
+ if target["backend"] in suite_backends
+ and target["sku"] in suite["platforms"]
+ and target["ep"] in suite["ep_degrees"]
+ and target["mode"] == mode
+ ]
+ if covered_candidates != candidates:
+ raise SystemExit(
+ f"suite {name} does not cover every declared precision target"
+ )
suite_workloads = _list(suite["workloads"], f"suite {name}.workloads", str)
unknown = sorted(set(suite_workloads) - set(registry))
if unknown:
@@ -230,6 +296,10 @@ def validate_config_documents(
raise SystemExit(f"suite {name}: EPLB is only valid for Zipf routing")
if suite["required_publication"] not in {"official", "comparable-experimental"}:
raise SystemExit(f"suite {name}.required_publication is invalid")
+ if suite["required_publication"] != contract["publication"]:
+ raise SystemExit(
+ f"suite {name}.required_publication differs from the frozen v1 catalog"
+ )
if suite["required_publication"] == "official":
unverified = [item for item in suite_workloads if not registry[item].get("verified_against")]
if unverified:
@@ -288,6 +358,8 @@ def _ladder(suite: dict[str, Any], phase: str) -> str:
def _v1_requested_ladder(case: dict[str, Any]) -> str:
"""Bind extracted controls to the frozen v1 suite and workload catalog."""
suite = V1_SUITE_CONTRACTS.get(case.get("suite"))
+ expected_profiles = None if suite is None else suite.get("precision_profiles")
+ precision_profile = case.get("precision_profile")
coordinate = (
case.get("mode"), case.get("phase"), case.get("routing"), case.get("eplb")
)
@@ -298,6 +370,8 @@ def _v1_requested_ladder(case: dict[str, Any]) -> str:
or (
case.get("workload"), case.get("hidden"), case.get("topk"), case.get("experts")
) != V1_WORKLOAD
+ or (expected_profiles is None and precision_profile is not None)
+ or (expected_profiles is not None and precision_profile not in expected_profiles)
):
raise MatrixError("case differs from the frozen v1 suite/workload catalog")
return " ".join(map(str, suite["ladders"][case["phase"]]))
@@ -307,17 +381,36 @@ def _expected_disposition(
sku: str, case: dict[str, Any]
) -> tuple[str, str | None, str | None]:
requested_ladder = _v1_requested_ladder(case)
- ok, detail = cap.resolve(
+ precision_profile = case.get("precision_profile")
+ if precision_profile is not None and not cap.precision_target_declared(
+ precision_profile,
+ sku=sku,
+ backend=case["backend"],
+ ep=case["ep"],
+ mode=case["mode"],
+ ):
+ raise MatrixError("precision case is not an exact native capability target")
+ disposition, detail = cap.resolve_disposition(
sku, case["backend"], ep=case["ep"], nodes=case["nodes"],
routing=case["routing"], eplb=case["eplb"], mode=case["mode"],
+ precision_profile=precision_profile,
)
- if ok:
+ if disposition == "supported":
if case["ladder"] != requested_ladder:
raise MatrixError("case ladder differs from the frozen v1 suite catalog")
return "runnable", None, None
if case["ladder"] != requested_ladder:
raise MatrixError("unsupported case ladder differs from the frozen v1 suite catalog")
- return "unsupported", "backend-platform-unsupported", detail
+ if disposition == "unsupported":
+ reason = (
+ "precision-profile-unsupported"
+ if precision_profile is not None
+ else "backend-platform-unsupported"
+ )
+ return "unsupported", reason, detail
+ if disposition == "provisional":
+ raise MatrixError("provisional precision target entered the executable matrix")
+ raise MatrixError("not-applicable precision tuple entered the requested matrix")
def _case_id(sku: str, case: dict[str, Any]) -> str:
@@ -380,14 +473,27 @@ def resolve_matrix(
suites_document = _load("suites.yaml")
validate_config_documents(suites_document, workloads)
registry = suites_document["suites"]
- names = list(registry) if suites == "all" else [
- value.strip() for value in suites.split(",") if value.strip()
- ]
+ select_all = suites == "all"
+ names = (
+ [name for name, suite in registry.items() if not suite.get("provisional", False)]
+ if select_all
+ else [value.strip() for value in suites.split(",") if value.strip()]
+ )
if not names or len(names) != len(set(names)):
raise SystemExit("suite selection must be non-empty and unique")
unknown = sorted(set(names) - set(registry))
if unknown:
raise SystemExit(f"unknown suites {unknown}; have {sorted(registry)}")
+ blocked = [name for name in names if registry[name].get("provisional", False)]
+ if blocked:
+ unresolved = sum(
+ len(cap.provisional_precision_targets(registry[name]["precision_profiles"]))
+ for name in blocked
+ )
+ raise SystemExit(
+ f"provisional precision suites cannot be scheduled: {blocked}; "
+ f"resolve {unresolved} capability targets first"
+ )
targets = _select_backends(backend, backends)
shards: dict[tuple[str, str, int], list[dict[str, Any]]] = {}
@@ -399,6 +505,7 @@ def resolve_matrix(
phases = suite["phases"]
routings = suite["routings"]
eplb_values = suite.get("eplb", [False])
+ precision_profiles = suite.get("precision_profiles", [None])
suite_backends = set(suite.get("backends", cap.SWEEP_BACKENDS))
suite_targets = [target for target in targets if target in suite_backends]
if not suite_targets:
@@ -407,9 +514,18 @@ def resolve_matrix(
if only_sku and platform_name != only_sku:
continue
ep_degrees = suite["ep_degrees"]
- for workload, ep, phase, routing, eplb, target in itertools.product(
- suite["workloads"], ep_degrees, phases, routings, eplb_values, suite_targets
+ for workload, ep, phase, routing, eplb, target, precision_profile in itertools.product(
+ suite["workloads"], ep_degrees, phases, routings, eplb_values,
+ suite_targets, precision_profiles,
):
+ if precision_profile is not None and not cap.precision_target_declared(
+ precision_profile,
+ sku=platform_name,
+ backend=target,
+ ep=ep,
+ mode=mode,
+ ):
+ continue
topology = cap.topology_for(platform_name, ep)
if topology is None:
raise SystemExit(
@@ -420,7 +536,7 @@ def resolve_matrix(
continue
if max_nodes and nodes > max_nodes:
continue
- ok, capability_detail = cap.resolve(
+ capability_disposition, capability_detail = cap.resolve_disposition(
platform_name,
target,
ep=ep,
@@ -428,6 +544,7 @@ def resolve_matrix(
routing=routing,
eplb=bool(eplb),
mode=mode,
+ precision_profile=precision_profile,
)
hidden, topk, experts = _dims(workloads, workload)
@@ -457,6 +574,8 @@ def add_case(
"canonical": True,
**{field: topology[field] for field in TOPOLOGY_FIELDS},
}
+ if precision_profile is not None:
+ case["precision_profile"] = precision_profile
for signature in _semantic_points(platform_name, case):
if signature in scheduled:
raise SystemExit(
@@ -477,14 +596,30 @@ def add_case(
shards.setdefault((platform_name, target, nodes), []).append(case)
requested_ladder = _ladder(suite, phase)
- if not ok:
+ if capability_disposition == "not-applicable":
+ continue
+ if capability_disposition == "provisional":
+ raise SystemExit(
+ f"suite {suite_name}: provisional target escaped its suite gate: "
+ f"{precision_profile} {target} {platform_name} EP{ep}"
+ )
+ if capability_disposition == "unsupported":
add_case(
requested_ladder,
"unsupported",
- "backend-platform-unsupported",
+ (
+ "precision-profile-unsupported"
+ if precision_profile is not None
+ else "backend-platform-unsupported"
+ ),
capability_detail,
)
continue
+ if capability_disposition != "supported":
+ raise SystemExit(
+ f"suite {suite_name}: invalid capability disposition "
+ f"{capability_disposition!r}"
+ )
add_case(requested_ladder, "runnable", None, None)
shards_by_sku: dict[str, list[dict[str, Any]]] = {}
@@ -503,6 +638,7 @@ def add_case(
"launcher": cap.PLATFORMS[sku]["launcher"],
**{field: chunk[0][field] for field in TOPOLOGY_FIELDS},
"n": len(chunk),
+ "execution_weight": execution_weight(chunk),
"case_ids": [case["case_id"] for case in chunk],
})
include = [
@@ -552,6 +688,161 @@ def _positive_int(value: Any, field: str) -> int:
return value
+def _qualification_index(value: Any, field: str = "qualification_index") -> int:
+ if type(value) is not int or value not in QUALIFICATION_INDICES:
+ raise MatrixError(f"{field} must be an integer in 1..3")
+ return value
+
+
+def _requested_qualification_index(value: int | None = None) -> int:
+ if value is not None:
+ return _qualification_index(value)
+ raw = os.environ.get("CX_QUALIFICATION_INDEX", "1")
+ if raw not in {"1", "2", "3"}:
+ raise MatrixError("CX_QUALIFICATION_INDEX must be an integer in 1..3")
+ return int(raw)
+
+
+def _case_precision_profile(case: dict[str, Any]) -> str:
+ profile = case.get("precision_profile", identity.V1_CONTROL_PRECISION_PROFILE)
+ if not isinstance(profile, str) or profile not in identity.V1_PRECISION_PROFILES:
+ raise MatrixError("qualification case has an invalid precision profile")
+ return profile
+
+
+def _qualification_digest(
+ shard_id: str,
+ case_id: str,
+ profile_id: str,
+ qualification_index: int,
+) -> bytes:
+ return hashlib.sha256(
+ "\0".join((shard_id, case_id, profile_id, str(qualification_index))).encode()
+ ).digest()
+
+
+def _rotate(values: list[Any], offset: int) -> list[Any]:
+ if not values:
+ return []
+ position = offset % len(values)
+ return values[position:] + values[:position]
+
+
+def _seeded_qualification_order(
+ shard_id: str,
+ cases: list[dict[str, Any]],
+ qualification_index: int,
+) -> list[dict[str, Any]]:
+ by_profile: dict[str, list[dict[str, Any]]] = {}
+ for case in cases:
+ by_profile.setdefault(_case_precision_profile(case), []).append(case)
+ profiles = sorted(
+ by_profile,
+ key=lambda profile: _qualification_digest(
+ shard_id, "profile-order", profile, 1
+ ),
+ )
+ profiles = _rotate(profiles, qualification_index - 1)
+ groups: dict[str, list[dict[str, Any]]] = {}
+ for profile in profiles:
+ group = sorted(
+ by_profile[profile],
+ key=lambda case: _qualification_digest(
+ shard_id, case["case_id"], profile, qualification_index
+ ),
+ )
+ groups[profile] = _rotate(group, qualification_index - 1)
+ interleaved = [
+ groups[profile][position]
+ for position in range(max(map(len, groups.values())))
+ for profile in profiles
+ if position < len(groups[profile])
+ ]
+ return interleaved
+
+
+def qualification_execution_order(
+ shard_id: str,
+ cases: list[dict[str, Any]],
+ qualification_index: int,
+) -> list[dict[str, Any]]:
+ """Return one deterministic, repeat-specific permutation of a shard's cases."""
+ index = _qualification_index(qualification_index)
+ if not isinstance(shard_id, str) or not IDENTIFIER.fullmatch(shard_id):
+ raise MatrixError("qualification shard ID is invalid")
+ if not isinstance(cases, list) or not cases:
+ raise MatrixError("qualification planning requires at least one case")
+ selected: list[dict[str, Any]] = []
+ seen: set[tuple[str, ...]] = set()
+ for current in range(1, index + 1):
+ candidate = _seeded_qualification_order(shard_id, cases, current)
+ signature = tuple(case["case_id"] for case in candidate)
+ if len(cases) >= current and signature in seen:
+ for offset in range(1, len(candidate)):
+ rotated = _rotate(candidate, offset)
+ rotated_signature = tuple(case["case_id"] for case in rotated)
+ if rotated_signature not in seen:
+ candidate = rotated
+ signature = rotated_signature
+ break
+ seen.add(signature)
+ selected = candidate
+ return selected
+
+
+def execution_plan_sha256(cases: list[dict[str, Any]]) -> str:
+ """Bind an execution plan to only its ordered case and precision-profile IDs."""
+ plan = [
+ [case["case_id"], _case_precision_profile(case)]
+ for case in cases
+ ]
+ payload = json.dumps(plan, ensure_ascii=True, separators=(",", ":")).encode()
+ return hashlib.sha256(payload).hexdigest()
+
+
+def execution_weight(cases: list[dict[str, Any]]) -> int:
+ """Return deterministic GPU-point work used to bound workflow parallelism."""
+ if not isinstance(cases, list) or not cases:
+ raise MatrixError("execution weight requires at least one case")
+ weight = 0
+ for case in cases:
+ ep = _positive_int(case.get("ep"), "execution-weight.ep")
+ ladder = case.get("ladder")
+ if not isinstance(ladder, str) or not ladder.split():
+ raise MatrixError("execution weight requires a token ladder")
+ weight += ep * len(ladder.split())
+ return weight
+
+
+def qualification_execution_plan_sha256(
+ matrix: dict[str, Any], qualification_index: int
+) -> str:
+ """Bind one qualification repeat to every shard's ordered case plan."""
+ index = _qualification_index(qualification_index)
+ document = validate_matrix_document(matrix)
+ requested = {
+ item["case"]["case_id"]: item["case"]
+ for item in document["requested_cases"]
+ }
+ plan = []
+ for shard in sorted(document["include"], key=lambda item: item["id"]):
+ ordered = qualification_execution_order(
+ shard["id"],
+ [requested[case_id] for case_id in shard["case_ids"]],
+ index,
+ )
+ plan.append([
+ shard["id"], shard["execution_weight"], execution_plan_sha256(ordered),
+ ])
+ payload = json.dumps(
+ {"qualification_index": index, "shards": plan},
+ ensure_ascii=True,
+ sort_keys=True,
+ separators=(",", ":"),
+ ).encode()
+ return hashlib.sha256(payload).hexdigest()
+
+
def validate_shard_control(
shard: dict[str, Any],
*,
@@ -559,13 +850,17 @@ def validate_shard_control(
backend: str,
nodes: int,
require_runnable: bool = True,
+ qualification_index: int | None = None,
) -> None:
"""Validate one shard against the workflow cell that requested it."""
if not isinstance(shard, dict):
raise MatrixError("shard must be a JSON object")
if sku not in cap.PLATFORMS or backend not in cap.SWEEP_BACKENDS:
raise MatrixError("shard platform/backend is not registered")
- top_fields = {"schema_version", "id", "sku", "backend", "nodes", "n", "cases"}
+ top_fields = {
+ "schema_version", "id", "sku", "backend", "nodes", "n", "cases",
+ "qualification_index", "execution_plan_sha256", "execution_weight",
+ }
if (
set(shard) != top_fields
or type(shard.get("schema_version")) is not int
@@ -574,6 +869,19 @@ def validate_shard_control(
raise MatrixError("shard fields or schema version differ from v1 contract")
if not isinstance(shard.get("id"), str) or not IDENTIFIER.fullmatch(shard["id"]):
raise MatrixError("shard has invalid id")
+ observed_qualification = _qualification_index(
+ shard.get("qualification_index"), "shard.qualification_index"
+ )
+ if (
+ qualification_index is not None
+ and observed_qualification != _qualification_index(qualification_index)
+ ):
+ raise MatrixError("shard qualification_index differs from the requested repeat")
+ if (
+ not isinstance(shard.get("execution_plan_sha256"), str)
+ or re.fullmatch(r"[0-9a-f]{64}", shard["execution_plan_sha256"]) is None
+ ):
+ raise MatrixError("shard execution_plan_sha256 is invalid")
for field, expected in (("sku", sku), ("backend", backend)):
if shard.get(field) != expected:
raise MatrixError(
@@ -589,7 +897,7 @@ def validate_shard_control(
if _positive_int(shard.get("n"), "shard.n") != len(cases):
raise MatrixError("shard.n does not match the number of cases")
seen: set[str] = set()
- required = {
+ base_required = {
"case_id", "suite", "workload", "required_publication", "backend", "routing",
"mode", "phase", "ep", "eplb", "hidden", "topk", "experts",
"samples_per_point",
@@ -598,6 +906,12 @@ def validate_shard_control(
for index, case in enumerate(cases):
if not isinstance(case, dict):
raise MatrixError(f"case {index} must be a JSON object")
+ suite_contract = V1_SUITE_CONTRACTS.get(case.get("suite"))
+ required = base_required | (
+ {"precision_profile"}
+ if suite_contract is not None and "precision_profiles" in suite_contract
+ else set()
+ )
fields = set(case)
if fields != required:
raise MatrixError(
@@ -610,13 +924,21 @@ def validate_shard_control(
if case_id in seen:
raise MatrixError(f"duplicate case_id {case_id}")
seen.add(case_id)
- for field in (
+ string_fields = [
"suite", "workload", "required_publication", "backend", "mode", "routing",
"phase", "warmup_semantics", "ladder", "timing",
- ):
+ ]
+ if "precision_profile" in required:
+ string_fields.append("precision_profile")
+ for field in string_fields:
if not isinstance(case[field], str) or not case[field]:
raise MatrixError(f"case {index}.{field} must be a non-empty string")
- for field in ("suite", "workload", "required_publication", "backend", "routing", "phase"):
+ identifier_fields = [
+ "suite", "workload", "required_publication", "backend", "routing", "phase",
+ ]
+ if "precision_profile" in required:
+ identifier_fields.append("precision_profile")
+ for field in identifier_fields:
if not IDENTIFIER.fullmatch(case[field]):
raise MatrixError(f"case {index}.{field} is not a safe identifier")
if case["required_publication"] not in {"official", "comparable-experimental"}:
@@ -680,6 +1002,19 @@ def validate_shard_control(
raise MatrixError(f"case {index} violates capability registry: {reason}")
else:
_v1_requested_ladder(case)
+ if _positive_int(
+ shard.get("execution_weight"), "shard.execution_weight"
+ ) != execution_weight(cases):
+ raise MatrixError("shard execution_weight differs from its cases")
+ expected_order = qualification_execution_order(
+ shard["id"], cases, observed_qualification
+ )
+ if [case["case_id"] for case in cases] != [
+ case["case_id"] for case in expected_order
+ ]:
+ raise MatrixError("shard cases differ from the qualification execution order")
+ if shard["execution_plan_sha256"] != execution_plan_sha256(cases):
+ raise MatrixError("shard execution_plan_sha256 differs from its ordered cases")
def validate_matrix_document(document: Any) -> dict[str, Any]:
@@ -734,6 +1069,7 @@ def validate_matrix_document(document: Any) -> dict[str, Any]:
nodes = case.get("nodes")
if not isinstance(backend, str) or type(nodes) is not int:
raise MatrixError(f"{path}.case backend/nodes are invalid")
+ requested_case_plan = [case]
validate_shard_control(
{
"schema_version": 1,
@@ -742,7 +1078,12 @@ def validate_matrix_document(document: Any) -> dict[str, Any]:
"backend": backend,
"nodes": nodes,
"n": 1,
- "cases": [case],
+ "execution_weight": execution_weight(requested_case_plan),
+ "qualification_index": 1,
+ "execution_plan_sha256": execution_plan_sha256(
+ requested_case_plan
+ ),
+ "cases": requested_case_plan,
},
sku=sku,
backend=backend,
@@ -768,7 +1109,7 @@ def validate_matrix_document(document: Any) -> dict[str, Any]:
for index, shard in enumerate(include):
path = f"matrix.include[{index}]"
expected = {
- "id", "sku", "backend", "launcher", "n", "case_ids",
+ "id", "sku", "backend", "launcher", "n", "execution_weight", "case_ids",
} | set(TOPOLOGY_FIELDS)
if not isinstance(shard, dict) or set(shard) != expected:
raise MatrixError(f"{path} fields differ from the v1 contract")
@@ -803,6 +1144,10 @@ def validate_matrix_document(document: Any) -> dict[str, Any]:
):
raise MatrixError(f"{path} case does not match shard coordinates")
assigned.append(case_id)
+ if shard["execution_weight"] != execution_weight(
+ [cases_by_id[case_id]["case"] for case_id in case_ids]
+ ):
+ raise MatrixError(f"{path}.execution_weight differs from its cases")
if len(assigned) != len(set(assigned)):
raise MatrixError("a runnable case is assigned to more than one shard")
if set(assigned) != runnable_ids:
@@ -818,8 +1163,10 @@ def extract_shard(
sku: str,
backend: str,
nodes: int,
+ qualification_index: int | None = None,
) -> dict[str, Any]:
"""Extract one strictly matched shard control file, writing it atomically."""
+ qualification = _requested_qualification_index(qualification_index)
document = validate_matrix_document(_strict_json_load(Path(matrix_path)))
include = document["include"]
matches = [item for item in include if isinstance(item, dict) and item.get("id") == shard_id]
@@ -830,7 +1177,11 @@ def extract_shard(
item["case"]["case_id"]: item
for item in document["requested_cases"]
}
- cases = [requested[case_id]["case"] for case_id in source["case_ids"]]
+ cases = qualification_execution_order(
+ source["id"],
+ [requested[case_id]["case"] for case_id in source["case_ids"]],
+ qualification,
+ )
control = {
"schema_version": 1,
"id": source.get("id"),
@@ -838,9 +1189,18 @@ def extract_shard(
"backend": source.get("backend"),
"nodes": source.get("nodes"),
"n": source.get("n"),
+ "execution_weight": source.get("execution_weight"),
+ "qualification_index": qualification,
+ "execution_plan_sha256": execution_plan_sha256(cases),
"cases": cases,
}
- validate_shard_control(control, sku=sku, backend=backend, nodes=nodes)
+ validate_shard_control(
+ control,
+ sku=sku,
+ backend=backend,
+ nodes=nodes,
+ qualification_index=qualification,
+ )
output = Path(output_path)
output.parent.mkdir(parents=True, exist_ok=True)
temporary = output.with_name(f".{output.name}.tmp-{os.getpid()}")
@@ -862,9 +1222,16 @@ def emit_unsupported(
document = validate_matrix_document(_strict_json_load(source))
control_sha256 = hashlib.sha256(source.read_bytes()).hexdigest()
generated_at = dt.datetime.now(dt.timezone.utc).isoformat()
+ try:
+ qualification_index = int(os.environ.get("CX_QUALIFICATION_INDEX", "1"))
+ except ValueError as exc:
+ raise MatrixError("CX_QUALIFICATION_INDEX must be an integer in 1..3") from exc
+ if qualification_index not in range(1, 4):
+ raise MatrixError("CX_QUALIFICATION_INDEX must be in 1..3")
git_run = {
"run_id": os.environ.get("GITHUB_RUN_ID"),
"run_attempt": os.environ.get("GITHUB_RUN_ATTEMPT"),
+ "qualification_index": qualification_index,
"ref": os.environ.get("GITHUB_REF_NAME") or os.environ.get("GITHUB_REF"),
"source_sha": os.environ.get("COLLECTIVEX_SOURCE_SHA") or os.environ.get("GITHUB_SHA"),
"repo": os.environ.get("GITHUB_REPOSITORY"),
@@ -875,6 +1242,7 @@ def emit_unsupported(
"artifact": git_run["artifact"],
"execution_id": os.environ.get("COLLECTIVEX_EXECUTION_ID"),
"job": git_run["job"],
+ "qualification_index": qualification_index,
"repo": git_run["repo"],
"run_attempt": git_run["run_attempt"],
"run_id": git_run["run_id"],
@@ -928,6 +1296,52 @@ def emit_unsupported(
return written
+def frontend_catalog(matrix: dict[str, Any]) -> dict[str, Any]:
+ """Project the validated requested graph into a compact frontend test fixture."""
+ document = validate_matrix_document(matrix)
+ matrix_bytes = contracts.canonical_json_bytes(document) + b"\n"
+ cases = []
+ for wrapper in document["requested_cases"]:
+ case = wrapper["case"]
+ precision_profile = case.get(
+ "precision_profile", identity.V1_CONTROL_PRECISION_PROFILE
+ )
+ cases.append({
+ "backend": case["backend"],
+ "case_id": case["case_id"],
+ "disposition": wrapper["disposition"],
+ "eplb": case["eplb"],
+ "mode": case["mode"],
+ "phase": case["phase"],
+ "precision_profile": precision_profile,
+ "publication_tier": case["required_publication"],
+ "reason": wrapper["reason"],
+ "routing": case["routing"],
+ "sku": wrapper["sku"],
+ "suite": case["suite"],
+ "topology": {
+ "ep_size": case["ep"],
+ **{field: case[field] for field in TOPOLOGY_FIELDS},
+ },
+ "workload": case["workload"],
+ "points": [
+ {
+ "global_tokens": int(token) * case["ep"],
+ "tokens_per_rank": int(token),
+ }
+ for token in case["ladder"].split()
+ ],
+ })
+ return {
+ "case_count": len(cases),
+ "format": "collectivex.frontend-catalog.v1",
+ "matrix_sha256": hashlib.sha256(matrix_bytes).hexdigest(),
+ "point_count": sum(len(case["points"]) for case in cases),
+ "schema_version": 1,
+ "cases": cases,
+ }
+
+
def main() -> int:
parser = argparse.ArgumentParser(description="CollectiveX v1 matrix resolver")
parser.add_argument("--suites", default="all", help="'all' or comma-list of suites")
@@ -941,10 +1355,12 @@ def main() -> int:
parser.add_argument("--validate-control", default="", metavar="SHARD")
parser.add_argument("--emit-unsupported-from", default="", metavar="MATRIX")
parser.add_argument("--out-dir", default="")
+ parser.add_argument("--frontend-catalog", action="store_true")
parser.add_argument("--shard-id", default="")
parser.add_argument("--expect-sku", default="")
parser.add_argument("--expect-backend", default="")
parser.add_argument("--expect-nodes", type=int, default=0)
+ parser.add_argument("--qualification-index", type=int, default=None)
parser.add_argument("--out", default="")
args = parser.parse_args()
@@ -965,11 +1381,15 @@ def main() -> int:
)
try:
control = _strict_json_load(Path(args.validate_control))
+ qualification = _requested_qualification_index(
+ args.qualification_index
+ )
validate_shard_control(
control,
sku=args.expect_sku,
backend=args.expect_backend,
nodes=args.expect_nodes,
+ qualification_index=qualification,
)
except MatrixError as exc:
parser.error(str(exc))
@@ -990,6 +1410,7 @@ def main() -> int:
sku=args.expect_sku,
backend=args.expect_backend,
nodes=args.expect_nodes,
+ qualification_index=args.qualification_index,
)
except MatrixError as exc:
parser.error(str(exc))
@@ -1010,9 +1431,10 @@ def main() -> int:
validate_matrix_document(matrix)
except MatrixError as exc:
parser.error(str(exc))
+ output_document = frontend_catalog(matrix) if args.frontend_catalog else matrix
if args.out:
with open(args.out, "w") as fh:
- json.dump(matrix, fh, sort_keys=True, separators=(",", ":"))
+ json.dump(output_document, fh, sort_keys=True, separators=(",", ":"))
fh.write("\n")
runnable = sum(
item["disposition"] == "runnable" for item in matrix["requested_cases"]
@@ -1023,7 +1445,7 @@ def main() -> int:
f"{runnable} runnable and {unsupported} unsupported cases",
file=sys.stderr,
)
- print(json.dumps(matrix))
+ print(json.dumps(output_document))
return 0
diff --git a/experimental/CollectiveX/tests/ep_deepep.py b/experimental/CollectiveX/tests/ep_deepep.py
index 26cc889b5d..5ca5f47ed9 100644
--- a/experimental/CollectiveX/tests/ep_deepep.py
+++ b/experimental/CollectiveX/tests/ep_deepep.py
@@ -1,5 +1,5 @@
#!/usr/bin/env python3
-"""CollectiveX DeepEP adapter for the v1 BF16 normal-mode workload."""
+"""CollectiveX DeepEP adapter for native V1 dispatch/combine precision profiles."""
from __future__ import annotations
import inspect
@@ -10,6 +10,7 @@
import torch
import torch.distributed as dist
import contracts
+import ep_precision
try:
import deep_ep
@@ -69,6 +70,7 @@ def _normal_buffer_sizes(hidden: int, world_size: int) -> tuple[int, int]:
class DeepEPBackend:
name = "deepep"
+ stage_device_work = False
combine_needs_redispatch = False
# DeepEP reduces activations and top-k weights independently. The activation
# tensor must therefore carry the complete local weighted expert sum.
@@ -84,11 +86,48 @@ def __init__(self, args, rank, world_size, local_rank, device):
self.mode = getattr(args, "mode", "normal")
if self.mode not in {"normal", "low-latency"}:
raise ValueError(f"unsupported DeepEP mode {self.mode!r}")
+ supported_profiles = {
+ "normal": {
+ "d-bf16.c-bf16",
+ "d-fp8-e4m3fn-b128-f32-prequantized.c-bf16",
+ },
+ "low-latency": {
+ "d-bf16.c-bf16",
+ "d-fp8-e4m3fn-b128-f32-fused.c-bf16",
+ "d-bf16.c-logfmt10-dynamic64",
+ "d-fp8-e4m3fn-b128-f32-fused.c-logfmt10-dynamic64",
+ },
+ }
+ self.precision_profile_id, self.communication_precision = (
+ ep_precision.resolve_precision(
+ args,
+ backend=self.name,
+ mode=self.mode,
+ supported_profiles=supported_profiles[self.mode],
+ )
+ )
+ self._fp8_dispatch = ep_precision.is_low_precision_dispatch(
+ self.communication_precision
+ )
+ self._use_logfmt = ep_precision.uses_logfmt_combine(
+ self.communication_precision
+ )
+ self.stage_device_work = self._fp8_dispatch
self.group = dist.group.WORLD
device_sms = torch.cuda.get_device_properties(device).multi_processor_count
mnnvl_kwargs, mnnvl_comm = _mnnvl_buffer_configuration()
if self.mode == "low-latency":
+ ep_precision.require_keyword(
+ Buffer.low_latency_dispatch,
+ "use_fp8",
+ api="deep_ep.Buffer.low_latency_dispatch",
+ )
+ ep_precision.require_keyword(
+ Buffer.low_latency_combine,
+ "use_logfmt",
+ api="deep_ep.Buffer.low_latency_combine",
+ )
if args.phase != "decode":
raise ValueError("DeepEP low-latency mode only supports the decode ladder")
if args.experts % world_size:
@@ -126,6 +165,16 @@ def __init__(self, args, rank, world_size, local_rank, device):
"num_qps_per_rank": num_qps_per_rank,
}
else:
+ ep_precision.require_keyword(
+ Buffer.dispatch,
+ "async_finish",
+ api="deep_ep.Buffer.dispatch",
+ )
+ ep_precision.require_keyword(
+ Buffer.combine,
+ "async_finish",
+ api="deep_ep.Buffer.combine",
+ )
num_nvl_bytes, num_rdma_bytes = _normal_buffer_sizes(args.hidden, world_size)
if world_size > args.scale_up_domain and num_rdma_bytes == 0:
raise RuntimeError("DeepEP scale-out configuration returned no RDMA buffer")
@@ -158,9 +207,13 @@ def __init__(self, args, rank, world_size, local_rank, device):
"deepep_commit": os.environ.get("DEEPEP_COMMIT") or f"pkg-{version}",
"backend_lineage": "deepep-v1",
"mode": self.mode,
- "dispatch_dtype": "bf16",
- "combine_dtype": "bf16",
- "resource_mode": "tuned",
+ "dispatch_dtype": ep_precision.communication_format(
+ self.communication_precision, "dispatch"
+ ),
+ "combine_dtype": ep_precision.communication_format(
+ self.communication_precision, "combine"
+ ),
+ "resource_mode": "fixed-profile",
"device_sms": device_sms,
"allow_mnnvl": bool(mnnvl_kwargs),
"mnnvl_comm": mnnvl_comm,
@@ -171,9 +224,15 @@ def buffer_cap(self, args):
return self.max_tokens_per_rank if self.mode == "low-latency" else None
def make_problem(self, T, idx, weights, x):
+ encoding = ep_precision.encode_dispatch(
+ torch, x, self.communication_precision
+ )
return types.SimpleNamespace(
T=T,
x=x,
+ dispatch_x=encoding.native_input,
+ oracle_x=encoding.semantic,
+ dispatch_precision_evidence=encoding.evidence,
topk_idx=idx.to(torch.int64),
topk_weights=weights.to(torch.float32),
)
@@ -185,7 +244,7 @@ def dispatch(self, p):
p.topk_idx,
self.max_tokens_per_rank,
self.args.experts,
- use_fp8=False,
+ use_fp8=self._fp8_dispatch, # BF16 control realizes use_fp8=False.
async_finish=False,
return_recv_hook=False,
)
@@ -202,13 +261,14 @@ def dispatch(self, p):
_,
) = self.buffer.get_dispatch_layout(p.topk_idx, self.args.experts)
recv_x, recv_topk_idx, recv_topk_weights, recv_counts, handle, _ = self.buffer.dispatch(
- p.x,
+ p.dispatch_x,
topk_idx=p.topk_idx,
topk_weights=p.topk_weights,
num_tokens_per_rank=num_tokens_per_rank,
num_tokens_per_rdma_rank=num_tokens_per_rdma_rank,
is_token_in_rank=is_token_in_rank,
num_tokens_per_expert=num_tokens_per_expert,
+ async_finish=False,
)
return types.SimpleNamespace(
recv_x=recv_x,
@@ -219,7 +279,7 @@ def dispatch(self, p):
)
def stage(self, p, h):
- h.combine_input = h.recv_x
+ h.combine_input = self._semantic_recv(h, p)
def combine(self, p, h):
if self.mode == "low-latency":
@@ -228,11 +288,14 @@ def combine(self, p, h):
p.topk_idx,
p.topk_weights,
h.handle,
+ use_logfmt=self._use_logfmt,
async_finish=False,
return_recv_hook=False,
)
return combined_x
- combined_x, _, _ = self.buffer.combine(h.combine_input, h.handle)
+ combined_x, _, _ = self.buffer.combine(
+ h.combine_input, h.handle, async_finish=False
+ )
return combined_x
def inspect_dispatch(self, p, h):
@@ -243,7 +306,9 @@ def inspect_dispatch(self, p, h):
h.recv_topk_idx,
)
return types.SimpleNamespace(
- payload=h.recv_x,
+ payload=self._semantic_recv(h, p),
+ encoded_payload=self._encoded_recv(h),
+ scales=self._recv_scales(h),
expert_ids=expert_ids,
weights=h.recv_topk_weights.masked_fill(~valid, 0),
local_expert_counts=torch.tensor(h.recv_counts, device=self.device, dtype=torch.int64),
@@ -253,8 +318,11 @@ def inspect_dispatch(self, p, h):
def inspect_expert_dispatch(self, p, h):
if self.mode != "low-latency":
raise RuntimeError("expert-packed inspection requires low-latency mode")
+ p.recv_counts = tuple(int(value) for value in h.recv_counts.tolist())
return types.SimpleNamespace(
- payload=h.recv_x,
+ payload=self._semantic_recv(h, p),
+ encoded_payload=self._encoded_recv(h),
+ scales=self._recv_scales(h),
local_expert_counts=h.recv_counts,
source_info=h.handle[0],
layout_range=h.handle[1],
@@ -262,26 +330,88 @@ def inspect_expert_dispatch(self, p, h):
def combine_transformed(self, p, h, transformed):
if self.mode == "low-latency":
- packed = torch.zeros_like(h.recv_x)
+ packed = torch.zeros(
+ self._encoded_recv(h).shape,
+ dtype=torch.bfloat16,
+ device=self._encoded_recv(h).device,
+ )
packed[h.oracle_local_expert_slots, h.oracle_packed_positions] = transformed.to(
- h.recv_x.dtype
+ packed.dtype
)
combined, _, _ = self.buffer.low_latency_combine(
packed,
p.topk_idx,
p.topk_weights,
h.handle,
+ use_logfmt=self._use_logfmt,
async_finish=False,
return_recv_hook=False,
)
return combined
- combined, _, _ = self.buffer.combine(transformed.to(h.recv_x.dtype), h.handle)
+ semantic = self._semantic_recv(h, p)
+ combined, _, _ = self.buffer.combine(
+ transformed.to(semantic.dtype), h.handle, async_finish=False
+ )
return combined
def recv_tokens(self, h):
if self.mode == "low-latency":
return int(h.recv_counts.to(torch.int64).sum().item())
- return int(h.recv_x.shape[0])
+ return int(self._encoded_recv(h).shape[0])
+
+ def _encoded_recv(self, h):
+ return h.recv_x[0] if isinstance(h.recv_x, tuple) else h.recv_x
+
+ def _recv_scales(self, h):
+ return h.recv_x[1] if isinstance(h.recv_x, tuple) else None
+
+ def _semantic_recv(self, h, problem=None):
+ if not self._fp8_dispatch:
+ return h.recv_x
+ if not hasattr(h, "recv_semantic"):
+ if self.mode == "low-latency":
+ counts = getattr(problem, "recv_counts", None)
+ if counts is None:
+ counts = tuple(int(value) for value in h.recv_counts.tolist())
+ if problem is not None:
+ problem.recv_counts = counts
+ workspace = getattr(self, "_ll_semantic_workspace", None)
+ if workspace is None:
+ encoded = self._encoded_recv(h)
+ workspace = torch.empty(
+ encoded.shape, dtype=torch.bfloat16, device=encoded.device
+ )
+ self._ll_semantic_workspace = workspace
+ h.recv_semantic = ep_precision.dequantize_expert_prefixes(
+ torch,
+ self._encoded_recv(h),
+ self._recv_scales(h),
+ self.communication_precision["dispatch"],
+ counts,
+ workspace,
+ )
+ else:
+ h.recv_semantic = ep_precision.dequantize_dispatch(
+ torch,
+ self._encoded_recv(h),
+ self._recv_scales(h),
+ self.communication_precision["dispatch"],
+ )
+ return h.recv_semantic
+
+ def oracle_dispatch_payload(self, payload):
+ return ep_precision.encode_dispatch(
+ torch, payload, self.communication_precision
+ ).semantic
+
+ def precision_evidence(self, problem, view=None):
+ return ep_precision.precision_evidence(
+ torch,
+ profile_id=self.precision_profile_id,
+ profile=self.communication_precision,
+ problem=problem,
+ view=view,
+ )
def finalize(self, rc):
try:
diff --git a/experimental/CollectiveX/tests/ep_deepep_hybrid.py b/experimental/CollectiveX/tests/ep_deepep_hybrid.py
index 70cd17b005..f48005e4c0 100644
--- a/experimental/CollectiveX/tests/ep_deepep_hybrid.py
+++ b/experimental/CollectiveX/tests/ep_deepep_hybrid.py
@@ -20,8 +20,8 @@
round trip gives relerr(combined, x) = 4.28, matching E[distinct ranks] ~ 5.26 exactly. So this uses
the same per-rank-sum combine contract (no gate re-weight). BF16 tolerance is 5e-2.
-STATUS: bf16 / normal / layout-and-dispatch-v1. The v1 scope covers one MNNVL domain or x86
-scale-out between two eight-GPU NVLink domains; fp8 and cross-NVL72 scale-out remain out of scope.
+STATUS: BF16 or native block-scaled FP8 dispatch, BF16 combine, normal mode. The v1 scope covers
+one MNNVL domain or x86 scale-out between two eight-GPU NVLink domains.
"""
from __future__ import annotations
@@ -39,6 +39,7 @@
import torch
import torch.distributed as dist
import contracts
+import ep_precision
try:
import deep_ep
@@ -191,6 +192,7 @@ def _hybrid_topology(args, world_size: int) -> dict[str, int | str]:
class DeepEPHybridBackend:
name = "deepep-hybrid"
+ stage_device_work = False
# HybridEPBuffer.combine consumes the recv payload + the dispatch handle (no re-dispatch needed
# before a timed combine); the harness times dispatch and combine separately (like ep_deepep).
combine_needs_redispatch = False
@@ -202,6 +204,31 @@ def __init__(self, args, rank, world_size, local_rank, device):
self.world_size = world_size
self.device = device
self.mode = "normal"
+ self.precision_profile_id, self.communication_precision = (
+ ep_precision.resolve_precision(
+ args,
+ backend=self.name,
+ mode=self.mode,
+ supported_profiles={
+ "d-bf16.c-bf16",
+ "d-fp8-e4m3fn-b128-f32-prequantized.c-bf16",
+ },
+ )
+ )
+ self._fp8_dispatch = ep_precision.is_low_precision_dispatch(
+ self.communication_precision
+ )
+ self.stage_device_work = self._fp8_dispatch
+ ep_precision.require_keyword(
+ HybridEPBuffer.__init__,
+ "use_fp8",
+ api="deep_ep.HybridEPBuffer.__init__",
+ )
+ ep_precision.require_keyword(
+ HybridEPBuffer.dispatch,
+ "scaling_factor",
+ api="deep_ep.HybridEPBuffer.dispatch",
+ )
self.group = dist.group.WORLD
self.tolerance = 5e-2
self.top_k = int(args.topk)
@@ -254,7 +281,9 @@ def __init__(self, args, rank, world_size, local_rank, device):
self.buffer = HybridEPBuffer(
self.group, hidden_dim=self.hidden,
max_num_of_tokens_per_rank=self.max_tokens,
- num_local_experts=self.local_experts, use_fp8=False)
+ num_local_experts=self.local_experts,
+ use_fp8=self._fp8_dispatch,
+ )
realized_geometry = (
int(self.buffer.num_of_hybrid_ep_ranks_per_nvlink_domain),
int(self.buffer.num_of_nodes),
@@ -300,6 +329,11 @@ def tracked_update_template_config(*call_args, **call_kwargs):
or realized["num_of_nodes"] != self.communication_domains
):
raise RuntimeError("DeepEP Hybrid realized topology changed within one case")
+ expected_token_type = "UINT8" if self._fp8_dispatch else "UINT16"
+ if realized["token_data_type"] != expected_token_type:
+ raise RuntimeError(
+ "DeepEP Hybrid realized token dtype differs from the precision profile"
+ )
if self._realized_config is not None and realized != self._realized_config:
raise RuntimeError("DeepEP Hybrid realized autotune config changed within one case")
self._realized_config = realized
@@ -322,7 +356,13 @@ def tracked_update_template_config(*call_args, **call_kwargs):
"loaded_libraries": loaded_libraries,
"impl": "deep_ep.HybridEPBuffer (NVIDIA TMA + warp-pipeline)",
"mode": "normal", "transport": topology["transport"],
- "resource_mode": "tuned",
+ "dispatch_dtype": ep_precision.communication_format(
+ self.communication_precision, "dispatch"
+ ),
+ "combine_dtype": ep_precision.communication_format(
+ self.communication_precision, "combine"
+ ),
+ "resource_mode": "fixed-profile",
"num_sms": None, "device_sms": dev_sms,
"tuned_source": "deepep-hybrid-configurer-autotune-v1",
"realized_config": None, "jit_kernel_keys": [], "jit_shared_objects": [],
@@ -335,15 +375,29 @@ def buffer_cap(self, args):
return self.max_tokens
def make_problem(self, T, idx, weights, x):
+ encoding = ep_precision.encode_dispatch(
+ torch, x, self.communication_precision
+ )
+ dispatch_x = (
+ encoding.encoded_payload.view(torch.uint8)
+ if self._fp8_dispatch
+ else encoding.native_input
+ )
return types.SimpleNamespace(
- T=int(T), x=x,
+ T=int(T),
+ x=x,
+ dispatch_x=dispatch_x,
+ dispatch_scales=encoding.scales,
+ oracle_x=encoding.semantic,
+ dispatch_precision_evidence=encoding.evidence,
topk_idx=idx.to(torch.int64),
topk_weights=weights.to(torch.float32),
)
def dispatch(self, p):
recv, recv_probs, _scales, handle = self.buffer.dispatch(
- p.x,
+ p.dispatch_x,
+ scaling_factor=p.dispatch_scales,
topk_idx=p.topk_idx,
topk_weights=p.topk_weights,
num_of_experts=self.num_experts,
@@ -351,6 +405,7 @@ def dispatch(self, p):
return types.SimpleNamespace(
recv=recv,
recv_payload=recv,
+ recv_scales=_scales,
recv_probs=recv_probs,
handle=handle,
combine_input=None,
@@ -358,7 +413,7 @@ def dispatch(self, p):
def stage(self, p, h):
# Identity expert: the recv hidden IS the "expert output". combine reduces it per source token.
- h.combine_input = h.recv_payload
+ h.combine_input = self._semantic_recv(h, p.recv_tokens)
return None
def combine(self, p, h):
@@ -421,7 +476,9 @@ def inspect_dispatch(self, p, h):
expert_ids[rows, positions] = local_expert_ids + self.rank * self.local_experts
weights[rows, positions] = h.recv_probs[:count][rows, probability_columns]
return types.SimpleNamespace(
- payload=h.recv_payload[:count],
+ payload=self._semantic_recv(h, count)[:count],
+ encoded_payload=h.recv_payload[:count],
+ scales=(h.recv_scales[:count] if h.recv_scales is not None else None),
expert_ids=expert_ids,
weights=weights,
local_expert_counts=routing_map.sum(dim=0, dtype=torch.int64),
@@ -430,13 +487,50 @@ def inspect_dispatch(self, p, h):
def combine_transformed(self, p, h, transformed):
combined = self.buffer.combine(
- transformed.to(h.recv_payload.dtype), handle=h.handle
+ transformed.to(torch.bfloat16), handle=h.handle
)
return combined[0] if isinstance(combined, (tuple, list)) else combined
def recv_tokens(self, h):
return int(h.handle[3].item())
+ def _semantic_recv(self, h, rows):
+ if not self._fp8_dispatch:
+ return h.recv_payload
+ if not hasattr(h, "recv_semantic"):
+ semantic = torch.empty(
+ h.recv_payload.shape,
+ dtype=torch.bfloat16,
+ device=h.recv_payload.device,
+ )
+ semantic[:rows].copy_(ep_precision.dequantize_dispatch(
+ torch,
+ h.recv_payload[:rows],
+ h.recv_scales[:rows],
+ self.communication_precision["dispatch"],
+ uint8_storage=True,
+ ))
+ h.recv_semantic = semantic
+ h.recv_semantic_rows = rows
+ elif h.recv_semantic_rows != rows:
+ raise RuntimeError("DeepEP Hybrid receive count changed for one dispatch handle")
+ return h.recv_semantic
+
+ def oracle_dispatch_payload(self, payload):
+ return ep_precision.encode_dispatch(
+ torch, payload, self.communication_precision
+ ).semantic
+
+ def precision_evidence(self, problem, view=None):
+ return ep_precision.precision_evidence(
+ torch,
+ profile_id=self.precision_profile_id,
+ profile=self.communication_precision,
+ problem=problem,
+ view=view,
+ uint8_storage=True,
+ )
+
def finalize(self, rc):
try:
dist.barrier()
diff --git a/experimental/CollectiveX/tests/ep_deepep_v2.py b/experimental/CollectiveX/tests/ep_deepep_v2.py
index c629b0294e..c6c30ad276 100644
--- a/experimental/CollectiveX/tests/ep_deepep_v2.py
+++ b/experimental/CollectiveX/tests/ep_deepep_v2.py
@@ -18,6 +18,7 @@
import torch.distributed as dist
import contracts
import ep_harness
+import ep_precision
try:
import deep_ep
@@ -160,8 +161,18 @@ def _jit_cache_key(
max_tokens: int,
allow_hybrid_mode: bool,
realized: dict[str, int | bool],
+ precision_profile_id: str = "d-bf16.c-bf16",
+ communication_precision: dict[str, object] | None = None,
) -> str:
"""Key generated kernels by codegen inputs, not routing data or case identity."""
+ if communication_precision is None:
+ communication_precision = {
+ "dispatch": {
+ "communication_format": "bf16",
+ "api_input_dtype": "bf16",
+ },
+ "combine": {"communication_format": "bf16"},
+ }
payload = {
"contract": "deepep-v2-jit-config-v3",
"runner": args.runner,
@@ -171,9 +182,10 @@ def _jit_cache_key(
"physical_experts": args.experts,
"tuning_experts": getattr(args, "num_logical_experts", args.experts),
"max_tokens": max_tokens,
- "dispatch_dtype": "bf16",
- "combine_dtype": "bf16",
- "input_layout": "bf16-no-sf",
+ "precision_profile": precision_profile_id,
+ "dispatch_dtype": communication_precision["dispatch"]["communication_format"],
+ "combine_dtype": communication_precision["combine"]["communication_format"],
+ "input_layout": communication_precision["dispatch"]["api_input_dtype"],
"expert_alignment": 1,
"do_cpu_sync": True,
"cached_mode": False,
@@ -291,6 +303,7 @@ def _require_runtime() -> tuple[str, str]:
class DeepEPV2Backend:
name = "deepep-v2"
+ stage_device_work = False
combine_needs_redispatch = False
combine_weight_semantics = "unweighted-rank-sum"
@@ -300,6 +313,26 @@ def __init__(self, args, rank, world_size, local_rank, device):
self.world_size = world_size
self.device = device
self.mode = "normal"
+ self.precision_profile_id, self.communication_precision = (
+ ep_precision.resolve_precision(
+ args,
+ backend=self.name,
+ mode=self.mode,
+ supported_profiles={
+ "d-bf16.c-bf16",
+ "d-fp8-e4m3fn-b128-f32-prequantized.c-bf16",
+ },
+ )
+ )
+ self._fp8_dispatch = ep_precision.is_low_precision_dispatch(
+ self.communication_precision
+ )
+ self.stage_device_work = self._fp8_dispatch
+ ep_precision.require_keyword(
+ ElasticBuffer.__init__,
+ "use_fp8_dispatch",
+ api="deep_ep.ElasticBuffer.__init__",
+ )
self.group = dist.group.WORLD
torch_version, nccl_runtime_version = _require_runtime()
ladder, _ = ep_harness.token_ladder(args.tokens_ladder, args.phase, None)
@@ -320,7 +353,7 @@ def __init__(self, args, rank, world_size, local_rank, device):
num_max_tokens_per_rank=self.max_tokens,
hidden=args.hidden,
num_topk=args.topk,
- use_fp8_dispatch=False,
+ use_fp8_dispatch=self._fp8_dispatch,
deterministic=False,
allow_hybrid_mode=allow_hybrid_mode,
allow_multiple_reduction=True,
@@ -356,7 +389,13 @@ def __init__(self, args, rank, world_size, local_rank, device):
):
raise RuntimeError("DeepEP V2 realized communication domains differ from topology")
self.jit_cache_key = _jit_cache_key(
- args, world_size, self.max_tokens, allow_hybrid_mode, jit_config
+ args,
+ world_size,
+ self.max_tokens,
+ allow_hybrid_mode,
+ jit_config,
+ self.precision_profile_id,
+ self.communication_precision,
)
os.environ["EP_JIT_CACHE_DIR"] = str(jit_root / self.jit_cache_key)
realized_config = {
@@ -397,10 +436,14 @@ def __init__(self, args, rank, world_size, local_rank, device):
"jit_random_seed": DEEPEP_V2_JIT_RANDOM_SEED,
"num_experts": int(args.experts),
"mode": "normal",
- "dispatch_dtype": "bf16",
- "combine_dtype": "bf16",
+ "dispatch_dtype": ep_precision.communication_format(
+ self.communication_precision, "dispatch"
+ ),
+ "combine_dtype": ep_precision.communication_format(
+ self.communication_precision, "combine"
+ ),
"deterministic": False,
- "resource_mode": "tuned",
+ "resource_mode": "fixed-profile",
"requested_num_sms": self.num_sms,
"tuning_num_experts": tuning_num_experts,
"num_sms": self.num_sms,
@@ -425,16 +468,22 @@ def buffer_cap(self, args):
return self.max_tokens
def make_problem(self, T, idx, weights, x):
+ encoding = ep_precision.encode_dispatch(
+ torch, x, self.communication_precision
+ )
return types.SimpleNamespace(
T=T,
x=x,
+ dispatch_x=encoding.native_input,
+ oracle_x=encoding.semantic,
+ dispatch_precision_evidence=encoding.evidence,
topk_idx=idx.to(deep_ep.topk_idx_t),
topk_weights=weights.to(torch.float32),
)
def dispatch(self, p):
recv_x, recv_topk_idx, recv_topk_weights, handle, _ = self.buffer.dispatch(
- p.x,
+ p.dispatch_x,
topk_idx=p.topk_idx,
topk_weights=p.topk_weights,
num_experts=self.args.experts,
@@ -455,7 +504,7 @@ def dispatch(self, p):
)
def stage(self, p, h):
- h.combine_input = h.recv_x
+ h.combine_input = self._semantic_recv(h, p.recv_tokens)
def combine(self, p, h):
combined_x, _, _ = self.buffer.combine(
@@ -493,7 +542,13 @@ def inspect_dispatch(self, p, h):
)
local = local_idx[valid].to(torch.int64)
return types.SimpleNamespace(
- payload=h.recv_x[:count],
+ payload=self._semantic_recv(h, count)[:count],
+ encoded_payload=self._encoded_recv(h)[:count],
+ scales=(
+ self._recv_scales(h)[:count]
+ if self._recv_scales(h) is not None
+ else None
+ ),
expert_ids=expert_ids,
weights=h.recv_topk_weights[:count].masked_fill(~valid, 0),
local_expert_counts=torch.bincount(
@@ -503,7 +558,8 @@ def inspect_dispatch(self, p, h):
)
def combine_transformed(self, p, h, transformed):
- combine_input = torch.zeros_like(h.recv_x)
+ semantic = self._semantic_recv(h, self.recv_tokens(h))
+ combine_input = torch.zeros_like(semantic)
combine_input[: transformed.shape[0]].copy_(transformed.to(combine_input.dtype))
combined, _, _ = self.buffer.combine(
combine_input,
@@ -517,6 +573,46 @@ def combine_transformed(self, p, h, transformed):
def recv_tokens(self, h):
return int(h.handle.psum_num_recv_tokens_per_scaleup_rank[-1].item())
+ def _encoded_recv(self, h):
+ return h.recv_x[0] if isinstance(h.recv_x, tuple) else h.recv_x
+
+ def _recv_scales(self, h):
+ return h.recv_x[1] if isinstance(h.recv_x, tuple) else None
+
+ def _semantic_recv(self, h, rows):
+ if not self._fp8_dispatch:
+ return h.recv_x
+ if not hasattr(h, "recv_semantic"):
+ encoded = self._encoded_recv(h)
+ semantic = torch.empty(
+ encoded.shape, dtype=torch.bfloat16, device=encoded.device
+ )
+ semantic[:rows].copy_(ep_precision.dequantize_dispatch(
+ torch,
+ encoded[:rows],
+ self._recv_scales(h)[:rows],
+ self.communication_precision["dispatch"],
+ ))
+ h.recv_semantic = semantic
+ h.recv_semantic_rows = rows
+ elif h.recv_semantic_rows != rows:
+ raise RuntimeError("DeepEP V2 receive count changed for one dispatch handle")
+ return h.recv_semantic
+
+ def oracle_dispatch_payload(self, payload):
+ return ep_precision.encode_dispatch(
+ torch, payload, self.communication_precision
+ ).semantic
+
+ def precision_evidence(self, problem, view=None):
+ return ep_precision.precision_evidence(
+ torch,
+ profile_id=self.precision_profile_id,
+ profile=self.communication_precision,
+ problem=problem,
+ view=view,
+ )
+
def finalize(self, rc):
try:
dist.barrier()
diff --git a/experimental/CollectiveX/tests/ep_harness.py b/experimental/CollectiveX/tests/ep_harness.py
index c9a022a484..af2ec98cae 100644
--- a/experimental/CollectiveX/tests/ep_harness.py
+++ b/experimental/CollectiveX/tests/ep_harness.py
@@ -27,7 +27,8 @@
buffer_cap(args) -> int|None
make_problem(T, idx, weights, x) -> problem # materialize this rank's trace slice
dispatch(problem) -> handle # pure dispatch comm (timed)
- stage(problem, handle) # untimed expert-output placement
+ stage(problem, handle) # expert-output placement
+ stage_device_work # true only when stage launches device work
combine(problem, handle) -> tensor # pure combine comm (timed)
inspect_dispatch(problem, handle) -> view # normalized payload/expert/weight metadata
combine_transformed(problem, handle, tensor) -> tensor
@@ -60,16 +61,17 @@
TRIALS_PER_POINT = 64
WARMUP_ITERS_PER_TRIAL = 32
WARMUP_SEMANTICS = "full-roundtrip-before-each-component-trial-point-v1"
+QUALIFICATION_RUNS = 3
ROUTING_SEED = 67
ROUTING_GENERATOR = workload_contract.GENERATOR_VERSION
-ACTIVATION_PROFILE = "canonical-counter-source-v3"
+ACTIVATION_PROFILE = "canonical-counter-source-v4"
ACTIVATION_GENERATOR = workload_contract.ACTIVATION_GENERATOR
PLACEMENT = "packed"
-COMPONENT_ORDER_CONTRACT = "roundtrip-dispatch-activation-only-combine-v2"
+COMPONENT_ORDER_CONTRACT = "qualification-hash-rotated-components-v1"
LOW_LATENCY_MODE = "low-latency"
LOW_LATENCY_MAX_TOKENS_PER_RANK = 128
LOW_LATENCY_MEASUREMENT_CONTRACT = "expert-packed-weighted-combine-v1"
-LOW_LATENCY_COMPONENT_ORDER_CONTRACT = "roundtrip-dispatch-gate-weighted-combine-v1"
+LOW_LATENCY_COMPONENT_ORDER_CONTRACT = "qualification-hash-rotated-components-v1"
LOW_LATENCY_ORACLE_CONTRACT = "expert-assignment-transform-v1"
LOW_LATENCY_CORRECTNESS_SCOPE = "expert-assignment-and-weighted-combine"
@@ -87,7 +89,6 @@
ORACLE_RTOL = 5e-2
ORACLE_ATOL = 2e-2
-BF16_BYTES = 2
EPLB_REDUNDANT_EXPERTS = 32
EPLB_REFERENCE_TOKENS_PER_RANK = 2048
EPLB_PLANNER = "greedy-rank-major-v1"
@@ -97,7 +98,7 @@
"combine_quant_mode": "none",
"mode": "normal",
"measurement_contract": "layout-and-dispatch-v1",
- "resource_mode": "tuned",
+ "resource_mode": "fixed-profile",
"placement": PLACEMENT,
"activation_profile": ACTIVATION_PROFILE,
"activation_generator": ACTIVATION_GENERATOR,
@@ -111,6 +112,37 @@
"num_sms": 24,
}
+
+def precision_byte_provenance(
+ axis: dict, logical_copies: int, hidden: int
+) -> dict[str, int | str]:
+ """Return comparable logical activation and required scale bytes for one direction."""
+ if logical_copies < 0 or hidden < 0:
+ raise ValueError("logical precision byte dimensions must be non-negative")
+ bits_per_value = {
+ "bf16": 16,
+ "fp8-e4m3fn": 8,
+ "fp8-e4m3fnuz": 8,
+ "logfmt10": 10,
+ }.get(axis["communication_format"])
+ if bits_per_value is None:
+ raise ValueError(f"unknown communication format {axis['communication_format']!r}")
+ activation_data_bytes = logical_copies * math.ceil(hidden * bits_per_value / 8)
+ scale_bytes_per_value = {None: 0, "f32": 4, "implicit-logfmt10": 0}.get(
+ axis["scale_dtype"]
+ )
+ if scale_bytes_per_value is None:
+ raise ValueError(f"unknown communication scale dtype {axis['scale_dtype']!r}")
+ group_size = axis["scale_group_size"]
+ scale_groups = math.ceil(hidden / group_size) if group_size is not None else 0
+ scale_bytes = logical_copies * scale_groups * scale_bytes_per_value
+ return {
+ "accounting_contract": "activation-data-plus-scales-v1",
+ "activation_data_bytes": activation_data_bytes,
+ "scale_bytes": scale_bytes,
+ "total_logical_bytes": activation_data_bytes + scale_bytes,
+ }
+
def format_collective_version(raw) -> str:
"""Normalize PyTorch's tuple or packed NCCL/RCCL version representation."""
if isinstance(raw, int):
@@ -126,6 +158,12 @@ def add_common_args(ap: argparse.ArgumentParser) -> None:
"""Add the varying v1 inputs; fixed profile values are not CLI axes."""
ap.set_defaults(**V1_PROFILE)
ap.add_argument("--mode", default="normal", choices=["normal", LOW_LATENCY_MODE])
+ ap.add_argument(
+ "--precision-profile",
+ default="",
+ choices=("", *identity.V1_PRECISION_PROFILES),
+ help="exact native dispatch/combine communication profile; blank selects BF16 control",
+ )
ap.add_argument("--phase", default="decode", choices=["decode", "prefill"],
help="token-size regime: decode (small T) / prefill (large T) — picks the default ladder")
ap.add_argument("--tokens-ladder", default="",
@@ -149,6 +187,13 @@ def add_common_args(ap: argparse.ArgumentParser) -> None:
ap.add_argument("--workload-name", default="")
ap.add_argument("--required-publication", default="")
ap.add_argument("--seed", type=int, default=ROUTING_SEED)
+ ap.add_argument(
+ "--qualification-index",
+ type=int,
+ choices=range(1, QUALIFICATION_RUNS + 1),
+ default=os.environ.get("COLLECTIVEX_QUALIFICATION_INDEX", "1"),
+ help="one-based qualification repeat used for deterministic measurement ordering",
+ )
# 32: B300/Blackwell needs ~30 untimed iters to reach steady-state GPU clocks +
# establish NVLink/NVSHMEM connections — at warmup=8 its dispatch read ~1787us
# (cold), at warmup>=30 it settles to ~85us (faster than H100, reproducible within
@@ -200,6 +245,67 @@ def sampling_contract_error(iters: int, trials: int, warmup: int) -> str | None:
return None
+def qualification_order(
+ values: list, qualification_index: int, trial_index: int, *, identity_key: str = ""
+) -> list:
+ """Return a deterministic, position-balanced order for one qualification trial.
+
+ Official runs bind the base permutation to the case identity. The cyclic schedule then gives
+ every value every position equally often over 64 trials while qualification repeats start at
+ different offsets. Keeping the empty-key behavior stable preserves local diagnostic fixtures.
+ """
+ if not values or len(values) != len(set(values)):
+ raise ValueError("qualification order requires non-empty unique values")
+ if qualification_index not in range(1, QUALIFICATION_RUNS + 1):
+ raise ValueError(f"qualification_index must be in 1..{QUALIFICATION_RUNS}")
+ if type(trial_index) is not int or trial_index < 0:
+ raise ValueError("trial_index must be a non-negative integer")
+ if not isinstance(identity_key, str):
+ raise ValueError("qualification identity_key must be a string")
+ base_values = list(values)
+ if identity_key:
+ base_values.sort(
+ key=lambda value: hashlib.sha256(
+ f"{identity_key}\0{qualification_index}\0{value}".encode("utf-8")
+ ).digest()
+ )
+ position = trial_index + qualification_index - 1
+ cycle, offset = divmod(position, len(values))
+ base = base_values if cycle % 2 == 0 else list(reversed(base_values))
+ return base[offset:] + base[:offset]
+
+
+def sampled_component_evidence(trials: list[list[float]]) -> dict:
+ """Validate and copy private 64x8 trial blocks without flattening their boundaries."""
+ if not trials:
+ return {"availability": "unavailable", "sample_count": 0, "trials": None}
+ if len(trials) != TRIALS_PER_POINT:
+ raise ValueError(
+ f"measured component needs {TRIALS_PER_POINT} trial blocks; got {len(trials)}"
+ )
+ normalized: list[list[float]] = []
+ for trial in trials:
+ if len(trial) != TIMED_ITERS_PER_TRIAL:
+ raise ValueError(
+ f"measured trial needs {TIMED_ITERS_PER_TRIAL} samples; got {len(trial)}"
+ )
+ block = []
+ for sample in trial:
+ if isinstance(sample, bool) or not isinstance(sample, (int, float)):
+ raise ValueError("measured samples must be numeric")
+ value = float(sample)
+ if not math.isfinite(value) or value < 0:
+ raise ValueError("measured samples must be finite and non-negative")
+ block.append(value)
+ normalized.append(block)
+ count = sum(map(len, normalized))
+ if count != TIMED_SAMPLES_PER_POINT:
+ raise ValueError(
+ f"measured component needs {TIMED_SAMPLES_PER_POINT} samples; got {count}"
+ )
+ return {"availability": "measured", "sample_count": count, "trials": normalized}
+
+
def _stats_vec(xs: list[int]) -> dict:
"""min/mean/max/CV (+ empty count) of a per-rank count vector — self-describing source-token
or load summary without dumping the full vector."""
@@ -504,10 +610,11 @@ def _expert_transform_expanded(torch, payload, expert_ids):
def _expected_transformed_combine(torch, problem):
"""Independently derive sum_i gate_i * expert_i(x) for each source token."""
- expected = torch.zeros_like(problem.x, dtype=torch.float32)
+ semantic_x = getattr(problem, "oracle_x", problem.x)
+ expected = torch.zeros_like(semantic_x, dtype=torch.float32)
expert_ids = problem.topk_idx.to(torch.int64)
weights = problem.topk_weights.to(torch.float32)
- columns = torch.arange(problem.x.shape[1], device=problem.x.device, dtype=torch.int64)
+ columns = torch.arange(semantic_x.shape[1], device=semantic_x.device, dtype=torch.int64)
pattern = (((columns * 13) % 17) - 8).to(torch.float32) / 8
for slot in range(expert_ids.shape[1]):
expert = expert_ids[:, slot]
@@ -515,11 +622,129 @@ def _expected_transformed_combine(torch, problem):
scale = (((expert * 17 + 5) % 31 + 1).to(torch.float32) / 32).unsqueeze(1)
offset_a = ((((expert * 29 + 7) % 37) - 18).to(torch.float32) / 64).unsqueeze(1)
offset_b = ((((expert * 43 + 11) % 41) - 20).to(torch.float32) / 128).unsqueeze(1)
- expert_output = problem.x.float() * scale + offset_a + offset_b * pattern.unsqueeze(0)
+ expert_output = semantic_x.float() * scale + offset_a + offset_b * pattern.unsqueeze(0)
expected.add_(gate * expert_output)
return expected
+def _baseline_precision_axis() -> dict:
+ return {
+ "encoded_payload_valid": True,
+ "scales_finite": None,
+ "scales_positive": None,
+ "dequantized_semantics": True,
+ "saturation_count": 0,
+ "saturation_rate": 0.0,
+ "max_abs_error": 0.0,
+ "max_rel_error": 0.0,
+ "passed": True,
+ }
+
+
+def _precision_evidence(backend, problem, view, combined, expected_combined) -> dict:
+ method = getattr(backend, "precision_evidence", None)
+ if method is not None:
+ evidence = method(problem, view)
+ combine_axis = backend.communication_precision["combine"]
+ if combine_axis["communication_format"] != "bf16":
+ if combined.shape == expected_combined.shape and combined.numel():
+ absolute = (combined.float() - expected_combined.float()).abs()
+ max_abs_error = float(absolute.max().item())
+ max_rel_error = max_abs_error / (
+ float(expected_combined.float().abs().max().item()) + 1e-6
+ )
+ tolerance = ORACLE_ATOL + float(
+ getattr(backend, "tolerance", ORACLE_RTOL)
+ ) * expected_combined.float().abs()
+ semantics = bool((absolute <= tolerance).all().item())
+ elif combined.shape == expected_combined.shape:
+ max_abs_error = max_rel_error = 0.0
+ semantics = True
+ else:
+ max_abs_error = max_rel_error = 1e30
+ semantics = False
+ direction = evidence["combine"]
+ direction.update({
+ "dequantized_semantics": semantics,
+ "max_abs_error": max_abs_error,
+ "max_rel_error": max_rel_error,
+ })
+ scale_ok = (
+ direction["scales_finite"] is not False
+ and direction["scales_positive"] is not False
+ )
+ direction["passed"] = bool(
+ direction["encoded_payload_valid"] and semantics and scale_ok
+ )
+ evidence["passed"] = bool(
+ evidence["dispatch"]["passed"] and direction["passed"]
+ )
+ return evidence
+ profile_id = getattr(backend, "precision_profile_id", None)
+ if profile_id != identity.V1_CONTROL_PRECISION_PROFILE:
+ failed = _baseline_precision_axis()
+ failed.update({"encoded_payload_valid": False, "dequantized_semantics": False,
+ "passed": False})
+ return {"profile_id": profile_id, "dispatch": failed, "combine": dict(failed),
+ "passed": False}
+ return {
+ "profile_id": profile_id,
+ "dispatch": _baseline_precision_axis(),
+ "combine": _baseline_precision_axis(),
+ "passed": True,
+ }
+
+
+def _failed_precision_evidence(backend) -> dict:
+ failed = _baseline_precision_axis()
+ failed.update({"encoded_payload_valid": False, "dequantized_semantics": False,
+ "passed": False})
+ return {
+ "profile_id": getattr(backend, "precision_profile_id", None),
+ "dispatch": failed,
+ "combine": dict(failed),
+ "passed": False,
+ }
+
+
+def aggregate_precision_evidence(evidence_by_rank: list[dict]) -> dict:
+ """Collapse pre/post rank evidence without hiding any direction's worst observation."""
+ records = [record[phase] for record in evidence_by_rank for phase in ("pre", "post")]
+ profile_ids = {record["profile_id"] for record in records}
+ if len(profile_ids) != 1:
+ raise ValueError("precision evidence profiles differ across ranks or oracle passes")
+ result = {"profile_id": profile_ids.pop()}
+ for direction in ("dispatch", "combine"):
+ axes = [record[direction] for record in records]
+ rank_counts = []
+ for rank_index in range(len(evidence_by_rank)):
+ rank_counts.append(max(
+ evidence_by_rank[rank_index][phase][direction]["saturation_count"]
+ for phase in ("pre", "post")
+ ))
+ scale_finite = [axis["scales_finite"] for axis in axes]
+ scale_positive = [axis["scales_positive"] for axis in axes]
+ result[direction] = {
+ "encoded_payload_valid": all(axis["encoded_payload_valid"] for axis in axes),
+ "scales_finite": (
+ None if all(value is None for value in scale_finite)
+ else all(value is True for value in scale_finite)
+ ),
+ "scales_positive": (
+ None if all(value is None for value in scale_positive)
+ else all(value is True for value in scale_positive)
+ ),
+ "dequantized_semantics": all(axis["dequantized_semantics"] for axis in axes),
+ "saturation_count": sum(rank_counts),
+ "saturation_rate": max(axis["saturation_rate"] for axis in axes),
+ "max_abs_error": max(axis["max_abs_error"] for axis in axes),
+ "max_rel_error": max(axis["max_rel_error"] for axis in axes),
+ "passed": all(axis["passed"] for axis in axes),
+ }
+ result["passed"] = all(result[direction]["passed"] for direction in ("dispatch", "combine"))
+ return result
+
+
def _run_expert_packed_oracle(
torch,
routing,
@@ -558,6 +783,7 @@ def _run_expert_packed_oracle(
except Exception as cleanup_error:
raise inspection_error from cleanup_error
return {
+ "_precision": _failed_precision_evidence(backend),
"contract": contract,
"passed": False,
"ordering_contract": "adapter-inspection-failed",
@@ -618,6 +844,9 @@ def _run_expert_packed_oracle(
if source_range
else torch.empty_like(view.payload)
)
+ normalize_payload = getattr(backend, "oracle_dispatch_payload", None)
+ if source_range and normalize_payload is not None:
+ expected_payload = normalize_payload(expected_payload)
payload_ok = bool(
source_range
and torch.equal(decoded_source_ids.to(torch.int64), view.source_ids)
@@ -708,6 +937,7 @@ def _run_expert_packed_oracle(
max_relative_error = None
combine_values_ok = False
tolerance = float(getattr(backend, "tolerance", ORACLE_RTOL))
+ precision = _precision_evidence(backend, problem, view, combined, expected_combined)
checks = {
"combine_values": combine_values_ok,
"counts": counts_ok,
@@ -718,9 +948,11 @@ def _run_expert_packed_oracle(
"weights": weights_ok,
}
return {
+ "_precision": precision,
"contract": contract,
"passed": bool(
all(checks.values())
+ and precision["passed"]
and ordering_contract
and max_relative_error is not None
and max_relative_error < tolerance
@@ -778,6 +1010,7 @@ def _run_expert_oracle(
except Exception as cleanup_error:
raise inspection_error from cleanup_error
return {
+ "_precision": _failed_precision_evidence(backend),
"contract": ORACLE_CONTRACT,
"passed": False,
"ordering_contract": "adapter-inspection-failed",
@@ -823,6 +1056,9 @@ def _run_expert_oracle(
expected_payload = routing.activations_for_source_ids(
source_ids, problem.x.shape[1], seed, problem.x.dtype
)
+ normalize_payload = getattr(backend, "oracle_dispatch_payload", None)
+ if normalize_payload is not None:
+ expected_payload = normalize_payload(expected_payload)
else:
expected_ids = torch.full_like(view.expert_ids, -1)
expected_weights = torch.zeros_like(view.weights)
@@ -901,6 +1137,7 @@ def _run_expert_oracle(
max_relative_error = None
combine_values_ok = False
tolerance = float(getattr(backend, "tolerance", 5e-2))
+ precision = _precision_evidence(backend, problem, view, combined, expected_combined)
checks = {
"combine_values": combine_values_ok,
"counts": counts_ok,
@@ -911,9 +1148,11 @@ def _run_expert_oracle(
"weights": weights_ok,
}
return {
+ "_precision": precision,
"contract": ORACLE_CONTRACT,
"passed": bool(
all(checks.values())
+ and precision["passed"]
and ordering_contract
and max_relative_error is not None
and max_relative_error < tolerance
@@ -963,8 +1202,14 @@ def _derive_publication_status(v: dict) -> str:
def run_sweep(args, backend, torch, dist, device, rank: int, world_size: int) -> int:
"""Drive the source-tokens-per-rank sweep for one fully-specified line."""
mode = getattr(args, "mode", "normal")
+ requested_precision = getattr(args, "precision_profile", "") or None
+ resolved_precision_id = requested_precision or identity.V1_CONTROL_PRECISION_PROFILE
try:
- case_profile = identity.case_profile(mode)
+ profile_case = {"mode": mode}
+ if requested_precision is not None:
+ profile_case["precision_profile"] = requested_precision
+ case_profile = identity.profile_for_case(profile_case)
+ communication_precision = identity.precision_profile(resolved_precision_id)
except identity.IdentityError as exc:
if rank == 0:
print(f"ERROR: {exc}")
@@ -993,6 +1238,13 @@ def run_sweep(args, backend, torch, dist, device, rank: int, world_size: int) ->
if rank == 0:
print(f"ERROR: backend mode {getattr(backend, 'mode', None)!r} != {mode!r}")
return 2
+ if (
+ getattr(backend, "precision_profile_id", None) != resolved_precision_id
+ or getattr(backend, "communication_precision", None) != communication_precision
+ ):
+ if rank == 0:
+ print("ERROR: backend did not realize the requested communication precision")
+ return 2
expected_weight_semantics = (
"gate-weighted-sum"
if case_profile["combine_semantics"] == "gate-weighted"
@@ -1034,14 +1286,34 @@ def run_sweep(args, backend, torch, dist, device, rank: int, world_size: int) ->
# plans from an observed load estimate). build_trace builds the LOGICAL trace and remaps
# to physical when the plan is present; otherwise it's the identity (logical == physical).
eplb_plan = None
+ eplb_calibration = None
if eplb_on:
- ref_idx, _ = routing.build_global_routing(
+ calibration_id, calibration_checksums, calibration_rows, _ = (
+ workload_contract.canonical_eplb_calibration_member(
+ args.routing,
+ args.hidden,
+ args.topk,
+ num_logical,
+ ep_size,
+ EPLB_REFERENCE_TOKENS_PER_RANK,
+ args.seed,
+ )
+ )
+ ref_idx = torch.tensor(
+ calibration_rows,
+ dtype=torch.int64,
+ )
+ eplb_calibration = {
+ "token_offset": workload_contract.EPLB_CALIBRATION_TOKEN_OFFSET,
+ "trace_sha256": calibration_checksums["trace"],
+ "window": workload_contract.EPLB_CALIBRATION_WINDOW,
+ "workload_id": calibration_id,
+ }
+ if ref_idx.shape != (
EPLB_REFERENCE_TOKENS_PER_RANK * ep_size,
- num_logical,
args.topk,
- args.routing,
- args.seed,
- )
+ ):
+ raise RuntimeError("EPLB calibration trace dimensions differ from the contract")
load = torch.bincount(ref_idx.reshape(-1), minlength=num_logical).float().tolist()
eplb_plan = eplb.build_plan(load, args.experts, ep_size)
if rank == 0:
@@ -1117,8 +1389,6 @@ def warm_roundtrips(problem, count):
f"in {backend.backend_provenance}"
)
return 4
- elem_dispatch = BF16_BYTES
-
# ---- Pass 1: build each deterministic problem and run the expert oracle. ----
problems, gate, gts, global_traces, input_snapshots = {}, {}, {}, {}, {}
routing_hashes = set()
@@ -1145,6 +1415,7 @@ def warm_roundtrips(problem, count):
torch, routing, backend, problem, idx_g, w_g, rank, experts_per_rank,
args.seed,
)
+ precision_pre = oracle.pop("_precision")
before_x, before_idx, before_weights = input_snapshots[T]
pre_input_unchanged = (
torch.equal(problem.x, before_x)
@@ -1159,19 +1430,38 @@ def warm_roundtrips(problem, count):
"max_rel": oracle["max_relative_error"] or 0.0,
"local_ok": int(oracle["passed"]),
"oracle_pre": oracle,
+ "precision_pre": precision_pre,
"pre_input_unchanged": pre_input_unchanged,
}
# ---- Pass 2: every backend uses the same ascending point order and conditioning ramp.
# Per-iteration cross-rank MAX samples are pooled across trials. ----
disp_pool = {T: [] for T in ladder} # pooled per-iteration cross-rank MAX (dispatch)
+ stage_pool = {T: [] for T in ladder} # measured only when stage launches device work
comb_pool = {T: [] for T in ladder} # ... combine
rt_pool = {T: [] for T in ladder} # independently measured round trip
disp_trials = {T: [] for T in ladder}
+ stage_trials = {T: [] for T in ladder}
comb_trials = {T: [] for T in ladder}
rt_trials = {T: [] for T in ladder}
- order = list(ladder)
- for _trial in range(args.trials):
+ stage_device_work = getattr(backend, "stage_device_work", False)
+ if type(stage_device_work) is not bool:
+ raise ValueError("backend.stage_device_work must be a boolean")
+ order_identity = args.case_id or _sha256_json({
+ "backend": backend.name,
+ "ep_size": ep_size,
+ "mode": mode,
+ "phase": args.phase,
+ "precision_profile": resolved_precision_id,
+ "runner": args.runner,
+ "suite": args.suite,
+ })
+ observed_component_orders = []
+ for trial_index in range(args.trials):
+ order = qualification_order(
+ list(ladder), args.qualification_index, trial_index,
+ identity_key=f"{order_identity}:tokens",
+ )
for T in order:
problem = problems[T]
# Stateful paired APIs may expose only a measured round trip.
@@ -1183,30 +1473,76 @@ def rt_once(p=problem):
backend.stage(p, hh)
return backend.combine(p, hh)
- # Every available component starts after the same synchronized full-roundtrip warmup.
- # Roundtrip is first on every backend because it is the comparison headline.
- warm_roundtrips(problem, args.warmup)
- rt_iters = time_us(torch, lambda p=problem: rt_once(p), 0, args.iters)
- if roundtrip_only:
- disp_iters = comb_iters = []
- else:
- warm_roundtrips(problem, args.warmup)
- disp_iters = time_us(torch, lambda p=problem: backend.dispatch(p),
- 0, args.iters)
+ available_components = ["roundtrip"]
+ if not roundtrip_only:
+ available_components.extend(["dispatch", "combine"])
+ if stage_device_work:
+ available_components.append("stage")
+ component_order = qualification_order(
+ available_components,
+ args.qualification_index,
+ trial_index,
+ identity_key=f"{order_identity}:components:{T}",
+ )
+ observed_component_orders.append({
+ "components": component_order,
+ "tokens_per_rank": T,
+ "trial_index": trial_index,
+ })
+ measured = {name: [] for name in ("dispatch", "stage", "combine", "roundtrip")}
+
+ def prep_stage(p=problem):
+ return backend.dispatch(p)
+
+ def prep_combine(p=problem):
+ hh = backend.dispatch(p)
+ backend.stage(p, hh)
+ return hh
- def prep(p=problem):
- hh = backend.dispatch(p)
- backend.stage(p, hh)
- return hh
+ for component_name in component_order:
+ # Every measured component receives the same 32 synchronized full-roundtrip
+ # warmups immediately before its timed trial.
warm_roundtrips(problem, args.warmup)
- if backend.combine_needs_redispatch:
- comb_iters = time_us(torch, lambda hh, p=problem: backend.combine(p, hh),
- 0, args.iters, pre=prep)
- else:
- hh = prep()
- torch.cuda.synchronize()
- comb_iters = time_us(torch, lambda p=problem, hx=hh: backend.combine(p, hx),
- 0, args.iters)
+ if component_name == "roundtrip":
+ measured[component_name] = time_us(
+ torch, lambda p=problem: rt_once(p), 0, args.iters
+ )
+ elif component_name == "dispatch":
+ measured[component_name] = time_us(
+ torch, lambda p=problem: backend.dispatch(p), 0, args.iters
+ )
+ elif component_name == "stage":
+ measured[component_name] = time_us(
+ torch,
+ lambda hh, p=problem: backend.stage(p, hh),
+ 0,
+ args.iters,
+ pre=prep_stage,
+ )
+ elif component_name == "combine":
+ if backend.combine_needs_redispatch:
+ measured[component_name] = time_us(
+ torch,
+ lambda hh, p=problem: backend.combine(p, hh),
+ 0,
+ args.iters,
+ pre=prep_combine,
+ )
+ else:
+ hh = prep_combine()
+ torch.cuda.synchronize()
+ measured[component_name] = time_us(
+ torch,
+ lambda p=problem, hx=hh: backend.combine(p, hx),
+ 0,
+ args.iters,
+ )
+ else: # pragma: no cover - generated from the fixed list above
+ raise RuntimeError(f"unknown timed component {component_name!r}")
+ disp_iters = measured["dispatch"]
+ stage_iters = measured["stage"]
+ comb_iters = measured["combine"]
+ rt_iters = measured["roundtrip"]
# per-iteration cross-rank MAX (the distributed-op latency per iter), pooled.
if disp_iters:
reduced_dispatch = _reduce_vec(torch, dist, device, disp_iters, MAX)
@@ -1215,6 +1551,10 @@ def prep(p=problem):
comb_trials[T].append(reduced_combine)
disp_pool[T] += reduced_dispatch
comb_pool[T] += reduced_combine
+ if stage_iters:
+ reduced_stage = _reduce_vec(torch, dist, device, stage_iters, MAX)
+ stage_trials[T].append(reduced_stage)
+ stage_pool[T] += reduced_stage
reduced_roundtrip = _reduce_vec(torch, dist, device, rt_iters, MAX)
rt_trials[T].append(reduced_roundtrip)
rt_pool[T] += reduced_roundtrip
@@ -1233,6 +1573,7 @@ def prep(p=problem):
torch, routing, backend, problem, idx_g, w_g, rank, experts_per_rank,
args.seed,
)
+ precision_post = post.pop("_precision")
pre = gate[T]["oracle_pre"]
order_stable = (
pre["ordering_contract"] == post["ordering_contract"]
@@ -1244,6 +1585,7 @@ def prep(p=problem):
"local_ok": int(pre["passed"] and post["passed"] and input_unchanged and order_stable),
"max_rel": max(pre["max_relative_error"] or 0.0, post["max_relative_error"] or 0.0),
"oracle_post": post,
+ "precision_post": precision_post,
"order_stable": order_stable,
})
@@ -1269,12 +1611,16 @@ def component(percentiles, count, *, derived=False):
gt = gts[T]
g = gate[T]
rstats = g["rstats"]
- d, c, rt = disp_pool[T], comb_pool[T], rt_pool[T]
- dp, cp, rtp = pcts(d), pcts(c), pcts(rt)
- # isolated_sum = SUM of the isolated dispatch+combine percentiles. NOT a measured op
+ d, s, c, rt = disp_pool[T], stage_pool[T], comb_pool[T], rt_pool[T]
+ dp, sp, cp, rtp = pcts(d), pcts(s), pcts(c), pcts(rt)
+ # isolated_sum = SUM of the isolated dispatch+stage+combine percentiles. Stage contributes
+ # zero when it is explicitly not applicable. This is NOT a measured chained operation
# (can't reveal shared sync / launch amortization / overlap) — do NOT use for throughput
# or SLO capacity. The MEASURED round trip (rtp) is the real chained latency.
- isum = {key: dp[key] + cp[key] for key in dp} if dp and cp else None
+ isum = (
+ {key: dp[key] + (sp[key] if sp is not None else 0.0) + cp[key] for key in dp}
+ if dp and cp else None
+ )
recv_total = _reduce_int(torch, dist, device, g["recv_local"], SUM)
recv_max = _reduce_int(torch, dist, device, g["recv_local"], MAX)
recv_min = _reduce_int(torch, dist, device, g["recv_local"], MIN)
@@ -1292,6 +1638,12 @@ def component(percentiles, count, *, derived=False):
"rank": rank,
},
)
+ precision_rank_evidence = [None] * world_size
+ dist.all_gather_object(
+ precision_rank_evidence,
+ {"pre": g["precision_pre"], "post": g["precision_post"]},
+ )
+ precision_evidence = aggregate_precision_evidence(precision_rank_evidence)
# Canonical LOGICAL payload byte contracts (from the routing trace, NOT backend recv
# tensors): token-rank = one copy per unique (token,dest-rank); token-expert = one copy
# per routed (token,expert). routed_copies = token-rank copies; gt*topk = token-expert.
@@ -1306,8 +1658,27 @@ def component(percentiles, count, *, derived=False):
percentile_name: gt / (latency_us * 1e-6)
for percentile_name, latency_us in rtp.items()
}
- disp_bytes_l = logical_copies * H * elem_dispatch
- comb_bytes_l = logical_copies * H * BF16_BYTES
+ dispatch_bytes = precision_byte_provenance(
+ communication_precision["dispatch"], logical_copies, H
+ )
+ combine_bytes = precision_byte_provenance(
+ communication_precision["combine"], logical_copies, H
+ )
+ stage_bytes = {
+ "accounting_contract": "activation-data-plus-scales-v1",
+ "activation_data_bytes": 0,
+ "scale_bytes": 0,
+ "total_logical_bytes": 0,
+ }
+ roundtrip_bytes = {
+ "accounting_contract": "activation-data-plus-scales-v1",
+ **{
+ field: dispatch_bytes[field] + combine_bytes[field]
+ for field in (
+ "activation_data_bytes", "scale_bytes", "total_logical_bytes"
+ )
+ },
+ }
# Contract-level anomalies are attached to the row and rolled into validity.
# roundtrip_gt_isolated_sum: measured RT p99 >> Σ(isolated dispatch+combine) p99.
# roundtrip_lt_component_floor: measured RT p50 < max(dispatch,combine) p50 — a chained
@@ -1317,7 +1688,10 @@ def component(percentiles, count, *, derived=False):
row_anoms.append({"type": "roundtrip_gt_isolated_sum", "T": T,
"roundtrip_p99": round(rtp["p99"], 2), "isolated_sum_p99": round(isum["p99"], 2),
"ratio": round(rtp["p99"] / isum["p99"], 2), "threshold": thr_rt})
- floor = max(dp["p50"], cp["p50"]) if dp and cp else None
+ floor = (
+ max(dp["p50"], cp["p50"], sp["p50"] if sp is not None else 0.0)
+ if dp and cp else None
+ )
if floor and rtp["p50"] > 0 and rtp["p50"] < 0.95 * floor:
row_anoms.append({"type": "roundtrip_lt_component_floor", "T": T,
"roundtrip_p50": round(rtp["p50"], 2), "component_floor_p50": round(floor, 2)})
@@ -1329,19 +1703,22 @@ def component(percentiles, count, *, derived=False):
"dispatch": component(dp, len(d)),
"isolated_sum": component(isum, 0, derived=True),
"roundtrip": component(rtp, len(rt)),
+ "stage": component(sp, len(s)),
},
"correctness": {
"contract": case_profile["oracle_contract"],
"max_relative_error": max_rel,
"passed": point_ok,
+ "precision": precision_evidence,
"rank_evidence": rank_evidence,
"scope": case_profile["correctness_scope"],
},
"global_tokens": gt,
- "logical_bytes": {
- "combine": comb_bytes_l,
- "dispatch": disp_bytes_l,
- "roundtrip": disp_bytes_l + comb_bytes_l,
+ "byte_provenance": {
+ "combine": combine_bytes,
+ "dispatch": dispatch_bytes,
+ "roundtrip": roundtrip_bytes,
+ "stage": stage_bytes,
},
"receive": {
"max": recv_max,
@@ -1372,6 +1749,7 @@ def component(percentiles, count, *, derived=False):
},
"sample_histograms": {
"dispatch": _histogram(d) if d else None,
+ "stage": _histogram(s) if s else None,
"combine": _histogram(c) if c else None,
"roundtrip": _histogram(rt),
},
@@ -1406,10 +1784,12 @@ def component(percentiles, count, *, derived=False):
# Adapters never self-label official; status is derived from these gates.
prov = backend.backend_provenance
+ allocation_stratum_sha256 = getattr(args, "allocation_stratum_sha256", None)
provenance_complete = contracts.provenance_complete(
prov,
backend.name,
getattr(args, "git_run", None),
+ allocation_stratum_sha256=allocation_stratum_sha256,
image_digest=getattr(args, "image_digest", None),
image_verified=getattr(args, "image_digest_verified", False),
squash_sha256=getattr(args, "squash_sha256", None),
@@ -1453,17 +1833,14 @@ def component(percentiles, count, *, derived=False):
shape = { # FIXED line identity (no T, no per-backend resource knobs)
"hidden": args.hidden, "topk": args.topk, "experts": args.experts,
- "experts_per_rank": experts_per_rank, "dispatch_dtype": "bf16",
+ "experts_per_rank": experts_per_rank,
+ "precision_profile": resolved_precision_id,
+ "dispatch_precision": communication_precision["dispatch"],
+ "combine_precision": communication_precision["combine"],
"routing": args.routing, "eplb": bool(eplb_plan), "num_logical_experts": num_logical,
# V2 is reserved for the PR #605 ElasticBuffer adapter; package versions never imply it.
"kernel_gen": kernel_generation(backend),
"activation_profile": ACTIVATION_PROFILE,
- "quant": {
- "combine_input_dtype": "bf16",
- "combine_accum_dtype": getattr(backend, "combine_accum_dtype", "fp32"),
- "combine_output_dtype": "bf16", "combine_quant_mode": "none",
- "scale_layout": None,
- },
}
generated_at = args.timestamp or _dt.datetime.now().astimezone().isoformat()
realized_placement = getattr(args, "realized_placement", None)
@@ -1472,8 +1849,7 @@ def component(percentiles, count, *, derived=False):
if realized_placement is not None
else int(os.environ.get("SLURM_NNODES", "1"))
)
- case_factors = {
- "case": {
+ scheduled_case = {
"backend": backend.name,
"canonical": canonical,
"eplb": bool(eplb_plan),
@@ -1499,7 +1875,11 @@ def component(percentiles, count, *, derived=False):
"transport": args.transport,
"warmup_semantics": WARMUP_SEMANTICS,
"workload": args.workload_name or "manual",
- },
+ }
+ if requested_precision is not None:
+ scheduled_case["precision_profile"] = requested_precision
+ case_factors = {
+ "case": scheduled_case,
"profile": case_profile,
"sku": args.runner,
}
@@ -1514,6 +1894,7 @@ def component(percentiles, count, *, derived=False):
"artifact": git_run.get("artifact"),
"execution_id": getattr(args, "allocation_execution_id", None),
"job": git_run.get("job"),
+ "qualification_index": args.qualification_index,
"repo": git_run.get("repo"),
"run_attempt": git_run.get("run_attempt"),
"run_id": git_run.get("run_id"),
@@ -1564,18 +1945,12 @@ def component(percentiles, count, *, derived=False):
for row in rows:
token_count = row["tokens_per_rank"]
- def sampled_component(trials):
- return {
- "availability": "measured" if trials else "unavailable",
- "sample_count": sum(len(trial) for trial in trials),
- "trials": trials if trials else None,
- }
-
sample_point = {
"components": {
- "combine": sampled_component(comb_trials[token_count]),
- "dispatch": sampled_component(disp_trials[token_count]),
- "roundtrip": sampled_component(rt_trials[token_count]),
+ "combine": sampled_component_evidence(comb_trials[token_count]),
+ "dispatch": sampled_component_evidence(disp_trials[token_count]),
+ "roundtrip": sampled_component_evidence(rt_trials[token_count]),
+ "stage": sampled_component_evidence(stage_trials[token_count]),
},
"tokens_per_rank": token_count,
}
@@ -1610,6 +1985,7 @@ def sampled_component(trials):
"case_id": case_identifier,
"format": "collectivex.samples.v1",
"points": sample_points,
+ "qualification_index": args.qualification_index,
"sampling": {
"iterations_per_trial": args.iters,
"reduction": case_profile["rank_reduction"],
@@ -1630,6 +2006,10 @@ def sampled_component(trials):
headline = next((r for r in rows if r["tokens_per_rank"] == 64), rows[len(rows) // 2])
eplb_record = (
{
+ "calibration_token_offset": eplb_calibration["token_offset"],
+ "calibration_trace_sha256": eplb_calibration["trace_sha256"],
+ "calibration_window": eplb_calibration["window"],
+ "calibration_workload_id": eplb_calibration["workload_id"],
"enabled": True,
"imbalance_after": eplb_plan["imbalance_after"],
"imbalance_before": eplb_plan["imbalance_before"],
@@ -1644,6 +2024,10 @@ def sampled_component(trials):
}
if eplb_plan
else {
+ "calibration_token_offset": None,
+ "calibration_trace_sha256": None,
+ "calibration_window": None,
+ "calibration_workload_id": None,
"enabled": False,
"imbalance_after": None,
"imbalance_before": None,
@@ -1680,7 +2064,7 @@ def sampled_component(trials):
"mode": mode,
"phase": args.phase,
"required_publication": args.required_publication or "diagnostic",
- "resource_mode": "tuned",
+ "resource_mode": "fixed-profile",
"runner": args.runner,
"shape": shape,
"suite": args.suite or "manual",
@@ -1707,6 +2091,8 @@ def sampled_component(trials):
"roundtrips_per_shape": CONDITIONING_ROUNDS_PER_SHAPE,
},
"contract": case_profile["contract"],
+ "execution_order_sha256": _sha256_json(observed_component_orders),
+ "qualification_index": args.qualification_index,
"rows": rows,
"sampling": {
"contract": SAMPLING_CONTRACT,
@@ -1743,6 +2129,7 @@ def sampled_component(trials):
},
"runtime_fingerprint": runtime_fingerprint,
"provenance": {
+ "allocation_stratum_sha256": allocation_stratum_sha256,
"command": getattr(args, "reproduction_command", ""),
"distributed_launcher": getattr(args, "distributed_launcher", None),
"git_run": getattr(args, "git_run", None),
diff --git a/experimental/CollectiveX/tests/ep_mori.py b/experimental/CollectiveX/tests/ep_mori.py
index 936425c5af..832a69ea28 100644
--- a/experimental/CollectiveX/tests/ep_mori.py
+++ b/experimental/CollectiveX/tests/ep_mori.py
@@ -1,5 +1,5 @@
#!/usr/bin/env python3
-"""CollectiveX MoRI adapter for the v1 BF16 normal-mode workload."""
+"""CollectiveX MoRI adapter for native V1 dispatch/combine precision profiles."""
from __future__ import annotations
import os
@@ -14,6 +14,7 @@
import torch
import torch.distributed as dist
+import ep_precision
try:
import mori # type: ignore
@@ -48,6 +49,7 @@ def _mori_source_commit() -> str:
class MoRIBackend:
name = "mori"
+ stage_device_work = False
combine_needs_redispatch = True
combine_weight_semantics = "unweighted-rank-sum"
@@ -57,10 +59,71 @@ def __init__(self, args, rank, world_size, local_rank, device):
self.world_size = world_size
self.device = device
self.mode = "normal"
+ runner = str(getattr(args, "runner", ""))
+ if runner.startswith("mi355x"):
+ fp8_format = "fp8-e4m3fn"
+ supported_profiles = {
+ "d-bf16.c-bf16",
+ "d-fp8-e4m3fn-b128-f32-prequantized.c-bf16",
+ }
+ if world_size == 8:
+ supported_profiles.update({
+ "d-bf16.c-fp8-e4m3fn-direct-cast-noscale",
+ "d-fp8-e4m3fn-b128-f32-prequantized.c-fp8-e4m3fn-direct-cast-noscale",
+ })
+ elif runner.startswith("mi325x"):
+ fp8_format = "fp8-e4m3fnuz"
+ supported_profiles = {
+ "d-bf16.c-bf16",
+ "d-fp8-e4m3fnuz-b128-f32-prequantized.c-bf16",
+ }
+ if world_size == 8:
+ supported_profiles.update({
+ "d-bf16.c-fp8-e4m3fnuz-direct-cast-noscale",
+ "d-fp8-e4m3fnuz-b128-f32-prequantized.c-fp8-e4m3fnuz-direct-cast-noscale",
+ })
+ else:
+ raise ep_precision.PrecisionError(
+ f"MoRI precision contract has no pinned FP8 format for runner {runner!r}"
+ )
+ self.precision_profile_id, self.communication_precision = (
+ ep_precision.resolve_precision(
+ args,
+ backend=self.name,
+ mode=self.mode,
+ supported_profiles=supported_profiles,
+ )
+ )
+ self._fp8_dispatch = ep_precision.is_low_precision_dispatch(
+ self.communication_precision
+ )
+ self._direct_cast_combine = ep_precision.uses_direct_cast_combine(
+ self.communication_precision
+ )
+ if self._fp8_dispatch and ep_precision.communication_format(
+ self.communication_precision, "dispatch"
+ ) != fp8_format:
+ raise ep_precision.PrecisionError(
+ "MoRI dispatch FP8 format differs from the pinned GPU architecture"
+ )
+ if self._direct_cast_combine:
+ quant_enum = getattr(mori.ops, "EpDispatchCombineQuantType", None)
+ if quant_enum is None or not hasattr(quant_enum, "Fp8DirectCast"):
+ raise ep_precision.PrecisionError(
+ "pinned MoRI API omits EpDispatchCombineQuantType.Fp8DirectCast"
+ )
self.ep_size = world_size
self.experts_per_rank = args.experts // self.ep_size
- device_cus = torch.cuda.get_device_properties(device).multi_processor_count
+ device_properties = torch.cuda.get_device_properties(device)
+ device_cus = device_properties.multi_processor_count
+ realized_arch = str(getattr(device_properties, "gcnArchName", "")).split(":", 1)[0]
+ expected_arch = "gfx950" if runner.startswith("mi355x") else "gfx942"
+ if realized_arch != expected_arch:
+ raise ep_precision.PrecisionError(
+ f"MoRI runner {runner!r} realized architecture {realized_arch!r}, "
+ f"expected {expected_arch!r}"
+ )
gpus_per_node = int(args.gpus_per_node or world_size)
scale_up_domain = int(args.scale_up_domain or gpus_per_node)
scale_out = world_size > scale_up_domain
@@ -151,7 +214,12 @@ def __init__(self, args, rank, world_size, local_rank, device):
else "async-ll" if self._async_ll
else "intranode"
)
- self._external_input = self._async_ll or self._inter_node
+ self._external_input = (
+ self._async_ll or self._inter_node or self._direct_cast_combine
+ )
+ # Registered-input MoRI copies expert output into a device-side symmetric buffer. External
+ # input kernels consume the dispatch output directly, so their stage is not applicable.
+ self.stage_device_work = self._fp8_dispatch or not self._external_input
world_group = torch.distributed.group.WORLD
torch._C._distributed_c10d._register_process_group("default", world_group)
@@ -163,13 +231,33 @@ def __init__(self, args, rank, world_size, local_rank, device):
)
self._cap = self.buffer_cap(args)
+ dispatch_dtype = (
+ getattr(
+ torch,
+ "float8_e4m3fn"
+ if fp8_format == "fp8-e4m3fn"
+ else "float8_e4m3fnuz",
+ None,
+ )
+ if self._fp8_dispatch
+ else torch.bfloat16
+ )
+ if dispatch_dtype is None:
+ raise ep_precision.PrecisionError(
+ f"active torch build does not expose {fp8_format}"
+ )
+ scale_dim = args.hidden // 128 if self._fp8_dispatch else 0
+ if self._fp8_dispatch and args.hidden % 128:
+ raise ep_precision.PrecisionError(
+ "MoRI native FP8 dispatch requires hidden divisible by 128"
+ )
config_kwargs = {
- "data_type": torch.bfloat16,
+ "data_type": dispatch_dtype,
"rank": rank,
"world_size": world_size,
"hidden_dim": args.hidden,
- "scale_dim": 0,
- "scale_type_size": 1,
+ "scale_dim": scale_dim,
+ "scale_type_size": 4 if self._fp8_dispatch else 1,
"max_token_type_size": (
torch.tensor([], dtype=torch.bfloat16).element_size()
if self._inter_node
@@ -179,7 +267,9 @@ def __init__(self, args, rank, world_size, local_rank, device):
"num_experts_per_rank": self.experts_per_rank,
"num_experts_per_token": args.topk,
"use_external_inp_buf": self._external_input,
- "quant_type": "none",
+ "quant_type": (
+ "fp8_direct_cast" if self._direct_cast_combine else "none"
+ ),
}
if self._kernel_type is not None:
config_kwargs["kernel_type"] = self._kernel_type
@@ -195,7 +285,13 @@ def __init__(self, args, rank, world_size, local_rank, device):
"num_qp_per_pe": self.num_qps,
})
self.config = mori.ops.EpDispatchCombineConfig(**config_kwargs)
- expected_config = {"use_external_inp_buf": self._external_input}
+ expected_config = {
+ "data_type": dispatch_dtype,
+ "scale_dim": scale_dim,
+ "scale_type_size": 4 if self._fp8_dispatch else 1,
+ "use_external_inp_buf": self._external_input,
+ "quant_type": config_kwargs["quant_type"],
+ }
if self._async_ll or self._inter_node:
expected_config.update({
"block_num": self.block_num,
@@ -228,8 +324,12 @@ def __init__(self, args, rank, world_size, local_rank, device):
else "mori.ops.EpDispatchCombineOp/registered-input"
),
"mode": "normal",
- "dispatch_dtype": "bf16",
- "combine_dtype": "bf16",
+ "dispatch_dtype": ep_precision.communication_format(
+ self.communication_precision, "dispatch"
+ ),
+ "combine_dtype": ep_precision.communication_format(
+ self.communication_precision, "combine"
+ ),
"kernel_type": self._kernel_type_label,
"enable_sdma": os.environ.get("MORI_ENABLE_SDMA"),
"heap_size": os.environ.get("MORI_SHMEM_HEAP_SIZE"),
@@ -239,7 +339,7 @@ def __init__(self, args, rank, world_size, local_rank, device):
"rdma_block_num": self.rdma_block_num,
"use_external_inp_buf": self._external_input,
"num_qps": self.num_qps,
- "resource_mode": "tuned",
+ "resource_mode": "fixed-profile",
"block_num": self.block_num,
"block_num_target": self._block_target,
"block_num_floored": self._block_floored,
@@ -254,22 +354,32 @@ def buffer_cap(self, args):
return 512
def make_problem(self, T, idx, weights, x):
+ encoding = ep_precision.encode_dispatch(
+ torch, x, self.communication_precision
+ )
indices = idx.to(torch.int32)
gate_weights = weights.to(torch.float32)
return types.SimpleNamespace(
T=T,
x=x,
+ dispatch_x=encoding.native_input[0] if self._fp8_dispatch else x,
+ oracle_x=encoding.semantic,
+ dispatch_precision_evidence=encoding.evidence,
topk_idx=indices,
topk_weights=gate_weights,
indices=indices,
weights=gate_weights,
- scales=torch.empty((T, 0), dtype=torch.uint8, device=self.device),
+ scales=(
+ encoding.scales
+ if encoding.scales is not None
+ else torch.empty((T, 0), dtype=torch.uint8, device=self.device)
+ ),
)
def dispatch(self, p):
dispatch_output, dispatch_weights, _scales, dispatch_indices, recv_num = (
self.op.dispatch(
- p.x,
+ p.dispatch_x,
p.weights,
p.scales,
p.indices,
@@ -283,15 +393,17 @@ def dispatch(self, p):
return types.SimpleNamespace(
dispatch_output=dispatch_output,
dispatch_weights=dispatch_weights,
+ dispatch_scales=_scales,
dispatch_indices=dispatch_indices,
recv_num=recv_num[0],
- combine_input=dispatch_output.to(torch.bfloat16),
+ combine_input=None,
)
def stage(self, p, h):
rows = getattr(p, "recv_tokens", None)
- if not isinstance(rows, int) or rows < 0 or rows > h.combine_input.size(0):
+ if not isinstance(rows, int) or rows < 0 or rows > h.dispatch_output.size(0):
raise RuntimeError("MoRI receive count was not validated before staging")
+ h.combine_input = self._semantic_recv(h, rows)
if self._external_input:
return None
buffer = self.op.get_registered_combine_input_buffer(
@@ -332,7 +444,13 @@ def inspect_dispatch(self, p, h):
self.experts_per_rank,
)
return types.SimpleNamespace(
- payload=h.dispatch_output[:count],
+ payload=self._semantic_recv(h, count)[:count],
+ encoded_payload=h.dispatch_output[:count],
+ scales=(
+ h.dispatch_scales[:count]
+ if h.dispatch_scales is not None
+ else None
+ ),
expert_ids=expert_ids,
weights=weights,
local_expert_counts=torch.bincount(
@@ -343,12 +461,59 @@ def inspect_dispatch(self, p, h):
def combine_transformed(self, p, h, transformed):
h.combine_input = transformed.to(torch.bfloat16)
- self.stage(p, h)
+ rows = getattr(p, "recv_tokens", None)
+ if not isinstance(rows, int) or rows < 0 or rows > h.combine_input.size(0):
+ raise RuntimeError("MoRI receive count was not validated before transformed combine")
+ if not self._external_input:
+ buffer = self.op.get_registered_combine_input_buffer(
+ torch.bfloat16, hidden_dim=h.combine_input.size(1)
+ )
+ buffer[:rows, :].copy_(h.combine_input[:rows, :])
+ h.combine_input = buffer
return self.combine(p, h)
def recv_tokens(self, h):
return int(h.recv_num.item())
+ def _semantic_recv(self, h, rows):
+ if not self._fp8_dispatch:
+ return h.dispatch_output
+ if not hasattr(h, "recv_semantic"):
+ if h.dispatch_scales is None:
+ raise ep_precision.PrecisionError(
+ "MoRI FP8 dispatch did not return scaling factors"
+ )
+ semantic = torch.empty(
+ h.dispatch_output.shape,
+ dtype=torch.bfloat16,
+ device=h.dispatch_output.device,
+ )
+ semantic[:rows].copy_(ep_precision.dequantize_dispatch(
+ torch,
+ h.dispatch_output[:rows],
+ h.dispatch_scales[:rows],
+ self.communication_precision["dispatch"],
+ ))
+ h.recv_semantic = semantic
+ h.recv_semantic_rows = rows
+ elif h.recv_semantic_rows != rows:
+ raise RuntimeError("MoRI receive count changed for one dispatch handle")
+ return h.recv_semantic
+
+ def oracle_dispatch_payload(self, payload):
+ return ep_precision.encode_dispatch(
+ torch, payload, self.communication_precision
+ ).semantic
+
+ def precision_evidence(self, problem, view=None):
+ return ep_precision.precision_evidence(
+ torch,
+ profile_id=self.precision_profile_id,
+ profile=self.communication_precision,
+ problem=problem,
+ view=view,
+ )
+
def finalize(self, rc):
try:
dist.barrier()
diff --git a/experimental/CollectiveX/tests/ep_nccl.py b/experimental/CollectiveX/tests/ep_nccl.py
index cab9cb8238..d440f8ecf9 100644
--- a/experimental/CollectiveX/tests/ep_nccl.py
+++ b/experimental/CollectiveX/tests/ep_nccl.py
@@ -20,6 +20,7 @@
import torch
import torch.distributed as dist
import contracts
+import ep_precision
def _runtime_collective(args, torch_module) -> tuple[str, str]:
@@ -38,6 +39,7 @@ def _runtime_collective(args, torch_module) -> tuple[str, str]:
class NCCLBackend:
name = "nccl-ep"
+ stage_device_work = False
combine_needs_redispatch = False # dispatch saves the permutation + splits
combine_weight_semantics = "unweighted-rank-sum"
@@ -47,6 +49,17 @@ def __init__(self, args, rank, world_size, local_rank, device):
self.world_size = world_size
self.device = device
self.experts = args.experts
+ self.mode = getattr(args, "mode", "normal")
+ if self.mode != "normal":
+ raise ep_precision.PrecisionError("NCCL/RCCL EP supports normal mode only")
+ self.precision_profile_id, self.communication_precision = (
+ ep_precision.resolve_precision(
+ args,
+ backend=self.name,
+ mode=self.mode,
+ supported_profiles={"d-bf16.c-bf16"},
+ )
+ )
if args.experts % world_size:
raise ValueError(f"experts({args.experts}) must divide world_size({world_size})")
self.experts_per_rank = args.experts // world_size
@@ -67,21 +80,33 @@ def __init__(self, args, rank, world_size, local_rank, device):
"collective_library": _library,
"nccl_version": _version,
"transport": f"{_library}-all_to_all_single",
- "resource_mode": "tuned",
+ "resource_mode": "fixed-profile",
"num_sms": None,
"device_sms": torch.cuda.get_device_properties(device).multi_processor_count,
"tuned_source": "nccl-collective",
"reference_semantics": "rank-deduplicated-payload-plus-routing-metadata-v2",
"routing_metadata": "expert-index-gate-weight-source-token",
+ "dispatch_dtype": "bf16",
+ "combine_dtype": "bf16",
}
def buffer_cap(self, args):
return None # no fixed pre-allocated buffer; all-to-all sizes itself per step
def make_problem(self, T, idx, weights, x):
+ encoding = ep_precision.encode_dispatch(
+ torch, x, self.communication_precision
+ )
# idx[T,topk] int64, weights[T,topk] f32, x[T,hidden] bf16 — the shared routing-trace slice.
- return types.SimpleNamespace(T=T, x=x, topk_idx=idx.to(torch.int64),
- topk_weights=weights.to(torch.float32), layout=None)
+ return types.SimpleNamespace(
+ T=T,
+ x=x,
+ oracle_x=encoding.semantic,
+ dispatch_precision_evidence=encoding.evidence,
+ topk_idx=idx.to(torch.int64),
+ topk_weights=weights.to(torch.float32),
+ layout=None,
+ )
def dispatch(self, p):
ws = self.world_size
@@ -177,6 +202,18 @@ def combine_transformed(self, p, h, transformed):
def recv_tokens(self, h):
return int(h.total_recv)
+ def oracle_dispatch_payload(self, payload):
+ return payload
+
+ def precision_evidence(self, problem, view=None):
+ return ep_precision.precision_evidence(
+ torch,
+ profile_id=self.precision_profile_id,
+ profile=self.communication_precision,
+ problem=problem,
+ view=view,
+ )
+
def finalize(self, rc):
try:
dist.barrier()
diff --git a/experimental/CollectiveX/tests/ep_precision.py b/experimental/CollectiveX/tests/ep_precision.py
new file mode 100644
index 0000000000..c4f19f54fe
--- /dev/null
+++ b/experimental/CollectiveX/tests/ep_precision.py
@@ -0,0 +1,371 @@
+#!/usr/bin/env python3
+"""Native communication-precision helpers for CollectiveX EP adapters."""
+from __future__ import annotations
+
+import inspect
+from copy import deepcopy
+from dataclasses import dataclass
+from typing import Any, Iterable
+
+import identity
+
+
+class PrecisionError(RuntimeError):
+ """A requested precision profile cannot be realized by the pinned API."""
+
+
+@dataclass(frozen=True)
+class DispatchEncoding:
+ """One dispatch input plus its post-codec semantic representation."""
+
+ native_input: Any
+ encoded_payload: Any | None
+ scales: Any | None
+ semantic: Any
+ evidence: dict[str, Any]
+
+
+def resolve_precision(
+ args,
+ *,
+ backend: str,
+ mode: str,
+ supported_profiles: Iterable[str],
+) -> tuple[str, dict[str, Any]]:
+ """Resolve and validate the exact profile requested for one adapter."""
+ profile_id = (
+ getattr(args, "precision_profile", "")
+ or identity.V1_CONTROL_PRECISION_PROFILE
+ )
+ try:
+ profile = identity.precision_profile(profile_id)
+ except identity.IdentityError as exc:
+ raise PrecisionError(str(exc)) from exc
+ if mode not in profile["modes"]:
+ raise PrecisionError(
+ f"precision profile {profile_id!r} is not valid in mode {mode!r}"
+ )
+ supported = frozenset(supported_profiles)
+ if profile_id not in supported:
+ raise PrecisionError(
+ f"{backend} does not realize precision profile {profile_id!r} in mode {mode!r}"
+ )
+ return profile_id, profile
+
+
+def require_keyword(callable_object, keyword: str, *, api: str) -> None:
+ """Fail closed when a pinned Python API does not expose a required control."""
+ try:
+ parameters = inspect.signature(callable_object).parameters
+ except (TypeError, ValueError) as exc:
+ raise PrecisionError(f"cannot inspect required precision API {api}") from exc
+ if keyword not in parameters:
+ raise PrecisionError(f"required precision API {api} omits {keyword!r}")
+
+
+def communication_format(profile: dict[str, Any], component: str) -> str:
+ """Return the exact wire format for dispatch or combine."""
+ return str(profile[component]["communication_format"])
+
+
+def is_low_precision_dispatch(profile: dict[str, Any]) -> bool:
+ return communication_format(profile, "dispatch").startswith("fp8-")
+
+
+def is_caller_prequantized(profile: dict[str, Any]) -> bool:
+ return profile["dispatch"]["quantization_origin"] == "caller-prequantized"
+
+
+def uses_logfmt_combine(profile: dict[str, Any]) -> bool:
+ return communication_format(profile, "combine") == "logfmt10"
+
+
+def uses_direct_cast_combine(profile: dict[str, Any]) -> bool:
+ return profile["combine"]["quantization_origin"] == "backend-internal-direct-cast"
+
+
+def _fp8_dtype(torch_module, axis: dict[str, Any]):
+ fmt = axis["communication_format"]
+ attribute = {
+ "fp8-e4m3fn": "float8_e4m3fn",
+ "fp8-e4m3fnuz": "float8_e4m3fnuz",
+ }.get(fmt)
+ if attribute is None:
+ raise PrecisionError(f"unsupported FP8 communication format {fmt!r}")
+ dtype = getattr(torch_module, attribute, None)
+ if dtype is None:
+ raise PrecisionError(f"active torch build does not expose torch.{attribute}")
+ return dtype
+
+
+def _axis_evidence(
+ *,
+ dequantized_semantics: bool,
+ encoded_payload_valid: bool,
+ max_abs_error: float,
+ max_rel_error: float,
+ saturation_count: int,
+ saturation_rate: float,
+ scales_finite: bool | None,
+ scales_positive: bool | None,
+ passed: bool,
+) -> dict[str, Any]:
+ return {
+ "encoded_payload_valid": bool(encoded_payload_valid),
+ "scales_finite": scales_finite,
+ "scales_positive": scales_positive,
+ "dequantized_semantics": bool(dequantized_semantics),
+ "saturation_count": int(saturation_count),
+ "saturation_rate": float(saturation_rate),
+ "max_abs_error": float(max_abs_error),
+ "max_rel_error": float(max_rel_error),
+ "passed": bool(passed),
+ }
+
+
+def exact_axis_evidence() -> dict[str, Any]:
+ """Evidence for an unquantized BF16 communication axis."""
+ return _axis_evidence(
+ encoded_payload_valid=True,
+ scales_finite=None,
+ scales_positive=None,
+ dequantized_semantics=True,
+ saturation_count=0,
+ saturation_rate=0.0,
+ max_abs_error=0.0,
+ max_rel_error=0.0,
+ passed=True,
+ )
+
+
+def _quantize_fp8(torch_module, x, axis: dict[str, Any]) -> DispatchEncoding:
+ group_size = axis["scale_group_size"]
+ if group_size != 128 or axis["scale_dtype"] != "f32":
+ raise PrecisionError("v1 FP8 dispatch requires block-128 FP32 scales")
+ if x.ndim != 2 or x.shape[1] % group_size:
+ raise PrecisionError(
+ "v1 native FP8 dispatch requires a 2D hidden dimension divisible by 128"
+ )
+ dtype = _fp8_dtype(torch_module, axis)
+ fp8_max = float(torch_module.finfo(dtype).max)
+ blocks = x.float().reshape(x.shape[0], x.shape[1] // group_size, group_size)
+ amax = blocks.abs().amax(dim=-1)
+ # Match the pinned DeepEP/HybridEP block codec, including its nonzero scale floor.
+ scales = (amax.clamp_min(1e-4) / fp8_max).to(torch_module.float32)
+ normalized = blocks / scales.unsqueeze(-1)
+ saturation_mask = normalized.abs() > fp8_max
+ encoded = normalized.clamp(min=-fp8_max, max=fp8_max).to(dtype).reshape_as(x).contiguous()
+ semantic = dequantize_dispatch(
+ torch_module, encoded, scales, axis
+ ).to(x.dtype).contiguous()
+ absolute = (semantic.float() - x.float()).abs()
+ max_abs = float(absolute.max().item()) if absolute.numel() else 0.0
+ reference_max = float(x.float().abs().max().item()) if x.numel() else 0.0
+ max_rel = max_abs / (reference_max + 1e-6)
+ saturation_count = int(saturation_mask.sum().item())
+ saturation_rate = saturation_count / max(1, int(x.numel()))
+ finite = bool(torch_module.isfinite(scales).all().item())
+ positive = bool((scales > 0).all().item())
+ semantic_ok = bool(
+ torch_module.isfinite(semantic.float()).all().item()
+ and torch_module.allclose(
+ semantic.float(), x.float(), rtol=0.05, atol=0.02
+ )
+ )
+ valid = encoded.dtype == dtype and encoded.shape == x.shape
+ evidence = _axis_evidence(
+ encoded_payload_valid=valid,
+ scales_finite=finite,
+ scales_positive=positive,
+ dequantized_semantics=semantic_ok,
+ saturation_count=saturation_count,
+ saturation_rate=saturation_rate,
+ max_abs_error=max_abs,
+ max_rel_error=max_rel,
+ passed=valid and finite and positive and semantic_ok,
+ )
+ return DispatchEncoding(
+ native_input=(encoded, scales),
+ encoded_payload=encoded,
+ scales=scales,
+ semantic=semantic,
+ evidence=evidence,
+ )
+
+
+def encode_dispatch(torch_module, x, profile: dict[str, Any]) -> DispatchEncoding:
+ """Build caller-prequantized input or a fused-codec oracle outside timing."""
+ axis = profile["dispatch"]
+ origin = axis["quantization_origin"]
+ if origin == "none":
+ return DispatchEncoding(
+ native_input=x,
+ encoded_payload=None,
+ scales=None,
+ semantic=x,
+ evidence=exact_axis_evidence(),
+ )
+ if origin not in {"caller-prequantized", "backend-fused"}:
+ raise PrecisionError(f"unsupported dispatch quantization origin {origin!r}")
+ encoded = _quantize_fp8(torch_module, x, axis)
+ if origin == "backend-fused":
+ return DispatchEncoding(
+ native_input=x,
+ encoded_payload=encoded.encoded_payload,
+ scales=encoded.scales,
+ semantic=encoded.semantic,
+ evidence=encoded.evidence,
+ )
+ return encoded
+
+
+def dequantize_dispatch(
+ torch_module,
+ encoded_payload,
+ scales,
+ axis: dict[str, Any],
+ *,
+ uint8_storage: bool = False,
+):
+ """Decode one native block-scaled FP8 payload to BF16 semantics."""
+ group_size = axis["scale_group_size"]
+ if group_size != 128 or scales is None:
+ raise PrecisionError("FP8 dispatch payload is missing block-128 scales")
+ dtype = _fp8_dtype(torch_module, axis)
+ payload = encoded_payload.view(dtype) if uint8_storage else encoded_payload
+ if payload.dtype != dtype or payload.ndim < 2 or payload.shape[-1] % group_size:
+ raise PrecisionError("native FP8 dispatch payload has an invalid dtype or shape")
+ expected_scale_shape = (*payload.shape[:-1], payload.shape[-1] // group_size)
+ if tuple(scales.shape) != expected_scale_shape or scales.dtype != torch_module.float32:
+ raise PrecisionError("native FP8 dispatch scales have an invalid dtype or shape")
+ values = payload.float().reshape(
+ *payload.shape[:-1], payload.shape[-1] // group_size, group_size
+ )
+ return (values * scales.float().reshape(*expected_scale_shape, 1)).reshape(
+ payload.shape
+ ).to(torch_module.bfloat16).contiguous()
+
+
+def validate_received_encoding(
+ torch_module,
+ *,
+ encoded_payload,
+ scales,
+ semantic,
+ axis: dict[str, Any],
+ uint8_storage: bool = False,
+) -> bool:
+ """Validate that received bytes/scales exactly decode to the semantic view."""
+ try:
+ decoded = dequantize_dispatch(
+ torch_module,
+ encoded_payload,
+ scales,
+ axis,
+ uint8_storage=uint8_storage,
+ )
+ except PrecisionError:
+ return False
+ return bool(torch_module.equal(decoded, semantic))
+
+
+def dequantize_expert_prefixes(
+ torch_module,
+ encoded_payload,
+ scales,
+ axis: dict[str, Any],
+ counts: tuple[int, ...],
+ output,
+):
+ """Decode only valid expert-packed prefixes into a reusable BF16 workspace."""
+ if encoded_payload.ndim != 3 or len(counts) != encoded_payload.shape[0]:
+ raise PrecisionError("expert-packed FP8 receive counts have an invalid shape")
+ if output.shape != encoded_payload.shape or output.dtype != torch_module.bfloat16:
+ raise PrecisionError("expert-packed BF16 stage workspace has an invalid shape")
+ capacity = encoded_payload.shape[1]
+ for expert, count in enumerate(counts):
+ if count < 0 or count > capacity:
+ raise PrecisionError("expert-packed FP8 receive count exceeds capacity")
+ if count:
+ output[expert, :count].copy_(dequantize_dispatch(
+ torch_module,
+ encoded_payload[expert, :count],
+ scales[expert, :count],
+ axis,
+ ))
+ return output
+
+
+def _direct_cast_saturation(torch_module, profile: dict[str, Any], view) -> tuple[int, float]:
+ """Count values clipped by MoRI's unscaled BF16-to-FP8 combine cast."""
+ if not all(hasattr(view, field) for field in ("payload", "expert_ids", "weights")):
+ return 0, 0.0
+ expert_ids = view.expert_ids
+ weights = view.weights
+ if expert_ids.ndim != 2 or weights.shape != expert_ids.shape:
+ return 0, 0.0
+ valid = expert_ids >= 0
+ expert = expert_ids.clamp(min=0).to(torch_module.int64)
+ gate = weights.to(torch_module.float32).masked_fill(~valid, 0)
+ scale = ((expert * 17 + 5) % 31 + 1).to(torch_module.float32) / 32
+ offset_a = (((expert * 29 + 7) % 37) - 18).to(torch_module.float32) / 64
+ offset_b = (((expert * 43 + 11) % 41) - 20).to(torch_module.float32) / 128
+ columns = torch_module.arange(
+ view.payload.shape[1], device=view.payload.device, dtype=torch_module.int64
+ )
+ pattern = (((columns * 13) % 17) - 8).to(torch_module.float32) / 8
+ transformed = (
+ view.payload.float() * (gate * scale).sum(dim=1, keepdim=True)
+ + (gate * offset_a).sum(dim=1, keepdim=True)
+ + (gate * offset_b).sum(dim=1, keepdim=True) * pattern.unsqueeze(0)
+ )
+ dtype = _fp8_dtype(torch_module, profile["combine"])
+ fp8_max = float(torch_module.finfo(dtype).max)
+ count = int((transformed.abs() > fp8_max).sum().item())
+ return count, count / max(1, int(transformed.numel()))
+
+
+def precision_evidence(
+ torch_module,
+ *,
+ profile_id: str,
+ profile: dict[str, Any],
+ problem,
+ view=None,
+ uint8_storage: bool = False,
+) -> dict[str, Any]:
+ """Return schema-shaped codec evidence; the harness adds combine-oracle errors."""
+ dispatch = deepcopy(problem.dispatch_precision_evidence)
+ if (
+ is_low_precision_dispatch(profile)
+ and view is not None
+ and all(hasattr(view, field) for field in ("encoded_payload", "scales", "payload"))
+ ):
+ valid = validate_received_encoding(
+ torch_module,
+ encoded_payload=view.encoded_payload,
+ scales=view.scales,
+ semantic=view.payload,
+ axis=profile["dispatch"],
+ uint8_storage=uint8_storage,
+ )
+ dispatch["encoded_payload_valid"] = (
+ dispatch["encoded_payload_valid"] and valid
+ )
+ dispatch["passed"] = dispatch["passed"] and valid
+ combine = exact_axis_evidence()
+ # Internal quantizers are validated by native configuration here. The harness
+ # replaces the error fields and pass bit with the transformed-combine oracle.
+ if communication_format(profile, "combine") != "bf16":
+ combine["scales_finite"] = None
+ combine["scales_positive"] = None
+ if uses_direct_cast_combine(profile) and view is not None:
+ count, rate = _direct_cast_saturation(torch_module, profile, view)
+ combine["saturation_count"] = count
+ combine["saturation_rate"] = rate
+ return {
+ "profile_id": profile_id,
+ "dispatch": dispatch,
+ "combine": combine,
+ "passed": bool(dispatch["passed"] and combine["passed"]),
+ }
diff --git a/experimental/CollectiveX/tests/ep_uccl.py b/experimental/CollectiveX/tests/ep_uccl.py
index 7d02bb2991..8eb5147416 100644
--- a/experimental/CollectiveX/tests/ep_uccl.py
+++ b/experimental/CollectiveX/tests/ep_uccl.py
@@ -1,5 +1,5 @@
#!/usr/bin/env python3
-"""CollectiveX UCCL adapter for the v1 BF16 normal-mode workload."""
+"""CollectiveX UCCL adapter for native V1 dispatch/combine precision profiles."""
from __future__ import annotations
import importlib.metadata as metadata
@@ -13,6 +13,7 @@
import torch
import torch.distributed as dist
import contracts
+import ep_precision
try:
import uccl
@@ -184,6 +185,7 @@ def _normal_buffer_sizes(hidden: int, world_size: int) -> tuple[int, int]:
class UCCLBackend:
name = "uccl"
+ stage_device_work = False
combine_needs_redispatch = False
combine_weight_semantics = "unweighted-rank-sum"
oracle_layout = "token-rank"
@@ -197,10 +199,47 @@ def __init__(self, args, rank, world_size, local_rank, device):
self.mode = getattr(args, "mode", "normal")
if self.mode not in {"normal", "low-latency"}:
raise ValueError(f"unsupported UCCL mode {self.mode!r}")
+ supported_profiles = {
+ "normal": {
+ "d-bf16.c-bf16",
+ "d-fp8-e4m3fn-b128-f32-prequantized.c-bf16",
+ },
+ "low-latency": {
+ "d-bf16.c-bf16",
+ "d-fp8-e4m3fn-b128-f32-fused.c-bf16",
+ "d-bf16.c-logfmt10-dynamic64",
+ "d-fp8-e4m3fn-b128-f32-fused.c-logfmt10-dynamic64",
+ },
+ }
+ self.precision_profile_id, self.communication_precision = (
+ ep_precision.resolve_precision(
+ args,
+ backend=self.name,
+ mode=self.mode,
+ supported_profiles=supported_profiles[self.mode],
+ )
+ )
+ self._fp8_dispatch = ep_precision.is_low_precision_dispatch(
+ self.communication_precision
+ )
+ self._use_logfmt = ep_precision.uses_logfmt_combine(
+ self.communication_precision
+ )
+ self.stage_device_work = self._fp8_dispatch
self.group = dist.group.WORLD
device_sms = torch.cuda.get_device_properties(device).multi_processor_count
if self.mode == "low-latency":
+ ep_precision.require_keyword(
+ Buffer.low_latency_dispatch,
+ "use_fp8",
+ api="uccl_deepep.Buffer.low_latency_dispatch",
+ )
+ ep_precision.require_keyword(
+ Buffer.low_latency_combine,
+ "use_logfmt",
+ api="uccl_deepep.Buffer.low_latency_combine",
+ )
if args.phase != "decode":
raise ValueError("UCCL low-latency mode only supports the decode ladder")
if args.experts % world_size:
@@ -235,6 +274,16 @@ def __init__(self, args, rank, world_size, local_rank, device):
"num_rdma_bytes": num_rdma_bytes,
}
else:
+ ep_precision.require_keyword(
+ Buffer.dispatch,
+ "async_finish",
+ api="uccl_deepep.Buffer.dispatch",
+ )
+ ep_precision.require_keyword(
+ Buffer.combine,
+ "async_finish",
+ api="uccl_deepep.Buffer.combine",
+ )
num_nvl_bytes, num_rdma_bytes = _normal_buffer_sizes(args.hidden, world_size)
if world_size > args.scale_up_domain and num_rdma_bytes == 0:
raise RuntimeError("UCCL scale-out configuration returned no RDMA buffer")
@@ -271,9 +320,13 @@ def __init__(self, args, rank, world_size, local_rank, device):
"uccl_dependency_versions": dependency_versions,
"loaded_libraries": loaded_libraries,
"mode": self.mode,
- "dispatch_dtype": "bf16",
- "combine_dtype": "bf16",
- "resource_mode": "tuned",
+ "dispatch_dtype": ep_precision.communication_format(
+ self.communication_precision, "dispatch"
+ ),
+ "combine_dtype": ep_precision.communication_format(
+ self.communication_precision, "combine"
+ ),
+ "resource_mode": "fixed-profile",
"device_sms": device_sms,
**resource_provenance,
}
@@ -282,9 +335,15 @@ def buffer_cap(self, args):
return self.max_tokens_per_rank if self.mode == "low-latency" else None
def make_problem(self, T, idx, weights, x):
+ encoding = ep_precision.encode_dispatch(
+ torch, x, self.communication_precision
+ )
return types.SimpleNamespace(
T=T,
x=x,
+ dispatch_x=encoding.native_input,
+ oracle_x=encoding.semantic,
+ dispatch_precision_evidence=encoding.evidence,
topk_idx=idx.to(torch.int64),
topk_weights=weights.to(torch.float32),
)
@@ -296,7 +355,7 @@ def dispatch(self, p):
p.topk_idx,
self.max_tokens_per_rank,
self.args.experts,
- use_fp8=False,
+ use_fp8=self._fp8_dispatch, # BF16 control realizes use_fp8=False.
async_finish=False,
return_recv_hook=False,
)
@@ -313,13 +372,14 @@ def dispatch(self, p):
_,
) = self.buffer.get_dispatch_layout(p.topk_idx, self.args.experts)
recv_x, recv_topk_idx, recv_topk_weights, recv_counts, handle, _ = self.buffer.dispatch(
- p.x,
+ p.dispatch_x,
topk_idx=p.topk_idx,
topk_weights=p.topk_weights,
num_tokens_per_rank=num_tokens_per_rank,
num_tokens_per_rdma_rank=num_tokens_per_rdma_rank,
is_token_in_rank=is_token_in_rank,
num_tokens_per_expert=num_tokens_per_expert,
+ async_finish=False,
)
return types.SimpleNamespace(
recv_x=recv_x,
@@ -330,7 +390,7 @@ def dispatch(self, p):
)
def stage(self, p, h):
- h.combine_input = h.recv_x
+ h.combine_input = self._semantic_recv(h, p)
def combine(self, p, h):
if self.mode == "low-latency":
@@ -339,11 +399,14 @@ def combine(self, p, h):
p.topk_idx,
p.topk_weights,
h.handle,
+ use_logfmt=self._use_logfmt,
async_finish=False,
return_recv_hook=False,
)
return combined_x
- combined_x, _, _ = self.buffer.combine(h.combine_input, h.handle)
+ combined_x, _, _ = self.buffer.combine(
+ h.combine_input, h.handle, async_finish=False
+ )
return combined_x
def inspect_dispatch(self, p, h):
@@ -354,7 +417,9 @@ def inspect_dispatch(self, p, h):
h.recv_topk_idx,
)
return types.SimpleNamespace(
- payload=h.recv_x,
+ payload=self._semantic_recv(h, p),
+ encoded_payload=self._encoded_recv(h),
+ scales=self._recv_scales(h),
expert_ids=expert_ids,
weights=h.recv_topk_weights.masked_fill(~valid, 0),
local_expert_counts=torch.tensor(h.recv_counts, device=self.device, dtype=torch.int64),
@@ -364,8 +429,11 @@ def inspect_dispatch(self, p, h):
def inspect_expert_dispatch(self, p, h):
if self.mode != "low-latency":
raise RuntimeError("expert-packed inspection requires low-latency mode")
+ p.recv_counts = tuple(int(value) for value in h.recv_counts.tolist())
return types.SimpleNamespace(
- payload=h.recv_x,
+ payload=self._semantic_recv(h, p),
+ encoded_payload=self._encoded_recv(h),
+ scales=self._recv_scales(h),
local_expert_counts=h.recv_counts,
source_info=h.handle[0],
layout_range=h.handle[1],
@@ -373,26 +441,88 @@ def inspect_expert_dispatch(self, p, h):
def combine_transformed(self, p, h, transformed):
if self.mode == "low-latency":
- packed = torch.zeros_like(h.recv_x)
+ packed = torch.zeros(
+ self._encoded_recv(h).shape,
+ dtype=torch.bfloat16,
+ device=self._encoded_recv(h).device,
+ )
packed[h.oracle_local_expert_slots, h.oracle_packed_positions] = transformed.to(
- h.recv_x.dtype
+ packed.dtype
)
combined, _, _ = self.buffer.low_latency_combine(
packed,
p.topk_idx,
p.topk_weights,
h.handle,
+ use_logfmt=self._use_logfmt,
async_finish=False,
return_recv_hook=False,
)
return combined
- combined, _, _ = self.buffer.combine(transformed.to(h.recv_x.dtype), h.handle)
+ semantic = self._semantic_recv(h, p)
+ combined, _, _ = self.buffer.combine(
+ transformed.to(semantic.dtype), h.handle, async_finish=False
+ )
return combined
def recv_tokens(self, h):
if self.mode == "low-latency":
return int(h.recv_counts.to(torch.int64).sum().item())
- return int(h.recv_x.shape[0])
+ return int(self._encoded_recv(h).shape[0])
+
+ def _encoded_recv(self, h):
+ return h.recv_x[0] if isinstance(h.recv_x, tuple) else h.recv_x
+
+ def _recv_scales(self, h):
+ return h.recv_x[1] if isinstance(h.recv_x, tuple) else None
+
+ def _semantic_recv(self, h, problem=None):
+ if not self._fp8_dispatch:
+ return h.recv_x
+ if not hasattr(h, "recv_semantic"):
+ if self.mode == "low-latency":
+ counts = getattr(problem, "recv_counts", None)
+ if counts is None:
+ counts = tuple(int(value) for value in h.recv_counts.tolist())
+ if problem is not None:
+ problem.recv_counts = counts
+ workspace = getattr(self, "_ll_semantic_workspace", None)
+ if workspace is None:
+ encoded = self._encoded_recv(h)
+ workspace = torch.empty(
+ encoded.shape, dtype=torch.bfloat16, device=encoded.device
+ )
+ self._ll_semantic_workspace = workspace
+ h.recv_semantic = ep_precision.dequantize_expert_prefixes(
+ torch,
+ self._encoded_recv(h),
+ self._recv_scales(h),
+ self.communication_precision["dispatch"],
+ counts,
+ workspace,
+ )
+ else:
+ h.recv_semantic = ep_precision.dequantize_dispatch(
+ torch,
+ self._encoded_recv(h),
+ self._recv_scales(h),
+ self.communication_precision["dispatch"],
+ )
+ return h.recv_semantic
+
+ def oracle_dispatch_payload(self, payload):
+ return ep_precision.encode_dispatch(
+ torch, payload, self.communication_precision
+ ).semantic
+
+ def precision_evidence(self, problem, view=None):
+ return ep_precision.precision_evidence(
+ torch,
+ profile_id=self.precision_profile_id,
+ profile=self.communication_precision,
+ problem=problem,
+ view=view,
+ )
def finalize(self, rc):
# UCCL's proxy teardown can crash after results are written; preserve the real rc.
diff --git a/experimental/CollectiveX/tests/probe_precision.py b/experimental/CollectiveX/tests/probe_precision.py
new file mode 100644
index 0000000000..2a92e70795
--- /dev/null
+++ b/experimental/CollectiveX/tests/probe_precision.py
@@ -0,0 +1,1162 @@
+#!/usr/bin/env python3
+"""Bounded real-hardware capability probe for provisional CollectiveX precision cells."""
+from __future__ import annotations
+
+import argparse
+import datetime as dt
+import hashlib
+import inspect
+import json
+import os
+import platform
+import re
+import socket
+import sys
+import tempfile
+from pathlib import Path
+from types import SimpleNamespace
+from typing import Any
+
+
+HERE = Path(__file__).resolve().parent
+ROOT = HERE.parent
+sys.path[:0] = [str(HERE), str(ROOT)]
+
+import artifact_safety # noqa: E402
+import capability # noqa: E402
+import ep_harness # noqa: E402
+
+
+FORMAT = "collectivex.precision-probe.v1"
+PLAN_FORMAT = "collectivex.precision-probe-plan.v1"
+CONTROL_FORMAT = "collectivex.precision-probe-control.v1"
+RECORD_TYPE = "precision-capability-probe"
+PROBE_CONTRACT = "bounded-native-cell-v1"
+FENCE_CONTRACT = "caller-event-cross-stream-v1"
+SUPPORTED_REASON = "native-probe-passed"
+UNSUPPORTED_REASONS = frozenset({
+ "backend-construction-failed",
+ "completion-fence-failed",
+ "cross-rank-evidence-mismatch",
+ "native-operation-failed",
+ "precision-contract-mismatch",
+ "runtime-identity-mismatch",
+ "target-not-provisional",
+ "transport-fallback-detected",
+ "unsupported-native-api",
+ "unverified-execution-identity",
+})
+BACKENDS = frozenset({
+ "deepep", "deepep-v2", "deepep-hybrid", "mori", "uccl",
+})
+SHA40 = re.compile(r"[0-9a-f]{40}")
+SHA256 = re.compile(r"[0-9a-f]{64}")
+IMAGE_DIGEST = re.compile(r"sha256:[0-9a-f]{64}")
+
+
+class ProbeError(RuntimeError):
+ """A provisional cell did not produce complete native runtime evidence."""
+
+ def __init__(self, reason: str):
+ if reason not in UNSUPPORTED_REASONS:
+ raise ValueError(f"unknown precision probe reason {reason!r}")
+ super().__init__(reason)
+ self.reason = reason
+
+
+def _canonical(value: Any) -> bytes:
+ return json.dumps(
+ value, allow_nan=False, ensure_ascii=True, sort_keys=True, separators=(",", ":")
+ ).encode("ascii")
+
+
+def _sha(value: Any) -> str:
+ return hashlib.sha256(_canonical(value)).hexdigest()
+
+
+def _exact_keys(value: Any, expected: set[str], path: str) -> dict[str, Any]:
+ if not isinstance(value, dict) or set(value) != expected:
+ raise ValueError(f"{path} fields differ from {FORMAT}")
+ return value
+
+
+def _text(value: Any, path: str, *, nullable: bool = False) -> str | None:
+ if nullable and value is None:
+ return None
+ if (
+ not isinstance(value, str)
+ or not value
+ or len(value) > 4096
+ or any(ord(character) < 0x20 or ord(character) > 0x7E for character in value)
+ ):
+ raise ValueError(f"{path} is not bounded printable ASCII")
+ return value
+
+
+def _boolean(value: Any, path: str) -> bool:
+ if type(value) is not bool:
+ raise ValueError(f"{path} is not boolean")
+ return value
+
+
+def _integer(value: Any, path: str, minimum: int = 0) -> int:
+ if type(value) is not int or value < minimum:
+ raise ValueError(f"{path} is not an integer >= {minimum}")
+ return value
+
+
+def validate_manifest(document: Any) -> dict[str, Any]:
+ """Validate the closed probe format without extending publication schemas."""
+ doc = _exact_keys(document, {
+ "evidence", "format", "generated_at", "privacy", "probe_contract",
+ "record_type", "result", "schema_version", "target", "topology",
+ }, "probe")
+ if (
+ doc["format"] != FORMAT
+ or doc["record_type"] != RECORD_TYPE
+ or doc["schema_version"] != 1
+ or doc["probe_contract"] != PROBE_CONTRACT
+ ):
+ raise ValueError("probe format, record type, schema, or contract differs")
+ _text(doc["generated_at"], "probe.generated_at")
+ target = _exact_keys(doc["target"], {
+ "backend", "basis", "ep", "mode", "precision_profile", "registry_disposition",
+ "sku",
+ }, "probe.target")
+ if (
+ target["backend"] not in BACKENDS
+ or target["registry_disposition"] != "provisional"
+ or target["mode"] not in {"normal", "low-latency"}
+ ):
+ raise ValueError("probe target is not a provisional native adapter cell")
+ for field in ("basis", "precision_profile", "sku"):
+ _text(target[field], f"probe.target.{field}")
+ _integer(target["ep"], "probe.target.ep", 1)
+ topology = _exact_keys(doc["topology"], {
+ "gpus_per_node", "nodes", "placement_valid", "scale_up_domain",
+ "scale_up_transport", "scale_out_transport", "scope", "topology_class",
+ "transport", "world_size",
+ }, "probe.topology")
+ for field in ("gpus_per_node", "nodes", "scale_up_domain", "world_size"):
+ _integer(topology[field], f"probe.topology.{field}", 1)
+ for field in ("scale_up_transport", "scope", "topology_class", "transport"):
+ _text(topology[field], f"probe.topology.{field}")
+ _text(topology["scale_out_transport"], "probe.topology.scale_out_transport", nullable=True)
+ _boolean(topology["placement_valid"], "probe.topology.placement_valid")
+ result = _exact_keys(doc["result"], {
+ "disposition", "reason", "registry_mutation", "runtime_executed",
+ "static_inspection_sufficient",
+ }, "probe.result")
+ if result["disposition"] not in {"supported", "unsupported"}:
+ raise ValueError("probe result disposition is invalid")
+ expected_reason = (
+ SUPPORTED_REASON if result["disposition"] == "supported" else result["reason"]
+ )
+ if result["reason"] != expected_reason or (
+ result["disposition"] == "unsupported"
+ and result["reason"] not in UNSUPPORTED_REASONS
+ ):
+ raise ValueError("probe result reason is invalid")
+ if result["registry_mutation"] is not False or result["static_inspection_sufficient"] is not False:
+ raise ValueError("probe must never mutate or statically promote the registry")
+ _boolean(result["runtime_executed"], "probe.result.runtime_executed")
+ privacy = _exact_keys(doc["privacy"], {"contract", "sanitized"}, "probe.privacy")
+ if privacy != {"contract": "artifact-safety-v1", "sanitized": True}:
+ raise ValueError("probe privacy contract differs")
+ if result["disposition"] == "supported":
+ _validate_evidence(doc["evidence"])
+ elif doc["evidence"] is not None:
+ _validate_evidence(doc["evidence"])
+ artifact_safety.assert_publication_safe([doc])
+ return doc
+
+
+def _validate_evidence(value: Any) -> None:
+ evidence = _exact_keys(value, {
+ "api", "completion", "identity", "precision", "transport",
+ }, "probe.evidence")
+ api = _exact_keys(evidence["api"], {"calls", "signature_sha256"}, "probe.evidence.api")
+ if not isinstance(api["calls"], list) or not api["calls"]:
+ raise ValueError("probe API calls are empty")
+ for index, call in enumerate(api["calls"]):
+ item = _exact_keys(call, {"name", "signature"}, f"probe.evidence.api.calls[{index}]")
+ _text(item["name"], "probe API name")
+ _text(item["signature"], "probe API signature")
+ if not isinstance(api["signature_sha256"], str) or not SHA256.fullmatch(api["signature_sha256"]):
+ raise ValueError("probe API signature digest is invalid")
+ completion = _exact_keys(evidence["completion"], {
+ "caller_event_complete", "contract", "mode", "output_finite",
+ "verifier_stream_complete",
+ }, "probe.evidence.completion")
+ if completion["contract"] != FENCE_CONTRACT:
+ raise ValueError("probe completion contract differs")
+ _text(completion["mode"], "probe completion mode")
+ if not all(
+ _boolean(completion[field], f"probe completion {field}")
+ for field in ("caller_event_complete", "output_finite", "verifier_stream_complete")
+ ):
+ raise ValueError("probe completion evidence did not pass")
+ identity_record = _exact_keys(evidence["identity"], {
+ "backend_components", "backend_provenance_sha256", "image_digest",
+ "image_digest_verified", "image_reference", "source_sha",
+ }, "probe.evidence.identity")
+ if not SHA40.fullmatch(str(identity_record["source_sha"])):
+ raise ValueError("probe source SHA is invalid")
+ if not IMAGE_DIGEST.fullmatch(str(identity_record["image_digest"])):
+ raise ValueError("probe image digest is invalid")
+ _text(identity_record["image_reference"], "probe image reference")
+ if identity_record["image_digest_verified"] is not True:
+ raise ValueError("probe image digest is unverified")
+ if not SHA256.fullmatch(str(identity_record["backend_provenance_sha256"])):
+ raise ValueError("probe backend provenance digest is invalid")
+ components = identity_record["backend_components"]
+ if not isinstance(components, list) or not components:
+ raise ValueError("probe backend component identity is empty")
+ for component in components:
+ item = _exact_keys(component, {"revision", "role", "version"}, "probe backend component")
+ _text(item["role"], "probe backend component role")
+ _text(item["revision"], "probe backend component revision", nullable=True)
+ _text(item["version"], "probe backend component version", nullable=True)
+ if item["revision"] is None and item["version"] is None:
+ raise ValueError("probe backend component has no identity")
+ precision = _exact_keys(evidence["precision"], {
+ "combine", "correctness", "dispatch", "profile_id",
+ }, "probe.evidence.precision")
+ _text(precision["profile_id"], "probe precision profile")
+ for direction in ("dispatch", "combine"):
+ axis = _exact_keys(precision[direction], {
+ "accumulator_dtype", "accumulator_evidence", "api_input_dtype",
+ "api_output_dtype", "communication_format", "runtime_input",
+ "runtime_output", "scale_contract", "semantic_output",
+ }, f"probe.evidence.precision.{direction}")
+ for field in (
+ "accumulator_dtype", "accumulator_evidence", "api_input_dtype",
+ "api_output_dtype", "communication_format",
+ ):
+ _text(axis[field], f"probe precision {direction} {field}")
+ _validate_tensor_summary(axis["runtime_input"], f"probe precision {direction} input")
+ _validate_tensor_summary(axis["runtime_output"], f"probe precision {direction} output")
+ _validate_tensor_summary(axis["semantic_output"], f"probe precision {direction} semantic")
+ scale = _exact_keys(axis["scale_contract"], {
+ "alignment", "dtype", "finite", "group_size", "layout", "padding",
+ "positive", "runtime_shapes", "runtime_storage_dtype",
+ }, f"probe precision {direction} scales")
+ for field in ("alignment", "layout", "padding"):
+ _text(scale[field], f"probe precision {direction} scale {field}")
+ _text(scale["dtype"], "probe scale dtype", nullable=True)
+ _text(scale["runtime_storage_dtype"], "probe scale storage", nullable=True)
+ if scale["group_size"] is not None:
+ _integer(scale["group_size"], "probe scale group", 1)
+ for field in ("finite", "positive"):
+ if scale[field] is not None:
+ _boolean(scale[field], f"probe scale {field}")
+ if not isinstance(scale["runtime_shapes"], list):
+ raise ValueError("probe scale shapes are invalid")
+ correctness = precision["correctness"]
+ if not isinstance(correctness, dict) or correctness.get("passed") is not True:
+ raise ValueError("probe precision correctness did not pass")
+ transport = _exact_keys(evidence["transport"], {
+ "evidence", "fallback_used", "native_backend", "requested", "runtime_route",
+ }, "probe.evidence.transport")
+ for field in ("native_backend", "requested", "runtime_route"):
+ _text(transport[field], f"probe transport {field}")
+ if transport["fallback_used"] is not False:
+ raise ValueError("probe transport fallback is present")
+ if not isinstance(transport["evidence"], list) or not transport["evidence"]:
+ raise ValueError("probe transport evidence is empty")
+ for item in transport["evidence"]:
+ _text(item, "probe transport evidence item")
+
+
+def _validate_tensor_summary(value: Any, path: str) -> None:
+ summary = _exact_keys(value, {"finite", "rank", "shapes", "storage_dtype"}, path)
+ _text(summary["storage_dtype"], f"{path}.storage_dtype")
+ _integer(summary["rank"], f"{path}.rank", 0)
+ if summary["finite"] is not True:
+ raise ValueError(f"{path} is not finite")
+ if not isinstance(summary["shapes"], list) or not summary["shapes"]:
+ raise ValueError(f"{path} shapes are empty")
+ for shape in summary["shapes"]:
+ if not isinstance(shape, list) or any(type(item) is not int or item < 0 for item in shape):
+ raise ValueError(f"{path} shape is invalid")
+
+
+def provisional_targets() -> list[dict[str, Any]]:
+ """Return deterministic probe cells without changing their dispositions."""
+ return sorted(
+ capability.provisional_precision_targets(),
+ key=lambda item: (
+ item["sku"], item["backend"], item["ep"], item["mode"],
+ item["precision_profile"],
+ ),
+ )
+
+
+def _probe_id(target: dict[str, Any]) -> str:
+ return f"probe-{_sha({key: target[key] for key in ('backend', 'sku', 'ep', 'mode', 'precision_profile')})[:20]}"
+
+
+def _workflow_row(target: dict[str, Any]) -> dict[str, Any]:
+ topology = capability.topology_for(target["sku"], target["ep"])
+ if topology is None:
+ raise ValueError("precision probe target has no registered topology")
+ return {
+ "backend": target["backend"],
+ "basis": target["basis"],
+ "disposition": target["disposition"],
+ "ep": target["ep"],
+ "execution_weight": target["ep"],
+ "gpus_per_node": topology["gpus_per_node"],
+ "id": _probe_id(target),
+ "launcher": capability.PLATFORMS[target["sku"]]["launcher"],
+ "mode": target["mode"],
+ "n": 1,
+ "nodes": topology["nodes"],
+ "precision_profile": target["precision_profile"],
+ "scale_up_domain": topology["scale_up_domain"],
+ "sku": target["sku"],
+ }
+
+
+def workflow_plan(*, backend: str = "all", only_sku: str = "") -> dict[str, Any]:
+ targets = [
+ target for target in provisional_targets()
+ if (backend == "all" or target["backend"] == backend)
+ and (not only_sku or target["sku"] == only_sku)
+ ]
+ if backend != "all" and backend not in BACKENDS:
+ raise ValueError("precision probe backend is not registered")
+ if only_sku and only_sku not in capability.PLATFORMS:
+ raise ValueError("precision probe SKU is not registered")
+ if not targets:
+ raise ValueError("precision probe filters select no provisional cells")
+ return {
+ "format": PLAN_FORMAT,
+ "include": [_workflow_row(target) for target in targets],
+ "schema_version": 1,
+ }
+
+
+def validate_workflow_plan(document: Any) -> dict[str, Any]:
+ plan = _exact_keys(document, {"format", "include", "schema_version"}, "probe plan")
+ if plan["format"] != PLAN_FORMAT or plan["schema_version"] != 1:
+ raise ValueError("precision probe plan format differs")
+ if not isinstance(plan["include"], list):
+ raise ValueError("precision probe plan include is not a list")
+ expected = {_probe_id(target): _workflow_row(target) for target in provisional_targets()}
+ seen: set[str] = set()
+ for row in plan["include"]:
+ if not isinstance(row, dict) or row.get("id") not in expected or row != expected[row["id"]]:
+ raise ValueError("precision probe plan row differs from the capability registry")
+ if row["id"] in seen:
+ raise ValueError("precision probe plan contains a duplicate row")
+ seen.add(row["id"])
+ return plan
+
+
+def extract_control(
+ plan: Any, *, probe_id: str, sku: str, backend: str, nodes: int,
+) -> dict[str, Any]:
+ rows = [row for row in validate_workflow_plan(plan)["include"] if row["id"] == probe_id]
+ if len(rows) != 1:
+ raise ValueError("precision probe ID is not unique in the plan")
+ row = rows[0]
+ if (row["sku"], row["backend"], row["nodes"]) != (sku, backend, nodes):
+ raise ValueError("precision probe control differs from the workflow matrix")
+ target = select_target(
+ backend=row["backend"], sku=row["sku"], ep=row["ep"], mode=row["mode"],
+ precision_profile=row["precision_profile"],
+ )
+ topology = capability.topology_for(row["sku"], row["ep"])
+ if topology is None:
+ raise ValueError("precision probe control has no topology")
+ return {
+ "format": CONTROL_FORMAT,
+ "id": row["id"],
+ "launcher": row["launcher"],
+ "schema_version": 1,
+ "target": target,
+ "topology": topology,
+ }
+
+
+def validate_control(
+ document: Any, *, sku: str, backend: str, nodes: int,
+) -> dict[str, Any]:
+ control = _exact_keys(
+ document, {"format", "id", "launcher", "schema_version", "target", "topology"},
+ "probe control",
+ )
+ if control["format"] != CONTROL_FORMAT or control["schema_version"] != 1:
+ raise ValueError("precision probe control format differs")
+ expected = extract_control(
+ {"format": PLAN_FORMAT, "include": [_workflow_row(control["target"])], "schema_version": 1},
+ probe_id=control["id"], sku=sku, backend=backend, nodes=nodes,
+ )
+ if control != expected:
+ raise ValueError("precision probe control differs from the capability registry")
+ return control
+
+
+def validate_bundle(plan: Any, manifests: list[Any]) -> None:
+ rows = validate_workflow_plan(plan)["include"]
+ expected = {
+ (row["backend"], row["sku"], row["ep"], row["mode"], row["precision_profile"])
+ for row in rows
+ }
+ observed = []
+ for manifest in manifests:
+ target = validate_manifest(manifest)["target"]
+ observed.append((
+ target["backend"], target["sku"], target["ep"], target["mode"],
+ target["precision_profile"],
+ ))
+ if len(observed) != len(set(observed)) or set(observed) != expected:
+ raise ValueError("precision probe bundle does not cover the exact workflow plan")
+
+
+def select_target(
+ *, backend: str, sku: str, ep: int, mode: str, precision_profile: str
+) -> dict[str, Any]:
+ matches = [
+ item for item in provisional_targets()
+ if item["backend"] == backend and item["sku"] == sku and item["ep"] == ep
+ and item["mode"] == mode and item["precision_profile"] == precision_profile
+ ]
+ if len(matches) != 1:
+ raise ProbeError("target-not-provisional")
+ return matches[0]
+
+
+def _target_record(target: dict[str, Any]) -> dict[str, Any]:
+ return {
+ "backend": target["backend"],
+ "basis": target["basis"],
+ "ep": target["ep"],
+ "mode": target["mode"],
+ "precision_profile": target["precision_profile"],
+ "registry_disposition": "provisional",
+ "sku": target["sku"],
+ }
+
+
+def build_manifest(
+ *, target: dict[str, Any], topology: dict[str, Any], disposition: str,
+ reason: str, runtime_executed: bool, evidence: dict[str, Any] | None,
+) -> dict[str, Any]:
+ document = {
+ "evidence": evidence,
+ "format": FORMAT,
+ "generated_at": dt.datetime.now(dt.timezone.utc).isoformat(),
+ "privacy": {"contract": "artifact-safety-v1", "sanitized": True},
+ "probe_contract": PROBE_CONTRACT,
+ "record_type": RECORD_TYPE,
+ "result": {
+ "disposition": disposition,
+ "reason": reason,
+ "registry_mutation": False,
+ "runtime_executed": runtime_executed,
+ "static_inspection_sufficient": False,
+ },
+ "schema_version": 1,
+ "target": _target_record(target),
+ "topology": topology,
+ }
+ return validate_manifest(document)
+
+
+def _write_atomic(path: Path, document: dict[str, Any]) -> None:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ descriptor, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent)
+ try:
+ os.fchmod(descriptor, 0o600)
+ with os.fdopen(descriptor, "wb") as handle:
+ handle.write(_canonical(document) + b"\n")
+ handle.flush()
+ os.fsync(handle.fileno())
+ os.replace(temporary, path)
+ finally:
+ if os.path.exists(temporary):
+ os.unlink(temporary)
+
+
+def _dtype_name(dtype: Any) -> str:
+ return str(dtype).removeprefix("torch.")
+
+
+def _local_tensor_summary(torch_module, tensor) -> dict[str, Any]:
+ return {
+ "finite": bool(torch_module.isfinite(tensor.float()).all().item()),
+ "rank": int(tensor.ndim),
+ "shape": [int(item) for item in tensor.shape],
+ "storage_dtype": _dtype_name(tensor.dtype),
+ }
+
+
+def _aggregate_tensor_summaries(records: list[dict[str, Any]]) -> dict[str, Any]:
+ dtypes = {record["storage_dtype"] for record in records}
+ ranks = {record["rank"] for record in records}
+ if len(dtypes) != 1 or len(ranks) != 1:
+ raise ProbeError("cross-rank-evidence-mismatch")
+ return {
+ "finite": all(record["finite"] for record in records),
+ "rank": ranks.pop(),
+ "shapes": sorted({tuple(record["shape"]) for record in records}),
+ "storage_dtype": dtypes.pop(),
+ }
+
+
+def _scale_contract(torch_module, axis: dict[str, Any], scales) -> dict[str, Any]:
+ return {
+ "alignment": axis["alignment_contract"],
+ "dtype": axis["scale_dtype"],
+ "finite": (
+ bool(torch_module.isfinite(scales.float()).all().item())
+ if scales is not None else None
+ ),
+ "group_size": axis["scale_group_size"],
+ "layout": axis["scale_layout"],
+ "padding": axis["padding_contract"],
+ "positive": bool((scales > 0).all().item()) if scales is not None else None,
+ "runtime_shape": [int(item) for item in scales.shape] if scales is not None else None,
+ "runtime_storage_dtype": _dtype_name(scales.dtype) if scales is not None else None,
+ }
+
+
+def _aggregate_scale_contracts(records: list[dict[str, Any]]) -> dict[str, Any]:
+ fixed_fields = (
+ "alignment", "dtype", "group_size", "layout", "padding", "runtime_storage_dtype",
+ )
+ result: dict[str, Any] = {}
+ for field in fixed_fields:
+ values = {_canonical(record[field]) for record in records}
+ if len(values) != 1:
+ raise ProbeError("cross-rank-evidence-mismatch")
+ result[field] = records[0][field]
+ for field in ("finite", "positive"):
+ values = [record[field] for record in records]
+ result[field] = None if all(value is None for value in values) else all(value is True for value in values)
+ result["runtime_shapes"] = sorted({
+ tuple(record["runtime_shape"] or ()) for record in records
+ }) if any(record["runtime_shape"] is not None for record in records) else []
+ return result
+
+
+def _signature(callable_object: Any, name: str) -> dict[str, str]:
+ try:
+ signature = str(inspect.signature(callable_object))
+ except (TypeError, ValueError) as exc:
+ raise ProbeError("unsupported-native-api") from exc
+ _text(signature, f"native API {name} signature")
+ return {"name": name, "signature": signature}
+
+
+def _api_evidence(backend_name: str, backend) -> dict[str, Any]:
+ if backend_name in {"deepep", "uccl"}:
+ native = type(backend.buffer)
+ dispatch_name = "low_latency_dispatch" if backend.mode == "low-latency" else "dispatch"
+ combine_name = "low_latency_combine" if backend.mode == "low-latency" else "combine"
+ calls = [
+ _signature(native.__init__, f"{native.__name__}.__init__"),
+ _signature(getattr(native, dispatch_name), f"{native.__name__}.{dispatch_name}"),
+ _signature(getattr(native, combine_name), f"{native.__name__}.{combine_name}"),
+ ]
+ elif backend_name in {"deepep-v2", "deepep-hybrid"}:
+ native = type(backend.buffer)
+ calls = [
+ _signature(native.__init__, f"{native.__name__}.__init__"),
+ _signature(native.dispatch, f"{native.__name__}.dispatch"),
+ _signature(native.combine, f"{native.__name__}.combine"),
+ ]
+ elif backend_name == "mori":
+ native = type(backend.op)
+ calls = [
+ _signature(type(backend.config).__init__, "EpDispatchCombineConfig.__init__"),
+ _signature(native.dispatch, f"{native.__name__}.dispatch"),
+ _signature(native.combine, f"{native.__name__}.combine"),
+ ]
+ else: # pragma: no cover - guarded by target registry
+ raise ProbeError("unsupported-native-api")
+ return {"calls": calls, "signature_sha256": _sha(calls)}
+
+
+def _completion_mode(backend_name: str, mode: str) -> str:
+ if backend_name in {"deepep", "uccl"}:
+ return "async_finish=false;return_recv_hook=false"
+ if backend_name == "deepep-v2":
+ return "async_with_compute_stream=false;do_cpu_sync=true"
+ if backend_name == "deepep-hybrid":
+ return "metadata-nonblocking=false;caller-stream-ordered"
+ if backend_name == "mori":
+ return "current-stream-ordered"
+ raise ProbeError("unsupported-native-api")
+
+
+def _transport_evidence(backend_name: str, backend, args) -> dict[str, Any]:
+ provenance = backend.backend_provenance
+ fallback = False
+ facts: list[str]
+ if backend_name == "deepep":
+ if args.scope == "scale-out" and int(provenance["num_rdma_bytes"]) <= 0:
+ fallback = True
+ if args.scale_up_transport == "mnnvl" and provenance["mnnvl_comm"] != "explicit-allow-mnnvl":
+ fallback = True
+ route = f"deepep-{backend.mode}"
+ facts = [
+ f"mnnvl={provenance['mnnvl_comm']}",
+ f"nvl-buffer={int(provenance['num_nvl_bytes']) > 0}",
+ f"rdma-buffer={int(provenance['num_rdma_bytes']) > 0}",
+ ]
+ elif backend_name == "uccl":
+ scratch_location = str(backend.buffer.scratch.device.type)
+ rdma_active = int(provenance["num_rdma_bytes"]) > 0
+ fallback = rdma_active and scratch_location != "cuda"
+ route = f"uccl-proxy-{backend.mode}"
+ facts = [f"rdma-buffer={rdma_active}", f"rdma-memory={scratch_location}"]
+ elif backend_name == "deepep-v2":
+ expected_gin = args.scope == "scale-out"
+ fallback = bool(provenance["gin_enabled"]) != expected_gin
+ route = str(provenance["communication_backend"])
+ facts = [
+ f"gin-enabled={bool(provenance['gin_enabled'])}",
+ f"nccl-communicator={provenance['nccl_communicator']}",
+ ]
+ elif backend_name == "deepep-hybrid":
+ route = str(provenance["transport"])
+ expected_build = "multinode-doca" if args.scope == "scale-out" else "intradomain"
+ realized_build = os.environ.get("DEEPEP_HYBRID_BUILD_MODE")
+ fallback = realized_build != expected_build
+ facts = [f"build-mode={realized_build or 'missing'}", f"domains={backend.communication_domains}"]
+ elif backend_name == "mori":
+ route = str(backend.kernel_generation)
+ expected_kernel = (
+ "inter-node-v1" if args.scope == "scale-out"
+ else "async-ll" if args.runner == "mi325x" else "intranode"
+ )
+ fallback = route != expected_kernel
+ facts = [
+ f"kernel={route}",
+ f"external-input={bool(provenance['use_external_inp_buf'])}",
+ f"qps={int(provenance['num_qps'])}",
+ ]
+ else: # pragma: no cover - guarded by registry
+ raise ProbeError("unsupported-native-api")
+ return {
+ "evidence": sorted(facts),
+ "fallback_used": fallback,
+ "native_backend": backend_name,
+ "requested": str(args.transport),
+ "runtime_route": route,
+ }
+
+
+def _component_identities(backend_name: str, provenance: dict[str, Any]) -> list[dict[str, Any]]:
+ if backend_name == "deepep":
+ values = [("deepep", provenance.get("deepep_commit"), provenance.get("deepep_version"))]
+ elif backend_name == "deepep-v2":
+ values = [
+ ("deepep-v2", provenance.get("deepep_commit"), provenance.get("deepep_version")),
+ ("deepep-tree", provenance.get("deepep_tree"), None),
+ ("fmt", provenance.get("fmt_commit"), None),
+ ]
+ elif backend_name == "deepep-hybrid":
+ values = [
+ ("deepep-hybrid", provenance.get("deepep_commit"), None),
+ ("deepep-tree", provenance.get("deepep_tree"), None),
+ ]
+ elif backend_name == "uccl":
+ values = [
+ ("uccl", provenance.get("uccl_commit"), provenance.get("uccl_version")),
+ ("uccl-wrapper", provenance.get("uccl_wrapper_commit"), None),
+ ]
+ elif backend_name == "mori":
+ values = [("mori", provenance.get("mori_commit"), None)]
+ else: # pragma: no cover
+ raise ProbeError("unsupported-native-api")
+ result = [
+ {"revision": revision, "role": role, "version": version}
+ for role, revision, version in values
+ ]
+ for item in result:
+ if item["revision"] is None and item["version"] is None:
+ raise ProbeError("unverified-execution-identity")
+ return result
+
+
+def _execution_identity(backend_name: str, backend) -> dict[str, Any]:
+ source_sha = os.environ.get("COLLECTIVEX_SOURCE_SHA") or os.environ.get("GITHUB_SHA")
+ image_reference = os.environ.get("COLLECTIVEX_IMAGE")
+ image_digest = os.environ.get("COLLECTIVEX_IMAGE_DIGEST")
+ verified = os.environ.get("COLLECTIVEX_IMAGE_DIGEST_VERIFIED") == "1"
+ if (
+ not isinstance(source_sha, str) or not SHA40.fullmatch(source_sha)
+ or not isinstance(image_reference, str) or not image_reference
+ or not isinstance(image_digest, str) or not IMAGE_DIGEST.fullmatch(image_digest)
+ or not verified
+ ):
+ raise ProbeError("unverified-execution-identity")
+ provenance = backend.backend_provenance
+ return {
+ "backend_components": _component_identities(backend_name, provenance),
+ "backend_provenance_sha256": _sha(provenance),
+ "image_digest": image_digest,
+ "image_digest_verified": True,
+ "image_reference": image_reference,
+ "source_sha": source_sha,
+ }
+
+
+def _correctness_aggregate(records: list[dict[str, Any]]) -> dict[str, Any]:
+ profile_ids = {record["profile_id"] for record in records}
+ if len(profile_ids) != 1:
+ raise ProbeError("cross-rank-evidence-mismatch")
+ result: dict[str, Any] = {"profile_id": profile_ids.pop()}
+ for direction in ("dispatch", "combine"):
+ axes = [record[direction] for record in records]
+ scale_finite = [axis["scales_finite"] for axis in axes]
+ scale_positive = [axis["scales_positive"] for axis in axes]
+ result[direction] = {
+ "dequantized_semantics": all(axis["dequantized_semantics"] for axis in axes),
+ "encoded_payload_valid": all(axis["encoded_payload_valid"] for axis in axes),
+ "max_abs_error": max(float(axis["max_abs_error"]) for axis in axes),
+ "max_rel_error": max(float(axis["max_rel_error"]) for axis in axes),
+ "passed": all(axis["passed"] for axis in axes),
+ "saturation_count": sum(int(axis["saturation_count"]) for axis in axes),
+ "saturation_rate": max(float(axis["saturation_rate"]) for axis in axes),
+ "scales_finite": (
+ None if all(value is None for value in scale_finite)
+ else all(value is True for value in scale_finite)
+ ),
+ "scales_positive": (
+ None if all(value is None for value in scale_positive)
+ else all(value is True for value in scale_positive)
+ ),
+ }
+ result["passed"] = all(record["passed"] for record in records) and all(
+ result[direction]["passed"] for direction in ("dispatch", "combine")
+ )
+ return result
+
+
+def _topology_record(topology: dict[str, Any], placement_valid: bool) -> dict[str, Any]:
+ return {
+ "gpus_per_node": topology["gpus_per_node"],
+ "nodes": topology["nodes"],
+ "placement_valid": placement_valid,
+ "scale_up_domain": topology["scale_up_domain"],
+ "scale_up_transport": topology["scale_up_transport"],
+ "scale_out_transport": topology["scale_out_transport"],
+ "scope": topology["scope"],
+ "topology_class": topology["topology_class"],
+ "transport": topology["transport"],
+ "world_size": topology["nodes"] * topology["gpus_per_node"],
+ }
+
+
+def _backend_class(name: str):
+ if name == "deepep":
+ from ep_deepep import DeepEPBackend
+ return DeepEPBackend
+ if name == "deepep-v2":
+ from ep_deepep_v2 import DeepEPV2Backend
+ return DeepEPV2Backend
+ if name == "deepep-hybrid":
+ from ep_deepep_hybrid import DeepEPHybridBackend
+ return DeepEPHybridBackend
+ if name == "uccl":
+ from ep_uccl import UCCLBackend
+ return UCCLBackend
+ if name == "mori":
+ from ep_mori import MoRIBackend
+ return MoRIBackend
+ raise ProbeError("unsupported-native-api")
+
+
+def _runtime_args(target: dict[str, Any], topology: dict[str, Any], fingerprint: dict[str, Any]):
+ return SimpleNamespace(
+ backend=target["backend"],
+ eplb=False,
+ experts=256,
+ gpus_per_node=topology["gpus_per_node"],
+ hidden=7168,
+ mode=target["mode"],
+ num_logical_experts=256,
+ num_sms=24,
+ phase="decode",
+ precision_profile=target["precision_profile"],
+ runner=target["sku"],
+ runtime_fingerprint=fingerprint,
+ scale_out_transport=topology["scale_out_transport"] or "",
+ scale_up_domain=topology["scale_up_domain"],
+ scale_up_transport=topology["scale_up_transport"],
+ scope=topology["scope"],
+ tokens_ladder="8",
+ topk=8,
+ topology_class=topology["topology_class"],
+ transport=topology["transport"],
+ )
+
+
+def _init_distributed(torch_module, dist, backend_name: str, device, rank: int, world_size: int) -> None:
+ if dist.is_initialized():
+ return
+ if backend_name == "mori":
+ dist.init_process_group(
+ backend="cpu:gloo,cuda:nccl", rank=rank, world_size=world_size, device_id=device
+ )
+ elif backend_name == "deepep-v2":
+ dist.init_process_group("nccl", device_id=device)
+ else:
+ dist.init_process_group("nccl")
+
+
+def _runtime_context(torch_module, dist, target: dict[str, Any], device, local_rank: int):
+ import run_ep
+
+ world_size = dist.get_world_size()
+ topology = capability.topology_for(target["sku"], target["ep"])
+ if topology is None or world_size != target["ep"]:
+ raise ProbeError("runtime-identity-mismatch")
+ machine = {"x86_64": "amd64", "aarch64": "arm64"}.get(
+ platform.machine(), platform.machine()
+ )
+ properties = torch_module.cuda.get_device_properties(device)
+ if torch_module.version.hip:
+ vendor = "amd"
+ arch = str(getattr(properties, "gcnArchName", "")).split(":", 1)[0]
+ else:
+ vendor = "nvidia"
+ major, minor = torch_module.cuda.get_device_capability(device)
+ arch = f"sm{major}{minor}"
+ fingerprint = run_ep._runtime_fingerprint(
+ torch_module, device, machine=machine, vendor=vendor, arch=arch
+ )
+ issues = capability.runtime_identity_issues(
+ target["sku"], vendor=vendor, arch=arch, machine=machine,
+ device_name=torch_module.cuda.get_device_name(device),
+ device_count=torch_module.cuda.device_count(), world_size=world_size,
+ )
+ records: list[Any] = [None] * world_size
+ dist.all_gather_object(records, (socket.gethostname(), local_rank, fingerprint, issues))
+ if any(record[3] for record in records):
+ raise ProbeError("runtime-identity-mismatch")
+ placement = run_ep._summarize_realized_placement(
+ [(record[0], record[1]) for record in records],
+ expected_nodes=topology["nodes"],
+ expected_gpus_per_node=topology["gpus_per_node"],
+ expected_world_size=world_size,
+ )
+ common_fingerprint = run_ep._common_runtime_fingerprint([record[2] for record in records])
+ return topology, placement, common_fingerprint
+
+
+def _local_probe(torch_module, dist, target: dict[str, Any], backend, args, rank: int):
+ import routing
+
+ tokens = 8
+ global_idx, global_weights = routing.build_global_routing(
+ tokens * target["ep"], args.experts, args.topk, "uniform", ep_harness.ROUTING_SEED
+ )
+ local_idx, local_weights = routing.rank_slice(
+ global_idx, global_weights, rank, tokens
+ )
+ x = routing.rank_activations(
+ tokens, args.hidden, ep_harness.ROUTING_SEED, rank,
+ torch_module.device(f"cuda:{int(os.environ.get('LOCAL_RANK', '0'))}"),
+ torch_module.bfloat16,
+ )
+ problem = backend.make_problem(
+ tokens, local_idx.to(x.device), local_weights.to(x.device), x
+ )
+ oracle = ep_harness._run_expert_oracle(
+ torch_module, routing, backend, problem, global_idx, global_weights, rank,
+ args.experts // target["ep"], ep_harness.ROUTING_SEED,
+ )
+ if not oracle["passed"] or not oracle["_precision"]["passed"]:
+ raise ProbeError("precision-contract-mismatch")
+
+ caller = torch_module.cuda.Stream(device=x.device)
+ verifier = torch_module.cuda.Stream(device=x.device)
+ completion_event = torch_module.cuda.Event()
+ with torch_module.cuda.stream(caller):
+ handle = backend.dispatch(problem)
+ problem.recv_tokens = backend.recv_tokens(handle)
+ view = (
+ backend.inspect_expert_dispatch(problem, handle)
+ if target["mode"] == "low-latency"
+ else backend.inspect_dispatch(problem, handle)
+ )
+ backend.stage(problem, handle)
+ combined = backend.combine(problem, handle)
+ completion_event.record(caller)
+ with torch_module.cuda.stream(verifier):
+ verifier.wait_event(completion_event)
+ verifier_sentinel = combined.float().abs().sum()
+ verifier.synchronize()
+ completion = {
+ "caller_event_complete": bool(completion_event.query()),
+ "contract": FENCE_CONTRACT,
+ "mode": _completion_mode(target["backend"], target["mode"]),
+ "output_finite": bool(torch_module.isfinite(combined.float()).all().item()),
+ "verifier_stream_complete": bool(torch_module.isfinite(verifier_sentinel).item()),
+ }
+ if not all(
+ completion[field]
+ for field in ("caller_event_complete", "output_finite", "verifier_stream_complete")
+ ):
+ raise ProbeError("completion-fence-failed")
+
+ deferred = getattr(backend, "capture_deferred_provenance", None)
+ if deferred is not None:
+ deferred()
+ dispatch_input = problem.dispatch_x[0] if isinstance(problem.dispatch_x, tuple) else problem.dispatch_x
+ dispatch_input_scales = (
+ problem.dispatch_x[1] if isinstance(problem.dispatch_x, tuple)
+ else getattr(problem, "dispatch_scales", None)
+ or getattr(problem, "scales", None)
+ )
+ dispatch_axis = backend.communication_precision["dispatch"]
+ combine_axis = backend.communication_precision["combine"]
+ local = {
+ "api": _api_evidence(target["backend"], backend),
+ "completion": completion,
+ "identity": _execution_identity(target["backend"], backend),
+ "precision": {
+ "profile_id": backend.precision_profile_id,
+ "correctness": oracle["_precision"],
+ "dispatch": {
+ "accumulator_dtype": "not-applicable",
+ "accumulator_evidence": "not-applicable",
+ "api_input_dtype": dispatch_axis["api_input_dtype"],
+ "api_output_dtype": dispatch_axis["api_output_dtype"],
+ "communication_format": dispatch_axis["communication_format"],
+ "runtime_input": _local_tensor_summary(torch_module, dispatch_input),
+ "runtime_output": _local_tensor_summary(torch_module, view.encoded_payload),
+ "scale_contract": _scale_contract(
+ torch_module, dispatch_axis,
+ view.scales if view.scales is not None else dispatch_input_scales,
+ ),
+ "semantic_output": _local_tensor_summary(torch_module, view.payload),
+ },
+ "combine": {
+ "accumulator_dtype": "fp32",
+ "accumulator_evidence": "pinned-source-image-plus-runtime-oracle",
+ "api_input_dtype": combine_axis["api_input_dtype"],
+ "api_output_dtype": combine_axis["api_output_dtype"],
+ "communication_format": combine_axis["communication_format"],
+ "runtime_input": _local_tensor_summary(torch_module, handle.combine_input),
+ "runtime_output": _local_tensor_summary(torch_module, combined),
+ "scale_contract": _scale_contract(torch_module, combine_axis, None),
+ "semantic_output": _local_tensor_summary(torch_module, combined),
+ },
+ },
+ "transport": _transport_evidence(target["backend"], backend, args),
+ }
+ if local["transport"]["fallback_used"]:
+ raise ProbeError("transport-fallback-detected")
+ return local
+
+
+def _aggregate_local(records: list[dict[str, Any]]) -> dict[str, Any]:
+ for field in ("api", "completion", "identity", "transport"):
+ values = {_canonical(record[field]) for record in records}
+ if len(values) != 1:
+ raise ProbeError("cross-rank-evidence-mismatch")
+ profile_ids = {record["precision"]["profile_id"] for record in records}
+ if len(profile_ids) != 1:
+ raise ProbeError("cross-rank-evidence-mismatch")
+ precision: dict[str, Any] = {
+ "profile_id": profile_ids.pop(),
+ "correctness": _correctness_aggregate([
+ record["precision"]["correctness"] for record in records
+ ]),
+ }
+ for direction in ("dispatch", "combine"):
+ axes = [record["precision"][direction] for record in records]
+ fixed = (
+ "accumulator_dtype", "accumulator_evidence", "api_input_dtype",
+ "api_output_dtype", "communication_format",
+ )
+ result: dict[str, Any] = {}
+ for field in fixed:
+ if len({axis[field] for axis in axes}) != 1:
+ raise ProbeError("cross-rank-evidence-mismatch")
+ result[field] = axes[0][field]
+ for field in ("runtime_input", "runtime_output", "semantic_output"):
+ result[field] = _aggregate_tensor_summaries([axis[field] for axis in axes])
+ result["scale_contract"] = _aggregate_scale_contracts([
+ axis["scale_contract"] for axis in axes
+ ])
+ precision[direction] = result
+ return {
+ "api": records[0]["api"],
+ "completion": records[0]["completion"],
+ "identity": records[0]["identity"],
+ "precision": precision,
+ "transport": records[0]["transport"],
+ }
+
+
+def _finalize(backend, dist) -> None:
+ if backend is not None:
+ backend.finalize(0)
+ return
+ if dist.is_initialized():
+ dist.destroy_process_group()
+
+
+def run_target(target: dict[str, Any], output: Path) -> int:
+ try:
+ import torch
+ import torch.distributed as dist
+ except Exception as exc: # pragma: no cover - diagnostic runtime requirement
+ raise ProbeError("runtime-identity-mismatch") from exc
+ rank = int(os.environ.get("RANK", "0"))
+ world_size = int(os.environ.get("WORLD_SIZE", "1"))
+ local_rank = int(os.environ.get("LOCAL_RANK", "0"))
+ os.environ.setdefault("MASTER_ADDR", "localhost")
+ os.environ.setdefault("MASTER_PORT", "12355")
+ torch.cuda.set_device(local_rank)
+ device = torch.device(f"cuda:{local_rank}")
+ _init_distributed(torch, dist, target["backend"], device, rank, world_size)
+ backend = None
+ topology = capability.topology_for(target["sku"], target["ep"])
+ topology_record = _topology_record(topology, False) if topology is not None else {
+ "gpus_per_node": 1, "nodes": target["ep"], "placement_valid": False,
+ "scale_up_domain": 1, "scale_up_transport": "unknown",
+ "scale_out_transport": None, "scope": "scale-out",
+ "topology_class": "unknown", "transport": "unknown", "world_size": target["ep"],
+ }
+ try:
+ topology, placement, fingerprint = _runtime_context(
+ torch, dist, target, device, local_rank
+ )
+ topology_record = _topology_record(topology, bool(placement["valid"]))
+ args = _runtime_args(target, topology, fingerprint)
+ try:
+ backend = _backend_class(target["backend"])(
+ args, rank, world_size, local_rank, device
+ )
+ construction = {"ok": True}
+ except Exception:
+ construction = {"ok": False, "reason": "backend-construction-failed"}
+ gathered: list[Any] = [None] * world_size
+ dist.all_gather_object(gathered, construction)
+ if not all(record.get("ok") is True for record in gathered):
+ manifest = build_manifest(
+ target=target, topology=topology_record, disposition="unsupported",
+ reason="backend-construction-failed", runtime_executed=True, evidence=None,
+ )
+ else:
+ try:
+ local = {"ok": True, "evidence": _local_probe(
+ torch, dist, target, backend, args, rank
+ )}
+ except ProbeError as exc:
+ local = {"ok": False, "reason": exc.reason}
+ except Exception:
+ local = {"ok": False, "reason": "native-operation-failed"}
+ gathered = [None] * world_size
+ dist.all_gather_object(gathered, local)
+ if not all(record.get("ok") is True for record in gathered):
+ reasons = {record.get("reason") for record in gathered}
+ reason = reasons.pop() if len(reasons) == 1 else "cross-rank-evidence-mismatch"
+ manifest = build_manifest(
+ target=target, topology=topology_record, disposition="unsupported",
+ reason=reason, runtime_executed=True, evidence=None,
+ )
+ else:
+ evidence = _aggregate_local([record["evidence"] for record in gathered])
+ manifest = build_manifest(
+ target=target, topology=topology_record, disposition="supported",
+ reason=SUPPORTED_REASON, runtime_executed=True, evidence=evidence,
+ )
+ except ProbeError as exc:
+ manifest = build_manifest(
+ target=target, topology=topology_record, disposition="unsupported",
+ reason=exc.reason, runtime_executed=False, evidence=None,
+ )
+ if rank == 0:
+ _write_atomic(output, manifest)
+ print(json.dumps(manifest, allow_nan=False, sort_keys=True, separators=(",", ":")))
+ dist.barrier()
+ _finalize(backend, dist)
+ return 0
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--list-targets", action="store_true")
+ parser.add_argument("--workflow-plan", action="store_true")
+ parser.add_argument("--extract-from", type=Path)
+ parser.add_argument("--probe-id")
+ parser.add_argument("--validate-control", type=Path)
+ parser.add_argument("--validate-manifest", type=Path, nargs="+")
+ parser.add_argument("--validate-bundle", type=Path)
+ parser.add_argument("--backend", choices=sorted(BACKENDS | {"all"}))
+ parser.add_argument("--sku")
+ parser.add_argument("--only-sku", default="")
+ parser.add_argument("--expect-sku")
+ parser.add_argument("--expect-backend")
+ parser.add_argument("--expect-nodes", type=int)
+ parser.add_argument("--ep", type=int)
+ parser.add_argument("--mode", choices=("normal", "low-latency"))
+ parser.add_argument("--precision-profile")
+ parser.add_argument("--out", type=Path)
+ args = parser.parse_args()
+ if args.list_targets:
+ print(json.dumps(provisional_targets(), allow_nan=False, sort_keys=True, separators=(",", ":")))
+ return 0
+ if args.workflow_plan:
+ plan = workflow_plan(backend=args.backend or "all", only_sku=args.only_sku)
+ if args.out is None:
+ print(json.dumps(plan, allow_nan=False, sort_keys=True, separators=(",", ":")))
+ else:
+ _write_atomic(args.out, plan)
+ return 0
+ if args.extract_from is not None:
+ if None in (args.probe_id, args.expect_sku, args.expect_backend, args.expect_nodes, args.out):
+ parser.error("probe extraction requires ID, expected placement, and --out")
+ control = extract_control(
+ json.loads(args.extract_from.read_text()), probe_id=args.probe_id,
+ sku=args.expect_sku, backend=args.expect_backend, nodes=args.expect_nodes,
+ )
+ _write_atomic(args.out, control)
+ return 0
+ if args.validate_control is not None:
+ if None in (args.expect_sku, args.expect_backend, args.expect_nodes):
+ parser.error("control validation requires expected placement")
+ validate_control(
+ json.loads(args.validate_control.read_text()), sku=args.expect_sku,
+ backend=args.expect_backend, nodes=args.expect_nodes,
+ )
+ return 0
+ if args.validate_bundle is not None:
+ if not args.validate_manifest:
+ parser.error("bundle validation requires manifest paths")
+ validate_bundle(
+ json.loads(args.validate_bundle.read_text()),
+ [json.loads(path.read_text()) for path in args.validate_manifest],
+ )
+ return 0
+ if args.validate_manifest is not None:
+ for path in args.validate_manifest:
+ validate_manifest(json.loads(path.read_text()))
+ return 0
+ if any(
+ value is None
+ for value in (args.backend, args.sku, args.ep, args.mode, args.precision_profile, args.out)
+ ):
+ parser.error("one exact --backend/--sku/--ep/--mode/--precision-profile/--out cell is required")
+ try:
+ target = select_target(
+ backend=args.backend, sku=args.sku, ep=args.ep, mode=args.mode,
+ precision_profile=args.precision_profile,
+ )
+ return run_target(target, args.out)
+ except ProbeError as exc:
+ parser.error(exc.reason)
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/experimental/CollectiveX/tests/routing.py b/experimental/CollectiveX/tests/routing.py
index 6065a06e43..ee4eb12a8c 100644
--- a/experimental/CollectiveX/tests/routing.py
+++ b/experimental/CollectiveX/tests/routing.py
@@ -24,19 +24,32 @@
import torch
-ACTIVATION_GENERATOR = "collectivex-activation-counter-v3"
-SOURCE_ID_BASE = 128
-SOURCE_ID_COLUMNS = 4
+ACTIVATION_GENERATOR = "collectivex-activation-counter-v4"
+SOURCE_ID_BITS = 32
+SOURCE_CHECKSUM_BITS = 16
+SOURCE_ID_COLUMNS = SOURCE_ID_BITS + SOURCE_CHECKSUM_BITS
+SOURCE_ID_CONTRACT = "bounded-sign-bit-source-v1"
def build_global_routing(
- global_tokens: int, experts: int, topk: int, routing: str, seed: int
+ global_tokens: int,
+ experts: int,
+ topk: int,
+ routing: str,
+ seed: int,
+ *,
+ token_offset: int = 0,
):
- """Return byte-stable counter-generated routing tensors on CPU."""
+ """Return one byte-stable counter-generated routing window on CPU."""
import workload
indices, weights = workload.canonical_routing_rows(
- int(global_tokens), int(experts), int(topk), routing, int(seed)
+ int(global_tokens),
+ int(experts),
+ int(topk),
+ routing,
+ int(seed),
+ token_offset=token_offset,
)
return (
torch.tensor(indices, dtype=torch.int64),
@@ -51,7 +64,7 @@ def rank_slice(idx, weights, rank: int, tokens_per_rank: int):
def rank_activations(tokens: int, hidden: int, seed: int, rank: int, device,
dtype=torch.bfloat16):
- """Exact counter-derived inputs with a reversible global source-token prefix."""
+ """Exact counter-derived inputs with a quantization-safe source-token prefix."""
source = torch.arange(tokens, device=device, dtype=torch.int64) + rank * tokens
return activations_for_source_ids(source, hidden, seed, dtype)
@@ -64,10 +77,19 @@ def activations_for_source_ids(source, hidden: int, seed: int, dtype=torch.bfloa
column = torch.arange(hidden, device=source.device, dtype=torch.int64)
values = (source[:, None] * 131 + column[None, :] * 17 + int(seed) * 19) % 257 - 128
output = values.to(dtype).mul_(1 / 64)
- output[:, 0] = source % SOURCE_ID_BASE
- output[:, 1] = (source // SOURCE_ID_BASE) % SOURCE_ID_BASE
- output[:, 2] = (source // (SOURCE_ID_BASE**2)) % SOURCE_ID_BASE
- output[:, 3] = (source * 29 + int(seed) * 7) % SOURCE_ID_BASE
+ if bool((source < 0).any().item()) or bool((source >= (1 << SOURCE_ID_BITS)).any().item()):
+ raise ValueError("source token ID is outside the bounded identity contract")
+ source_columns = torch.arange(SOURCE_ID_BITS, device=source.device, dtype=torch.int64)
+ source_bits = ((source[:, None] >> source_columns[None, :]) & 1) * 2 - 1
+ checksum = (source * 0x9E37 + int(seed) * 0xA24B) & ((1 << SOURCE_CHECKSUM_BITS) - 1)
+ checksum_columns = torch.arange(
+ SOURCE_CHECKSUM_BITS, device=source.device, dtype=torch.int64
+ )
+ checksum_bits = ((checksum[:, None] >> checksum_columns[None, :]) & 1) * 2 - 1
+ # Magnitude one sits inside the ordinary [-2, 2] activation range, so the identity cannot set
+ # an FP8 block scale. Decode depends only on sign and remains stable after dequantization.
+ output[:, :SOURCE_ID_BITS] = source_bits.to(dtype)
+ output[:, SOURCE_ID_BITS:SOURCE_ID_COLUMNS] = checksum_bits.to(dtype)
return output
@@ -76,14 +98,21 @@ def decode_source_ids(payload, seed: int):
if payload.ndim != 2 or payload.shape[1] < SOURCE_ID_COLUMNS:
raise ValueError("received payload cannot carry the source-token prefix")
prefix = payload[:, :SOURCE_ID_COLUMNS].float()
- digits = prefix.round().to(torch.int64)
- if not torch.equal(prefix, digits.float()):
- raise ValueError("received source-token prefix is not exact")
- if bool(((digits < 0) | (digits >= SOURCE_ID_BASE)).any().item()):
- raise ValueError("received source-token prefix is out of range")
- source = digits[:, 0] + SOURCE_ID_BASE * digits[:, 1] + SOURCE_ID_BASE**2 * digits[:, 2]
- checksum = (source * 29 + int(seed) * 7) % SOURCE_ID_BASE
- if not torch.equal(checksum, digits[:, 3]):
+ if not bool(torch.isfinite(prefix).all().item()) or bool((prefix.abs() < 0.25).any().item()):
+ raise ValueError("received source-token prefix is not quantization-stable")
+ bits = prefix >= 0
+ powers = 1 << torch.arange(SOURCE_ID_BITS, device=payload.device, dtype=torch.int64)
+ source = (bits[:, :SOURCE_ID_BITS].to(torch.int64) * powers).sum(dim=1)
+ checksum_powers = 1 << torch.arange(
+ SOURCE_CHECKSUM_BITS, device=payload.device, dtype=torch.int64
+ )
+ observed_checksum = (
+ bits[:, SOURCE_ID_BITS:SOURCE_ID_COLUMNS].to(torch.int64) * checksum_powers
+ ).sum(dim=1)
+ checksum = (source * 0x9E37 + int(seed) * 0xA24B) & (
+ (1 << SOURCE_CHECKSUM_BITS) - 1
+ )
+ if not torch.equal(checksum, observed_checksum):
raise ValueError("received source-token checksum differs")
return source
diff --git a/experimental/CollectiveX/tests/run_ep.py b/experimental/CollectiveX/tests/run_ep.py
index 45d7dd60c9..7f3ca79d0b 100644
--- a/experimental/CollectiveX/tests/run_ep.py
+++ b/experimental/CollectiveX/tests/run_ep.py
@@ -5,6 +5,8 @@
import argparse
import ctypes
+import hashlib
+import hmac
import json
import os
import platform
@@ -23,6 +25,15 @@
import identity # noqa: E402
+ALLOCATION_STRATUM_CONTRACT = "collectivex-allocation-stratum-v1"
+PRIVATE_FABRIC_ENV = {
+ "ib_gid_index": "CX_IB_GID_INDEX",
+ "rdma_devices": "CX_RDMA_DEVICES",
+ "rdma_service_level": "CX_RDMA_SERVICE_LEVEL",
+ "socket_ifname": "CX_SOCKET_IFNAME",
+}
+
+
def _numeric_version(command: list[str]) -> str | None:
try:
result = subprocess.run(
@@ -155,6 +166,71 @@ def _common_runtime_fingerprint(records: list[dict]) -> dict:
return records[0]
+def _allocation_stratum_sha256(
+ physical_hosts: list[str],
+ *,
+ audit_salt: str | None,
+ fabric_selectors: dict[str, str | None],
+ required: bool,
+) -> str | None:
+ """Commit private allocation/fabric identity without exposing its inputs."""
+ if audit_salt in (None, ""):
+ if required:
+ raise ValueError("canonical execution requires a private allocation audit salt")
+ return None
+ if not isinstance(audit_salt, str) or not re.fullmatch(r"[0-9a-f]{64}", audit_salt):
+ raise ValueError("allocation audit salt is invalid")
+ if set(fabric_selectors) != set(PRIVATE_FABRIC_ENV):
+ raise ValueError("private fabric selector set differs from the stratum contract")
+ for value in fabric_selectors.values():
+ if value is not None and (
+ not isinstance(value, str)
+ or not value
+ or len(value) > 512
+ or any(ord(char) < 32 or ord(char) == 127 for char in value)
+ ):
+ raise ValueError("private fabric selector is invalid")
+ if not physical_hosts or any(
+ not isinstance(host, str)
+ or not host
+ or len(host) > 255
+ or any(ord(char) < 32 or ord(char) == 127 for char in host)
+ for host in physical_hosts
+ ):
+ raise ValueError("physical allocation host evidence is invalid")
+ payload = json.dumps(
+ {
+ "contract": ALLOCATION_STRATUM_CONTRACT,
+ "fabric_selectors": fabric_selectors,
+ "physical_hosts": sorted(set(physical_hosts)),
+ },
+ allow_nan=False,
+ ensure_ascii=False,
+ separators=(",", ":"),
+ sort_keys=True,
+ ).encode("utf-8")
+ return hmac.new(bytes.fromhex(audit_salt), payload, hashlib.sha256).hexdigest()
+
+
+def _common_allocation_stratum(
+ records: list[str | None], *, required: bool
+) -> str | None:
+ """Require every distributed rank to derive the same private stratum."""
+ if not records or any(
+ value is not None
+ and (not isinstance(value, str) or not re.fullmatch(r"[0-9a-f]{64}", value))
+ for value in records
+ ):
+ raise ValueError("allocation stratum evidence is invalid")
+ distinct = set(records)
+ if len(distinct) != 1:
+ raise ValueError("allocation stratum differs across distributed ranks")
+ value = records[0]
+ if required and value is None:
+ raise ValueError("canonical execution requires an allocation stratum")
+ return value
+
+
def main() -> int:
ap = argparse.ArgumentParser(description="CollectiveX EP dispatch/combine sweep")
ap.add_argument(
@@ -191,6 +267,12 @@ def main() -> int:
file=sys.stderr,
)
return 2
+ if args.qualification_index not in range(1, ep_harness.QUALIFICATION_RUNS + 1):
+ print(
+ f"ERROR: qualification index must be in 1..{ep_harness.QUALIFICATION_RUNS}",
+ file=sys.stderr,
+ )
+ return 2
sampling_error = ep_harness.sampling_contract_error(
args.iters, args.trials, args.warmup
@@ -336,7 +418,11 @@ def main() -> int:
"job": os.environ.get("GITHUB_JOB"),
"artifact": os.environ.get("COLLECTIVEX_ARTIFACT_NAME"),
}
- args.git_run = _run if any(_run.values()) else None
+ if any(_run.values()):
+ _run["qualification_index"] = args.qualification_index
+ args.git_run = _run
+ else:
+ args.git_run = None
# Import the backend class only after torch initializes. The selected mode is an
# explicit case dimension; adapters do not infer it from the token ladder.
@@ -395,6 +481,21 @@ def main() -> int:
args.runtime_fingerprint = _common_runtime_fingerprint(
[record[2] for record in complete_records]
)
+ canonical = bool(args.workload_dir)
+ local_stratum = _allocation_stratum_sha256(
+ [record[0] for record in complete_records],
+ audit_salt=os.environ.get("CX_AUDIT_SALT"),
+ fabric_selectors={
+ field: os.environ.get(environment) or None
+ for field, environment in PRIVATE_FABRIC_ENV.items()
+ },
+ required=canonical,
+ )
+ stratum_records: list[str | None] = [None] * world_size
+ dist.all_gather_object(stratum_records, local_stratum)
+ args.allocation_stratum_sha256 = _common_allocation_stratum(
+ stratum_records, required=canonical
+ )
# Construct + run inside a try so a backend exception (esp. a new adapter on GPU) prints its
# FULL traceback to STDOUT — torchrun captures per-rank stdout but only summarizes stderr, so an
@@ -406,7 +507,8 @@ def main() -> int:
f"[run_ep] backend={args.backend} phase={args.phase} mode={args.mode} "
f"world={world_size} ep_size={world_size} hidden={args.hidden} "
f"topk={args.topk} experts={args.experts} dtype=bf16 "
- f"routing={args.routing} seed={args.seed}"
+ f"routing={args.routing} seed={args.seed} "
+ f"qualification_index={args.qualification_index}"
)
rc = ep_harness.run_sweep(args, backend, torch, dist, device, rank, world_size)
except Exception:
diff --git a/experimental/CollectiveX/tests/test_deepep_v2_contract.py b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
index 082cd491a6..c9f65c2c30 100644
--- a/experimental/CollectiveX/tests/test_deepep_v2_contract.py
+++ b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
@@ -112,7 +112,7 @@ def operator_config(root: Path) -> dict[str, object]:
"partition": "test", "squash_dir": path, "stage_dir": path, **network,
},
}
- return {"schema_version": 1, "runners": runners}
+ return {"schema_version": 1, "audit_salt": "a" * 64, "runners": runners}
class DeepEPV2ContractTests(unittest.TestCase):
@@ -483,7 +483,7 @@ def test_low_latency_mode_parser_and_profile_are_explicit(self) -> None:
self.assertEqual(profile["contract"], "expert-packed-weighted-combine-v1")
self.assertEqual(
profile["component_order_contract"],
- "roundtrip-dispatch-gate-weighted-combine-v1",
+ "qualification-hash-rotated-components-v1",
)
self.assertEqual(
profile["correctness_scope"],
diff --git a/experimental/CollectiveX/tests/test_ep_precision_adapters.py b/experimental/CollectiveX/tests/test_ep_precision_adapters.py
new file mode 100644
index 0000000000..a9655cdcdb
--- /dev/null
+++ b/experimental/CollectiveX/tests/test_ep_precision_adapters.py
@@ -0,0 +1,130 @@
+#!/usr/bin/env python3
+"""CPU-only contract tests for native EP precision adapter wiring."""
+from __future__ import annotations
+
+import ast
+import sys
+import unittest
+from pathlib import Path
+from types import SimpleNamespace
+
+
+ROOT = Path(__file__).resolve().parents[1]
+TESTS = ROOT / "tests"
+sys.path.insert(0, str(ROOT))
+sys.path.insert(0, str(TESTS))
+
+import ep_precision # noqa: E402
+
+
+class PrecisionResolutionTests(unittest.TestCase):
+ def test_blank_profile_resolves_to_bf16_control(self):
+ profile_id, profile = ep_precision.resolve_precision(
+ SimpleNamespace(precision_profile=""),
+ backend="nccl-ep",
+ mode="normal",
+ supported_profiles={"d-bf16.c-bf16"},
+ )
+ self.assertEqual(profile_id, "d-bf16.c-bf16")
+ self.assertEqual(profile["dispatch"]["communication_format"], "bf16")
+ self.assertEqual(profile["combine"]["communication_format"], "bf16")
+
+ def test_adapter_profile_rejection_is_fail_closed(self):
+ with self.assertRaisesRegex(
+ ep_precision.PrecisionError,
+ "nccl-ep does not realize precision profile",
+ ):
+ ep_precision.resolve_precision(
+ SimpleNamespace(
+ precision_profile=(
+ "d-fp8-e4m3fn-b128-f32-prequantized.c-bf16"
+ )
+ ),
+ backend="nccl-ep",
+ mode="normal",
+ supported_profiles={"d-bf16.c-bf16"},
+ )
+
+ def test_profile_mode_mismatch_is_rejected(self):
+ with self.assertRaisesRegex(ep_precision.PrecisionError, "not valid in mode"):
+ ep_precision.resolve_precision(
+ SimpleNamespace(
+ precision_profile="d-bf16.c-logfmt10-dynamic64"
+ ),
+ backend="deepep",
+ mode="normal",
+ supported_profiles={"d-bf16.c-logfmt10-dynamic64"},
+ )
+
+ def test_required_native_keyword_is_checked(self):
+ def native_api(*, use_fp8=False):
+ return use_fp8
+
+ ep_precision.require_keyword(native_api, "use_fp8", api="native_api")
+ with self.assertRaisesRegex(ep_precision.PrecisionError, "omits 'use_logfmt'"):
+ ep_precision.require_keyword(
+ native_api, "use_logfmt", api="native_api"
+ )
+
+ def test_bf16_evidence_is_exact_and_has_no_scale_checks(self):
+ evidence = ep_precision.exact_axis_evidence()
+ self.assertTrue(evidence["passed"])
+ self.assertEqual(evidence["max_abs_error"], 0.0)
+ self.assertEqual(evidence["max_rel_error"], 0.0)
+ self.assertEqual(evidence["saturation_count"], 0)
+ self.assertEqual(evidence["saturation_rate"], 0.0)
+ self.assertIsNone(evidence["scales_finite"])
+ self.assertIsNone(evidence["scales_positive"])
+
+
+class NativeAdapterWiringTests(unittest.TestCase):
+ @staticmethod
+ def _tree(name: str) -> ast.Module:
+ return ast.parse((TESTS / name).read_text(encoding="utf-8"))
+
+ @staticmethod
+ def _keywords(tree: ast.AST, attribute: str) -> list[set[str]]:
+ result = []
+ for node in ast.walk(tree):
+ if not isinstance(node, ast.Call):
+ continue
+ function = node.func
+ if (
+ isinstance(function, ast.Attribute) and function.attr == attribute
+ ) or (
+ isinstance(function, ast.Name) and function.id == attribute
+ ):
+ result.append({keyword.arg for keyword in node.keywords})
+ return result
+
+ def test_deepep_and_uccl_wire_native_low_latency_controls(self):
+ for filename in ("ep_deepep.py", "ep_uccl.py"):
+ with self.subTest(filename=filename):
+ tree = self._tree(filename)
+ dispatch_calls = self._keywords(tree, "low_latency_dispatch")
+ combine_calls = self._keywords(tree, "low_latency_combine")
+ self.assertTrue(any("use_fp8" in call for call in dispatch_calls))
+ self.assertTrue(any("use_logfmt" in call for call in combine_calls))
+
+ def test_elastic_and_hybrid_constructors_enable_native_fp8(self):
+ v2 = self._tree("ep_deepep_v2.py")
+ hybrid = self._tree("ep_deepep_hybrid.py")
+ self.assertTrue(
+ any("use_fp8_dispatch" in call for call in self._keywords(v2, "ElasticBuffer"))
+ )
+ self.assertTrue(
+ any("use_fp8" in call for call in self._keywords(hybrid, "HybridEPBuffer"))
+ )
+ self.assertTrue(
+ any("scaling_factor" in call for call in self._keywords(hybrid, "dispatch"))
+ )
+
+ def test_mori_wires_dispatch_scales_and_direct_cast_config(self):
+ source = (TESTS / "ep_mori.py").read_text(encoding="utf-8")
+ self.assertIn('"fp8_direct_cast" if self._direct_cast_combine', source)
+ self.assertIn("p.scales,", source)
+ self.assertIn("dispatch_scales=_scales", source)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
new file mode 100644
index 0000000000..f103855378
--- /dev/null
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -0,0 +1,347 @@
+#!/usr/bin/env python3
+"""CPU-only tests for CollectiveX communication-precision scheduling."""
+from __future__ import annotations
+
+import copy
+from pathlib import Path
+import sys
+import unittest
+from unittest import mock
+
+
+HERE = Path(__file__).resolve().parent
+ROOT = HERE.parent
+sys.path[:0] = [str(ROOT), str(HERE)]
+
+import capability # noqa: E402
+import identity # noqa: E402
+import probe_precision # noqa: E402
+import sweep_matrix # noqa: E402
+
+
+class PrecisionSchedulingTest(unittest.TestCase):
+ def test_precision_probe_inventory_is_exact_and_non_mutating(self) -> None:
+ before = copy.deepcopy(capability.PRECISION_CAPABILITIES)
+ targets = probe_precision.provisional_targets()
+ key = lambda item: (
+ item["sku"], item["backend"], item["ep"], item["mode"],
+ item["precision_profile"],
+ )
+ self.assertEqual(targets, sorted(capability.provisional_precision_targets(), key=key))
+ self.assertEqual(len(targets), 94)
+ self.assertEqual(capability.PRECISION_CAPABILITIES, before)
+ self.assertEqual(
+ len({
+ (item["backend"], item["sku"], item["ep"], item["mode"],
+ item["precision_profile"])
+ for item in targets
+ }),
+ len(targets),
+ )
+
+ def test_precision_probe_selects_only_one_exact_provisional_cell(self) -> None:
+ target = probe_precision.provisional_targets()[0]
+ selected = probe_precision.select_target(
+ backend=target["backend"], sku=target["sku"], ep=target["ep"],
+ mode=target["mode"], precision_profile=target["precision_profile"],
+ )
+ self.assertEqual(selected, target)
+ with self.assertRaisesRegex(probe_precision.ProbeError, "target-not-provisional"):
+ probe_precision.select_target(
+ backend=target["backend"], sku=target["sku"], ep=target["ep"],
+ mode=target["mode"], precision_profile="d-bf16.c-bf16",
+ )
+
+ def test_precision_probe_workflow_plan_binds_exact_controls(self) -> None:
+ plan = probe_precision.workflow_plan(backend="deepep", only_sku="b200-dgxc")
+ self.assertTrue(plan["include"])
+ self.assertTrue(all(
+ row["backend"] == "deepep" and row["sku"] == "b200-dgxc"
+ for row in plan["include"]
+ ))
+ row = plan["include"][0]
+ control = probe_precision.extract_control(
+ plan, probe_id=row["id"], sku=row["sku"], backend=row["backend"],
+ nodes=row["nodes"],
+ )
+ self.assertEqual(
+ probe_precision.validate_control(
+ control, sku=row["sku"], backend=row["backend"], nodes=row["nodes"],
+ ),
+ control,
+ )
+ with self.assertRaisesRegex(ValueError, "workflow matrix"):
+ probe_precision.extract_control(
+ plan, probe_id=row["id"], sku=row["sku"], backend=row["backend"],
+ nodes=row["nodes"] + 1,
+ )
+ with self.assertRaisesRegex(ValueError, "select no provisional"):
+ probe_precision.workflow_plan(backend="mori", only_sku="b200-dgxc")
+
+ def test_precision_probe_manifest_is_sanitized_and_runtime_evidence_is_required(self) -> None:
+ target = probe_precision.provisional_targets()[0]
+ topology = capability.topology_for(target["sku"], target["ep"])
+ self.assertIsNotNone(topology)
+ topology_record = probe_precision._topology_record(topology, False)
+ document = probe_precision.build_manifest(
+ target=target, topology=topology_record, disposition="unsupported",
+ reason="unsupported-native-api", runtime_executed=True, evidence=None,
+ )
+ self.assertEqual(document["result"], {
+ "disposition": "unsupported",
+ "reason": "unsupported-native-api",
+ "registry_mutation": False,
+ "runtime_executed": True,
+ "static_inspection_sufficient": False,
+ })
+ with self.assertRaises((TypeError, ValueError)):
+ probe_precision.build_manifest(
+ target=target, topology=topology_record, disposition="supported",
+ reason=probe_precision.SUPPORTED_REASON, runtime_executed=True,
+ evidence=None,
+ )
+
+ def test_precision_profiles_bind_exact_formats_and_timing_boundaries(self) -> None:
+ scheduled = set(identity.V1_NORMAL_PRECISION_PROFILE_IDS) | set(
+ identity.V1_LOW_LATENCY_PRECISION_PROFILE_IDS
+ )
+ self.assertEqual(
+ set(identity.V1_PRECISION_PROFILES),
+ scheduled | {identity.V1_CONTROL_PRECISION_PROFILE},
+ )
+ self.assertNotIn(identity.V1_CONTROL_PRECISION_PROFILE, scheduled)
+ self.assertNotIn("fp4", repr(identity.V1_PRECISION_PROFILES).lower())
+
+ required_axis_fields = {
+ "api_input_dtype",
+ "api_output_dtype",
+ "communication_format",
+ "scale_dtype",
+ "scale_layout",
+ "scale_group_size",
+ "padding_contract",
+ "alignment_contract",
+ "quantization_origin",
+ "conversion_boundary",
+ }
+ for name in identity.V1_PRECISION_PROFILES:
+ with self.subTest(profile=name):
+ profile = identity.precision_profile(name)
+ self.assertEqual(profile["profile_id"], name)
+ self.assertEqual(set(profile["dispatch"]), required_axis_fields)
+ self.assertEqual(set(profile["combine"]), required_axis_fields)
+ self.assertRegex(name, r"^[a-z0-9][a-z0-9.-]*$")
+
+ prequantized = identity.precision_profile(
+ "d-fp8-e4m3fn-b128-f32-prequantized.c-bf16"
+ )["dispatch"]
+ fused = identity.precision_profile(
+ "d-fp8-e4m3fn-b128-f32-fused.c-bf16"
+ )["dispatch"]
+ self.assertEqual(prequantized["conversion_boundary"], "before-dispatch-timing")
+ self.assertEqual(fused["conversion_boundary"], "inside-dispatch-timing")
+ self.assertEqual(prequantized["scale_group_size"], 128)
+
+ mi325 = identity.precision_profile(
+ "d-fp8-e4m3fnuz-b128-f32-prequantized.c-bf16"
+ )["dispatch"]
+ self.assertEqual(mi325["communication_format"], "fp8-e4m3fnuz")
+ logfmt = identity.precision_profile("d-bf16.c-logfmt10-dynamic64")["combine"]
+ self.assertEqual(
+ (logfmt["communication_format"], logfmt["scale_group_size"]),
+ ("logfmt10", 64),
+ )
+
+ base = {"mode": "normal"}
+ precision_case = {
+ **base,
+ "precision_profile": "d-fp8-e4m3fn-b128-f32-prequantized.c-bf16",
+ }
+ self.assertIs(identity.profile_for_case(base), identity.V1_NORMAL_CASE_PROFILE)
+ self.assertIn("communication_precision", identity.profile_for_case(precision_case))
+ self.assertNotEqual(
+ identity.digest("case", identity.profile_for_case(base)),
+ identity.digest("case", identity.profile_for_case(precision_case)),
+ )
+
+ def test_capability_registry_uses_exact_native_targets(self) -> None:
+ targets = capability.precision_targets()
+ self.assertTrue(targets)
+ self.assertTrue(all(item["disposition"] == "provisional" for item in targets))
+ self.assertEqual(targets, capability.provisional_precision_targets())
+ keys = {
+ (
+ item["precision_profile"],
+ item["backend"],
+ item["sku"],
+ item["ep"],
+ item["mode"],
+ )
+ for item in targets
+ }
+ self.assertEqual(len(keys), len(targets))
+
+ normal = "d-fp8-e4m3fn-b128-f32-prequantized.c-bf16"
+ direct = "d-bf16.c-fp8-e4m3fn-direct-cast-noscale"
+ fnuz_direct = "d-bf16.c-fp8-e4m3fnuz-direct-cast-noscale"
+ low_latency = "d-bf16.c-logfmt10-dynamic64"
+ cases = (
+ (("h200-dgxc", "deepep-v2", 8, "normal", normal), "provisional"),
+ (("h100-dgxc", "deepep-v2", 8, "normal", normal), "not-applicable"),
+ (("h200-dgxc", "nccl-ep", 8, "normal", normal), "not-applicable"),
+ (("mi355x", "mori", 8, "normal", direct), "provisional"),
+ (("mi355x", "mori", 16, "normal", direct), "not-applicable"),
+ (("mi325x", "mori", 8, "normal", fnuz_direct), "provisional"),
+ (("h200-dgxc", "deepep", 8, "low-latency", low_latency), "provisional"),
+ (("h200-dgxc", "deepep-hybrid", 8, "low-latency", low_latency),
+ "not-applicable"),
+ )
+ for (sku, backend, ep, mode, profile), expected in cases:
+ with self.subTest(sku=sku, backend=backend, profile=profile):
+ topology = capability.topology_for(sku, ep)
+ self.assertIsNotNone(topology)
+ disposition, _ = capability.resolve_disposition(
+ sku,
+ backend,
+ ep=ep,
+ nodes=topology["nodes"], # type: ignore[index]
+ mode=mode,
+ precision_profile=profile,
+ )
+ self.assertEqual(disposition, expected)
+
+ control, _ = capability.resolve_disposition(
+ "h200-dgxc",
+ "deepep",
+ ep=8,
+ nodes=1,
+ precision_profile=identity.V1_CONTROL_PRECISION_PROFILE,
+ )
+ self.assertEqual(control, "supported")
+
+ def test_split_suites_are_provisional_and_do_not_duplicate_bf16(self) -> None:
+ suites = sweep_matrix._load("suites.yaml")
+ workloads = sweep_matrix._load("workloads.yaml")
+ sweep_matrix.validate_config_documents(suites, workloads)
+ normal = suites["suites"]["ep-precision-normal-v1"]
+ low_latency = suites["suites"]["ep-precision-low-latency-v1"]
+ self.assertEqual(
+ (
+ normal["mode"],
+ normal["phases"],
+ normal["token_points_decode"],
+ normal["token_points_prefill"],
+ ),
+ ("normal", ["decode", "prefill"], [128], [512]),
+ )
+ self.assertEqual(
+ (
+ low_latency["mode"],
+ low_latency["phases"],
+ low_latency["token_points_decode"],
+ ),
+ ("low-latency", ["decode"], [128]),
+ )
+ self.assertTrue(normal["provisional"])
+ self.assertTrue(low_latency["provisional"])
+ self.assertEqual(
+ normal["required_publication"], "comparable-experimental"
+ )
+ self.assertEqual(
+ low_latency["required_publication"], "comparable-experimental"
+ )
+ listed = normal["precision_profiles"] + low_latency["precision_profiles"]
+ self.assertNotIn(identity.V1_CONTROL_PRECISION_PROFILE, listed)
+ self.assertEqual(len(listed), len(set(listed)))
+
+ matrix = sweep_matrix.validate_matrix_document(
+ sweep_matrix.resolve_matrix(backends="all")
+ )
+ self.assertFalse(any("precision_profile" in item["case"] for item in matrix["requested_cases"]))
+ with self.assertRaisesRegex(SystemExit, "provisional precision suites"):
+ sweep_matrix.resolve_matrix(
+ suites="ep-precision-normal-v1", backends="all"
+ )
+
+ stale = copy.deepcopy(suites)
+ stale["suites"]["ep-precision-normal-v1"]["provisional"] = False
+ with self.assertRaisesRegex(SystemExit, "must track unresolved"):
+ sweep_matrix.validate_config_documents(stale, workloads)
+
+ def test_resolved_profiles_schedule_without_cartesian_fill(self) -> None:
+ suites = sweep_matrix._load("suites.yaml")
+ workloads = sweep_matrix._load("workloads.yaml")
+ promoted = copy.deepcopy(capability.PRECISION_CAPABILITIES)
+ for rules in promoted.values():
+ for rule in rules:
+ rule["disposition"] = "supported"
+ normal_profile = "d-fp8-e4m3fn-b128-f32-prequantized.c-bf16"
+ promoted[normal_profile][0]["disposition"] = "unsupported"
+ resolved_suites = copy.deepcopy(suites)
+ for name in ("ep-precision-normal-v1", "ep-precision-low-latency-v1"):
+ resolved_suites["suites"][name]["provisional"] = False
+
+ def load_config(name: str) -> dict[str, object]:
+ if name == "suites.yaml":
+ return resolved_suites
+ if name == "workloads.yaml":
+ return workloads
+ raise AssertionError(name)
+
+ suite_names = "ep-precision-normal-v1,ep-precision-low-latency-v1"
+ expected_cases = sum(
+ len(resolved_suites["suites"][
+ "ep-precision-normal-v1"
+ if target["mode"] == "normal"
+ else "ep-precision-low-latency-v1"
+ ]["phases"])
+ for target in capability.precision_targets()
+ )
+ unsupported_targets = [
+ target for target in capability.precision_targets([normal_profile])
+ if target["backend"] == "deepep"
+ ]
+ with mock.patch.object(capability, "PRECISION_CAPABILITIES", promoted):
+ with self.assertRaisesRegex(SystemExit, "must track unresolved"):
+ sweep_matrix.validate_config_documents(suites, workloads)
+ with mock.patch.object(sweep_matrix, "_load", side_effect=load_config):
+ matrix = sweep_matrix.validate_matrix_document(
+ sweep_matrix.resolve_matrix(suites=suite_names, backends="all")
+ )
+
+ unsupported = [
+ item for item in matrix["requested_cases"]
+ if item["disposition"] == "unsupported"
+ ]
+ self.assertEqual(
+ len(unsupported),
+ len(unsupported_targets)
+ * len(resolved_suites["suites"]["ep-precision-normal-v1"]["phases"]),
+ )
+ self.assertTrue(all(
+ item["reason"] == "precision-profile-unsupported" for item in unsupported
+ ))
+ self.assertTrue(any(
+ item["disposition"] == "runnable" for item in matrix["requested_cases"]
+ ))
+
+ self.assertEqual(len(matrix["requested_cases"]), expected_cases)
+ self.assertEqual(
+ {item["case"]["precision_profile"] for item in matrix["requested_cases"]},
+ set(identity.V1_NORMAL_PRECISION_PROFILE_IDS)
+ | set(identity.V1_LOW_LATENCY_PRECISION_PROFILE_IDS),
+ )
+ self.assertFalse(any(
+ item["case"]["precision_profile"] == identity.V1_CONTROL_PRECISION_PROFILE
+ for item in matrix["requested_cases"]
+ ))
+ direct_cases = [
+ item for item in matrix["requested_cases"]
+ if "direct-cast" in item["case"]["precision_profile"]
+ ]
+ self.assertTrue(direct_cases)
+ self.assertEqual({item["case"]["ep"] for item in direct_cases}, {8})
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/experimental/CollectiveX/tests/test_publisher.py b/experimental/CollectiveX/tests/test_publisher.py
index 5624fbcf68..abc43c6ec3 100644
--- a/experimental/CollectiveX/tests/test_publisher.py
+++ b/experimental/CollectiveX/tests/test_publisher.py
@@ -31,6 +31,7 @@
"repository": "SemiAnalysisAI/InferenceX",
"run_id": "12345",
"run_attempt": 1,
+ "qualification_index": 1,
"source_sha": "a" * 40,
}
@@ -53,6 +54,7 @@ def _unsupported_delivery(
"job": "setup",
"ref": "collectivex",
"repo": run["repository"],
+ "qualification_index": run["qualification_index"],
"run_attempt": str(run["run_attempt"]),
"run_id": run["run_id"],
"source_sha": run["source_sha"],
@@ -61,6 +63,7 @@ def _unsupported_delivery(
"artifact": artifact_name,
"execution_id": f"{run['run_id']}_{run['run_attempt']}_unsupported",
"job": "setup",
+ "qualification_index": run["qualification_index"],
"repo": run["repository"],
"run_attempt": str(run["run_attempt"]),
"run_id": run["run_id"],
@@ -96,6 +99,7 @@ def _args(
repository=run["repository"],
run_id=run["run_id"],
run_attempt=run["run_attempt"],
+ qualification_index=run["qualification_index"],
source_sha=run["source_sha"],
)
@@ -115,18 +119,54 @@ def _ids(seed: str) -> tuple[str, str, str, str, str, str]:
def _component(scale: float = 1.0) -> dict:
latency = {"p50": 10.0 * scale, "p90": 12.0 * scale,
"p95": 14.0 * scale, "p99": 20.0 * scale}
- logical_bytes = 100_000
+ byte_provenance = {
+ "accounting_contract": "activation-data-plus-scales-v1",
+ "activation_data_bytes": 100_000,
+ "scale_bytes": 0,
+ "total_logical_bytes": 100_000,
+ }
return {
"origin": "measured",
"latency_us": latency,
- "logical_bytes": logical_bytes,
- "logical_payload_rate_gbps_at_latency_percentile": {
- name: logical_bytes / (value * 1000.0) for name, value in latency.items()
+ "byte_provenance": byte_provenance,
+ "activation_data_rate_gbps_at_latency_percentile": {
+ name: byte_provenance["activation_data_bytes"] / (value * 1000.0)
+ for name, value in latency.items()
+ },
+ "total_logical_data_rate_gbps_at_latency_percentile": {
+ name: byte_provenance["total_logical_bytes"] / (value * 1000.0)
+ for name, value in latency.items()
},
"sample_count": 512,
}
+def _precision_axis_evidence() -> dict:
+ return {
+ "encoded_payload_valid": True,
+ "scales_finite": None,
+ "scales_positive": None,
+ "dequantized_semantics": True,
+ "saturation_count": 0,
+ "saturation_rate": 0.0,
+ "max_abs_error": 0.0,
+ "max_rel_error": 0.0,
+ "passed": True,
+ }
+
+
+def _precision_evidence(
+ profile_id: str = identity.V1_CONTROL_PRECISION_PROFILE,
+) -> dict:
+ axis = _precision_axis_evidence()
+ return {
+ "profile_id": profile_id,
+ "dispatch": axis,
+ "combine": copy.deepcopy(axis),
+ "passed": True,
+ }
+
+
def _hybrid_provenance(ep_size: int = 1) -> dict:
realized = {field: 1 for field in contracts.HYBRID_REALIZED_CONFIG_FIELDS}
for field in contracts.HYBRID_REALIZED_BOOL_FIELDS:
@@ -158,7 +198,7 @@ def _hybrid_provenance(ep_size: int = 1) -> dict:
{"name": "hybrid_ep_cpp", "role": "deepep-hybrid-extension", "sha256": "5" * 64},
],
"realized_config": realized,
- "resource_mode": "tuned",
+ "resource_mode": "fixed-profile",
"tuned_source": "deepep-hybrid-configurer-autotune-v1",
}
@@ -179,18 +219,23 @@ def digest(value: object) -> str:
"warmup_semantics": "full-roundtrip-before-each-component-trial-point-v1",
"workload": "deepseek-v3-v1",
}
+ communication_precision = identity.precision_profile(
+ identity.V1_CONTROL_PRECISION_PROFILE
+ )
case_factors = {
"case": scheduled, "profile": identity.V1_NORMAL_CASE_PROFILE, "sku": "fixture"
}
case_id = identity.digest("case", case_factors)
git_run = {
"artifact": "cxshard-fixture-999-1", "job": "sweep", "ref": "collectivex",
- "repo": RUN["repository"], "run_attempt": "1", "run_id": "999",
+ "repo": RUN["repository"], "qualification_index": 1,
+ "run_attempt": "1", "run_id": "999",
"source_sha": RUN["source_sha"],
}
allocation_factors = {
"artifact": git_run["artifact"], "execution_id": "999_1_fixture",
- "job": git_run["job"], "repo": git_run["repo"], "run_attempt": "1",
+ "job": git_run["job"], "qualification_index": 1,
+ "repo": git_run["repo"], "run_attempt": "1",
"run_id": "999", "runner": "fixture", "source_sha": git_run["source_sha"],
}
allocation_id = identity.allocation_id(allocation_factors)
@@ -235,7 +280,7 @@ def digest(value: object) -> str:
kernel_generation=implementation["kernel_generation"],
provenance=implementation_provenance,
resource_profile=implementation["resource_profile"],
- resource_mode="tuned",
+ resource_mode="fixed-profile",
device_product=runtime["device"]["product"],
)
series_factors = {
@@ -262,6 +307,9 @@ def digest(value: object) -> str:
}
for name, latency in (("combine", 20.0), ("dispatch", 10.0), ("roundtrip", 40.0))
}
+ sample_components["stage"] = {
+ "availability": "unavailable", "sample_count": 0, "trials": None,
+ }
sample_sha = digest({"components": sample_components, "tokens_per_rank": 1})
evidence_id = identity.evidence_id(
point=point_id, allocation=allocation_id, attempt=attempt_id,
@@ -278,7 +326,7 @@ def digest(value: object) -> str:
"iterations_per_trial": 8, "reduction": "cross-rank-max-per-iteration",
"trials": 64,
},
- "schema_version": 1, "series_id": series_id,
+ "qualification_index": 1, "schema_version": 1, "series_id": series_id,
}
sample_bytes = contracts.canonical_json_bytes(samples)
oracle = {
@@ -311,10 +359,14 @@ def measured(value: float) -> dict:
"percentiles_us": pct(30.0), "sample_count": 0,
},
"roundtrip": measured(40.0),
+ "stage": {
+ "availability": "unavailable", "origin": None,
+ "percentiles_us": None, "sample_count": 0,
+ },
},
"correctness": {
"contract": "expert-specific-transform-v1", "max_relative_error": 0.0,
- "passed": True,
+ "passed": True, "precision": _precision_evidence(),
"rank_evidence": [{
"input_unchanged": True, "order_stable": True,
"post_timing": copy.deepcopy(oracle), "pre_timing": copy.deepcopy(oracle),
@@ -323,7 +375,28 @@ def measured(value: float) -> dict:
"scope": "dispatch-metadata-and-transformed-combine",
},
"evidence_id": evidence_id, "global_tokens": 1,
- "logical_bytes": {"combine": 2, "dispatch": 2, "roundtrip": 4},
+ "byte_provenance": {
+ "combine": {
+ "accounting_contract": "activation-data-plus-scales-v1",
+ "activation_data_bytes": 2, "scale_bytes": 0,
+ "total_logical_bytes": 2,
+ },
+ "dispatch": {
+ "accounting_contract": "activation-data-plus-scales-v1",
+ "activation_data_bytes": 2, "scale_bytes": 0,
+ "total_logical_bytes": 2,
+ },
+ "roundtrip": {
+ "accounting_contract": "activation-data-plus-scales-v1",
+ "activation_data_bytes": 4, "scale_bytes": 0,
+ "total_logical_bytes": 4,
+ },
+ "stage": {
+ "accounting_contract": "activation-data-plus-scales-v1",
+ "activation_data_bytes": 0, "scale_bytes": 0,
+ "total_logical_bytes": 0,
+ },
+ },
"point_id": point_id,
"receive": {"max": 1, "mean": 1.0, "min": 1, "total": 1},
"routing": contracts._expected_routing_summary(
@@ -342,26 +415,27 @@ def measured(value: float) -> dict:
"sample_sha256": sample_sha,
"token_rate_at_latency_percentile": pct(25_000.0), "tokens_per_rank": 1,
}
+ row["sample_histograms"]["stage"] = None
raw = {
"case": {
"attempt_ordinal": 1, "backend": backend,
"eplb": {
+ "calibration_token_offset": None, "calibration_trace_sha256": None,
+ "calibration_window": None, "calibration_workload_id": None,
"enabled": False, "imbalance_after": None, "imbalance_before": None,
"mapping_hash": None, "max_replicas": None, "num_logical_experts": 1,
"num_physical_experts": 1, "num_redundant": 0, "planner": None,
"reference_tokens_per_rank": None, "replicated_experts": 0,
},
"ep_size": 1, "mode": "normal", "phase": "decode",
- "required_publication": "official", "resource_mode": "tuned", "runner": "fixture",
+ "required_publication": "official", "resource_mode": "fixed-profile", "runner": "fixture",
"shape": {
- "activation_profile": "canonical-counter-source-v3", "dispatch_dtype": "bf16",
+ "activation_profile": "canonical-counter-source-v4",
+ "precision_profile": identity.V1_CONTROL_PRECISION_PROFILE,
+ "dispatch_precision": communication_precision["dispatch"],
+ "combine_precision": communication_precision["combine"],
"eplb": False, "experts": 1, "experts_per_rank": 1, "hidden": 1,
"kernel_gen": kernel_generation, "num_logical_experts": 1,
- "quant": {
- "combine_accum_dtype": "fp32", "combine_input_dtype": "bf16",
- "combine_output_dtype": "bf16", "combine_quant_mode": "none",
- "scale_layout": None,
- },
"routing": "uniform", "topk": 1,
},
"suite": "ep-core-v1", "workload_name": "deepseek-v3-v1",
@@ -374,13 +448,15 @@ def measured(value: float) -> dict:
},
"implementation": implementation,
"measurement": {
- "component_order_contract": "roundtrip-dispatch-activation-only-combine-v2",
+ "component_order_contract": "qualification-hash-rotated-components-v1",
"conditioning": {
"contract": "fixed-phase-ramp-8-roundtrips-v1",
"ladder": [1, 2, 4, 8, 16, 32, 64, 128],
"roundtrips_per_shape": 8,
},
"contract": "layout-and-dispatch-v1",
+ "execution_order_sha256": "9" * 64,
+ "qualification_index": 1,
"rows": [row],
"sampling": {
"contract": "fixed-512-v1", "iterations_per_trial": 8,
@@ -404,6 +480,7 @@ def measured(value: float) -> dict:
},
},
"provenance": {
+ "allocation_stratum_sha256": "f" * 64,
"command": "run_ep", "distributed_launcher": "torchrun", "git_run": git_run,
"image": {
"arch": "amd64", "digest": "sha256:" + "d" * 64,
@@ -431,11 +508,11 @@ def measured(value: float) -> dict:
"world_size": 1,
},
"workload": {
- "activation_generator": "collectivex-activation-counter-v3",
+ "activation_generator": "collectivex-activation-counter-v4",
"activation_identity": hashlib.sha256(
- b"counter|seed=67|hidden=1|gen=collectivex-activation-counter-v3"
+ b"counter|seed=67|hidden=1|gen=collectivex-activation-counter-v4"
).hexdigest(),
- "activation_profile": "canonical-counter-source-v3", "cross_rank_consistent": True,
+ "activation_profile": "canonical-counter-source-v4", "cross_rank_consistent": True,
"manifest_checksums": {member_id: member_checksums}, "members": [member_id],
"routing_generator": "collectivex-routing-counter-v3", "source": "canonical-serialized",
"trace_hashes": [routing_hash],
@@ -459,6 +536,10 @@ def _series(seed: str, backend: str, *, decision_grade: bool = False) -> tuple[d
p99_ratio=1.02 if decision_grade else None,
)
component = _component(1.0 if backend == "deepep" else 1.2)
+ qualification_indices = [1, 2, 3] if decision_grade else [1]
+ communication_precision = identity.precision_profile(
+ identity.V1_CONTROL_PRECISION_PROFILE
+ )
item = {
"series_id": series_id,
"label": f"H100 / {backend}",
@@ -496,35 +577,65 @@ def _series(seed: str, backend: str, *, decision_grade: bool = False) -> tuple[d
"workload_id": identity.workload_id({"shape": "shared"}),
"hidden": 7168, "top_k": 8, "experts": 256,
"routing": "uniform", "eplb": False,
- "dispatch_dtype": "bf16", "combine_dtype": "bf16",
- "activation_profile": "canonical-counter-source-v3",
+ "precision_profile": identity.V1_CONTROL_PRECISION_PROFILE,
+ "dispatch_precision": communication_precision["dispatch"],
+ "combine_precision": communication_precision["combine"],
+ "activation_profile": "canonical-counter-source-v4",
},
"eplb": {
- "enabled": False, "planner": None, "mapping_sha256": None,
+ "enabled": False,
+ "calibration_workload_id": None, "calibration_trace_sha256": None,
+ "calibration_window": None, "calibration_token_offset": None,
+ "planner": None, "mapping_sha256": None,
"logical_experts": 256, "physical_experts": 256,
"redundant_experts": 0, "reference_tokens_per_rank": None,
"replicated_experts": 0, "max_replicas": None,
"imbalance_before": None, "imbalance_after": None,
},
- "resource": {"mode": "tuned", "profile": "profile-1", "comm_units_kind": "sm", "configured_units": 24},
+ "resource": {"mode": "fixed-profile", "profile": "profile-1", "comm_units_kind": "sm", "configured_units": 24},
"measurement": {
"contract": "layout-and-dispatch-v1",
- "component_order_contract": "roundtrip-dispatch-activation-only-combine-v2",
+ "component_order_contract": "qualification-hash-rotated-components-v1",
"combine_semantics": "activation-only", "payload_unit": "token-rank",
"sampling_contract": "fixed-512-v1",
"iters": 8, "trials": 64, "warmups": 32, "samples_per_component": 512,
+ "qualification_indices": qualification_indices,
"headline_component": "roundtrip", "headline_percentile": "p99",
},
"points": [{
"point_id": point_id, "tokens_per_rank": 8, "global_tokens": 64,
- "correct": True,
+ "anomalies": [],
+ "correctness": {"semantic_pass": True, "precision": _precision_evidence()},
+ "stability": {
+ "complete": decision_grade,
+ "qualification_indices": qualification_indices,
+ "p50_max_min_ratio": 1.02 if decision_grade else None,
+ "p99_max_min_ratio": 1.02 if decision_grade else None,
+ "stable_p50": decision_grade, "stable_p99": decision_grade,
+ },
+ "trial_diagnostics": {
+ "flagged": False,
+ "reasons": [],
+ "components": {
+ "dispatch": None,
+ "stage": None,
+ "combine": None,
+ "roundtrip": {
+ "drift_flagged": False,
+ "first_last_median_ratio": 1.0,
+ "outlier_flagged": False,
+ "robust_outlier_fraction": 0.0,
+ "trial_count": 192,
+ },
+ },
+ },
"routing": {
"fanout_mean": 4.0, "recv_tokens_max": 64,
"expert_load_cv": 0.5, "payload_rank_cv": 0.25,
"hotspot_ratio": 2.0, "empty_expert_count": 0,
"empty_rank_count": 0, "routed_copies": 256,
},
- "components": {"dispatch": None, "combine": None,
+ "components": {"dispatch": None, "stage": None, "combine": None,
"roundtrip": component, "isolated_sum": None},
"roundtrip_token_rate_at_latency_percentile": {
name: 64 / (latency * 1e-6)
@@ -567,14 +678,81 @@ def _series(seed: str, backend: str, *, decision_grade: bool = False) -> tuple[d
statistic: component["latency_us"][statistic] * (1 + run / 100)
for statistic in ("p50", "p99")
},
- "logical_payload_rate_gbps_at_latency_percentile": {
- statistic: component["logical_payload_rate_gbps_at_latency_percentile"][statistic] / (1 + run / 100)
+ "activation_data_rate_gbps_at_latency_percentile": {
+ statistic: component["activation_data_rate_gbps_at_latency_percentile"][statistic] / (1 + run / 100)
+ for statistic in ("p50", "p99")
+ },
+ "total_logical_data_rate_gbps_at_latency_percentile": {
+ statistic: component["total_logical_data_rate_gbps_at_latency_percentile"][statistic] / (1 + run / 100)
for statistic in ("p50", "p99")
},
}}
for run in range(3)
}
- internal = {"run_metrics": runs}
+ trial_blocks = {
+ run_id: {8: {
+ "dispatch": None,
+ "stage": None,
+ "combine": None,
+ "roundtrip": tuple(
+ tuple(metrics[8]["latency_us"]["p99"] for _ in range(8))
+ for _ in range(64)
+ ),
+ }}
+ for run_id, metrics in runs.items()
+ }
+ internal = {"run_metrics": runs, "trial_blocks": trial_blocks}
+ return item, internal
+
+
+def _precision_series(
+ seed: str,
+ precision_profile: str,
+ *,
+ tokens: tuple[int, ...] = (128,),
+) -> tuple[dict, dict]:
+ item, internal = _series(seed, "deepep", decision_grade=True)
+ precision = identity.precision_profile(precision_profile)
+ if precision_profile != identity.V1_CONTROL_PRECISION_PROFILE:
+ item["suite"] = "ep-precision-normal-v1"
+ item["publication_tier"] = "comparable-experimental"
+ item["workload"].update({
+ "precision_profile": precision_profile,
+ "dispatch_precision": precision["dispatch"],
+ "combine_precision": precision["combine"],
+ })
+ template = item["points"][0]
+ old_token = template["tokens_per_rank"]
+ old_metrics = {
+ run_id: metrics[old_token]
+ for run_id, metrics in internal["run_metrics"].items()
+ }
+ old_trials = {
+ run_id: metrics[old_token]
+ for run_id, metrics in internal["trial_blocks"].items()
+ }
+ item["points"] = []
+ for token in tokens:
+ point = copy.deepcopy(template)
+ point["tokens_per_rank"] = token
+ point["global_tokens"] = token * item["system"]["ep_size"]
+ point["point_id"] = identity.point_id(
+ series=item["series_id"], tokens_per_rank=token
+ )
+ point["correctness"]["precision"] = _precision_evidence(precision_profile)
+ point["roundtrip_token_rate_at_latency_percentile"] = {
+ name: point["global_tokens"] / (latency * 1e-6)
+ for name, latency in point["components"]["roundtrip"]["latency_us"].items()
+ }
+ item["points"].append(point)
+ internal["run_metrics"] = {
+ run_id: {token: copy.deepcopy(metrics) for token in tokens}
+ for run_id, metrics in old_metrics.items()
+ }
+ internal["trial_blocks"] = {
+ run_id: {token: copy.deepcopy(metrics) for token in tokens}
+ for run_id, metrics in old_trials.items()
+ }
return item, internal
@@ -590,13 +768,31 @@ def _dataset() -> dict:
"promotion": {
"status": "diagnostic", "reason": None, "matrix_id": "d" * 64,
"allocation_ids": [allocation], "required_allocations": 3,
+ "qualification_indices": [1],
"requested_cases": 1, "terminal_cases": 1,
+ "measured_cases": 1, "unsupported_cases": 0,
+ "requested_points": 1, "terminal_points": 1,
+ "measured_points": 1, "unsupported_points": 0,
"policy": "collectivex-decision-grade-v1",
},
"coverage": [{
"case_id": case, "label": "case", "required": True, "sku": "h100-dgxc",
- "backend": "deepep", "mode": "normal", "phase": "decode",
+ "suite": item["suite"], "workload": item["model"],
+ "publication_tier": item["publication_tier"],
+ "backend": "deepep", "backend_generation": item["backend"]["generation"],
+ "mode": "normal", "phase": "decode",
+ "routing": item["workload"]["routing"], "eplb": False,
+ "precision_profile": item["workload"]["precision_profile"],
+ "dispatch_precision": item["workload"]["dispatch_precision"],
+ "combine_precision": item["workload"]["combine_precision"],
+ "resource": item["resource"],
"topology": publisher._coverage_topology(item["system"]),
+ "points": [{
+ "point_id": item["points"][0]["point_id"],
+ "series_id": item["series_id"],
+ "tokens_per_rank": 8, "global_tokens": 64,
+ "terminal_status": "measured", "reason": None,
+ }],
"disposition": "runnable",
"selected_attempt_id": attempt,
"outcome": "success", "failure_mode": None, "reason": None,
@@ -608,6 +804,7 @@ def _dataset() -> dict:
"point_id": item["points"][0]["point_id"]}],
"case_id": case,
"allocation_id": allocation, "run_id": "1", "run_attempt": 1,
+ "qualification_index": 1,
"attempt_index": 1,
"selected": True, "outcome": "success", "failure_mode": None, "reason": None,
"series_id": item["series_id"],
@@ -618,21 +815,25 @@ def _dataset() -> dict:
}
-def _promoted_dataset() -> dict:
- specifications = (
- ("library-fast", "deepep", None, False),
- ("library-slow", "uccl", None, False),
- ("chip-peer", "deepep", "h200-dgxc", False),
- ("system-one", "nccl-ep", None, True),
- ("system-two", "nccl-ep", "h200-dgxc", True),
- ("routing-zipf", "deepep", None, False),
- ("routing-zipf-eplb", "deepep", None, False),
+def _promoted_dataset(*, precision_profiles: tuple[str, ...] = ()) -> dict:
+ specifications = [
+ ("library-fast", "deepep", None, False, None),
+ ("library-slow", "uccl", None, False, None),
+ ("chip-peer", "deepep", "h200-dgxc", False, None),
+ ("system-one", "nccl-ep", None, True, None),
+ ("system-two", "nccl-ep", "h200-dgxc", True, None),
+ ("routing-zipf", "deepep", None, False, None),
+ ("routing-zipf-eplb", "deepep", None, False, None),
+ ]
+ specifications.extend(
+ (f"precision-{index}", "deepep", None, False, precision_profile)
+ for index, precision_profile in enumerate(precision_profiles)
)
series = []
internals = {}
attempts = []
coverage = []
- for seed, backend, peer_sku, reference in specifications:
+ for seed, backend, peer_sku, reference, precision_profile in specifications:
item, internal = _series(seed, backend, decision_grade=True)
if peer_sku:
platform = publisher.capability.PLATFORMS[peer_sku]
@@ -650,11 +851,11 @@ def _promoted_dataset() -> dict:
item["workload"]["routing"] = "zipf"
if seed == "routing-zipf-eplb":
item["workload"]["eplb"] = True
- plan = contracts._expected_eplb_plan(
- "zipf", 8, 256, 288, item["system"]["ep_size"], 67, 2048
+ plan, calibration = contracts._expected_eplb_calibration(
+ "zipf", 7168, 8, 256, 288, item["system"]["ep_size"], 67, 2048
)
item["eplb"] = {
- "enabled": True, "planner": "greedy-rank-major-v1",
+ "enabled": True, **calibration, "planner": "greedy-rank-major-v1",
"mapping_sha256": contracts.eplb_contract.mapping_hash(plan),
"logical_experts": 256, "physical_experts": 288,
"redundant_experts": 32, "reference_tokens_per_rank": 2048,
@@ -664,6 +865,18 @@ def _promoted_dataset() -> dict:
"imbalance_after": plan["imbalance_after"],
}
item["build"]["implementation_contract_sha256"] = "8" * 64
+ if precision_profile is not None:
+ precision = identity.precision_profile(precision_profile)
+ item["suite"] = "ep-precision-normal-v1"
+ item["publication_tier"] = "comparable-experimental"
+ item["workload"].update({
+ "precision_profile": precision_profile,
+ "dispatch_precision": precision["dispatch"],
+ "combine_precision": precision["combine"],
+ })
+ item["points"][0]["correctness"]["precision"] = _precision_evidence(
+ precision_profile
+ )
case_id = identity.digest("case", publisher._public_case_factors(item))
item["case_ids"] = [case_id]
build = item["build"]
@@ -702,6 +915,7 @@ def _promoted_dataset() -> dict:
"evidence": [{"evidence_id": evidence_id, "point_id": point["point_id"]}],
"case_id": case_id, "allocation_id": allocation_id,
"run_id": str(run_id), "run_attempt": 1,
+ "qualification_index": run_id,
"attempt_index": 1, "selected": True,
"outcome": "success", "failure_mode": None, "reason": None,
"series_id": item["series_id"],
@@ -712,17 +926,34 @@ def _promoted_dataset() -> dict:
point["evidence_ids"] = evidence_ids
coverage.append({
"case_id": case_id, "label": seed, "required": True,
- "sku": item["system"]["sku"], "backend": backend,
- "mode": item["mode"], "phase": item["phase"], "disposition": "runnable",
+ "sku": item["system"]["sku"], "suite": item["suite"],
+ "workload": item["model"], "publication_tier": item["publication_tier"],
+ "backend": backend, "backend_generation": item["backend"]["generation"],
+ "mode": item["mode"], "phase": item["phase"],
+ "routing": item["workload"]["routing"], "eplb": item["workload"]["eplb"],
+ "precision_profile": item["workload"]["precision_profile"],
+ "dispatch_precision": item["workload"]["dispatch_precision"],
+ "combine_precision": item["workload"]["combine_precision"],
+ "resource": item["resource"], "disposition": "runnable",
"topology": publisher._coverage_topology(item["system"]),
+ "points": [{
+ "point_id": point["point_id"], "series_id": item["series_id"],
+ "tokens_per_rank": point["tokens_per_rank"],
+ "global_tokens": point["global_tokens"],
+ "terminal_status": "measured", "reason": None,
+ }],
"selected_attempt_id": case_attempts[-1], "outcome": "success",
"failure_mode": None, "reason": None, "attempt_ids": case_attempts,
})
series.append(item)
internals[item["series_id"]] = internal
- unsupported_case = (
- "cxcase-v1-f5d0b1c21df4eea0f5b74da25a0b85da7388a0dd22c8d81564c4bfb4e4465a34"
+ unsupported_case, unsupported = next(
+ (case_id, case)
+ for case_id, case in publisher._canonical_coverage_cases().items()
+ if case["sku"] == "mi355x" and case["backend"] == "deepep"
+ and case["phase"] == "decode" and case["routing"] == "uniform"
+ and not case["eplb"] and case["ep"] == 8
)
unsupported_attempts = []
for run_id in range(1, 4):
@@ -735,7 +966,7 @@ def _promoted_dataset() -> dict:
attempts.append({
"attempt_id": attempt_id, "evidence": [], "case_id": unsupported_case,
"allocation_id": allocation_id, "run_id": str(run_id),
- "run_attempt": 1,
+ "run_attempt": 1, "qualification_index": run_id,
"attempt_index": 1, "selected": True, "outcome": "unsupported",
"failure_mode": "capability", "reason": "backend-platform-unsupported",
"series_id": None, "completed_at": "2026-07-04T00:00:00Z",
@@ -743,13 +974,30 @@ def _promoted_dataset() -> dict:
unsupported_attempts.append(attempt_id)
coverage.append({
"case_id": unsupported_case, "label": "planned unsupported", "required": True,
- "sku": "mi355x", "backend": "deepep", "mode": "normal", "phase": "decode",
- "topology": {
- "ep_size": 8, "nodes": 1, "gpus_per_node": 8, "scale_up_domain": 8,
- "scope": "scale-up", "scale_up_transport": "xgmi",
- "scale_out_transport": None, "transport": "xgmi",
- "topology_class": "mi355x-xgmi",
+ "sku": unsupported["sku"], "suite": unsupported["suite"],
+ "workload": unsupported["workload"],
+ "publication_tier": unsupported["required_publication"],
+ "backend": unsupported["backend"], "backend_generation": None,
+ "mode": unsupported["mode"], "phase": unsupported["phase"],
+ "routing": unsupported["routing"], "eplb": unsupported["eplb"],
+ "precision_profile": identity.V1_CONTROL_PRECISION_PROFILE,
+ "dispatch_precision": identity.precision_profile(
+ identity.V1_CONTROL_PRECISION_PROFILE
+ )["dispatch"],
+ "combine_precision": identity.precision_profile(
+ identity.V1_CONTROL_PRECISION_PROFILE
+ )["combine"],
+ "resource": {
+ "mode": "fixed-profile", "profile": None,
+ "comm_units_kind": None, "configured_units": None,
},
+ "topology": publisher._coverage_topology(unsupported),
+ "points": [{
+ "point_id": None, "series_id": None,
+ "tokens_per_rank": token, "global_tokens": token * unsupported["ep"],
+ "terminal_status": "unsupported",
+ "reason": "backend-platform-unsupported",
+ } for token in map(int, unsupported["ladder"].split())],
"disposition": "unsupported", "selected_attempt_id": unsupported_attempts[-1],
"outcome": "unsupported", "failure_mode": "capability",
"reason": "backend-platform-unsupported", "attempt_ids": unsupported_attempts,
@@ -765,8 +1013,20 @@ def _promoted_dataset() -> dict:
"status": "promoted", "reason": None,
"matrix_id": publisher.CANONICAL_FULL_V1_MATRIX_SHA256,
"allocation_ids": sorted({item["allocation_id"] for item in attempts}),
- "required_allocations": 3, "requested_cases": len(coverage),
- "terminal_cases": len(coverage), "policy": "collectivex-decision-grade-v1",
+ "required_allocations": 3, "qualification_indices": [1, 2, 3],
+ "requested_cases": len(coverage), "terminal_cases": len(coverage),
+ "measured_cases": len(coverage) - 1, "unsupported_cases": 1,
+ "requested_points": sum(len(item["points"]) for item in coverage),
+ "terminal_points": sum(len(item["points"]) for item in coverage),
+ "measured_points": sum(
+ point["terminal_status"] == "measured"
+ for item in coverage for point in item["points"]
+ ),
+ "unsupported_points": sum(
+ point["terminal_status"] == "unsupported"
+ for item in coverage for point in item["points"]
+ ),
+ "policy": "collectivex-decision-grade-v1",
},
"coverage": sorted(coverage, key=lambda item: item["case_id"]),
"attempts": sorted(attempts, key=lambda item: item["attempt_id"]),
@@ -784,6 +1044,40 @@ def _cohort_counts(dataset: dict) -> dict[str, int]:
class PublisherTest(unittest.TestCase):
+ def test_trial_diagnostics_flag_drift_and_robust_outliers(self) -> None:
+ def runs() -> dict[str, dict[int, dict[str, object]]]:
+ return {
+ str(index): {
+ 8: {
+ "dispatch": tuple(tuple([10.0] * 8) for _ in range(64)),
+ "stage": None,
+ "combine": tuple(tuple([10.0] * 8) for _ in range(64)),
+ "roundtrip": tuple(tuple([10.0] * 8) for _ in range(64)),
+ }
+ }
+ for index in range(1, 4)
+ }
+
+ stable = publisher._trial_diagnostics(runs(), 8)
+ self.assertFalse(stable["flagged"])
+
+ drift = runs()
+ drift["1"][8]["roundtrip"] = tuple(
+ tuple([12.0 if trial >= 56 else 10.0] * 8) for trial in range(64)
+ )
+ self.assertEqual(publisher._trial_diagnostics(drift, 8)["reasons"], ["trial-drift"])
+
+ outliers = runs()
+ outliers["1"][8]["roundtrip"] = tuple(
+ tuple([100.0 if 20 <= trial < 36 else 10.0] * 8) for trial in range(64)
+ )
+ summary = publisher._trial_diagnostics(outliers, 8)
+ self.assertEqual(summary["reasons"], ["trial-outliers"])
+ self.assertGreater(
+ summary["components"]["roundtrip"]["robust_outlier_fraction"],
+ publisher.TRIAL_OUTLIER_FRACTION_LIMIT,
+ )
+
def test_terminal_allocation_and_source_status_are_bound(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary).resolve()
@@ -927,6 +1221,27 @@ def test_native_raw_and_sample_schema_match_semantic_validator(self) -> None:
publisher._schema("samples-v1.schema.json", samples)
publisher._schema("raw-case-v1.schema.json", raw)
self.assertIs(contracts.validate_raw_document(raw, samples), raw)
+ provenance = raw["provenance"]
+ image = provenance["image"]
+ self.assertTrue(contracts.provenance_complete(
+ raw["implementation"]["provenance"], raw["case"]["backend"],
+ provenance["git_run"],
+ allocation_stratum_sha256=provenance["allocation_stratum_sha256"],
+ image_digest=image["digest"], image_verified=image["digest_verified"],
+ squash_sha256=image["squash_sha256"],
+ ))
+ self.assertFalse(contracts.provenance_complete(
+ raw["implementation"]["provenance"], raw["case"]["backend"],
+ provenance["git_run"], allocation_stratum_sha256=None,
+ image_digest=image["digest"], image_verified=image["digest_verified"],
+ squash_sha256=image["squash_sha256"],
+ ))
+ missing_stratum = copy.deepcopy(raw)
+ missing_stratum["provenance"]["allocation_stratum_sha256"] = None
+ with self.assertRaises(publisher.PublisherError):
+ publisher._schema("raw-case-v1.schema.json", missing_stratum)
+ with self.assertRaisesRegex(contracts.ContractError, "allocation stratum"):
+ contracts.validate_raw_document(missing_stratum, samples)
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary).resolve()
(root / "samples.json").write_bytes(contracts.canonical_json_bytes(samples))
@@ -1009,7 +1324,9 @@ def test_native_contract_recomputes_routing_receive_histograms_and_anomalies(sel
changed = tampered["measurement"]["rows"][0]
changed["routing"]["routed_copies"] *= 2
for name in ("combine", "dispatch", "roundtrip"):
- changed["logical_bytes"][name] *= 2
+ byte_provenance = changed["byte_provenance"][name]
+ byte_provenance["activation_data_bytes"] *= 2
+ byte_provenance["total_logical_bytes"] *= 2
with self.assertRaisesRegex(contracts.ContractError, "routing.routed_copies"):
contracts.validate_raw_document(tampered, samples)
@@ -1208,25 +1525,21 @@ def test_invalid_retry_is_quarantined_before_valid_retry_upload(self) -> None:
self.assertTrue((root / "a01.json.quarantine").is_file())
self.assertTrue((root / "a01.samples.json.quarantine").is_file())
- def test_ingest_archives_first_and_publishes_latest_attempt(self) -> None:
+ def test_ingest_archives_without_publishing_a_channel(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary).resolve()
matrix, artifact = _unsupported_delivery(root)
self.assertEqual(len(summarize.load_results(str(artifact), None, None)), 1)
result = publisher.ingest_command(_args(root / "store", matrix, artifact))
store = publisher.Store(root / "store")
- pointer = store.verify_channel("latest-attempt")
self.assertEqual(result["status"], "accepted")
- self.assertEqual(pointer["dataset"]["sha256"], result["dataset_sha256"])
self.assertTrue((store.incoming / result["incoming_id"] / "COMPLETE").is_file())
self.assertTrue((store.bundles / result["bundle_id"] / "COMPLETE").is_file())
- self.assertFalse((store.channels / "dev-latest.json").exists())
+ self.assertEqual(list(store.channels.iterdir()), [])
+ self.assertEqual(list(store.datasets.iterdir()), [])
self.assertEqual(os.stat(store.private).st_mode & 0o777, 0o700)
self.assertEqual(os.stat(store.public).st_mode & 0o777, 0o755)
self.assertEqual(os.stat(store.bundles / result["bundle_id"]).st_mode & 0o777, 0o500)
- dataset_dir = store.datasets / result["dataset_sha256"]
- self.assertEqual(os.stat(dataset_dir).st_mode & 0o777, 0o555)
- self.assertEqual(os.stat(dataset_dir / "dataset.json").st_mode & 0o777, 0o444)
def test_repeated_ingest_is_content_idempotent(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
@@ -1235,15 +1548,12 @@ def test_repeated_ingest_is_content_idempotent(self) -> None:
args = _args(root / "store", matrix, artifact)
first = publisher.ingest_command(args)
store = publisher.Store(root / "store")
- pointer_before = (store.channels / "latest-attempt.json").read_bytes()
second = publisher.ingest_command(args)
self.assertEqual(second, first)
- self.assertEqual(
- (store.channels / "latest-attempt.json").read_bytes(), pointer_before
- )
self.assertEqual(len(list(store.incoming.iterdir())), 1)
self.assertEqual(len(list(store.bundles.iterdir())), 1)
- self.assertEqual(len(list(store.datasets.iterdir())), 1)
+ self.assertEqual(len(list(store.datasets.iterdir())), 0)
+ self.assertEqual(len(list(store.channels.iterdir())), 0)
bundle = publisher.strict_load(
store.bundles / first["bundle_id"] / "bundle.json"
)
@@ -1441,10 +1751,13 @@ def test_promotion_requires_an_eligible_cohort_for_every_comparison_kind(self) -
unstable_fast_internal["run_metrics"]["1"][8]["latency_us"][statistic]
/ 2
)
- unstable_slow_internal["run_metrics"]["1"][8]["logical_payload_rate_gbps_at_latency_percentile"][statistic] = (
- unstable_fast_internal["run_metrics"]["1"][8]["logical_payload_rate_gbps_at_latency_percentile"][statistic]
- * 2
- )
+ for field in (
+ "activation_data_rate_gbps_at_latency_percentile",
+ "total_logical_data_rate_gbps_at_latency_percentile",
+ ):
+ unstable_slow_internal["run_metrics"]["1"][8][field][statistic] = (
+ unstable_fast_internal["run_metrics"]["1"][8][field][statistic] * 2
+ )
series = [stable_fast, stable_slow, unstable_fast, unstable_slow]
internals = {
stable_fast["series_id"]: stable_fast_internal,
@@ -1567,14 +1880,17 @@ def test_promotion_rejects_more_than_three_bundles(self) -> None:
"id": str(run_id), "cases": [],
"manifest": {
"matrix": {"sha256": publisher.CANONICAL_FULL_V1_MATRIX_SHA256},
- "run": {"run_id": str(run_id), "run_attempt": 1},
+ "run": {
+ "run_id": str(run_id), "run_attempt": 1,
+ "qualification_index": min(run_id, 3),
+ },
},
}
for run_id in range(1, 5)
}
with mock.patch.object(
publisher, "load_bundle", side_effect=lambda _, bundle_id: bundles[bundle_id]
- ), self.assertRaisesRegex(publisher.PublisherError, "three independent"):
+ ), self.assertRaisesRegex(publisher.PublisherError, "qualification indices"):
publisher.build_dataset(object(), list(bundles), promote=True)
dataset = _promoted_dataset()
@@ -1644,7 +1960,9 @@ def test_standalone_promotion_binds_matrix_and_requested_dispositions(self) -> N
"CANONICAL_FULL_V1_CASE_CATALOG_SHA256",
publisher._case_disposition_catalog_sha256(broken["coverage"]),
), self.assertRaisesRegex(
- publisher.PublisherError, "requested dispositions"
+ publisher.PublisherError,
+ "requested dispositions" if original == "runnable"
+ else "coverage dimensions",
):
publisher.validate_public_dataset(broken)
@@ -1661,6 +1979,15 @@ def test_workflow_matrix_and_catalog_digests_do_not_drift(self) -> None:
capture_output=True,
)
self.assertEqual(result.returncode, 0, result.stderr)
+ if publisher.capability.provisional_precision_targets():
+ workflow = (
+ ROOT.parent.parent / ".github" / "workflows" / "collectivex-sweep.yml"
+ ).read_text()
+ self.assertIn(
+ "V1 sweeps require every precision capability cell to be resolved",
+ workflow,
+ )
+ return
self.assertEqual(
hashlib.sha256(matrix_path.read_bytes()).hexdigest(),
publisher.CANONICAL_FULL_V1_MATRIX_SHA256,
@@ -1741,7 +2068,10 @@ def test_build_promotion_requires_canonical_full_matrix(self) -> None:
"id": str(run_id), "cases": [],
"manifest": {
"matrix": {"sha256": "d" * 64},
- "run": {"run_id": str(run_id), "run_attempt": 1},
+ "run": {
+ "run_id": str(run_id), "run_attempt": 1,
+ "qualification_index": run_id,
+ },
},
}
for run_id in range(1, 4)
@@ -1751,7 +2081,7 @@ def test_build_promotion_requires_canonical_full_matrix(self) -> None:
), self.assertRaisesRegex(publisher.PublisherError, "canonical full-v1 matrix"):
publisher.build_dataset(object(), list(bundles), promote=True)
- def test_rejection_updates_latest_but_never_dev_latest(self) -> None:
+ def test_rejection_is_quarantined_without_updating_dev_latest(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary).resolve()
matrix, artifact = _unsupported_delivery(root)
@@ -1762,9 +2092,8 @@ def test_rejection_updates_latest_but_never_dev_latest(self) -> None:
with self.assertRaises(publisher.PublisherError):
publisher.ingest_command(_args(store.root, matrix, artifact))
self.assertEqual((store.channels / "dev-latest.json").read_bytes(), sentinel)
- pointer = store.verify_channel("latest-attempt")
- dataset = publisher.strict_load(store.public / pointer["dataset"]["path"])
- self.assertEqual(dataset["promotion"]["status"], "quarantined")
+ self.assertFalse((store.channels / "latest-attempt.json").exists())
+ self.assertEqual(list(store.datasets.iterdir()), [])
self.assertTrue(any(store.quarantine.iterdir()))
def test_repeated_rejection_is_content_idempotent(self) -> None:
@@ -1775,23 +2104,23 @@ def test_repeated_rejection_is_content_idempotent(self) -> None:
(artifact / "unknown.json").write_text('{"format":"unknown"}')
with self.assertRaises(publisher.PublisherError):
publisher.ingest_command(_args(store.root, matrix, artifact))
- pointer = (store.channels / "latest-attempt.json").read_bytes()
counts = tuple(
len(list(path.iterdir()))
- for path in (store.incoming, store.quarantine, store.datasets)
+ for path in (store.incoming, store.quarantine, store.datasets, store.channels)
)
with self.assertRaises(publisher.PublisherError):
publisher.ingest_command(_args(store.root, matrix, artifact))
- self.assertEqual((store.channels / "latest-attempt.json").read_bytes(), pointer)
self.assertEqual(
tuple(
len(list(path.iterdir()))
- for path in (store.incoming, store.quarantine, store.datasets)
+ for path in (
+ store.incoming, store.quarantine, store.datasets, store.channels
+ )
),
counts,
)
- def test_distinct_rejections_advance_latest_attempt(self) -> None:
+ def test_distinct_rejections_create_distinct_quarantine_objects(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary).resolve()
matrix, artifact = _unsupported_delivery(root)
@@ -1800,13 +2129,15 @@ def test_distinct_rejections_advance_latest_attempt(self) -> None:
unknown.write_text('{"format":"unknown-one"}')
with self.assertRaises(publisher.PublisherError):
publisher.ingest_command(_args(store.root, matrix, artifact))
- first = (store.channels / "latest-attempt.json").read_bytes()
+ first = {path.name for path in store.quarantine.iterdir()}
unknown.write_text('{"format":"unknown-two"}')
with self.assertRaises(publisher.PublisherError):
publisher.ingest_command(_args(store.root, matrix, artifact))
- second = (store.channels / "latest-attempt.json").read_bytes()
+ second = {path.name for path in store.quarantine.iterdir()}
self.assertNotEqual(second, first)
- self.assertEqual(len(list(store.datasets.iterdir())), 2)
+ self.assertEqual(len(second), 2)
+ self.assertEqual(list(store.datasets.iterdir()), [])
+ self.assertEqual(list(store.channels.iterdir()), [])
def test_zip_traversal_is_rejected(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
@@ -1903,7 +2234,9 @@ def test_delivery_rejects_extra_archive_and_non_native_member(self) -> None:
def test_rates_invert_latency_and_global_tokens_use_ep_size(self) -> None:
dataset = _dataset()
publisher.validate_public_dataset(dataset)
- rates = dataset["series"][0]["points"][0]["components"]["roundtrip"]["logical_payload_rate_gbps_at_latency_percentile"]
+ rates = dataset["series"][0]["points"][0]["components"]["roundtrip"][
+ "activation_data_rate_gbps_at_latency_percentile"
+ ]
self.assertGreater(rates["p50"], rates["p99"])
broken = copy.deepcopy(dataset)
broken["series"][0]["points"][0]["global_tokens"] = 128
@@ -1925,9 +2258,8 @@ def test_rates_invert_latency_and_global_tokens_use_ep_size(self) -> None:
publisher.validate_public_dataset(broken)
broken = copy.deepcopy(dataset)
component = broken["series"][0]["points"][0]["components"]["roundtrip"]
- component["logical_bytes"] = None
- component["logical_payload_rate_gbps_at_latency_percentile"] = None
- with self.assertRaisesRegex(publisher.PublisherError, "logical bandwidth is missing"):
+ component["activation_data_rate_gbps_at_latency_percentile"] = None
+ with self.assertRaisesRegex(publisher.PublisherError, "measured data rates are missing"):
publisher.validate_public_dataset(broken)
for mutate in (
@@ -2007,6 +2339,7 @@ def test_cohort_contract_and_labels_name_mode_explicitly(self) -> None:
"controlled_factors": cohort["controlled_factors"],
"varying_factors": cohort["varying_factors"],
})
+ broken["cohorts"].sort(key=lambda item: item["cohort_id"])
with self.assertRaisesRegex(publisher.PublisherError, "cohort factors"):
publisher.validate_public_dataset(broken)
@@ -2162,11 +2495,14 @@ def test_routing_implementation_mismatch_blocks_all_decisions(self) -> None:
"latency_us": {
name: roundtrip["latency_us"][name] for name in ("p50", "p99")
},
- "logical_payload_rate_gbps_at_latency_percentile": {
- name: roundtrip[
- "logical_payload_rate_gbps_at_latency_percentile"
- ][name]
- for name in ("p50", "p99")
+ **{
+ field: {
+ name: roundtrip[field][name] for name in ("p50", "p99")
+ }
+ for field in (
+ "activation_data_rate_gbps_at_latency_percentile",
+ "total_logical_data_rate_gbps_at_latency_percentile",
+ )
},
}
internals[member["series_id"]] = {
@@ -2234,7 +2570,7 @@ def test_promoted_series_fields_are_bound_to_case_and_series_identities(self) ->
(lambda item: item["resource"].update({
"profile": "profile-fabricated", "configured_units": 99,
}), "configuration"),
- (lambda item: item["system"].update({"label": "Fabricated H100"}), "commit"),
+ (lambda item: item["system"].update({"label": "Fabricated H100"}), "projection|commit"),
):
changed = copy.deepcopy(dataset)
mutate(changed["series"][0])
@@ -2261,20 +2597,24 @@ def test_all_decision_metrics_require_stable_repeat_ordering(self) -> None:
self.assertTrue(library["eligibility"]["decision_grade"])
self.assertEqual(
len([item for item in rankings if item["cohort_id"] == library["cohort_id"]]),
- 4,
+ 6,
)
self.assertEqual(
len([
item for item in recommendations
if item["cohort_id"] == library["cohort_id"]
]),
- 4,
+ 1,
)
for statistic in ("p50", "p99"):
- slow_internal["run_metrics"]["1"][8]["logical_payload_rate_gbps_at_latency_percentile"][statistic] = (
- fast_internal["run_metrics"]["1"][8]["logical_payload_rate_gbps_at_latency_percentile"][statistic] * 2
- )
+ for field in (
+ "activation_data_rate_gbps_at_latency_percentile",
+ "total_logical_data_rate_gbps_at_latency_percentile",
+ ):
+ slow_internal["run_metrics"]["1"][8][field][statistic] = (
+ fast_internal["run_metrics"]["1"][8][field][statistic] * 2
+ )
cohorts, rankings, recommendations, _ = publisher.build_decisions(
[fast, slow], internals
)
@@ -2288,6 +2628,216 @@ def test_all_decision_metrics_require_stable_repeat_ordering(self) -> None:
item["cohort_id"] == library["cohort_id"] for item in recommendations
))
+ def test_p99_bootstrap_is_deterministic_and_dataset_bound(self) -> None:
+ fast, fast_internal = _series("bootstrap-fast", "deepep", decision_grade=True)
+ slow, slow_internal = _series("bootstrap-slow", "uccl", decision_grade=True)
+ internals = {
+ fast["series_id"]: fast_internal,
+ slow["series_id"]: slow_internal,
+ }
+
+ first = publisher._hierarchical_p99_ratio(
+ fast["series_id"], slow["series_id"], 8, internals, "a" * 64
+ )
+ repeated = publisher._hierarchical_p99_ratio(
+ fast["series_id"], slow["series_id"], 8, internals, "a" * 64
+ )
+ rebound = publisher._hierarchical_p99_ratio(
+ fast["series_id"], slow["series_id"], 8, internals, "b" * 64
+ )
+
+ self.assertEqual(first, repeated)
+ self.assertEqual(first["resamples"], 10_000)
+ self.assertEqual(first["confidence"], 0.95)
+ self.assertEqual(first["equivalence_band"], 0.05)
+ self.assertTrue(first["all_runs_agree"])
+ self.assertTrue(first["baseline_wins"])
+ self.assertGreater(first["ci95"][0], 1.05)
+ self.assertNotEqual(first["seed_sha256"], rebound["seed_sha256"])
+
+ def test_p99_equivalence_band_emits_competition_tie_without_recommendation(self) -> None:
+ fast, fast_internal = _series("tie-fast", "deepep", decision_grade=True)
+ near, near_internal = _series("tie-near", "uccl", decision_grade=True)
+ fast_point = fast["points"][0]
+ near_point = near["points"][0]
+ fast_component = fast_point["components"]["roundtrip"]
+ near_component = near_point["components"]["roundtrip"]
+ for statistic, latency in fast_component["latency_us"].items():
+ near_latency = latency * 1.03
+ near_component["latency_us"][statistic] = near_latency
+ for field, byte_field in (
+ ("activation_data_rate_gbps_at_latency_percentile", "activation_data_bytes"),
+ ("total_logical_data_rate_gbps_at_latency_percentile", "total_logical_bytes"),
+ ):
+ near_component[field][statistic] = (
+ near_component["byte_provenance"][byte_field]
+ / (near_latency * 1000.0)
+ )
+ near_point["roundtrip_token_rate_at_latency_percentile"][statistic] = (
+ near_point["global_tokens"] / (near_latency * 1e-6)
+ )
+ for run_id, fast_metrics in fast_internal["run_metrics"].items():
+ for statistic in ("p50", "p99"):
+ latency = fast_metrics[8]["latency_us"][statistic] * 1.03
+ near_internal["run_metrics"][run_id][8]["latency_us"][statistic] = latency
+ for field, byte_field in (
+ ("activation_data_rate_gbps_at_latency_percentile", "activation_data_bytes"),
+ ("total_logical_data_rate_gbps_at_latency_percentile", "total_logical_bytes"),
+ ):
+ near_internal["run_metrics"][run_id][8][field][statistic] = (
+ near_component["byte_provenance"][byte_field]
+ / (latency * 1000.0)
+ )
+ near_internal["trial_blocks"][run_id][8]["roundtrip"] = tuple(
+ tuple(sample * 1.03 for sample in block)
+ for block in fast_internal["trial_blocks"][run_id][8]["roundtrip"]
+ )
+ internals = {
+ fast["series_id"]: fast_internal,
+ near["series_id"]: near_internal,
+ }
+
+ cohorts, rankings, recommendations, _ = publisher.build_decisions(
+ [fast, near], internals, dataset_binding="c" * 64
+ )
+ library = next(item for item in cohorts if item["kind"] == "library")
+ ranking = next(
+ item for item in rankings
+ if item["cohort_id"] == library["cohort_id"]
+ and item["metric"]["measure"] == "latency_us"
+ and item["metric"]["statistic"] == "p99"
+ )
+ self.assertEqual([entry["rank"] for entry in ranking["entries"]], [1, 1])
+ self.assertFalse(any(
+ item["cohort_id"] == library["cohort_id"]
+ for item in recommendations
+ ))
+ self.assertNotIn(
+ "trial_blocks", json.dumps({"series": [fast, near], "rankings": rankings})
+ )
+
+ def test_p99_winner_requires_every_run_to_agree(self) -> None:
+ fast, fast_internal = _series("run-fast", "deepep", decision_grade=True)
+ slow, slow_internal = _series("run-slow", "uccl", decision_grade=True)
+ ratios = {"0": 0.98, "1": 1.20, "2": 1.20}
+ for run_id, ratio in ratios.items():
+ slow_internal["trial_blocks"][run_id][8]["roundtrip"] = tuple(
+ tuple(sample * ratio for sample in block)
+ for block in fast_internal["trial_blocks"][run_id][8]["roundtrip"]
+ )
+ result = publisher._hierarchical_p99_ratio(
+ fast["series_id"], slow["series_id"], 8,
+ {
+ fast["series_id"]: fast_internal,
+ slow["series_id"]: slow_internal,
+ },
+ "d" * 64,
+ )
+ self.assertFalse(result["all_runs_agree"])
+ self.assertFalse(result["baseline_wins"])
+ self.assertTrue(result["tie"])
+
+ def test_precision_cohorts_isolate_axes_and_never_recommend(self) -> None:
+ profiles = (
+ identity.V1_CONTROL_PRECISION_PROFILE,
+ "d-fp8-e4m3fn-b128-f32-fused.c-bf16",
+ "d-bf16.c-logfmt10-dynamic64",
+ "d-fp8-e4m3fn-b128-f32-fused.c-logfmt10-dynamic64",
+ )
+ series = []
+ internals = {}
+ for index, profile_id in enumerate(profiles):
+ item, internal = _series(
+ f"precision-{index}", "deepep", decision_grade=True
+ )
+ precision = identity.precision_profile(profile_id)
+ item["suite"] = (
+ "ep-low-latency-v1"
+ if index == 0
+ else "ep-precision-low-latency-v1"
+ )
+ item["mode"] = "low-latency"
+ item["publication_tier"] = (
+ "official" if index == 0 else "comparable-experimental"
+ )
+ item["workload"].update({
+ "precision_profile": profile_id,
+ "dispatch_precision": precision["dispatch"],
+ "combine_precision": precision["combine"],
+ })
+ item["series_id"] = identity.series_id({"precision-fixture": profile_id})
+ series.append(item)
+ internals[item["series_id"]] = internal
+
+ cohorts, rankings, recommendations, sensitivities = publisher.build_decisions(
+ series, internals, dataset_binding="e" * 64
+ )
+ precision_cohorts = [
+ cohort for cohort in cohorts
+ if cohort["kind"] in publisher.PRECISION_COHORT_KINDS
+ ]
+ self.assertEqual(
+ {kind: sum(cohort["kind"] == kind for cohort in precision_cohorts)
+ for kind in publisher.PRECISION_COHORT_KINDS},
+ {"dispatch-precision": 2, "combine-precision": 2, "precision-pair": 1},
+ )
+ self.assertTrue(all(
+ cohort["publication_tier"] == "comparable-experimental"
+ and cohort["eligibility"]["decision_grade"]
+ for cohort in precision_cohorts
+ ))
+ self.assertEqual(len(rankings), 30)
+ self.assertEqual(len(sensitivities), 24)
+ self.assertEqual(recommendations, [])
+ pair = next(
+ cohort for cohort in precision_cohorts
+ if cohort["kind"] == "precision-pair"
+ )
+ self.assertEqual(
+ pair["varying_factors"],
+ [
+ "dispatch-precision", "combine-precision", "precision-profile",
+ "resource",
+ ],
+ )
+ self.assertNotIn("resource", pair["controlled_factors"])
+ self.assertFalse(any(
+ sensitivity["cohort_id"] == pair["cohort_id"]
+ for sensitivity in sensitivities
+ ))
+
+ def test_private_trial_copy_is_component_extensible(self) -> None:
+ blocks = [[float(trial + iteration + 1) for iteration in range(8)]
+ for trial in range(64)]
+ copied = publisher._private_trial_components({
+ "points": [{
+ "tokens_per_rank": 8,
+ "components": {
+ "roundtrip": {"availability": "measured", "trials": blocks},
+ "stage": {"availability": "measured", "trials": blocks},
+ "combine": {"availability": "not-applicable", "trials": None},
+ },
+ }],
+ })
+ self.assertEqual(set(copied[8]), {"roundtrip", "stage", "combine"})
+ self.assertEqual(len(copied[8]["stage"]), 64)
+ self.assertIsNone(copied[8]["combine"])
+
+ def test_missing_private_trials_blocks_decision_grade(self) -> None:
+ fast, fast_internal = _series("trials-fast", "deepep", decision_grade=True)
+ slow, slow_internal = _series("trials-slow", "uccl", decision_grade=True)
+ del slow_internal["trial_blocks"]
+ cohorts, rankings, recommendations, sensitivities = publisher.build_decisions(
+ [fast, slow], {
+ fast["series_id"]: fast_internal,
+ slow["series_id"]: slow_internal,
+ }
+ )
+ library = next(item for item in cohorts if item["kind"] == "library")
+ self.assertFalse(library["eligibility"]["decision_grade"])
+ self.assertIn("missing-trial-blocks", library["eligibility"]["reasons"])
+ self.assertEqual((rankings, recommendations, sensitivities), ([], [], []))
+
def test_extra_eligibility_reason_blocks_decision_grade(self) -> None:
allocations = [identity.allocation_id({"run": run}) for run in range(3)]
eligibility = publisher._eligibility_record(
@@ -2313,54 +2863,69 @@ def test_schema_is_strict_and_channel_target_must_be_complete(self) -> None:
publisher.validate_public_dataset(_dataset())
with tempfile.TemporaryDirectory() as temporary:
store = publisher.Store(Path(temporary).resolve())
- dataset = _dataset()
- digest, size = store.install_dataset(dataset)
- store.update_channel("latest-attempt", digest, size, dataset["generated_at"])
- self.assertEqual(store.verify_channel("latest-attempt")["dataset"]["sha256"], digest)
- channel_path = store.channels / "latest-attempt.json"
- pointer = publisher.strict_load(channel_path)
- pointer["generated_at"] = "2099-01-01T00:00:00Z"
- channel_path.write_bytes(contracts.canonical_json_bytes(pointer))
- with self.assertRaisesRegex(publisher.PublisherError, "metadata differs"):
- store.verify_channel("latest-attempt")
- store.update_channel("latest-attempt", digest, size, dataset["generated_at"])
- with self.assertRaisesRegex(publisher.PublisherError, "metadata differs"):
- store.update_channel(
- "latest-attempt", digest, size + 1, dataset["generated_at"]
- )
- with self.assertRaisesRegex(publisher.PublisherError, "metadata differs"):
- store.update_channel(
- "latest-attempt", digest, size, "2026-07-05T00:00:00Z"
+ dataset = _promoted_dataset()
+ with mock.patch.object(
+ publisher, "CANONICAL_FULL_V1_CASE_CATALOG_SHA256",
+ publisher._case_disposition_catalog_sha256(dataset["coverage"]),
+ ), mock.patch.object(
+ publisher, "REQUIRED_PROMOTION_COHORT_COUNTS", _cohort_counts(dataset),
+ ):
+ digest, size = store.install_dataset(dataset)
+ store.update_channel("dev-latest", digest, size, dataset["generated_at"])
+ self.assertEqual(
+ store.verify_channel("dev-latest")["dataset"]["sha256"], digest
)
- os.chmod(channel_path, 0o666)
- with self.assertRaisesRegex(publisher.PublisherError, "regular 644"):
- store.verify_channel("latest-attempt")
- os.chmod(channel_path, 0o644)
- dataset_dir = store.datasets / digest
- os.chmod(dataset_dir, 0o755)
- with self.assertRaisesRegex(publisher.PublisherError, "mode differs"):
- store.verify_channel("latest-attempt")
- os.chmod(dataset_dir, 0o555)
- os.chmod(dataset_dir / "dataset.json", 0o644)
- with self.assertRaisesRegex(publisher.PublisherError, "mode differs"):
- store.verify_channel("latest-attempt")
- os.chmod(dataset_dir / "dataset.json", 0o444)
- os.chmod(dataset_dir, 0o755)
- (dataset_dir / "COMPLETE").unlink()
- os.chmod(dataset_dir, 0o555)
- with self.assertRaisesRegex(publisher.PublisherError, "incomplete"):
- store.verify_channel("latest-attempt")
+ channel_path = store.channels / "dev-latest.json"
+ pointer = publisher.strict_load(channel_path)
+ pointer["generated_at"] = "2099-01-01T00:00:00Z"
+ channel_path.write_bytes(contracts.canonical_json_bytes(pointer))
+ with self.assertRaisesRegex(publisher.PublisherError, "metadata differs"):
+ store.verify_channel("dev-latest")
+ store.update_channel("dev-latest", digest, size, dataset["generated_at"])
+ with self.assertRaisesRegex(publisher.PublisherError, "metadata differs"):
+ store.update_channel(
+ "dev-latest", digest, size + 1, dataset["generated_at"]
+ )
+ with self.assertRaisesRegex(publisher.PublisherError, "metadata differs"):
+ store.update_channel(
+ "dev-latest", digest, size, "2026-07-05T00:00:00Z"
+ )
+ os.chmod(channel_path, 0o666)
+ with self.assertRaisesRegex(publisher.PublisherError, "regular 644"):
+ store.verify_channel("dev-latest")
+ os.chmod(channel_path, 0o644)
+ dataset_dir = store.datasets / digest
+ os.chmod(dataset_dir, 0o755)
+ with self.assertRaisesRegex(publisher.PublisherError, "mode differs"):
+ store.verify_channel("dev-latest")
+ os.chmod(dataset_dir, 0o555)
+ os.chmod(dataset_dir / "dataset.json", 0o644)
+ with self.assertRaisesRegex(publisher.PublisherError, "mode differs"):
+ store.verify_channel("dev-latest")
+ os.chmod(dataset_dir / "dataset.json", 0o444)
+ os.chmod(dataset_dir, 0o755)
+ (dataset_dir / "COMPLETE").unlink()
+ os.chmod(dataset_dir, 0o555)
+ with self.assertRaisesRegex(publisher.PublisherError, "incomplete"):
+ store.verify_channel("dev-latest")
def test_store_modes_do_not_depend_on_process_umask(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
previous = os.umask(0o077)
try:
store = publisher.Store(Path(temporary).resolve())
- dataset = _dataset()
- digest, size = store.install_dataset(dataset)
- store.update_channel(
- "latest-attempt", digest, size, dataset["generated_at"]
- )
+ dataset = _promoted_dataset()
+ with mock.patch.object(
+ publisher, "CANONICAL_FULL_V1_CASE_CATALOG_SHA256",
+ publisher._case_disposition_catalog_sha256(dataset["coverage"]),
+ ), mock.patch.object(
+ publisher, "REQUIRED_PROMOTION_COHORT_COUNTS",
+ _cohort_counts(dataset),
+ ):
+ digest, size = store.install_dataset(dataset)
+ store.update_channel(
+ "dev-latest", digest, size, dataset["generated_at"]
+ )
with store.locked():
pass
finally:
@@ -2370,7 +2935,7 @@ def test_store_modes_do_not_depend_on_process_umask(self) -> None:
0o750,
)
self.assertEqual(
- (store.channels / "latest-attempt.json").stat().st_mode & 0o777,
+ (store.channels / "dev-latest.json").stat().st_mode & 0o777,
0o644,
)
self.assertEqual(
@@ -2382,7 +2947,7 @@ def test_store_modes_do_not_depend_on_process_umask(self) -> None:
0o600,
)
- def test_verify_requires_bootstrap_but_dev_latest_is_optional(self) -> None:
+ def test_verify_requires_a_promoted_dev_latest_channel(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary).resolve()
args = types.SimpleNamespace(
@@ -2391,27 +2956,30 @@ def test_verify_requires_bootstrap_but_dev_latest_is_optional(self) -> None:
with self.assertRaises(publisher.PublisherError):
publisher.verify_command(args)
store = publisher.Store(args.store_root)
- dataset = publisher._quarantine_dataset(
- "awaiting-v1-runs", "2026-07-04T00:00:00Z"
- )
- digest, size = store.install_dataset(dataset)
- store.update_channel(
- "latest-attempt", digest, size, "2026-07-04T00:00:00Z"
- )
- result = publisher.verify_command(args)
- self.assertEqual(set(result["channels"]), {"latest-attempt"})
- explicit = types.SimpleNamespace(
- store_root=args.store_root, channel=["dev-latest"], bundle=[]
- )
- with self.assertRaises(publisher.PublisherError):
- publisher.verify_command(explicit)
- dev_pointer = copy.deepcopy(store.verify_channel("latest-attempt"))
- dev_pointer["channel"] = "dev-latest"
- (store.channels / "dev-latest.json").write_bytes(
- contracts.canonical_json_bytes(dev_pointer)
+ dataset = _promoted_dataset()
+ with mock.patch.object(
+ publisher, "CANONICAL_FULL_V1_CASE_CATALOG_SHA256",
+ publisher._case_disposition_catalog_sha256(dataset["coverage"]),
+ ), mock.patch.object(
+ publisher, "REQUIRED_PROMOTION_COHORT_COUNTS", _cohort_counts(dataset),
+ ):
+ digest, size = store.install_dataset(dataset)
+ store.update_channel(
+ "dev-latest", digest, size, dataset["generated_at"]
+ )
+ result = publisher.verify_command(args)
+ self.assertEqual(set(result["channels"]), {"dev-latest"})
+ explicit = types.SimpleNamespace(
+ store_root=args.store_root, channel=["dev-latest"], bundle=[]
+ )
+ self.assertEqual(
+ publisher.verify_command(explicit)["channels"], result["channels"]
+ )
+ unknown = types.SimpleNamespace(
+ store_root=args.store_root, channel=["latest-attempt"], bundle=[]
)
- with self.assertRaisesRegex(publisher.PublisherError, "non-promoted"):
- publisher.verify_command(args)
+ with self.assertRaisesRegex(publisher.PublisherError, "unknown channel"):
+ publisher.verify_command(unknown)
if __name__ == "__main__":
diff --git a/experimental/CollectiveX/tests/test_qualification_planning.py b/experimental/CollectiveX/tests/test_qualification_planning.py
new file mode 100644
index 0000000000..4718818ed9
--- /dev/null
+++ b/experimental/CollectiveX/tests/test_qualification_planning.py
@@ -0,0 +1,247 @@
+#!/usr/bin/env python3
+"""CPU-only tests for qualification-specific shard execution planning."""
+from __future__ import annotations
+
+import copy
+import hashlib
+import json
+import os
+from pathlib import Path
+import sys
+import tempfile
+import unittest
+from unittest import mock
+
+
+HERE = Path(__file__).resolve().parent
+ROOT = HERE.parent
+sys.path[:0] = [str(ROOT), str(HERE)]
+
+import identity # noqa: E402
+import sweep_matrix # noqa: E402
+
+
+def _canonical(value: object) -> bytes:
+ return json.dumps(
+ value, ensure_ascii=True, sort_keys=True, separators=(",", ":")
+ ).encode()
+
+
+class QualificationPlanningTest(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls) -> None:
+ cls.matrix = sweep_matrix.validate_matrix_document(
+ sweep_matrix.resolve_matrix(
+ backend="deepep", only_sku="h100-dgxc", max_cases=128
+ )
+ )
+ cls.shard = next(item for item in cls.matrix["include"] if item["n"] >= 3)
+
+ def test_matrix_semantics_do_not_depend_on_qualification_index(self) -> None:
+ expected = _canonical(self.matrix)
+ for qualification_index in (1, 2, 3):
+ with mock.patch.dict(
+ os.environ,
+ {"CX_QUALIFICATION_INDEX": str(qualification_index)},
+ ):
+ observed = sweep_matrix.validate_matrix_document(
+ sweep_matrix.resolve_matrix(
+ backend="deepep", only_sku="h100-dgxc", max_cases=128
+ )
+ )
+ self.assertEqual(_canonical(observed), expected)
+
+ def test_extract_shard_has_deterministic_distinct_exact_plans(self) -> None:
+ matrix_bytes = _canonical(self.matrix) + b"\n"
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary)
+ matrix_path = root / "matrix.json"
+ matrix_path.write_bytes(matrix_bytes)
+ original_digest = hashlib.sha256(matrix_path.read_bytes()).hexdigest()
+ controls = []
+ for qualification_index in (1, 2, 3):
+ first = sweep_matrix.extract_shard(
+ matrix_path,
+ self.shard["id"],
+ root / f"q{qualification_index}-first.json",
+ sku=self.shard["sku"],
+ backend=self.shard["backend"],
+ nodes=self.shard["nodes"],
+ qualification_index=qualification_index,
+ )
+ repeated = sweep_matrix.extract_shard(
+ matrix_path,
+ self.shard["id"],
+ root / f"q{qualification_index}-repeated.json",
+ sku=self.shard["sku"],
+ backend=self.shard["backend"],
+ nodes=self.shard["nodes"],
+ qualification_index=qualification_index,
+ )
+ self.assertEqual(first, repeated)
+ self.assertEqual(first["qualification_index"], qualification_index)
+ self.assertEqual(
+ {case["case_id"] for case in first["cases"]},
+ set(self.shard["case_ids"]),
+ )
+ plan = [
+ [
+ case["case_id"],
+ case.get(
+ "precision_profile",
+ identity.V1_CONTROL_PRECISION_PROFILE,
+ ),
+ ]
+ for case in first["cases"]
+ ]
+ self.assertEqual(
+ first["execution_plan_sha256"],
+ hashlib.sha256(
+ json.dumps(plan, separators=(",", ":")).encode()
+ ).hexdigest(),
+ )
+ sweep_matrix.validate_shard_control(
+ first,
+ sku=self.shard["sku"],
+ backend=self.shard["backend"],
+ nodes=self.shard["nodes"],
+ qualification_index=qualification_index,
+ )
+ controls.append(first)
+ self.assertEqual(
+ len({control["execution_plan_sha256"] for control in controls}), 3
+ )
+ self.assertEqual(
+ len({tuple(case["case_id"] for case in control["cases"])
+ for control in controls}),
+ 3,
+ )
+ self.assertEqual(
+ hashlib.sha256(matrix_path.read_bytes()).hexdigest(), original_digest
+ )
+
+ def test_precision_profiles_and_cases_rotate_across_repeats(self) -> None:
+ profiles = list(identity.V1_PRECISION_PROFILES)[:3]
+ cases = [
+ {
+ "case_id": identity.digest("case", {"fixture": index}),
+ "precision_profile": profiles[index % len(profiles)],
+ }
+ for index in range(9)
+ ]
+ plans = [
+ sweep_matrix.qualification_execution_order(
+ "qualification-fixture", cases, qualification_index
+ )
+ for qualification_index in (1, 2, 3)
+ ]
+ expected_ids = {case["case_id"] for case in cases}
+ self.assertTrue(all(
+ {case["case_id"] for case in plan} == expected_ids for plan in plans
+ ))
+ self.assertEqual(
+ len({tuple(case["case_id"] for case in plan) for plan in plans}), 3
+ )
+ self.assertEqual(
+ len({tuple(case["precision_profile"] for case in plan) for plan in plans}),
+ 3,
+ )
+
+ def test_matrix_execution_plan_digest_is_repeat_specific_and_stable(self) -> None:
+ digests = [
+ sweep_matrix.qualification_execution_plan_sha256(self.matrix, index)
+ for index in (1, 2, 3)
+ ]
+ self.assertEqual(len(set(digests)), 3)
+ self.assertTrue(all(len(digest) == 64 for digest in digests))
+ self.assertEqual(
+ digests,
+ [
+ sweep_matrix.qualification_execution_plan_sha256(
+ copy.deepcopy(self.matrix), index
+ )
+ for index in (1, 2, 3)
+ ],
+ )
+ self.assertTrue(all(shard["execution_weight"] > 0 for shard in self.matrix["include"]))
+ tampered = copy.deepcopy(self.matrix)
+ tampered["include"][0]["execution_weight"] += 1
+ with self.assertRaisesRegex(
+ sweep_matrix.MatrixError, "execution_weight differs from its cases"
+ ):
+ sweep_matrix.qualification_execution_plan_sha256(tampered, 1)
+
+ def test_frontend_catalog_covers_every_requested_case_and_point(self) -> None:
+ catalog = sweep_matrix.frontend_catalog(self.matrix)
+ self.assertEqual(catalog["format"], "collectivex.frontend-catalog.v1")
+ self.assertEqual(catalog["case_count"], len(self.matrix["requested_cases"]))
+ self.assertEqual(
+ catalog["point_count"],
+ sum(
+ len(item["case"]["ladder"].split())
+ for item in self.matrix["requested_cases"]
+ ),
+ )
+ self.assertEqual(
+ {item["case_id"] for item in catalog["cases"]},
+ {item["case"]["case_id"] for item in self.matrix["requested_cases"]},
+ )
+ self.assertLess(len(_canonical(catalog)) + 1, 1024 * 1024)
+
+ def test_invalid_qualification_controls_are_rejected(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary)
+ matrix_path = root / "matrix.json"
+ matrix_path.write_bytes(_canonical(self.matrix) + b"\n")
+ arguments = {
+ "sku": self.shard["sku"],
+ "backend": self.shard["backend"],
+ "nodes": self.shard["nodes"],
+ }
+ for invalid in (0, 4, True):
+ with self.subTest(qualification_index=invalid), self.assertRaisesRegex(
+ sweep_matrix.MatrixError, "integer in 1..3"
+ ):
+ sweep_matrix.extract_shard(
+ matrix_path,
+ self.shard["id"],
+ root / "invalid.json",
+ qualification_index=invalid,
+ **arguments,
+ )
+ with mock.patch.dict(os.environ, {"CX_QUALIFICATION_INDEX": "invalid"}):
+ with self.assertRaisesRegex(
+ sweep_matrix.MatrixError, "integer in 1..3"
+ ):
+ sweep_matrix.extract_shard(
+ matrix_path,
+ self.shard["id"],
+ root / "invalid-env.json",
+ **arguments,
+ )
+
+ with mock.patch.dict(os.environ, {}, clear=True):
+ control = sweep_matrix.extract_shard(
+ matrix_path,
+ self.shard["id"],
+ root / "default.json",
+ **arguments,
+ )
+ self.assertEqual(control["qualification_index"], 1)
+
+ invalid_control = copy.deepcopy(control)
+ invalid_control["qualification_index"] = 4
+ with self.assertRaisesRegex(
+ sweep_matrix.MatrixError, "integer in 1..3"
+ ):
+ sweep_matrix.validate_shard_control(invalid_control, **arguments)
+ tampered = copy.deepcopy(control)
+ tampered["execution_plan_sha256"] = "0" * 64
+ with self.assertRaisesRegex(
+ sweep_matrix.MatrixError, "differs from its ordered cases"
+ ):
+ sweep_matrix.validate_shard_control(tampered, **arguments)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 50de2b1ef5..51d06ee94d 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -53,10 +53,10 @@ def test_identity_and_fixed_sampling_profile(self) -> None:
),
(8, 64, 512, 32),
)
- self.assertEqual(identity.V1_CASE_PROFILE["activation_profile"], "canonical-counter-source-v3")
+ self.assertEqual(identity.V1_CASE_PROFILE["activation_profile"], "canonical-counter-source-v4")
self.assertEqual(
identity.V1_CASE_PROFILE["activation_generator"],
- "collectivex-activation-counter-v3",
+ "collectivex-activation-counter-v4",
)
self.assertEqual(identity.V1_CASE_PROFILE["sampling_contract"], "fixed-512-v1")
self.assertEqual(identity.V1_CASE_PROFILE["percentile_method"], "nearest-rank")
@@ -88,6 +88,7 @@ def test_identity_and_fixed_sampling_profile(self) -> None:
]
)
self.assertEqual((args.iters, args.trials, args.warmup), (8, 64, 32))
+ self.assertEqual(args.qualification_index, 1)
for profile in ((8, 64, 32), (128, 4, 32), (8, 1, 4), (0, 64, 32)):
with self.subTest(profile=profile):
self.assertEqual(
@@ -100,6 +101,53 @@ def test_nearest_rank_percentiles_use_all_512_samples(self) -> None:
self.assertEqual(ep_harness.percentile(samples, 50), 256)
self.assertEqual(ep_harness.percentile(samples, 99), 507)
+ def test_qualification_order_is_deterministic_and_position_balanced(self) -> None:
+ values = [1, 2, 4, 8, 16, 32, 64, 128]
+ for qualification_index in range(1, ep_harness.QUALIFICATION_RUNS + 1):
+ orders = [
+ ep_harness.qualification_order(values, qualification_index, trial)
+ for trial in range(64)
+ ]
+ self.assertEqual(
+ orders,
+ [
+ ep_harness.qualification_order(values, qualification_index, trial)
+ for trial in range(64)
+ ],
+ )
+ self.assertTrue(all(sorted(order) == values for order in orders))
+ for position in range(len(values)):
+ self.assertEqual(
+ {value: sum(order[position] == value for order in orders) for value in values},
+ {value: 8 for value in values},
+ )
+ with self.assertRaises(ValueError):
+ ep_harness.qualification_order(values, 0, 0)
+ with self.assertRaises(ValueError):
+ ep_harness.qualification_order([1, 1], 1, 0)
+
+ def test_sample_evidence_preserves_exact_trial_blocks(self) -> None:
+ trials = [
+ [float(trial * 8 + sample) for sample in range(8)]
+ for trial in range(64)
+ ]
+ evidence = ep_harness.sampled_component_evidence(trials)
+ self.assertEqual(evidence["availability"], "measured")
+ self.assertEqual(evidence["sample_count"], 512)
+ self.assertEqual(evidence["trials"], trials)
+ self.assertIsNot(evidence["trials"], trials)
+ self.assertEqual(
+ ep_harness.sampled_component_evidence([]),
+ {"availability": "unavailable", "sample_count": 0, "trials": None},
+ )
+ for malformed in (trials[:-1], [*trials[:-1], trials[-1][:-1]]):
+ with self.assertRaises(ValueError):
+ ep_harness.sampled_component_evidence(malformed)
+ invalid = copy.deepcopy(trials)
+ invalid[0][0] = float("nan")
+ with self.assertRaises(ValueError):
+ ep_harness.sampled_component_evidence(invalid)
+
def test_terminal_summary_uses_bound_sku_and_route(self) -> None:
terminal = {
"format": contracts.TERMINAL_FORMAT,
@@ -446,6 +494,72 @@ def test_only_three_shared_launchers_are_registered(self) -> None:
self.assertNotIn("flashinfer", capability.BACKENDS)
self.assertFalse((HERE / "ep_flashinfer.py").exists())
+ def test_canonical_operator_config_requires_a_private_audit_salt(self) -> None:
+ salt = "a" * 64
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary).resolve()
+ config = root / "operator.json"
+ document = {
+ "schema_version": 1,
+ "audit_salt": salt,
+ "runners": {
+ "h100-dgxc": {
+ "partition": "test", "account": "test",
+ "squash_dir": str(root), "stage_dir": str(root),
+ },
+ },
+ }
+ command = (
+ 'source "$1"; export COLLECTIVEX_EXECUTION_ID="audit-config-$$"; '
+ "trap 'cx_cleanup_private_logs 0' EXIT; cx_load_operator_config; "
+ 'test "$CX_AUDIT_SALT" = "$EXPECTED_AUDIT_SALT"'
+ )
+
+ def invoke(
+ value: dict, *, canonical: bool, expect_salt: bool = True
+ ) -> subprocess.CompletedProcess[str]:
+ config.write_text(json.dumps(value))
+ config.chmod(0o600)
+ environment = {
+ **os.environ,
+ "CX_RUNNER": "h100-dgxc",
+ "COLLECTIVEX_OPERATOR_CONFIG": str(config),
+ "EXPECTED_AUDIT_SALT": salt,
+ }
+ if canonical:
+ environment["COLLECTIVEX_CANONICAL_GHA"] = "1"
+ invocation = command if expect_salt else (
+ 'source "$1"; export COLLECTIVEX_EXECUTION_ID="audit-config-$$"; '
+ "trap 'cx_cleanup_private_logs 0' EXIT; cx_load_operator_config; "
+ 'test -z "${CX_AUDIT_SALT+x}"'
+ )
+ return subprocess.run(
+ ["bash", "-c", invocation, "_", str(ROOT / "runtime" / "common.sh")],
+ text=True,
+ capture_output=True,
+ env=environment,
+ )
+
+ accepted = invoke(document, canonical=True)
+ self.assertEqual(accepted.returncode, 0, accepted.stderr)
+ self.assertNotIn(salt, accepted.stdout + accepted.stderr)
+
+ missing = copy.deepcopy(document)
+ del missing["audit_salt"]
+ rejected = invoke(missing, canonical=True)
+ self.assertNotEqual(rejected.returncode, 0)
+ self.assertNotIn(salt, rejected.stdout + rejected.stderr)
+
+ manual = invoke(missing, canonical=False, expect_salt=False)
+ self.assertEqual(manual.returncode, 0, manual.stderr)
+ self.assertNotIn(salt, manual.stdout + manual.stderr)
+
+ malformed = copy.deepcopy(document)
+ malformed["audit_salt"] = "A" * 64
+ rejected = invoke(malformed, canonical=False)
+ self.assertNotEqual(rejected.returncode, 0)
+ self.assertNotIn("A" * 64, rejected.stdout + rejected.stderr)
+
def test_scaleout_network_profile_is_explicit_and_allowlisted(self) -> None:
command = r'''
set -euo pipefail
@@ -634,7 +748,10 @@ def test_image_pinned_deepep_and_input_integrity_order_are_explicit(self) -> Non
pass_one.index("oracle = _run_expert_oracle"),
)
self.assertIn("pre_input_unchanged", pass_one)
- self.assertIn("hh = prep()\n torch.cuda.synchronize()", harness)
+ self.assertIn(
+ "hh = prep_combine()\n torch.cuda.synchronize()",
+ harness,
+ )
def test_squash_imports_are_reproducible_and_use_a_fresh_cache_key(self) -> None:
common = (ROOT / "runtime" / "common.sh").read_text()
@@ -1085,26 +1202,32 @@ def test_allocation_cleanup_fails_closed_when_scheduler_queries_fail(self) -> No
def test_v1_publication_requires_explicit_release_markers(self) -> None:
workflows = ROOT.parent.parent / ".github" / "workflows"
sweep = (workflows / "collectivex-sweep.yml").read_text()
- publish = (workflows / "collectivex-publish.yml").read_text()
+ self.assertFalse((workflows / "collectivex-publish.yml").exists())
+ self.assertIn("options: [sweep, probe-precision, publish-v1, refresh-v1]", sweep)
+ self.assertIn("collectivex.precision-probe-plan.v1", (ROOT / "tests" / "probe_precision.py").read_text())
+ self.assertIn("cxprecision-probes-${{ github.run_id }}-${{ github.run_attempt }}", sweep)
+ self.assertIn("--validate-bundle", sweep)
self.assertIn("release_tag:", sweep)
self.assertIn("default: unversioned", sweep)
self.assertIn("options: [unversioned, v1]", sweep)
+ self.assertIn("qualification_index:", sweep)
self.assertIn("inputs.release_tag == 'v1'", sweep)
self.assertIn("collectivex.release-tag.v1", sweep)
self.assertIn("V1 release tag requires the locked full matrix", sweep)
self.assertIn("EXPECTED_MATRIX_SHA256", sweep)
self.assertIn("cxrelease-v1-${{ github.run_id }}-${{ github.run_attempt }}", sweep)
- self.assertIn("run_ids must contain exactly three IDs", publish)
- self.assertIn("source runs do not share one source SHA", publish)
- self.assertIn("cxrelease-v1-$run_id-$attempt/release.json", publish)
- self.assertIn("run $run_id is not tagged for V1 publication", publish)
- self.assertIn("collectivex.release-tag.v1", publish)
- self.assertIn("ref: ${{ steps.runs.outputs.source_sha }}", publish)
- self.assertIn("cxpublication-v1-${{ github.run_id }}-${{ github.run_attempt }}", publish)
- self.assertIn("retention-days: 90", publish)
- self.assertNotIn("workflow_run:", publish)
+ self.assertIn("publish_run_ids must contain exactly three IDs", sweep)
+ self.assertIn("source runs do not share one source SHA", sweep)
+ self.assertIn("cxrelease-v1-$run_id-$attempt/release.json", sweep)
+ self.assertIn("run $run_id is not tagged for V1 publication", sweep)
+ self.assertIn("ref: ${{ steps.runs.outputs.source_sha }}", sweep)
+ self.assertIn("[ \"$attempt\" = 1 ]", sweep)
+ self.assertIn("cxpublication-v1-${{ github.run_id }}-${{ github.run_attempt }}", sweep)
+ self.assertIn("refresh source bytes differ from their requested digest", sweep)
+ self.assertIn("retention-days: 90", sweep)
+ self.assertNotIn("workflow_run:", sweep)
def test_source_archive_preserves_only_contained_leaf_symlinks(self) -> None:
selected = "deepep-hybrid-pinned"
@@ -1391,6 +1514,66 @@ def test_runtime_identity_and_realized_placement_are_behavioral(self) -> None:
expected_world_size=8,
)
+ def test_private_allocation_stratum_is_salted_ordered_and_rank_consistent(self) -> None:
+ salt = "a" * 64
+ hosts = ["private-node-b", "private-node-a", "private-node-a"]
+ selectors = {
+ "ib_gid_index": "3",
+ "rdma_devices": "private-hca0:1,private-hca1:1",
+ "rdma_service_level": "2",
+ "socket_ifname": "private-if0",
+ }
+ digest = run_ep._allocation_stratum_sha256(
+ hosts, audit_salt=salt, fabric_selectors=selectors, required=True
+ )
+ self.assertRegex(digest or "", r"^[0-9a-f]{64}$")
+ self.assertEqual(
+ digest,
+ run_ep._allocation_stratum_sha256(
+ list(reversed(hosts)),
+ audit_salt=salt,
+ fabric_selectors=selectors,
+ required=True,
+ ),
+ )
+ for changed_hosts, changed_salt, changed_selectors in (
+ (hosts + ["private-node-c"], salt, selectors),
+ (hosts, "b" * 64, selectors),
+ (hosts, salt, {**selectors, "ib_gid_index": "4"}),
+ ):
+ self.assertNotEqual(
+ digest,
+ run_ep._allocation_stratum_sha256(
+ changed_hosts,
+ audit_salt=changed_salt,
+ fabric_selectors=changed_selectors,
+ required=True,
+ ),
+ )
+ serialized = json.dumps({"allocation_stratum_sha256": digest})
+ private_literals = [
+ salt,
+ *hosts,
+ selectors["rdma_devices"],
+ selectors["socket_ifname"],
+ ]
+ self.assertFalse(any(value in serialized for value in private_literals))
+ self.assertNotIn("physical_hosts", serialized)
+ self.assertNotIn("fabric_selectors", serialized)
+ self.assertEqual(
+ run_ep._common_allocation_stratum([digest, digest, digest], required=True),
+ digest,
+ )
+ with self.assertRaisesRegex(ValueError, "differs across distributed ranks"):
+ run_ep._common_allocation_stratum([digest, "b" * 64], required=True)
+ with self.assertRaisesRegex(ValueError, "requires"):
+ run_ep._allocation_stratum_sha256(
+ hosts, audit_salt=None, fabric_selectors=selectors, required=True
+ )
+ self.assertIsNone(run_ep._allocation_stratum_sha256(
+ hosts, audit_salt=None, fabric_selectors=selectors, required=False
+ ))
+
def test_collective_version_and_rccl_fingerprint_are_normalized(self) -> None:
self.assertEqual(ep_harness.format_collective_version(23004), "2.30.4")
self.assertEqual(ep_harness.format_collective_version(21805), "2.18.5")
@@ -1471,9 +1654,30 @@ def test_workloads_bind_generator_activation_and_trace(self) -> None:
self.assertEqual(member, manifest["workload_id"])
self.assertEqual(checksums, manifest["checksums"])
+ def test_eplb_calibration_window_is_disjoint_and_identity_bound(self) -> None:
+ evaluation = workload.canonical_member("zipf", 8, 2, 8, 2, 4, 67)
+ calibration = workload.canonical_eplb_calibration_member(
+ "zipf", 8, 2, 8, 2, 4, 67
+ )
+ self.assertNotEqual(evaluation[0], calibration[0])
+ self.assertNotEqual(evaluation[1]["trace"], calibration[1]["trace"])
+ self.assertGreater(
+ workload.EPLB_CALIBRATION_TOKEN_OFFSET,
+ 2 * 4,
+ )
+ repeated = workload.canonical_eplb_calibration_member(
+ "zipf", 8, 2, 8, 2, 4, 67
+ )
+ self.assertEqual(calibration, repeated)
+ with self.assertRaises(ValueError):
+ workload.canonical_routing_rows(
+ 8, 8, 2, "zipf", 67, token_offset=-1
+ )
+
def test_canonical_members_are_bound_to_each_scheduled_row(self) -> None:
case = {
"routing": "uniform", "hidden": 8, "topk": 2, "experts": 4, "ep": 2,
+ "mode": "normal",
}
eplb_record = {
"enabled": False, "mapping_hash": None, "num_physical_experts": 4,
@@ -1537,7 +1741,10 @@ def replace_member(document: dict, replacement: tuple[str, dict[str, str], str])
)
def test_eplb_row_hash_is_bound_to_the_frozen_remap(self) -> None:
- case = {"routing": "zipf", "hidden": 8, "topk": 2, "experts": 4, "ep": 2}
+ case = {
+ "routing": "zipf", "hidden": 8, "topk": 2, "experts": 4, "ep": 2,
+ "mode": "normal",
+ }
physical = eplb.physical_count(4, 32, 2)
plan = contracts._expected_eplb_plan("zipf", 2, 4, physical, 2, 67, 2048)
eplb_record = {
@@ -1635,11 +1842,13 @@ def test_terminal_fail_safe_fills_only_missing_shard_cases(self) -> None:
"artifact": "artifact", "job": "job", "ref": "collectivex",
"repo": "SemiAnalysisAI/InferenceX", "run_attempt": "1",
"run_id": "123", "source_sha": "a" * 40,
+ "qualification_index": 1,
}
allocation = {
"artifact": "artifact", "execution_id": "execution", "job": "job",
"repo": "SemiAnalysisAI/InferenceX", "run_attempt": "1", "run_id": "123",
"runner": shard["sku"], "source_sha": "a" * 40,
+ "qualification_index": 1,
}
out_dir.mkdir()
existing = contracts.make_terminal_document(
@@ -1672,6 +1881,7 @@ def test_terminal_fail_safe_fills_only_missing_shard_cases(self) -> None:
"GITHUB_REPOSITORY": "SemiAnalysisAI/InferenceX",
"GITHUB_RUN_ATTEMPT": "1", "GITHUB_RUN_ID": "123",
"GITHUB_SHA": "a" * 40,
+ "CX_QUALIFICATION_INDEX": "1",
}
subprocess.run(
[
@@ -1824,7 +2034,8 @@ def test_runtime_identity_mismatch_is_failed_not_unsupported(self) -> None:
manual["identity"]["allocation_factors"],
{
"artifact": None, "execution_id": "manual-execution", "job": None,
- "repo": None, "run_attempt": None, "run_id": None,
+ "qualification_index": 1, "repo": None,
+ "run_attempt": None, "run_id": None,
"runner": "manual-runner", "source_sha": None,
},
)
@@ -2034,7 +2245,7 @@ def test_canonical_gha_environment_is_locked_but_manual_overrides_survive(self)
export CX_PREPARED_BACKEND_CACHE=/untrusted CX_BACKEND_SOURCE_ROOT=/untrusted
! (cx_lock_canonical_gha_env mi325x)
export COLLECTIVEX_OPERATOR_CONFIG_LOADED=$$
- export CX_STAGE_DIR="$GITHUB_WORKSPACE"
+ export CX_STAGE_DIR="$GITHUB_WORKSPACE" CX_AUDIT_SALT="$(printf 'a%.0s' {1..64})"
unset CX_LOCK_DIR
cx_lock_canonical_gha_env mi325x
test "$CX_IMAGE" = "$CX_IMAGE_AMD_MORI_MI325"
@@ -2064,6 +2275,7 @@ def test_canonical_gha_environment_is_locked_but_manual_overrides_survive(self)
export COLLECTIVEX_OPERATOR_CONFIG_LOADED=$$
export CX_SHARD_SKU=mi355x CX_NODES=1 CX_GPUS_PER_NODE=8
export CX_LOCK_DIR=/validated/amd-locks CX_STAGE_DIR=/validated/amd-stage
+ export CX_AUDIT_SALT="$(printf 'a%.0s' {1..64})"
cx_lock_canonical_gha_env mi355x
test "$CX_LOCK_DIR" = /validated/amd-locks
test "$CX_STAGE_DIR" = /validated/amd-stage
@@ -2099,6 +2311,7 @@ def test_canonical_amd_stage_uses_config_not_world_writable_workspace(self) -> N
export CX_SHARD_FILE=.shards/test.json CX_SHARD_SKU=mi325x
export CX_NODES=1 CX_GPUS_PER_NODE=8 CX_SQUASH_DIR=/shared/containers
export COLLECTIVEX_OPERATOR_CONFIG_LOADED=$$ CX_STAGE_DIR=/shared/amd-stage
+ export CX_AUDIT_SALT="$(printf 'a%.0s' {1..64})"
cx_lock_canonical_gha_env mi325x
printf '%s' "$CX_STAGE_DIR"
'''
@@ -2128,6 +2341,7 @@ def test_canonical_amd_stage_uses_config_not_symlinked_workspace(self) -> None:
export CX_SHARD_FILE=.shards/test.json CX_SHARD_SKU=mi325x
export CX_NODES=1 CX_GPUS_PER_NODE=8 CX_SQUASH_DIR=/shared/containers
export COLLECTIVEX_OPERATOR_CONFIG_LOADED=$$ CX_STAGE_DIR=/shared/amd-stage
+ export CX_AUDIT_SALT="$(printf 'a%.0s' {1..64})"
cx_lock_canonical_gha_env mi325x
printf '%s' "$CX_STAGE_DIR"
'''
diff --git a/experimental/CollectiveX/tests/test_schema_v1_contract.py b/experimental/CollectiveX/tests/test_schema_v1_contract.py
new file mode 100644
index 0000000000..a517cd5d56
--- /dev/null
+++ b/experimental/CollectiveX/tests/test_schema_v1_contract.py
@@ -0,0 +1,436 @@
+#!/usr/bin/env python3
+"""Focused structural tests for the fail-closed CollectiveX V1 schemas."""
+from __future__ import annotations
+
+import copy
+import json
+import sys
+import unittest
+from pathlib import Path
+
+import jsonschema
+
+ROOT = Path(__file__).resolve().parents[1]
+if str(ROOT) not in sys.path:
+ sys.path.insert(0, str(ROOT))
+
+import identity # noqa: E402
+
+
+def _load(name: str) -> dict:
+ return json.loads((ROOT / "schemas" / name).read_text())
+
+
+def _definition_validator(schema: dict, name: str) -> jsonschema.Validator:
+ return jsonschema.Draft202012Validator(
+ {
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$defs": schema["$defs"],
+ "$ref": f"#/$defs/{name}",
+ }
+ )
+
+
+class CollectiveXV1SchemaContractTest(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls) -> None:
+ cls.raw = _load("raw-case-v1.schema.json")
+ cls.samples = _load("samples-v1.schema.json")
+ cls.public = _load("public-dataset-v1.schema.json")
+ cls.bundle = _load("private-bundle-v1.schema.json")
+ cls.terminal = _load("terminal-outcome-v1.schema.json")
+
+ def test_all_checked_in_schemas_are_draft_2020_12_valid(self) -> None:
+ for path in sorted((ROOT / "schemas").glob("*.schema.json")):
+ with self.subTest(path=path.name):
+ schema = json.loads(path.read_text())
+ jsonschema.Draft202012Validator.check_schema(schema)
+ self.assertFalse(schema["additionalProperties"])
+
+ def test_precision_catalog_and_axes_are_exact_and_strict(self) -> None:
+ expected = set(identity.V1_PRECISION_PROFILES)
+ self.assertEqual(set(self.raw["$defs"]["precision_profile"]["enum"]), expected)
+ self.assertEqual(set(self.public["$defs"]["precisionProfile"]["enum"]), expected)
+ self.assertEqual(set(self.terminal["$defs"]["precisionProfile"]["enum"]), expected)
+
+ axis_keys = {
+ "alignment_contract",
+ "api_input_dtype",
+ "api_output_dtype",
+ "communication_format",
+ "conversion_boundary",
+ "padding_contract",
+ "quantization_origin",
+ "scale_dtype",
+ "scale_group_size",
+ "scale_layout",
+ }
+ for schema, name in (
+ (self.raw, "communication_axis"),
+ (self.public, "communicationAxis"),
+ (self.terminal, "communicationAxis"),
+ ):
+ with self.subTest(schema=schema["$id"]):
+ axis = schema["$defs"][name]
+ self.assertFalse(axis["additionalProperties"])
+ self.assertEqual(set(axis["required"]), axis_keys)
+ self.assertEqual(set(axis["properties"]), axis_keys)
+
+ axis_validator = _definition_validator(self.raw, "communication_axis")
+ profile_validator = _definition_validator(self.raw, "communication_precision")
+ for name in sorted(expected):
+ profile = identity.precision_profile(name)
+ with self.subTest(profile=name):
+ axis_validator.validate(profile["dispatch"])
+ axis_validator.validate(profile["combine"])
+ profile_validator.validate(profile)
+ raw_case_profile = _definition_validator(self.raw, "case_profile")
+ terminal_case_profile = _definition_validator(self.terminal, "caseProfile")
+ for case in (
+ {"mode": "normal"},
+ {
+ "mode": "normal",
+ "precision_profile": "d-fp8-e4m3fn-b128-f32-prequantized.c-bf16",
+ },
+ {
+ "mode": "low-latency",
+ "precision_profile": "d-fp8-e4m3fn-b128-f32-fused.c-bf16",
+ },
+ ):
+ resolved = identity.profile_for_case(case)
+ raw_case_profile.validate(resolved)
+ terminal_case_profile.validate(resolved)
+
+ shape = self.raw["properties"]["case"]["properties"]["shape"]
+ self.assertIn("precision_profile", shape["required"])
+ self.assertIn("dispatch_precision", shape["required"])
+ self.assertIn("combine_precision", shape["required"])
+ self.assertNotIn("dispatch_dtype", shape["properties"])
+ self.assertNotIn("quant", shape["properties"])
+
+ workload = self.public["$defs"]["series"]["properties"]["workload"]
+ self.assertIn("precision_profile", workload["required"])
+ self.assertIn("dispatch_precision", workload["required"])
+ self.assertIn("combine_precision", workload["required"])
+ self.assertNotIn("dispatch_dtype", workload["properties"])
+ self.assertNotIn("combine_dtype", workload["properties"])
+
+ profile = self.raw["$defs"]["case_profile"]
+ self.assertEqual(
+ profile["properties"]["activation_generator"]["const"],
+ identity.V1_NORMAL_CASE_PROFILE["activation_generator"],
+ )
+ self.assertEqual(
+ profile["properties"]["activation_profile"]["const"],
+ identity.V1_NORMAL_CASE_PROFILE["activation_profile"],
+ )
+ self.assertEqual(
+ profile["properties"]["source_identity_contract"]["const"],
+ identity.V1_NORMAL_CASE_PROFILE["source_identity_contract"],
+ )
+
+ def test_qualification_index_is_bound_across_private_and_public_records(self) -> None:
+ paths = (
+ self.raw["properties"]["measurement"]["properties"]["qualification_index"],
+ self.raw["properties"]["identity"]["properties"]["allocation_factors"]["properties"]["qualification_index"],
+ self.raw["$defs"]["git_run"]["properties"]["qualification_index"],
+ self.samples["properties"]["qualification_index"],
+ self.bundle["properties"]["run"]["properties"]["qualification_index"],
+ self.terminal["$defs"]["allocationFactors"]["properties"]["qualification_index"],
+ self.terminal["$defs"]["gitRun"]["properties"]["qualification_index"],
+ self.public["$defs"]["attempt"]["properties"]["qualification_index"],
+ )
+ for value in paths:
+ self.assertEqual((value["minimum"], value["maximum"]), (1, 3))
+ promotion_indices = self.public["properties"]["promotion"]["properties"]["qualification_indices"]
+ series_indices = self.public["$defs"]["series"]["properties"]["measurement"]["properties"]["qualification_indices"]
+ for schema, valid_values in (
+ (promotion_indices, ([], [1], [1, 2, 3])),
+ (series_indices, ([1], [2, 3], [1, 2, 3])),
+ ):
+ validator = jsonschema.Draft202012Validator(schema)
+ for value in valid_values:
+ validator.validate(value)
+ for value in ([0], [4], [1, 1], [1, 2, 3, 1]):
+ with self.assertRaises(jsonschema.ValidationError):
+ validator.validate(value)
+ measurement = self.raw["properties"]["measurement"]
+ self.assertIn("execution_order_sha256", measurement["required"])
+ self.assertEqual(
+ measurement["properties"]["execution_order_sha256"]["pattern"],
+ "^[0-9a-f]{64}$",
+ )
+
+ def test_private_allocation_stratum_is_required_only_in_raw_canonical_evidence(self) -> None:
+ provenance = self.raw["properties"]["provenance"]
+ self.assertIn("allocation_stratum_sha256", provenance["required"])
+ stratum = provenance["properties"]["allocation_stratum_sha256"]
+ jsonschema.Draft202012Validator(stratum).validate(None)
+ jsonschema.Draft202012Validator(stratum).validate("a" * 64)
+ with self.assertRaises(jsonschema.ValidationError):
+ jsonschema.Draft202012Validator(stratum).validate("A" * 64)
+ conditional = self.raw["allOf"][0]
+ self.assertEqual(
+ conditional["if"]["properties"]["workload"]["properties"]["source"],
+ {"const": "canonical-serialized"},
+ )
+ canonical_stratum = conditional["then"]["properties"]["provenance"][
+ "properties"
+ ]["allocation_stratum_sha256"]
+ with self.assertRaises(jsonschema.ValidationError):
+ jsonschema.Draft202012Validator(canonical_stratum).validate(None)
+ self.assertNotIn("allocation_stratum_sha256", json.dumps(self.public))
+
+ def test_stage_samples_are_absent_or_exactly_512(self) -> None:
+ validator = _definition_validator(self.samples, "component")
+ measured = {
+ "availability": "measured",
+ "sample_count": 512,
+ "trials": [[1.0] * 8 for _ in range(64)],
+ }
+ validator.validate(measured)
+ unavailable = {
+ "availability": "unavailable",
+ "sample_count": 0,
+ "trials": None,
+ }
+ validator.validate(unavailable)
+ for mutate in (
+ lambda value: value.update(sample_count=511),
+ lambda value: value["trials"].pop(),
+ lambda value: value["trials"][0].pop(),
+ ):
+ broken = copy.deepcopy(measured)
+ mutate(broken)
+ with self.assertRaises(jsonschema.ValidationError):
+ validator.validate(broken)
+
+ raw_components = self.raw["properties"]["measurement"]["properties"]["rows"]["items"]["properties"]["components"]
+ public_components = self.public["$defs"]["point"]["properties"]["components"]
+ self.assertIn("stage", raw_components["required"])
+ self.assertIn("stage", public_components["required"])
+
+ def test_byte_provenance_supports_precision_aware_rates(self) -> None:
+ expected = {
+ "accounting_contract",
+ "activation_data_bytes",
+ "scale_bytes",
+ "total_logical_bytes",
+ }
+ for schema, name in (
+ (self.raw, "byte_accounting"),
+ (self.public, "byteAccounting"),
+ ):
+ definition = schema["$defs"][name]
+ self.assertFalse(definition["additionalProperties"])
+ self.assertEqual(set(definition["required"]), expected)
+ self.assertEqual(set(definition["properties"]), expected)
+ component = self.public["$defs"]["component"]
+ self.assertIn("byte_provenance", component["required"])
+ self.assertIn("activation_data_rate_gbps_at_latency_percentile", component["required"])
+ self.assertIn("total_logical_data_rate_gbps_at_latency_percentile", component["required"])
+
+ def test_raw_correctness_carries_directional_precision_evidence(self) -> None:
+ correctness = self.raw["properties"]["measurement"]["properties"]["rows"]["items"]["properties"]["correctness"]
+ self.assertIn("precision", correctness["required"])
+ evidence = self.raw["$defs"]["precision_evidence"]
+ self.assertFalse(evidence["additionalProperties"])
+ self.assertEqual(
+ set(evidence["required"]),
+ {"profile_id", "dispatch", "combine", "passed"},
+ )
+ axis = self.raw["$defs"]["precision_axis_evidence"]
+ self.assertFalse(axis["additionalProperties"])
+ self.assertEqual(
+ set(axis["required"]),
+ {
+ "encoded_payload_valid",
+ "scales_finite",
+ "scales_positive",
+ "dequantized_semantics",
+ "saturation_count",
+ "saturation_rate",
+ "max_abs_error",
+ "max_rel_error",
+ "passed",
+ },
+ )
+
+ def test_eplb_calibration_provenance_is_explicit(self) -> None:
+ fields = {
+ "calibration_workload_id",
+ "calibration_trace_sha256",
+ "calibration_window",
+ "calibration_token_offset",
+ }
+ raw = self.raw["properties"]["case"]["properties"]["eplb"]
+ public = self.public["$defs"]["series"]["properties"]["eplb"]
+ for descriptor in (raw, public):
+ self.assertTrue(fields <= set(descriptor["required"]))
+ self.assertTrue(fields <= set(descriptor["properties"]))
+ self.assertFalse(descriptor["additionalProperties"])
+
+ def test_public_coverage_is_a_full_case_and_point_inventory(self) -> None:
+ coverage = self.public["$defs"]["coverage"]
+ dimensions = {
+ "sku",
+ "suite",
+ "workload",
+ "publication_tier",
+ "backend",
+ "backend_generation",
+ "resource",
+ "topology",
+ "phase",
+ "mode",
+ "routing",
+ "eplb",
+ "precision_profile",
+ "dispatch_precision",
+ "combine_precision",
+ "points",
+ }
+ self.assertTrue(dimensions <= set(coverage["required"]))
+ self.assertFalse(coverage["additionalProperties"])
+ point = self.public["$defs"]["coveragePoint"]
+ self.assertEqual(
+ set(point["properties"]["terminal_status"]["enum"]),
+ {"measured", "unsupported", "failed", "invalid", "diagnostic"},
+ )
+ self.assertIn("tokens_per_rank", point["required"])
+ self.assertIn("global_tokens", point["required"])
+ self.assertIn("reason", point["required"])
+
+ promotion = self.public["properties"]["promotion"]
+ counts = {
+ "requested_cases",
+ "terminal_cases",
+ "measured_cases",
+ "unsupported_cases",
+ "requested_points",
+ "terminal_points",
+ "measured_points",
+ "unsupported_points",
+ }
+ self.assertTrue(counts <= set(promotion["required"]))
+ self.assertFalse(promotion["additionalProperties"])
+
+ def test_public_coverage_point_reason_tracks_terminal_status(self) -> None:
+ validator = _definition_validator(self.public, "coveragePoint")
+ point_id = "cxpoint-v1-" + "1" * 64
+ series_id = "cxseries-v1-" + "2" * 64
+ measured = {
+ "point_id": point_id,
+ "series_id": series_id,
+ "tokens_per_rank": 8,
+ "global_tokens": 64,
+ "terminal_status": "measured",
+ "reason": None,
+ }
+ validator.validate(measured)
+ unsupported = {
+ **measured,
+ "point_id": None,
+ "series_id": None,
+ "terminal_status": "unsupported",
+ "reason": "backend-platform-unsupported",
+ }
+ validator.validate(unsupported)
+ failed = {**unsupported, "terminal_status": "failed", "reason": "execution-failed"}
+ validator.validate(failed)
+
+ for broken in (
+ {**measured, "reason": "unexpected-reason"},
+ {**unsupported, "reason": None},
+ {**failed, "reason": None},
+ {**unsupported, "reason": "contains spaces"},
+ ):
+ with self.assertRaises(jsonschema.ValidationError):
+ validator.validate(broken)
+
+ def test_public_measured_point_has_bounded_detail_and_three_run_stability(self) -> None:
+ point = self.public["$defs"]["point"]
+ self.assertFalse(point["additionalProperties"])
+ self.assertTrue({"anomalies", "correctness", "stability"} <= set(point["required"]))
+ self.assertNotIn("correct", point["properties"])
+
+ axis = {
+ "encoded_payload_valid": True,
+ "scales_finite": None,
+ "scales_positive": None,
+ "dequantized_semantics": True,
+ "saturation_count": 0,
+ "saturation_rate": 0.0,
+ "max_abs_error": 0.0,
+ "max_rel_error": 0.0,
+ "passed": True,
+ }
+ correctness = {
+ "semantic_pass": True,
+ "precision": {
+ "profile_id": identity.V1_CONTROL_PRECISION_PROFILE,
+ "dispatch": axis,
+ "combine": copy.deepcopy(axis),
+ "passed": True,
+ },
+ }
+ correctness_validator = _definition_validator(self.public, "pointCorrectness")
+ correctness_validator.validate(correctness)
+ broken_correctness = copy.deepcopy(correctness)
+ broken_correctness["precision"]["dispatch"]["unexpected"] = True
+ with self.assertRaises(jsonschema.ValidationError):
+ correctness_validator.validate(broken_correctness)
+
+ stability_validator = _definition_validator(self.public, "pointStability")
+ stability_validator.validate({
+ "complete": True,
+ "qualification_indices": [1, 2, 3],
+ "p50_max_min_ratio": 1.02,
+ "p99_max_min_ratio": 1.04,
+ "stable_p50": True,
+ "stable_p99": True,
+ })
+ stability_validator.validate({
+ "complete": False,
+ "qualification_indices": [2],
+ "p50_max_min_ratio": None,
+ "p99_max_min_ratio": None,
+ "stable_p50": False,
+ "stable_p99": False,
+ })
+ for broken in (
+ {
+ "complete": True,
+ "qualification_indices": [1, 2],
+ "p50_max_min_ratio": 1.0,
+ "p99_max_min_ratio": 1.0,
+ "stable_p50": True,
+ "stable_p99": True,
+ },
+ {
+ "complete": False,
+ "qualification_indices": [1],
+ "p50_max_min_ratio": 1.0,
+ "p99_max_min_ratio": None,
+ "stable_p50": False,
+ "stable_p99": False,
+ },
+ ):
+ with self.assertRaises(jsonschema.ValidationError):
+ stability_validator.validate(broken)
+
+ anomalies = point["properties"]["anomalies"]
+ self.assertEqual(anomalies["maxItems"], 16)
+ self.assertTrue(anomalies["uniqueItems"])
+ anomaly_validator = jsonschema.Draft202012Validator({
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$defs": self.public["$defs"],
+ "$ref": "#/$defs/point/properties/anomalies",
+ })
+ anomaly_validator.validate(["roundtrip-gt-isolated-sum"])
+ with self.assertRaises(jsonschema.ValidationError):
+ anomaly_validator.validate([f"anomaly-{index}" for index in range(17)])
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/experimental/CollectiveX/tests/workload.py b/experimental/CollectiveX/tests/workload.py
index 89a6b46052..74860be803 100644
--- a/experimental/CollectiveX/tests/workload.py
+++ b/experimental/CollectiveX/tests/workload.py
@@ -29,7 +29,9 @@
# Bump when the counter or byte encoding changes. The workload ID binds parameters and trace bytes.
GENERATOR_VERSION = "collectivex-routing-counter-v3"
GATE_WEIGHT_FORMAT = "counter-u16-normalized-f32"
-ACTIVATION_GENERATOR = "collectivex-activation-counter-v3"
+ACTIVATION_GENERATOR = "collectivex-activation-counter-v4"
+EPLB_CALIBRATION_WINDOW = "collectivex-eplb-calibration-window-v1"
+EPLB_CALIBRATION_TOKEN_OFFSET = 1 << 32
_MASK64 = (1 << 64) - 1
@@ -56,13 +58,21 @@ def _counter(seed: int, token: int, slot: int, attempt: int, stream: int) -> int
def canonical_routing_rows(
- global_tokens: int, experts: int, topk: int, routing: str, seed: int
+ global_tokens: int,
+ experts: int,
+ topk: int,
+ routing: str,
+ seed: int,
+ *,
+ token_offset: int = 0,
) -> tuple[list[list[int]], list[list[float]]]:
- """Generate distinct experts and normalized weights using exact integer counters."""
+ """Generate a deterministic routing window from exact integer counters."""
if routing not in {"uniform", "zipf"}:
raise ValueError(f"unknown routing {routing!r} (uniform|zipf)")
if global_tokens <= 0 or experts <= 0 or topk <= 0 or topk > experts:
raise ValueError("global_tokens/experts/topk must be positive and topk <= experts")
+ if type(token_offset) is not int or token_offset < 0:
+ raise ValueError("token_offset must be a non-negative integer")
cumulative: list[int] | None = None
if routing == "zipf":
@@ -74,7 +84,8 @@ def canonical_routing_rows(
indices: list[list[int]] = []
weights: list[list[float]] = []
- for token in range(global_tokens):
+ for local_token in range(global_tokens):
+ token = token_offset + local_token
selected: list[int] = []
used: set[int] = set()
for slot in range(topk):
@@ -133,10 +144,19 @@ def canonical_member(
ep_size: int,
tokens_per_rank: int,
seed: int,
+ *,
+ token_offset: int = 0,
) -> tuple[str, dict[str, str], list[list[int]], list[list[float]]]:
"""Derive one canonical manifest member and retain its rows for proof checks."""
global_tokens = ep_size * tokens_per_rank
- indices, weights = canonical_routing_rows(global_tokens, experts, topk, routing, seed)
+ indices, weights = canonical_routing_rows(
+ global_tokens,
+ experts,
+ topk,
+ routing,
+ seed,
+ token_offset=token_offset,
+ )
checksums = trace_checksums(indices, weights)
member = compute_workload_id(
routing,
@@ -147,19 +167,52 @@ def canonical_member(
global_tokens,
seed,
trace_checksum=checksums["trace"],
+ token_offset=token_offset,
)
return member, checksums, indices, weights
+def canonical_eplb_calibration_member(
+ routing: str,
+ hidden: int,
+ topk: int,
+ experts: int,
+ ep_size: int,
+ tokens_per_rank: int,
+ seed: int,
+) -> tuple[str, dict[str, str], list[list[int]], list[list[float]]]:
+ """Return the EPLB calibration trace from a disjoint global-token window."""
+ return canonical_member(
+ routing,
+ hidden,
+ topk,
+ experts,
+ ep_size,
+ tokens_per_rank,
+ seed,
+ token_offset=EPLB_CALIBRATION_TOKEN_OFFSET,
+ )
+
+
def compute_workload_id(routing: str, hidden: int, topk: int, experts: int,
ep_size: int, global_tokens: int, seed: int,
generator: str = GENERATOR_VERSION,
- trace_checksum: str | None = None) -> str:
+ trace_checksum: str | None = None,
+ token_offset: int = 0) -> str:
"""Deterministic ID over parameters and canonical trace bytes."""
if generator != GENERATOR_VERSION:
raise ValueError(f"unsupported workload generator {generator!r}")
+ if type(token_offset) is not int or token_offset < 0:
+ raise ValueError("token_offset must be a non-negative integer")
if trace_checksum is None:
- indices, weights = canonical_routing_rows(global_tokens, experts, topk, routing, seed)
+ indices, weights = canonical_routing_rows(
+ global_tokens,
+ experts,
+ topk,
+ routing,
+ seed,
+ token_offset=token_offset,
+ )
idx_bytes, weight_bytes = _canonical_bytes(indices, weights)
trace_checksum = _sha256(idx_bytes + weight_bytes)
key = {
@@ -169,6 +222,11 @@ def compute_workload_id(routing: str, hidden: int, topk: int, experts: int,
"activation_generator": ACTIVATION_GENERATOR,
"activation_identity": compute_activation_identity(seed, hidden),
}
+ if token_offset:
+ key.update({
+ "routing_window": EPLB_CALIBRATION_WINDOW,
+ "token_offset": token_offset,
+ })
return identity.workload_id(key)
From eb6776d7422b29171e557f0b501994780db501ff Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 15:52:36 +0800
Subject: [PATCH 003/210] fix(collectivex): provision audit salt independently
---
.github/workflows/collectivex-sweep.yml | 1 +
experimental/CollectiveX/runtime/common.sh | 14 ++++++++++----
.../CollectiveX/tests/test_sampling_contract.py | 17 ++++++++++++++++-
3 files changed, 27 insertions(+), 5 deletions(-)
diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml
index 47de47a8ce..9fd32e1dea 100644
--- a/.github/workflows/collectivex-sweep.yml
+++ b/.github/workflows/collectivex-sweep.yml
@@ -435,6 +435,7 @@ jobs:
id: sweep_shard
env:
COLLECTIVEX_OPERATOR_CONFIG_CONTENT: ${{ secrets.COLLECTIVEX_OPERATOR_CONFIG_V1 }}
+ COLLECTIVEX_OPERATOR_AUDIT_SALT: ${{ secrets.COLLECTIVEX_AUDIT_SALT_V1 }}
COLLECTIVEX_OPERATOR_CONFIG_REQUIRED: '1'
run: |
set -euo pipefail
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 772ac8b0a1..da85bfea23 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -89,7 +89,9 @@ cx_fail_stage() {
cx_load_operator_config() {
[ -n "${COLLECTIVEX_OPERATOR_CONFIG_LOADED:-}" ] \
&& [ "$COLLECTIVEX_OPERATOR_CONFIG_LOADED" = "$$" ] && return 0
- local config_path generated=0 parsed_path config_log key value
+ local config_path generated=0 parsed_path config_log key value audit_salt_override
+ audit_salt_override="${COLLECTIVEX_OPERATOR_AUDIT_SALT:-}"
+ unset COLLECTIVEX_OPERATOR_AUDIT_SALT
unset CX_PARTITION CX_ACCOUNT CX_SQUASH_DIR CX_STAGE_DIR CX_ENROOT_CACHE_PATH
unset ENROOT_CACHE_PATH
unset CX_EXCLUDE_NODES CX_NODELIST CX_LOCK_DIR CX_MASTER_PORT
@@ -135,7 +137,7 @@ cx_load_operator_config() {
}
config_log="$(cx_private_log_path operator-config)"
if ! python3 - "$config_path" "${CX_RUNNER:-${CX_SHARD_SKU:-${CX_PUBLIC_RUNNER:-}}}" \
- "${COLLECTIVEX_CANONICAL_GHA:-0}" \
+ "${COLLECTIVEX_CANONICAL_GHA:-0}" "$audit_salt_override" \
> "$parsed_path" 2> "$config_log" <<'PY'
import json
import os
@@ -209,7 +211,7 @@ def valid_path(value):
)
try:
- path, runner, audit_required = sys.argv[1:]
+ path, runner, audit_required, audit_override = sys.argv[1:]
if runner not in RUNNERS or audit_required not in {"0", "1"}:
raise ValueError
metadata = os.lstat(path)
@@ -251,9 +253,13 @@ try:
(audit_salt is not None and (
not isinstance(audit_salt, str) or not AUDIT_SALT.fullmatch(audit_salt)
))
- or (audit_required == "1" and audit_salt is None)
+ or (audit_override and not AUDIT_SALT.fullmatch(audit_override))
+ or (audit_salt is not None and audit_override and audit_salt != audit_override)
):
raise ValueError
+ audit_salt = audit_salt or audit_override or None
+ if audit_required == "1" and audit_salt is None:
+ raise ValueError
runners = document["runners"]
if (
not isinstance(runners, dict) or not runners or set(runners) - RUNNERS
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 51d06ee94d..ba06612436 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -516,7 +516,8 @@ def test_canonical_operator_config_requires_a_private_audit_salt(self) -> None:
)
def invoke(
- value: dict, *, canonical: bool, expect_salt: bool = True
+ value: dict, *, canonical: bool, expect_salt: bool = True,
+ audit_override: str | None = None,
) -> subprocess.CompletedProcess[str]:
config.write_text(json.dumps(value))
config.chmod(0o600)
@@ -528,6 +529,8 @@ def invoke(
}
if canonical:
environment["COLLECTIVEX_CANONICAL_GHA"] = "1"
+ if audit_override is not None:
+ environment["COLLECTIVEX_OPERATOR_AUDIT_SALT"] = audit_override
invocation = command if expect_salt else (
'source "$1"; export COLLECTIVEX_EXECUTION_ID="audit-config-$$"; '
"trap 'cx_cleanup_private_logs 0' EXIT; cx_load_operator_config; "
@@ -550,6 +553,18 @@ def invoke(
self.assertNotEqual(rejected.returncode, 0)
self.assertNotIn(salt, rejected.stdout + rejected.stderr)
+ separate = invoke(missing, canonical=True, audit_override=salt)
+ self.assertEqual(separate.returncode, 0, separate.stderr)
+ self.assertNotIn(salt, separate.stdout + separate.stderr)
+
+ conflicting = invoke(document, canonical=True, audit_override="b" * 64)
+ self.assertNotEqual(conflicting.returncode, 0)
+ self.assertNotIn(salt, conflicting.stdout + conflicting.stderr)
+
+ malformed_override = invoke(missing, canonical=True, audit_override="A" * 64)
+ self.assertNotEqual(malformed_override.returncode, 0)
+ self.assertNotIn("A" * 64, malformed_override.stdout + malformed_override.stderr)
+
manual = invoke(missing, canonical=False, expect_salt=False)
self.assertEqual(manual.returncode, 0, manual.stderr)
self.assertNotIn(salt, manual.stdout + manual.stderr)
From 867f5d51e5d0e9501c8cc07d644b84a1314fef13 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 15:54:11 +0800
Subject: [PATCH 004/210] Revert "fix(collectivex): provision audit salt
independently"
This reverts commit eb6776d7422b29171e557f0b501994780db501ff.
---
.github/workflows/collectivex-sweep.yml | 1 -
experimental/CollectiveX/runtime/common.sh | 14 ++++----------
.../CollectiveX/tests/test_sampling_contract.py | 17 +----------------
3 files changed, 5 insertions(+), 27 deletions(-)
diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml
index 9fd32e1dea..47de47a8ce 100644
--- a/.github/workflows/collectivex-sweep.yml
+++ b/.github/workflows/collectivex-sweep.yml
@@ -435,7 +435,6 @@ jobs:
id: sweep_shard
env:
COLLECTIVEX_OPERATOR_CONFIG_CONTENT: ${{ secrets.COLLECTIVEX_OPERATOR_CONFIG_V1 }}
- COLLECTIVEX_OPERATOR_AUDIT_SALT: ${{ secrets.COLLECTIVEX_AUDIT_SALT_V1 }}
COLLECTIVEX_OPERATOR_CONFIG_REQUIRED: '1'
run: |
set -euo pipefail
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index da85bfea23..772ac8b0a1 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -89,9 +89,7 @@ cx_fail_stage() {
cx_load_operator_config() {
[ -n "${COLLECTIVEX_OPERATOR_CONFIG_LOADED:-}" ] \
&& [ "$COLLECTIVEX_OPERATOR_CONFIG_LOADED" = "$$" ] && return 0
- local config_path generated=0 parsed_path config_log key value audit_salt_override
- audit_salt_override="${COLLECTIVEX_OPERATOR_AUDIT_SALT:-}"
- unset COLLECTIVEX_OPERATOR_AUDIT_SALT
+ local config_path generated=0 parsed_path config_log key value
unset CX_PARTITION CX_ACCOUNT CX_SQUASH_DIR CX_STAGE_DIR CX_ENROOT_CACHE_PATH
unset ENROOT_CACHE_PATH
unset CX_EXCLUDE_NODES CX_NODELIST CX_LOCK_DIR CX_MASTER_PORT
@@ -137,7 +135,7 @@ cx_load_operator_config() {
}
config_log="$(cx_private_log_path operator-config)"
if ! python3 - "$config_path" "${CX_RUNNER:-${CX_SHARD_SKU:-${CX_PUBLIC_RUNNER:-}}}" \
- "${COLLECTIVEX_CANONICAL_GHA:-0}" "$audit_salt_override" \
+ "${COLLECTIVEX_CANONICAL_GHA:-0}" \
> "$parsed_path" 2> "$config_log" <<'PY'
import json
import os
@@ -211,7 +209,7 @@ def valid_path(value):
)
try:
- path, runner, audit_required, audit_override = sys.argv[1:]
+ path, runner, audit_required = sys.argv[1:]
if runner not in RUNNERS or audit_required not in {"0", "1"}:
raise ValueError
metadata = os.lstat(path)
@@ -253,13 +251,9 @@ try:
(audit_salt is not None and (
not isinstance(audit_salt, str) or not AUDIT_SALT.fullmatch(audit_salt)
))
- or (audit_override and not AUDIT_SALT.fullmatch(audit_override))
- or (audit_salt is not None and audit_override and audit_salt != audit_override)
+ or (audit_required == "1" and audit_salt is None)
):
raise ValueError
- audit_salt = audit_salt or audit_override or None
- if audit_required == "1" and audit_salt is None:
- raise ValueError
runners = document["runners"]
if (
not isinstance(runners, dict) or not runners or set(runners) - RUNNERS
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index ba06612436..51d06ee94d 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -516,8 +516,7 @@ def test_canonical_operator_config_requires_a_private_audit_salt(self) -> None:
)
def invoke(
- value: dict, *, canonical: bool, expect_salt: bool = True,
- audit_override: str | None = None,
+ value: dict, *, canonical: bool, expect_salt: bool = True
) -> subprocess.CompletedProcess[str]:
config.write_text(json.dumps(value))
config.chmod(0o600)
@@ -529,8 +528,6 @@ def invoke(
}
if canonical:
environment["COLLECTIVEX_CANONICAL_GHA"] = "1"
- if audit_override is not None:
- environment["COLLECTIVEX_OPERATOR_AUDIT_SALT"] = audit_override
invocation = command if expect_salt else (
'source "$1"; export COLLECTIVEX_EXECUTION_ID="audit-config-$$"; '
"trap 'cx_cleanup_private_logs 0' EXIT; cx_load_operator_config; "
@@ -553,18 +550,6 @@ def invoke(
self.assertNotEqual(rejected.returncode, 0)
self.assertNotIn(salt, rejected.stdout + rejected.stderr)
- separate = invoke(missing, canonical=True, audit_override=salt)
- self.assertEqual(separate.returncode, 0, separate.stderr)
- self.assertNotIn(salt, separate.stdout + separate.stderr)
-
- conflicting = invoke(document, canonical=True, audit_override="b" * 64)
- self.assertNotEqual(conflicting.returncode, 0)
- self.assertNotIn(salt, conflicting.stdout + conflicting.stderr)
-
- malformed_override = invoke(missing, canonical=True, audit_override="A" * 64)
- self.assertNotEqual(malformed_override.returncode, 0)
- self.assertNotIn("A" * 64, malformed_override.stdout + malformed_override.stderr)
-
manual = invoke(missing, canonical=False, expect_salt=False)
self.assertEqual(manual.returncode, 0, manual.stderr)
self.assertNotIn(salt, manual.stdout + manual.stderr)
From 81c1ce80a78c63752666b479ee18ff2b3c969086 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 15:54:50 +0800
Subject: [PATCH 005/210] fix(collectivex): isolate precision probe failures
---
.github/workflows/collectivex-sweep.yml | 2 +-
experimental/CollectiveX/runtime/common.sh | 3 ++-
experimental/CollectiveX/tests/test_sampling_contract.py | 6 ++++++
3 files changed, 9 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml
index 47de47a8ce..91bb3c7273 100644
--- a/.github/workflows/collectivex-sweep.yml
+++ b/.github/workflows/collectivex-sweep.yml
@@ -453,7 +453,7 @@ jobs:
|| { echo "CollectiveX allocation cleanup was not confirmed" >&2; exit 1; }
- name: Validate shard artifact safety
id: artifact_safety
- if: ${{ always() && steps.allocation_cleanup.outcome == 'success' }}
+ if: ${{ always() && steps.allocation_cleanup.outcome == 'success' && (inputs.operation != 'probe-precision' || steps.sweep_shard.outcome == 'success') }}
run: |
cd "$CX_SOURCE_ROOT" 2>/dev/null \
|| { echo "CollectiveX source is unavailable" >&2; exit 1; }
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 772ac8b0a1..13d8bbf044 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -2409,7 +2409,8 @@ cx_launcher_cleanup() {
cx_write_cleanup_guard unsafe || true
fi
[ "$allocation_stopped" = 1 ] || source_root="${REPO_ROOT:-$source_root}"
- if [ "$rc" != 0 ] && [ -n "${REPO_ROOT:-}" ] && [ -n "${CX_BENCH:-}" ]; then
+ if [ "$rc" != 0 ] && [ "${CX_PRECISION_PROBE:-0}" != 1 ] \
+ && [ -n "${REPO_ROOT:-}" ] && [ -n "${CX_BENCH:-}" ]; then
cx_log "ERROR: terminal-failure-class=${CX_FAILSAFE_MODE:-setup}"
[ -d "$source_root/experimental/CollectiveX" ] || source_root="$REPO_ROOT"
out_dir="$source_root/experimental/CollectiveX/results"
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 51d06ee94d..e0c10225f0 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -1169,6 +1169,12 @@ def test_allocation_cleanup_fails_closed_when_scheduler_queries_fail(self) -> No
self.assertIn('python3 "$destination/source_archive.py"', workflow)
artifact_validation = workflow[workflow.index("- name: Validate shard artifact safety"):]
self.assertIn("steps.allocation_cleanup.outcome == 'success'", artifact_validation)
+ self.assertIn(
+ "inputs.operation != 'probe-precision' || steps.sweep_shard.outcome == 'success'",
+ artifact_validation,
+ )
+ cleanup_function = (ROOT / "runtime" / "common.sh").read_text()
+ self.assertIn('[ "${CX_PRECISION_PROBE:-0}" != 1 ]', cleanup_function)
sweep_workflow = workflow[workflow.index(" sweep:"):]
self.assertNotIn("GITHUB_WORKSPACE", sweep_workflow)
self.assertNotIn("RUNNER_WORKSPACE", sweep_workflow)
From 8a6409d9dd3591566b73a3d481dcc3bd6e9d8710 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 16:54:56 +0800
Subject: [PATCH 006/210] fix(collectivex): report sanitized config validation
codes
---
experimental/CollectiveX/runtime/common.sh | 10 +++++++++-
.../CollectiveX/tests/test_sampling_contract.py | 1 +
2 files changed, 10 insertions(+), 1 deletion(-)
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 13d8bbf044..bcc1c8e800 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -333,14 +333,22 @@ try:
key.encode() + b"\0" + str(value).encode() + b"\0"
)
except (KeyError, OSError, TypeError, UnicodeError, ValueError):
+ import traceback
+
+ location = traceback.extract_tb(sys.exc_info()[2])[-1].lineno
+ print(f"validation-line-{location}", file=sys.stderr)
raise SystemExit(1)
PY
then
+ local validation_code
+ validation_code="$(head -n 1 "$config_log" 2>/dev/null || true)"
rm -f -- "$parsed_path"
[ "$generated" = 0 ] || rm -f -- "$config_path"
unset COLLECTIVEX_EPHEMERAL_CONFIG_PATH
unset COLLECTIVEX_OPERATOR_CONFIG COLLECTIVEX_OPERATOR_CONFIG_EPHEMERAL
- cx_die "runner-local configuration failed"
+ [[ "$validation_code" =~ ^validation-line-[0-9]+$ ]] \
+ || validation_code="validation-unknown"
+ cx_die "runner-local configuration failed ($validation_code)"
fi
while IFS= read -r -d '' key && IFS= read -r -d '' value; do
printf -v "$key" '%s' "$value"
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index e0c10225f0..62b6de7dda 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -548,6 +548,7 @@ def invoke(
del missing["audit_salt"]
rejected = invoke(missing, canonical=True)
self.assertNotEqual(rejected.returncode, 0)
+ self.assertRegex(rejected.stderr, r"validation-line-[0-9]+")
self.assertNotIn(salt, rejected.stdout + rejected.stderr)
manual = invoke(missing, canonical=False, expect_salt=False)
From e3533bd7d276006bd91a1703c70983a28de57e0a Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 16:57:24 +0800
Subject: [PATCH 007/210] fix(collectivex): unblock bounded precision probes
---
.github/workflows/collectivex-sweep.yml | 3 +++
experimental/CollectiveX/runtime/common.sh | 14 ++++++++----
experimental/CollectiveX/sweep_matrix.py | 15 +++++++++++++
.../CollectiveX/tests/probe_precision.py | 22 +++++++++++++++++--
.../tests/test_precision_scheduling.py | 7 ++++++
.../tests/test_qualification_planning.py | 10 +++++++++
.../tests/test_sampling_contract.py | 13 ++++++++++-
7 files changed, 77 insertions(+), 7 deletions(-)
diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml
index 91bb3c7273..d66593256f 100644
--- a/.github/workflows/collectivex-sweep.yml
+++ b/.github/workflows/collectivex-sweep.yml
@@ -122,6 +122,8 @@ jobs:
if [ "$INPUT_OPERATION" = probe-precision ]; then
args=(--workflow-plan --backend "$INPUT_BACKEND" --out matrix_full.json)
[ -n "$INPUT_ONLY_SKU" ] && args+=(--only-sku "$INPUT_ONLY_SKU")
+ [ -n "$INPUT_MIN_NODES" ] && args+=(--min-nodes "$INPUT_MIN_NODES")
+ [ -n "$INPUT_MAX_NODES" ] && args+=(--max-nodes "$INPUT_MAX_NODES")
python3 tests/probe_precision.py "${args[@]}"
else
args=(--suites "$INPUT_SUITES" --max-cases "$INPUT_MAX_CASES")
@@ -435,6 +437,7 @@ jobs:
id: sweep_shard
env:
COLLECTIVEX_OPERATOR_CONFIG_CONTENT: ${{ secrets.COLLECTIVEX_OPERATOR_CONFIG_V1 }}
+ COLLECTIVEX_OPERATOR_AUDIT_SALT: ${{ secrets.COLLECTIVEX_AUDIT_SALT_V1 }}
COLLECTIVEX_OPERATOR_CONFIG_REQUIRED: '1'
run: |
set -euo pipefail
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index bcc1c8e800..805ecaff54 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -90,6 +90,9 @@ cx_load_operator_config() {
[ -n "${COLLECTIVEX_OPERATOR_CONFIG_LOADED:-}" ] \
&& [ "$COLLECTIVEX_OPERATOR_CONFIG_LOADED" = "$$" ] && return 0
local config_path generated=0 parsed_path config_log key value
+ local audit_salt_override validation_code
+ audit_salt_override="${COLLECTIVEX_OPERATOR_AUDIT_SALT:-}"
+ unset COLLECTIVEX_OPERATOR_AUDIT_SALT
unset CX_PARTITION CX_ACCOUNT CX_SQUASH_DIR CX_STAGE_DIR CX_ENROOT_CACHE_PATH
unset ENROOT_CACHE_PATH
unset CX_EXCLUDE_NODES CX_NODELIST CX_LOCK_DIR CX_MASTER_PORT
@@ -135,7 +138,7 @@ cx_load_operator_config() {
}
config_log="$(cx_private_log_path operator-config)"
if ! python3 - "$config_path" "${CX_RUNNER:-${CX_SHARD_SKU:-${CX_PUBLIC_RUNNER:-}}}" \
- "${COLLECTIVEX_CANONICAL_GHA:-0}" \
+ "${COLLECTIVEX_CANONICAL_GHA:-0}" "$audit_salt_override" \
> "$parsed_path" 2> "$config_log" <<'PY'
import json
import os
@@ -209,7 +212,7 @@ def valid_path(value):
)
try:
- path, runner, audit_required = sys.argv[1:]
+ path, runner, audit_required, audit_override = sys.argv[1:]
if runner not in RUNNERS or audit_required not in {"0", "1"}:
raise ValueError
metadata = os.lstat(path)
@@ -251,9 +254,13 @@ try:
(audit_salt is not None and (
not isinstance(audit_salt, str) or not AUDIT_SALT.fullmatch(audit_salt)
))
- or (audit_required == "1" and audit_salt is None)
+ or (audit_override and not AUDIT_SALT.fullmatch(audit_override))
+ or (audit_salt is not None and audit_override and audit_salt != audit_override)
):
raise ValueError
+ audit_salt = audit_salt or audit_override or None
+ if audit_required == "1" and audit_salt is None:
+ raise ValueError
runners = document["runners"]
if (
not isinstance(runners, dict) or not runners or set(runners) - RUNNERS
@@ -340,7 +347,6 @@ except (KeyError, OSError, TypeError, UnicodeError, ValueError):
raise SystemExit(1)
PY
then
- local validation_code
validation_code="$(head -n 1 "$config_log" 2>/dev/null || true)"
rm -f -- "$parsed_path"
[ "$generated" = 0 ] || rm -f -- "$config_path"
diff --git a/experimental/CollectiveX/sweep_matrix.py b/experimental/CollectiveX/sweep_matrix.py
index 309199fe2a..840fc93760 100644
--- a/experimental/CollectiveX/sweep_matrix.py
+++ b/experimental/CollectiveX/sweep_matrix.py
@@ -1306,16 +1306,31 @@ def frontend_catalog(matrix: dict[str, Any]) -> dict[str, Any]:
precision_profile = case.get(
"precision_profile", identity.V1_CONTROL_PRECISION_PROFILE
)
+ precision = identity.precision_profile(precision_profile)
cases.append({
"backend": case["backend"],
+ "backend_generation": None,
"case_id": case["case_id"],
"disposition": wrapper["disposition"],
+ "dispatch_precision": precision["dispatch"],
+ "combine_precision": precision["combine"],
"eplb": case["eplb"],
+ "label": (
+ f"{wrapper['sku'].upper()} / {case['backend']} / EP{case['ep']} / "
+ f"{case['mode']} / {case['phase']} / {case['routing']}"
+ ),
"mode": case["mode"],
"phase": case["phase"],
"precision_profile": precision_profile,
"publication_tier": case["required_publication"],
"reason": wrapper["reason"],
+ "required": True,
+ "resource": {
+ "mode": "fixed-profile",
+ "profile": None,
+ "comm_units_kind": None,
+ "configured_units": None,
+ },
"routing": case["routing"],
"sku": wrapper["sku"],
"suite": case["suite"],
diff --git a/experimental/CollectiveX/tests/probe_precision.py b/experimental/CollectiveX/tests/probe_precision.py
index 2a92e70795..ec1afec2a1 100644
--- a/experimental/CollectiveX/tests/probe_precision.py
+++ b/experimental/CollectiveX/tests/probe_precision.py
@@ -318,11 +318,24 @@ def _workflow_row(target: dict[str, Any]) -> dict[str, Any]:
}
-def workflow_plan(*, backend: str = "all", only_sku: str = "") -> dict[str, Any]:
+def workflow_plan(
+ *, backend: str = "all", only_sku: str = "", min_nodes: int = 0,
+ max_nodes: int = 0,
+) -> dict[str, Any]:
+ if min_nodes < 0 or max_nodes < 0 or (min_nodes and max_nodes and min_nodes > max_nodes):
+ raise ValueError("precision probe node filters are invalid")
targets = [
target for target in provisional_targets()
if (backend == "all" or target["backend"] == backend)
and (not only_sku or target["sku"] == only_sku)
+ and (
+ not min_nodes
+ or capability.topology_for(target["sku"], target["ep"])["nodes"] >= min_nodes
+ )
+ and (
+ not max_nodes
+ or capability.topology_for(target["sku"], target["ep"])["nodes"] <= max_nodes
+ )
]
if backend != "all" and backend not in BACKENDS:
raise ValueError("precision probe backend is not registered")
@@ -1096,6 +1109,8 @@ def main() -> int:
parser.add_argument("--backend", choices=sorted(BACKENDS | {"all"}))
parser.add_argument("--sku")
parser.add_argument("--only-sku", default="")
+ parser.add_argument("--min-nodes", type=int, default=0)
+ parser.add_argument("--max-nodes", type=int, default=0)
parser.add_argument("--expect-sku")
parser.add_argument("--expect-backend")
parser.add_argument("--expect-nodes", type=int)
@@ -1108,7 +1123,10 @@ def main() -> int:
print(json.dumps(provisional_targets(), allow_nan=False, sort_keys=True, separators=(",", ":")))
return 0
if args.workflow_plan:
- plan = workflow_plan(backend=args.backend or "all", only_sku=args.only_sku)
+ plan = workflow_plan(
+ backend=args.backend or "all", only_sku=args.only_sku,
+ min_nodes=args.min_nodes, max_nodes=args.max_nodes,
+ )
if args.out is None:
print(json.dumps(plan, allow_nan=False, sort_keys=True, separators=(",", ":")))
else:
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
index f103855378..4441e820f0 100644
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -77,6 +77,13 @@ def test_precision_probe_workflow_plan_binds_exact_controls(self) -> None:
)
with self.assertRaisesRegex(ValueError, "select no provisional"):
probe_precision.workflow_plan(backend="mori", only_sku="b200-dgxc")
+ ep8 = probe_precision.workflow_plan(
+ backend="deepep", only_sku="b200-dgxc", max_nodes=1,
+ )
+ self.assertTrue(ep8["include"])
+ self.assertEqual({row["ep"] for row in ep8["include"]}, {8})
+ with self.assertRaisesRegex(ValueError, "node filters"):
+ probe_precision.workflow_plan(min_nodes=2, max_nodes=1)
def test_precision_probe_manifest_is_sanitized_and_runtime_evidence_is_required(self) -> None:
target = probe_precision.provisional_targets()[0]
diff --git a/experimental/CollectiveX/tests/test_qualification_planning.py b/experimental/CollectiveX/tests/test_qualification_planning.py
index 4718818ed9..708363135b 100644
--- a/experimental/CollectiveX/tests/test_qualification_planning.py
+++ b/experimental/CollectiveX/tests/test_qualification_planning.py
@@ -186,6 +186,16 @@ def test_frontend_catalog_covers_every_requested_case_and_point(self) -> None:
{item["case_id"] for item in catalog["cases"]},
{item["case"]["case_id"] for item in self.matrix["requested_cases"]},
)
+ self.assertTrue(all(item["required"] for item in catalog["cases"]))
+ self.assertTrue(all(item["backend_generation"] is None for item in catalog["cases"]))
+ self.assertEqual(
+ {item["resource"]["mode"] for item in catalog["cases"]},
+ {"fixed-profile"},
+ )
+ for item in catalog["cases"]:
+ profile = identity.precision_profile(item["precision_profile"])
+ self.assertEqual(item["dispatch_precision"], profile["dispatch"])
+ self.assertEqual(item["combine_precision"], profile["combine"])
self.assertLess(len(_canonical(catalog)) + 1, 1024 * 1024)
def test_invalid_qualification_controls_are_rejected(self) -> None:
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 62b6de7dda..d70c7b51a1 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -516,7 +516,8 @@ def test_canonical_operator_config_requires_a_private_audit_salt(self) -> None:
)
def invoke(
- value: dict, *, canonical: bool, expect_salt: bool = True
+ value: dict, *, canonical: bool, expect_salt: bool = True,
+ audit_override: str | None = None,
) -> subprocess.CompletedProcess[str]:
config.write_text(json.dumps(value))
config.chmod(0o600)
@@ -528,6 +529,8 @@ def invoke(
}
if canonical:
environment["COLLECTIVEX_CANONICAL_GHA"] = "1"
+ if audit_override is not None:
+ environment["COLLECTIVEX_OPERATOR_AUDIT_SALT"] = audit_override
invocation = command if expect_salt else (
'source "$1"; export COLLECTIVEX_EXECUTION_ID="audit-config-$$"; '
"trap 'cx_cleanup_private_logs 0' EXIT; cx_load_operator_config; "
@@ -551,6 +554,14 @@ def invoke(
self.assertRegex(rejected.stderr, r"validation-line-[0-9]+")
self.assertNotIn(salt, rejected.stdout + rejected.stderr)
+ separate = invoke(missing, canonical=True, audit_override=salt)
+ self.assertEqual(separate.returncode, 0, separate.stderr)
+ self.assertNotIn(salt, separate.stdout + separate.stderr)
+
+ malformed_override = invoke(missing, canonical=True, audit_override="A" * 64)
+ self.assertNotEqual(malformed_override.returncode, 0)
+ self.assertNotIn("A" * 64, malformed_override.stdout + malformed_override.stderr)
+
manual = invoke(missing, canonical=False, expect_salt=False)
self.assertEqual(manual.returncode, 0, manual.stderr)
self.assertNotIn(salt, manual.stdout + manual.stderr)
From 903285e95411765995363a36fb448bd2de03ff53 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 16:59:18 +0800
Subject: [PATCH 008/210] fix(collectivex): refine operator config diagnostics
---
experimental/CollectiveX/runtime/common.sh | 7 +++----
experimental/CollectiveX/sweep_matrix.py | 8 ++++++--
.../CollectiveX/tests/test_qualification_planning.py | 9 +++++++--
3 files changed, 16 insertions(+), 8 deletions(-)
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 805ecaff54..2d47d156ac 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -269,11 +269,10 @@ try:
raise ValueError
selected = None
for name, config in runners.items():
- if (
- not isinstance(config, dict)
- or (name == runner and not REQUIRED[name].issubset(config))
- ):
+ if not isinstance(config, dict):
raise ValueError
+ if name == runner and not REQUIRED[name].issubset(config):
+ raise KeyError("selected-runner-missing-required-fields")
if set(config) - ALLOWED[name]:
raise ValueError
for field, value in config.items():
diff --git a/experimental/CollectiveX/sweep_matrix.py b/experimental/CollectiveX/sweep_matrix.py
index 840fc93760..2cebf3be6e 100644
--- a/experimental/CollectiveX/sweep_matrix.py
+++ b/experimental/CollectiveX/sweep_matrix.py
@@ -1301,19 +1301,22 @@ def frontend_catalog(matrix: dict[str, Any]) -> dict[str, Any]:
document = validate_matrix_document(matrix)
matrix_bytes = contracts.canonical_json_bytes(document) + b"\n"
cases = []
+ precision_profiles: dict[str, dict[str, Any]] = {}
for wrapper in document["requested_cases"]:
case = wrapper["case"]
precision_profile = case.get(
"precision_profile", identity.V1_CONTROL_PRECISION_PROFILE
)
precision = identity.precision_profile(precision_profile)
+ precision_profiles[precision_profile] = {
+ "dispatch": precision["dispatch"],
+ "combine": precision["combine"],
+ }
cases.append({
"backend": case["backend"],
"backend_generation": None,
"case_id": case["case_id"],
"disposition": wrapper["disposition"],
- "dispatch_precision": precision["dispatch"],
- "combine_precision": precision["combine"],
"eplb": case["eplb"],
"label": (
f"{wrapper['sku'].upper()} / {case['backend']} / EP{case['ep']} / "
@@ -1352,6 +1355,7 @@ def frontend_catalog(matrix: dict[str, Any]) -> dict[str, Any]:
"format": "collectivex.frontend-catalog.v1",
"matrix_sha256": hashlib.sha256(matrix_bytes).hexdigest(),
"point_count": sum(len(case["points"]) for case in cases),
+ "precision_profiles": dict(sorted(precision_profiles.items())),
"schema_version": 1,
"cases": cases,
}
diff --git a/experimental/CollectiveX/tests/test_qualification_planning.py b/experimental/CollectiveX/tests/test_qualification_planning.py
index 708363135b..9cb1fd1e87 100644
--- a/experimental/CollectiveX/tests/test_qualification_planning.py
+++ b/experimental/CollectiveX/tests/test_qualification_planning.py
@@ -186,6 +186,10 @@ def test_frontend_catalog_covers_every_requested_case_and_point(self) -> None:
{item["case_id"] for item in catalog["cases"]},
{item["case"]["case_id"] for item in self.matrix["requested_cases"]},
)
+ self.assertEqual(
+ set(catalog["precision_profiles"]),
+ {item["precision_profile"] for item in catalog["cases"]},
+ )
self.assertTrue(all(item["required"] for item in catalog["cases"]))
self.assertTrue(all(item["backend_generation"] is None for item in catalog["cases"]))
self.assertEqual(
@@ -194,8 +198,9 @@ def test_frontend_catalog_covers_every_requested_case_and_point(self) -> None:
)
for item in catalog["cases"]:
profile = identity.precision_profile(item["precision_profile"])
- self.assertEqual(item["dispatch_precision"], profile["dispatch"])
- self.assertEqual(item["combine_precision"], profile["combine"])
+ projected = catalog["precision_profiles"][item["precision_profile"]]
+ self.assertEqual(projected["dispatch"], profile["dispatch"])
+ self.assertEqual(projected["combine"], profile["combine"])
self.assertLess(len(_canonical(catalog)) + 1, 1024 * 1024)
def test_invalid_qualification_controls_are_rejected(self) -> None:
From d63948576f33a0a8c1658913661fcb34366ba633 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 17:01:00 +0800
Subject: [PATCH 009/210] fix(collectivex): identify missing operator fields
safely
---
experimental/CollectiveX/runtime/common.sh | 12 +++++++++---
.../CollectiveX/tests/test_sampling_contract.py | 6 ++++++
2 files changed, 15 insertions(+), 3 deletions(-)
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 2d47d156ac..3f1f2d8215 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -271,8 +271,14 @@ try:
for name, config in runners.items():
if not isinstance(config, dict):
raise ValueError
- if name == runner and not REQUIRED[name].issubset(config):
- raise KeyError("selected-runner-missing-required-fields")
+ if name == runner:
+ missing = sorted(REQUIRED[name] - set(config))
+ if missing:
+ print(
+ "validation-missing-required-" + "-".join(missing),
+ file=sys.stderr,
+ )
+ raise SystemExit(1)
if set(config) - ALLOWED[name]:
raise ValueError
for field, value in config.items():
@@ -351,7 +357,7 @@ PY
[ "$generated" = 0 ] || rm -f -- "$config_path"
unset COLLECTIVEX_EPHEMERAL_CONFIG_PATH
unset COLLECTIVEX_OPERATOR_CONFIG COLLECTIVEX_OPERATOR_CONFIG_EPHEMERAL
- [[ "$validation_code" =~ ^validation-line-[0-9]+$ ]] \
+ [[ "$validation_code" =~ ^validation-(line-[0-9]+|missing-required-[a-z0-9_-]+)$ ]] \
|| validation_code="validation-unknown"
cx_die "runner-local configuration failed ($validation_code)"
fi
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index d70c7b51a1..af8b035008 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -572,6 +572,12 @@ def invoke(
self.assertNotEqual(rejected.returncode, 0)
self.assertNotIn("A" * 64, rejected.stdout + rejected.stderr)
+ missing_field = copy.deepcopy(document)
+ del missing_field["runners"]["h100-dgxc"]["stage_dir"]
+ rejected = invoke(missing_field, canonical=True)
+ self.assertNotEqual(rejected.returncode, 0)
+ self.assertIn("validation-missing-required-stage_dir", rejected.stderr)
+
def test_scaleout_network_profile_is_explicit_and_allowlisted(self) -> None:
command = r'''
set -euo pipefail
From d8f12e4a543532282b13aa93b90fed8f6c32a121 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 17:03:16 +0800
Subject: [PATCH 010/210] fix(collectivex): use isolated shared-stage fallback
---
experimental/CollectiveX/README.md | 22 ++++++++-------
experimental/CollectiveX/docs/methodology.md | 28 +++++++++++--------
experimental/CollectiveX/runtime/common.sh | 10 +++----
.../tests/test_deepep_v2_contract.py | 6 ++--
.../tests/test_sampling_contract.py | 7 ++---
5 files changed, 39 insertions(+), 34 deletions(-)
diff --git a/experimental/CollectiveX/README.md b/experimental/CollectiveX/README.md
index b58a5af61d..03169b3bb1 100644
--- a/experimental/CollectiveX/README.md
+++ b/experimental/CollectiveX/README.md
@@ -12,14 +12,15 @@ combine, and paired roundtrip latency across EP libraries and accelerator system
> Publication hold: historical schema 3-5 data is diagnostic. No current dataset is approved for
> rankings, recommendations, or regression baselines.
-> Development status: the sections below document the implemented BF16 pre-V1 baseline, not the
-> final V1 qualification contract. Precision profiles, full point-level publication, and branch-only
-> publication are under active implementation; case counts and digests are not frozen.
+> Development status: the V1 precision, point-level publication, branch-only delivery, and frontend
+> contracts are implemented. Native precision capability probes are still resolving provisional
+> cells, so the BF16 counts below remain a planning baseline and V1 counts/digests are not frozen.
## Implemented Pre-V1 Execution Profile
-Every scheduled case is BF16 with backend-tuned resources and packed placement. The explicit mode
-selects one of two contracts:
+The BF16/BF16 control and endpoint precision suites use packed placement and one pinned
+`fixed-profile` resource configuration per backend/topology/precision. V1 performs no tuning sweep.
+The explicit mode selects one of two contracts:
- Normal mode uses `layout-and-dispatch-v1`, rank-deduplicated token payloads, and activation-only
combine. Uniform core coverage and one Zipf sensitivity remain; EPLB is measured only as the Zipf
@@ -33,8 +34,8 @@ warmups before each measured component at every trial/point. Roundtrip is measur
iteration takes the cross-rank maximum before nearest-rank p50/p90/p95/p99, and roundtrip p99 is the
headline latency. A stdlib integer counter produces byte-identical routing and gate weights.
-The implemented baseline matrix covers H100, H200, B200, B300, GB200, GB300, MI325X, and MI355X.
-It requests
+The BF16 planning baseline covers H100, H200, B200, B300, GB200, GB300, MI325X, and MI355X. It
+requests
608 cases / 1,600 token points: 364 runnable cases / 940 points, emitted as 58 executable workflow
shards/allocation cells, plus 244 explicit unsupported cases / 660 points. `sweep_matrix.py`
materializes every token ladder and rejects missing, stale, malformed, or altered shard controls.
@@ -88,9 +89,10 @@ raw GitHub artifacts. Runs default to `release_tag=unversioned` and are diagnost
must explicitly select `release_tag=v1`; setup then requires the locked full-matrix digest and emits
a run/attempt/source-bound `cxrelease-v1-*` marker. Partial and filtered runs cannot receive it.
-The main-registered `.github/workflows/collectivex-sweep.yml` provides `sweep`, `publish-v1`, and
-`refresh-v1` operations, so its branch revision can be dispatched with `--ref collectivex` without
-a standalone branch-only workflow. Publication accepts exactly three successful first-attempt tagged
+The main-registered `.github/workflows/collectivex-sweep.yml` provides `probe-precision`, `sweep`,
+`publish-v1`, and `refresh-v1` operations, so its branch revision can be dispatched with
+`--ref collectivex` without a standalone branch-only workflow. Probes are unversioned, bounded,
+native-runtime checks and cannot publish. Publication accepts exactly three successful first-attempt tagged
sweep run IDs from one source SHA, revalidates their metadata and release markers, and runs
`publisher.py` in a disposable runner-local workspace. Refresh revalidates and reuploads only the
exact content-addressed sanitized dataset. Raw artifacts and the private publisher workspace are
diff --git a/experimental/CollectiveX/docs/methodology.md b/experimental/CollectiveX/docs/methodology.md
index 396a705b6e..440f882a78 100644
--- a/experimental/CollectiveX/docs/methodology.md
+++ b/experimental/CollectiveX/docs/methodology.md
@@ -23,15 +23,20 @@ It does not predict serving throughput without a separate correlation study.
## Implemented Matrix
-The implemented workload is `deepseek-v3-v1`: hidden 7168, top-k 8, 256 routed experts, BF16 dispatch
-and combine, packed placement, and backend-tuned resources. Each case explicitly selects normal
-`layout-and-dispatch-v1` or low-latency `expert-packed-weighted-combine-v1` semantics.
+The implemented workload is `deepseek-v3-v1`: hidden 7168, top-k 8, 256 routed experts, packed
+placement, and one pinned fixed resource profile per backend/topology/precision. The BF16/BF16
+control is portable; endpoint precision suites schedule only allowlisted native communication-format
+profiles after real-hardware probes resolve their capability cells. Each case explicitly selects
+normal `layout-and-dispatch-v1` or low-latency `expert-packed-weighted-combine-v1` semantics.
- `ep-core-v1`: uniform routing; decode T=1..128 powers of two; prefill T=256/512.
- `ep-routing-v1`: Zipf with EPLB off/on; decode T=128; prefill T=512.
- `ep-low-latency-v1`: DeepEP V1/UCCL native low-latency APIs; uniform decode T=1..128 powers of
two; the capability contract rejects every other backend instead of fabricating a low-latency path.
-- Implemented baseline surface: 608 requested cases / 1,600 token points; 364 runnable cases / 940
+- `ep-precision-normal-v1`: nonbaseline native dispatch/combine profiles at decode T=128 and prefill
+ T=512; BF16/BF16 endpoint controls are referenced rather than duplicated.
+- `ep-precision-low-latency-v1`: nonbaseline native low-latency profiles at decode T=128.
+- BF16 planning baseline: 608 requested cases / 1,600 token points; 364 runnable cases / 940
points in
58 executable workflow shards/allocation cells; 244 unsupported cases / 660 points.
@@ -181,9 +186,9 @@ frozen percentile, rank-reduction, conditioning, warmup, and correctness semanti
A controlled comparison declares one contrast:
-- `library`: backend implementation and its tuned resource profile may differ; the realized system,
+- `library`: backend implementation and its pinned fixed resource profile may differ; the realized system,
workload, EP, resource policy, source, and measurement remain matched;
-- `chip`: a controlled platform contrast. The full realized system/topology and tuned resource
+- `chip`: a controlled platform contrast. The full realized system/topology and pinned resource
profile may differ while workload, EP, placement class, resource policy, backend lineage, source,
and measurement remain matched. It is not a silicon-only comparison;
- `system`: all hardware/backend differences stay visible while workload, EP, and measurement match;
@@ -247,7 +252,7 @@ $COLLECTIVEX_STORE_ROOT/
private/bundles// # immutable source archives, native results/samples, matrix, checksums
private/quarantine/ # rejected attempts plus machine-readable reasons
public/datasets// # immutable sanitized frontend datasets
- public/channels/ # small atomic pointers: latest-attempt, dev-latest
+ public/channels/ # promoted dev-latest pointer; never served from persistent storage
locks/
```
@@ -278,13 +283,14 @@ Publication is fail-closed:
6. build and validate the sanitized content-addressed dataset, fsync, then atomically rename it;
7. atomically replace `dev-latest.json` only when every promotion gate passes.
-Rejected attempts may update workspace `latest-attempt` but never `dev-latest`. The workspace is
-destroyed with the publication runner and is never attached to the frontend. No artifact is emitted
-unless all three selected bundles advance `dev-latest`.
+Rejected attempts remain only in the disposable private workspace and short-lived source artifacts;
+they never advance `dev-latest` or enter a production channel. The workspace is destroyed with the
+publication runner and is never attached to the frontend. No publication artifact is emitted unless
+all three selected bundles advance `dev-latest`.
`publisher.py ingest` accepts the exact matrix plus one `--artifact` directory or ZIP per GitHub
artifact. `promote` accepts explicit immutable bundle IDs. Default `verify` requires
-`latest-attempt`; it also verifies `dev-latest` when present, while an explicit
+the private workspace; it also verifies `dev-latest` when present, while an explicit
`--channel dev-latest` requires it. The workflow copies only the verified sanitized dataset to a
one-record `collectivex_public_v1_.ndjson` artifact. Raw artifacts and private workspace
content are never bundled into the application.
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 3f1f2d8215..bbf8b5272a 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -169,9 +169,9 @@ NETWORK_FIELDS = {
"socket_ifname", "rdma_devices", "ib_gid_index", "rdma_service_level",
}
REQUIRED = {
- "h100-dgxc": {"partition", "account", "squash_dir", "stage_dir"},
- "h200-dgxc": {"partition", "squash_dir", "stage_dir"},
- "b200-dgxc": {"partition", "account", "squash_dir", "stage_dir"},
+ "h100-dgxc": {"partition", "account", "squash_dir"},
+ "h200-dgxc": {"partition", "squash_dir"},
+ "b200-dgxc": {"partition", "account", "squash_dir"},
"b300": {
"partition", "account", "squash_dir", "stage_dir",
},
@@ -1795,10 +1795,8 @@ cx_stage_path() {
|| cx_die "invalid staging execution identity"
safe_tag="$(printf '%s' "$tag" | tr -c 'A-Za-z0-9._-' '_')"
if [ -z "$stage_base" ] || [ "$stage_base" = "$repo_root" ]; then
- [ "${COLLECTIVEX_CANONICAL_GHA:-0}" != 1 ] \
- || cx_die "canonical CollectiveX execution requires compute-visible staging"
[ -n "${CX_SQUASH_DIR:-}" ] \
- || cx_die "manual CollectiveX staging requires CX_SQUASH_DIR"
+ || cx_die "CollectiveX staging requires CX_STAGE_DIR or CX_SQUASH_DIR"
stage_base="$CX_SQUASH_DIR"
stage_path="${stage_base%/}/.collectivex-stage-$safe_tag"
else
diff --git a/experimental/CollectiveX/tests/test_deepep_v2_contract.py b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
index c9f65c2c30..994a17e3f7 100644
--- a/experimental/CollectiveX/tests/test_deepep_v2_contract.py
+++ b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
@@ -2061,9 +2061,9 @@ def test_operator_config_is_strict_per_runner_json(self) -> None:
del missing_rdma["runners"]["mi355x"]["rdma_devices"]
missing_amd_stage = json.loads(json.dumps(document))
del missing_amd_stage["runners"]["mi325x"]["stage_dir"]
- missing_nvidia_stage = json.loads(json.dumps(document))
- del missing_nvidia_stage["runners"]["h100-dgxc"]["stage_dir"]
- for invalid in (rejected, boolean_version, missing_nvidia_stage):
+ missing_nvidia_account = json.loads(json.dumps(document))
+ del missing_nvidia_account["runners"]["h100-dgxc"]["account"]
+ for invalid in (rejected, boolean_version, missing_nvidia_account):
config.write_text(json.dumps(invalid))
config.chmod(0o600)
result = subprocess.run(
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index af8b035008..b4909bb0b1 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -573,10 +573,10 @@ def invoke(
self.assertNotIn("A" * 64, rejected.stdout + rejected.stderr)
missing_field = copy.deepcopy(document)
- del missing_field["runners"]["h100-dgxc"]["stage_dir"]
+ del missing_field["runners"]["h100-dgxc"]["account"]
rejected = invoke(missing_field, canonical=True)
self.assertNotEqual(rejected.returncode, 0)
- self.assertIn("validation-missing-required-stage_dir", rejected.stderr)
+ self.assertIn("validation-missing-required-account", rejected.stderr)
def test_scaleout_network_profile_is_explicit_and_allowlisted(self) -> None:
command = r'''
@@ -2440,8 +2440,7 @@ def test_canonical_gha_requires_compute_visible_staging(self) -> None:
set -euo pipefail
source "$1"
unset CX_SHARD_FILE CX_STAGE_DIR
- ! (COLLECTIVEX_CANONICAL_GHA=1; cx_stage_path "$2" "")
- staged="$(COLLECTIVEX_CANONICAL_GHA=0; cx_stage_path "$2" "")"
+ staged="$(COLLECTIVEX_CANONICAL_GHA=1; cx_stage_path "$2" "")"
cx_stage_repo "$2" "$staged"
test "$staged" != "$2"
test -f "$staged/experimental/CollectiveX/public.py"
From 8887ed5333e5f907bd864b31d2ffb8a930930d8f Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 17:06:21 +0800
Subject: [PATCH 011/210] fix(collectivex): align canonical staging fallback
---
experimental/CollectiveX/runtime/common.sh | 8 ++++++--
experimental/CollectiveX/tests/test_sampling_contract.py | 6 ++++++
2 files changed, 12 insertions(+), 2 deletions(-)
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index bbf8b5272a..daffb417a3 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -1476,8 +1476,12 @@ cx_lock_canonical_gha_env() {
[ -n "${CX_SQUASH_DIR:-}" ] \
|| cx_die "canonical CollectiveX execution requires shared container storage"
- [ -n "$trusted_stage_dir" ] \
- || cx_die "canonical CollectiveX execution requires a configured shared stage directory"
+ if [ -z "$trusted_stage_dir" ]; then
+ case "$runner" in
+ h100-dgxc|h200-dgxc|b200-dgxc) ;;
+ *) cx_die "canonical CollectiveX execution requires a configured shared stage directory" ;;
+ esac
+ fi
[[ "$trusted_audit_salt" =~ ^[0-9a-f]{64}$ ]] \
|| cx_die "canonical CollectiveX execution requires a private audit salt"
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index b4909bb0b1..aca9ce628c 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -2284,6 +2284,12 @@ def test_canonical_gha_environment_is_locked_but_manual_overrides_survive(self)
test -z "${CX_PREPARED_BACKEND_CACHE+x}${CX_BACKEND_SOURCE_ROOT+x}"
test -z "${CX_DRYRUN+x}"
+ export COLLECTIVEX_OPERATOR_CONFIG_LOADED=$$
+ export CX_SHARD_SKU=b200-dgxc CX_NODES=1 CX_GPUS_PER_NODE=8
+ unset CX_STAGE_DIR
+ cx_lock_canonical_gha_env b200-dgxc
+ test -z "$CX_STAGE_DIR"
+
export CX_STAGE_DIR=/shared/gb-stage
export CX_SHARD_SKU=gb300 CX_NODES=2 CX_GPUS_PER_NODE=4
export CX_IMAGE=untrusted CX_NGPUS=1 CX_MORI_KERNEL_TYPE=untrusted
From a1502cf0502ae2691567e83f7d6cf4373a48c101 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 17:07:58 +0800
Subject: [PATCH 012/210] fix(collectivex): report probe failure stages
---
experimental/CollectiveX/runtime/common.sh | 3 +++
experimental/CollectiveX/tests/test_sampling_contract.py | 1 +
2 files changed, 4 insertions(+)
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index daffb417a3..b433ace9fa 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -2430,6 +2430,9 @@ cx_launcher_cleanup() {
cx_write_cleanup_guard unsafe || true
fi
[ "$allocation_stopped" = 1 ] || source_root="${REPO_ROOT:-$source_root}"
+ if [ "$rc" != 0 ] && [ "${CX_PRECISION_PROBE:-0}" = 1 ]; then
+ cx_log "ERROR: precision-probe-failure-class=${CX_FAILSAFE_MODE:-setup}"
+ fi
if [ "$rc" != 0 ] && [ "${CX_PRECISION_PROBE:-0}" != 1 ] \
&& [ -n "${REPO_ROOT:-}" ] && [ -n "${CX_BENCH:-}" ]; then
cx_log "ERROR: terminal-failure-class=${CX_FAILSAFE_MODE:-setup}"
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index aca9ce628c..83a2d2a3a2 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -1193,6 +1193,7 @@ def test_allocation_cleanup_fails_closed_when_scheduler_queries_fail(self) -> No
)
cleanup_function = (ROOT / "runtime" / "common.sh").read_text()
self.assertIn('[ "${CX_PRECISION_PROBE:-0}" != 1 ]', cleanup_function)
+ self.assertIn("precision-probe-failure-class=", cleanup_function)
sweep_workflow = workflow[workflow.index(" sweep:"):]
self.assertNotIn("GITHUB_WORKSPACE", sweep_workflow)
self.assertNotIn("RUNNER_WORKSPACE", sweep_workflow)
From 8eba0bd8069650cd617018e9583a5580a9eaebb9 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 17:09:59 +0800
Subject: [PATCH 013/210] fix(collectivex): classify stage validation failures
---
experimental/CollectiveX/runtime/common.sh | 37 ++++++++++++++--------
1 file changed, 24 insertions(+), 13 deletions(-)
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index b433ace9fa..3f508565bc 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -1813,6 +1813,10 @@ import stat
import sys
repo, base, child, job_root, workspace = sys.argv[1:]
+def reject(reason):
+ print(f"[collectivex] FATAL: stage-base-validation={reason}", file=sys.stderr)
+ raise SystemExit(1)
+
try:
if (
not os.path.isabs(repo)
@@ -1821,25 +1825,32 @@ try:
or os.path.realpath(base) != base
or not os.path.isabs(child)
or os.path.dirname(child) != base.rstrip("/")
- or os.path.lexists(child)
):
- raise OSError
- metadata = os.stat(base, follow_symlinks=False)
+ reject("path-shape")
+ if os.path.lexists(child):
+ reject("child-exists")
+ try:
+ metadata = os.stat(base, follow_symlinks=False)
+ except OSError:
+ reject("base-stat")
excluded = [repo]
excluded.extend(path for path in (job_root, workspace) if path)
for path in excluded:
resolved = os.path.realpath(path)
if os.path.commonpath((base, resolved)) == resolved:
- raise OSError
- if (
- not stat.S_ISDIR(metadata.st_mode)
- or metadata.st_uid != os.getuid()
- or stat.S_IMODE(metadata.st_mode) & (stat.S_IWGRP | stat.S_IWOTH)
- or not os.access(base, os.W_OK | os.X_OK)
- ):
- raise OSError
-except OSError:
- raise SystemExit(1)
+ reject("overlap")
+ if not stat.S_ISDIR(metadata.st_mode):
+ reject("not-directory")
+ if metadata.st_uid != os.getuid():
+ reject("owner")
+ if stat.S_IMODE(metadata.st_mode) & stat.S_IWGRP:
+ reject("group-writable")
+ if stat.S_IMODE(metadata.st_mode) & stat.S_IWOTH:
+ reject("world-writable")
+ if not os.access(base, os.W_OK | os.X_OK):
+ reject("access")
+except ValueError:
+ reject("path-shape")
print(child, end="")
PY
}
From 6278cb460a7c1fd538af9a674921c099dd31664d Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 17:17:08 +0800
Subject: [PATCH 014/210] fix(collectivex): isolate shared staging fallback
---
experimental/CollectiveX/README.md | 9 +++++++--
experimental/CollectiveX/runtime/common.sh | 9 +++++----
experimental/CollectiveX/tests/test_sampling_contract.py | 3 ++-
3 files changed, 14 insertions(+), 7 deletions(-)
diff --git a/experimental/CollectiveX/README.md b/experimental/CollectiveX/README.md
index 03169b3bb1..6cde01c1dc 100644
--- a/experimental/CollectiveX/README.md
+++ b/experimental/CollectiveX/README.md
@@ -118,8 +118,8 @@ temporary copy before allocation. Required JSON fields are:
| SKU | Variables |
|---|---|
-| `h100-dgxc`, `b200-dgxc` | `partition`, `account`, `squash_dir`, `stage_dir` |
-| `h200-dgxc` | `partition`, `squash_dir`, `stage_dir` |
+| `h100-dgxc`, `b200-dgxc` | `partition`, `account`, `squash_dir` |
+| `h200-dgxc` | `partition`, `squash_dir` |
| `b300` | `partition`, `account`, `squash_dir`, `stage_dir` |
| `gb200` | `partition`, `account`, ordered `storage_roots` |
| `gb300` | `partition`, `account`, `squash_dir`, `stage_dir`, `enroot_cache_path` |
@@ -139,6 +139,11 @@ allocated node. Jobs create only a marked mode-0700 execution child, prove cross
visibility, and remove that exact child after allocation teardown; they never mount the runner
checkout or create a stage beneath image storage on AMD.
+H100, H200, and B200 runners may omit `stage_dir`. Their isolated execution child is then created
+directly under the runner-owned `squash_dir`; that shared parent may be group-writable, but the child
+is mode 0700 and is still validated, marked, cross-node checked, and removed using the same contract.
+All other runners require the stricter dedicated `stage_dir` above.
+
Before import, each Docker Hub tag is resolved with bounded registry requests and must match its
pinned digest; digest-qualified overrides are rejected. Enroot imports use a fixed filesystem epoch
and a versioned, registry-digest-bound cache key. Every mounted squash is freshly hashed. The
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 3f508565bc..07143e8948 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -1793,7 +1793,7 @@ BASH
# EXIT trap can remove an interrupted partial stage. The configured base must
# already exist on compute-visible storage and must not traverse symlinks.
cx_stage_path() {
- local repo_root="$1" stage_base="${2:-}" tag safe_tag stage_path
+ local repo_root="$1" stage_base="${2:-}" tag safe_tag stage_path stage_fallback=0
tag="${COLLECTIVEX_EXECUTION_ID:-${GITHUB_RUN_ID:-manual-$$}}"
[[ "$tag" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] \
|| cx_die "invalid staging execution identity"
@@ -1803,16 +1803,17 @@ cx_stage_path() {
|| cx_die "CollectiveX staging requires CX_STAGE_DIR or CX_SQUASH_DIR"
stage_base="$CX_SQUASH_DIR"
stage_path="${stage_base%/}/.collectivex-stage-$safe_tag"
+ stage_fallback=1
else
stage_path="${stage_base%/}/job_$safe_tag"
fi
python3 - "$repo_root" "$stage_base" "$stage_path" \
- "${CX_JOB_ROOT:-}" "${GITHUB_WORKSPACE:-}" <<'PY'
+ "${CX_JOB_ROOT:-}" "${GITHUB_WORKSPACE:-}" "$stage_fallback" <<'PY'
import os
import stat
import sys
-repo, base, child, job_root, workspace = sys.argv[1:]
+repo, base, child, job_root, workspace, fallback = sys.argv[1:]
def reject(reason):
print(f"[collectivex] FATAL: stage-base-validation={reason}", file=sys.stderr)
raise SystemExit(1)
@@ -1843,7 +1844,7 @@ try:
reject("not-directory")
if metadata.st_uid != os.getuid():
reject("owner")
- if stat.S_IMODE(metadata.st_mode) & stat.S_IWGRP:
+ if stat.S_IMODE(metadata.st_mode) & stat.S_IWGRP and fallback != "1":
reject("group-writable")
if stat.S_IMODE(metadata.st_mode) & stat.S_IWOTH:
reject("world-writable")
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 83a2d2a3a2..7ef43680ee 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -2440,7 +2440,8 @@ def test_canonical_gha_requires_compute_visible_staging(self) -> None:
squash = root / "squash"
source = repo / "experimental" / "CollectiveX"
source.mkdir(parents=True)
- squash.mkdir()
+ squash.mkdir(mode=0o770)
+ squash.chmod(0o770)
(source / "public.py").write_text("public\n")
(source / "private-infra.md").write_text("private\n")
command = r'''
From 33607028eac936db042def96fb3ce7d3b84cf833 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 17:18:56 +0800
Subject: [PATCH 015/210] fix(collectivex): permit sticky shared staging
---
experimental/CollectiveX/README.md | 7 ++++---
experimental/CollectiveX/runtime/common.sh | 5 ++++-
experimental/CollectiveX/tests/test_sampling_contract.py | 4 ++--
3 files changed, 10 insertions(+), 6 deletions(-)
diff --git a/experimental/CollectiveX/README.md b/experimental/CollectiveX/README.md
index 6cde01c1dc..7c0ee09448 100644
--- a/experimental/CollectiveX/README.md
+++ b/experimental/CollectiveX/README.md
@@ -140,9 +140,10 @@ visibility, and remove that exact child after allocation teardown; they never mo
checkout or create a stage beneath image storage on AMD.
H100, H200, and B200 runners may omit `stage_dir`. Their isolated execution child is then created
-directly under the runner-owned `squash_dir`; that shared parent may be group-writable, but the child
-is mode 0700 and is still validated, marked, cross-node checked, and removed using the same contract.
-All other runners require the stricter dedicated `stage_dir` above.
+directly under the runner-owned `squash_dir`; that shared parent may be group-writable or use the
+sticky bit for shared writes, but a non-sticky world-writable parent is rejected. The child is mode
+0700 and is still validated, marked, cross-node checked, and removed using the same contract. All
+other runners require the stricter dedicated `stage_dir` above.
Before import, each Docker Hub tag is resolved with bounded registry requests and must match its
pinned digest; digest-qualified overrides are rejected. Enroot imports use a fixed filesystem epoch
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 07143e8948..080082e26c 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -1846,7 +1846,10 @@ try:
reject("owner")
if stat.S_IMODE(metadata.st_mode) & stat.S_IWGRP and fallback != "1":
reject("group-writable")
- if stat.S_IMODE(metadata.st_mode) & stat.S_IWOTH:
+ if (
+ stat.S_IMODE(metadata.st_mode) & stat.S_IWOTH
+ and not (fallback == "1" and metadata.st_mode & stat.S_ISVTX)
+ ):
reject("world-writable")
if not os.access(base, os.W_OK | os.X_OK):
reject("access")
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 7ef43680ee..930b0bece6 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -2440,8 +2440,8 @@ def test_canonical_gha_requires_compute_visible_staging(self) -> None:
squash = root / "squash"
source = repo / "experimental" / "CollectiveX"
source.mkdir(parents=True)
- squash.mkdir(mode=0o770)
- squash.chmod(0o770)
+ squash.mkdir(mode=0o1777)
+ squash.chmod(0o1777)
(source / "public.py").write_text("public\n")
(source / "private-infra.md").write_text("private\n")
command = r'''
From 7cc75ca1b24a6e0d37173df4358cedb24d1aa8e1 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 17:21:26 +0800
Subject: [PATCH 016/210] fix(collectivex): stage from private runner home
---
experimental/CollectiveX/README.md | 8 +--
experimental/CollectiveX/runtime/common.sh | 59 +++++++++++++++----
.../tests/test_sampling_contract.py | 17 ++++--
3 files changed, 64 insertions(+), 20 deletions(-)
diff --git a/experimental/CollectiveX/README.md b/experimental/CollectiveX/README.md
index 7c0ee09448..92c216ae51 100644
--- a/experimental/CollectiveX/README.md
+++ b/experimental/CollectiveX/README.md
@@ -140,10 +140,10 @@ visibility, and remove that exact child after allocation teardown; they never mo
checkout or create a stage beneath image storage on AMD.
H100, H200, and B200 runners may omit `stage_dir`. Their isolated execution child is then created
-directly under the runner-owned `squash_dir`; that shared parent may be group-writable or use the
-sticky bit for shared writes, but a non-sticky world-writable parent is rejected. The child is mode
-0700 and is still validated, marked, cross-node checked, and removed using the same contract. All
-other runners require the stricter dedicated `stage_dir` above.
+under a runner-owned mode-0700 base in the validated runner home. The workflow still proves that
+base is visible from every allocated node before launch. The child is validated, marked, cross-node
+checked, and removed using the same contract. All other runners require the dedicated `stage_dir`
+above; no canonical run stages source beneath shared container storage.
Before import, each Docker Hub tag is resolved with bounded registry requests and must match its
pinned digest; digest-qualified overrides are rejected. Enroot imports use a fixed filesystem epoch
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 080082e26c..8141870053 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -1430,6 +1430,46 @@ cx_materialize_backend_source() {
return 0
}
+cx_prepare_implicit_stage_base() {
+ python3 - "${HOME:-}" <<'PY'
+import os
+from pathlib import Path
+import stat
+import sys
+
+home = Path(sys.argv[1])
+try:
+ if not home.is_absolute() or Path(os.path.realpath(home)) != home:
+ raise OSError
+ metadata = os.stat(home, follow_symlinks=False)
+ if (
+ not stat.S_ISDIR(metadata.st_mode)
+ or metadata.st_uid != os.getuid()
+ or stat.S_IMODE(metadata.st_mode) & (stat.S_IWGRP | stat.S_IWOTH)
+ ):
+ raise OSError
+ current = home
+ for component in (".cache", "inferencex", "collectivex-stage"):
+ current /= component
+ try:
+ os.mkdir(current, mode=0o700)
+ except FileExistsError:
+ pass
+ metadata = os.stat(current, follow_symlinks=False)
+ if (
+ not stat.S_ISDIR(metadata.st_mode)
+ or metadata.st_uid != os.getuid()
+ or Path(os.path.realpath(current)) != current
+ or stat.S_IMODE(metadata.st_mode) & (stat.S_IWGRP | stat.S_IWOTH)
+ ):
+ raise OSError
+ os.chmod(current, 0o700)
+except (OSError, ValueError):
+ raise SystemExit(1)
+print(current, end="")
+PY
+}
+
cx_lock_canonical_gha_env() {
local runner="$1" expected_nodes expected_gpn expected_world trusted_lock_dir=""
local trusted_stage_dir=""
@@ -1478,7 +1518,10 @@ cx_lock_canonical_gha_env() {
|| cx_die "canonical CollectiveX execution requires shared container storage"
if [ -z "$trusted_stage_dir" ]; then
case "$runner" in
- h100-dgxc|h200-dgxc|b200-dgxc) ;;
+ h100-dgxc|h200-dgxc|b200-dgxc)
+ trusted_stage_dir="$(cx_prepare_implicit_stage_base)" \
+ || cx_die "canonical CollectiveX execution cannot create an isolated stage directory"
+ ;;
*) cx_die "canonical CollectiveX execution requires a configured shared stage directory" ;;
esac
fi
@@ -1793,7 +1836,7 @@ BASH
# EXIT trap can remove an interrupted partial stage. The configured base must
# already exist on compute-visible storage and must not traverse symlinks.
cx_stage_path() {
- local repo_root="$1" stage_base="${2:-}" tag safe_tag stage_path stage_fallback=0
+ local repo_root="$1" stage_base="${2:-}" tag safe_tag stage_path
tag="${COLLECTIVEX_EXECUTION_ID:-${GITHUB_RUN_ID:-manual-$$}}"
[[ "$tag" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] \
|| cx_die "invalid staging execution identity"
@@ -1803,17 +1846,16 @@ cx_stage_path() {
|| cx_die "CollectiveX staging requires CX_STAGE_DIR or CX_SQUASH_DIR"
stage_base="$CX_SQUASH_DIR"
stage_path="${stage_base%/}/.collectivex-stage-$safe_tag"
- stage_fallback=1
else
stage_path="${stage_base%/}/job_$safe_tag"
fi
python3 - "$repo_root" "$stage_base" "$stage_path" \
- "${CX_JOB_ROOT:-}" "${GITHUB_WORKSPACE:-}" "$stage_fallback" <<'PY'
+ "${CX_JOB_ROOT:-}" "${GITHUB_WORKSPACE:-}" <<'PY'
import os
import stat
import sys
-repo, base, child, job_root, workspace, fallback = sys.argv[1:]
+repo, base, child, job_root, workspace = sys.argv[1:]
def reject(reason):
print(f"[collectivex] FATAL: stage-base-validation={reason}", file=sys.stderr)
raise SystemExit(1)
@@ -1844,12 +1886,9 @@ try:
reject("not-directory")
if metadata.st_uid != os.getuid():
reject("owner")
- if stat.S_IMODE(metadata.st_mode) & stat.S_IWGRP and fallback != "1":
+ if stat.S_IMODE(metadata.st_mode) & stat.S_IWGRP:
reject("group-writable")
- if (
- stat.S_IMODE(metadata.st_mode) & stat.S_IWOTH
- and not (fallback == "1" and metadata.st_mode & stat.S_ISVTX)
- ):
+ if stat.S_IMODE(metadata.st_mode) & stat.S_IWOTH:
reject("world-writable")
if not os.access(base, os.W_OK | os.X_OK):
reject("access")
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 930b0bece6..dd528cc8c0 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -2289,7 +2289,8 @@ def test_canonical_gha_environment_is_locked_but_manual_overrides_survive(self)
export CX_SHARD_SKU=b200-dgxc CX_NODES=1 CX_GPUS_PER_NODE=8
unset CX_STAGE_DIR
cx_lock_canonical_gha_env b200-dgxc
- test -z "$CX_STAGE_DIR"
+ test "$CX_STAGE_DIR" = "$HOME/.cache/inferencex/collectivex-stage"
+ test "$(stat -c '%a' "$CX_STAGE_DIR" 2>/dev/null || stat -f '%Lp' "$CX_STAGE_DIR")" = 700
export CX_STAGE_DIR=/shared/gb-stage
export CX_SHARD_SKU=gb300 CX_NODES=2 CX_GPUS_PER_NODE=4
@@ -2319,15 +2320,20 @@ def test_canonical_gha_environment_is_locked_but_manual_overrides_survive(self)
cx_lock_canonical_gha_env mi355x
test "$CX_IMAGE:$CX_IMAGE_DIGEST:$CX_NGPUS:$CX_MORI_KERNEL_TYPE" = manual:manual:3:manual
'''
- with tempfile.TemporaryDirectory(dir=Path.home()) as workspace:
- Path(workspace).chmod(0o720)
+ with tempfile.TemporaryDirectory(dir=Path.home()) as temporary:
+ root = Path(temporary)
+ home = root / "home"
+ workspace = root / "workspace"
+ home.mkdir(mode=0o700)
+ workspace.mkdir(mode=0o720)
subprocess.run(
["bash", "-c", command, "_", str(common)],
check=True,
env={
**os.environ,
+ "HOME": str(home),
"COLLECTIVEX_OPERATOR_CONFIG": "/dev/null",
- "GITHUB_WORKSPACE": workspace,
+ "GITHUB_WORKSPACE": str(workspace),
},
)
self.assertEqual(list(Path(workspace).iterdir()), [])
@@ -2440,8 +2446,7 @@ def test_canonical_gha_requires_compute_visible_staging(self) -> None:
squash = root / "squash"
source = repo / "experimental" / "CollectiveX"
source.mkdir(parents=True)
- squash.mkdir(mode=0o1777)
- squash.chmod(0o1777)
+ squash.mkdir(mode=0o700)
(source / "public.py").write_text("public\n")
(source / "private-infra.md").write_text("private\n")
command = r'''
From 0715bb659b7cdbe8d5bd941ea3cb5984f6fb5b5d Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 17:23:13 +0800
Subject: [PATCH 017/210] fix(collectivex): use runner account stage
---
experimental/CollectiveX/README.md | 9 +++--
experimental/CollectiveX/runtime/common.sh | 5 ++-
.../tests/test_sampling_contract.py | 38 +++++++++++++++++--
3 files changed, 43 insertions(+), 9 deletions(-)
diff --git a/experimental/CollectiveX/README.md b/experimental/CollectiveX/README.md
index 92c216ae51..1969b88767 100644
--- a/experimental/CollectiveX/README.md
+++ b/experimental/CollectiveX/README.md
@@ -140,10 +140,11 @@ visibility, and remove that exact child after allocation teardown; they never mo
checkout or create a stage beneath image storage on AMD.
H100, H200, and B200 runners may omit `stage_dir`. Their isolated execution child is then created
-under a runner-owned mode-0700 base in the validated runner home. The workflow still proves that
-base is visible from every allocated node before launch. The child is validated, marked, cross-node
-checked, and removed using the same contract. All other runners require the dedicated `stage_dir`
-above; no canonical run stages source beneath shared container storage.
+under a runner-owned mode-0700 base in the validated operating-system account home, independent of
+the workflow's temporary `HOME`. The workflow still proves that base is visible from every allocated
+node before launch. The child is validated, marked, cross-node checked, and removed using the same
+contract. All other runners require the dedicated `stage_dir` above; no canonical run stages source
+beneath shared container storage.
Before import, each Docker Hub tag is resolved with bounded registry requests and must match its
pinned digest; digest-qualified overrides are rejected. Enroot imports use a fixed filesystem epoch
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 8141870053..17011d5a4c 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -1431,13 +1431,14 @@ cx_materialize_backend_source() {
}
cx_prepare_implicit_stage_base() {
- python3 - "${HOME:-}" <<'PY'
+ python3 - "${1:-}" <<'PY'
import os
from pathlib import Path
+import pwd
import stat
import sys
-home = Path(sys.argv[1])
+home = Path(sys.argv[1] or pwd.getpwuid(os.getuid()).pw_dir)
try:
if not home.is_absolute() or Path(os.path.realpath(home)) != home:
raise OSError
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index dd528cc8c0..d7a67cf00e 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -2285,12 +2285,12 @@ def test_canonical_gha_environment_is_locked_but_manual_overrides_survive(self)
test -z "${CX_PREPARED_BACKEND_CACHE+x}${CX_BACKEND_SOURCE_ROOT+x}"
test -z "${CX_DRYRUN+x}"
+ cx_prepare_implicit_stage_base() { printf '%s' "$TEST_IMPLICIT_STAGE"; }
export COLLECTIVEX_OPERATOR_CONFIG_LOADED=$$
export CX_SHARD_SKU=b200-dgxc CX_NODES=1 CX_GPUS_PER_NODE=8
unset CX_STAGE_DIR
cx_lock_canonical_gha_env b200-dgxc
- test "$CX_STAGE_DIR" = "$HOME/.cache/inferencex/collectivex-stage"
- test "$(stat -c '%a' "$CX_STAGE_DIR" 2>/dev/null || stat -f '%Lp' "$CX_STAGE_DIR")" = 700
+ test "$CX_STAGE_DIR" = "$TEST_IMPLICIT_STAGE"
export CX_STAGE_DIR=/shared/gb-stage
export CX_SHARD_SKU=gb300 CX_NODES=2 CX_GPUS_PER_NODE=4
@@ -2324,19 +2324,51 @@ def test_canonical_gha_environment_is_locked_but_manual_overrides_survive(self)
root = Path(temporary)
home = root / "home"
workspace = root / "workspace"
+ implicit_stage = root / "implicit-stage"
home.mkdir(mode=0o700)
workspace.mkdir(mode=0o720)
+ implicit_stage.mkdir(mode=0o700)
subprocess.run(
["bash", "-c", command, "_", str(common)],
check=True,
env={
**os.environ,
"HOME": str(home),
+ "TEST_IMPLICIT_STAGE": str(implicit_stage),
"COLLECTIVEX_OPERATOR_CONFIG": "/dev/null",
"GITHUB_WORKSPACE": str(workspace),
},
)
- self.assertEqual(list(Path(workspace).iterdir()), [])
+ self.assertEqual(list(workspace.iterdir()), [])
+
+ def test_implicit_stage_base_is_private_and_non_symlinked(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary).resolve()
+ safe_home = root / "safe-home"
+ unsafe_home = root / "unsafe-home"
+ linked_home = root / "linked-home"
+ safe_home.mkdir(mode=0o700)
+ unsafe_home.mkdir(mode=0o770)
+ unsafe_home.chmod(0o770)
+ linked_home.symlink_to(safe_home, target_is_directory=True)
+ command = r'''
+ set -euo pipefail
+ source "$1"
+ base="$(cx_prepare_implicit_stage_base "$2")"
+ test "$base" = "$2/.cache/inferencex/collectivex-stage"
+ test "$(stat -c '%a' "$base" 2>/dev/null || stat -f '%Lp' "$base")" = 700
+ ! cx_prepare_implicit_stage_base "$3"
+ ! cx_prepare_implicit_stage_base "$4"
+ '''
+ subprocess.run(
+ [
+ "bash", "-c", command, "_",
+ str(ROOT / "runtime" / "common.sh"), str(safe_home),
+ str(unsafe_home), str(linked_home),
+ ],
+ check=True,
+ env={**os.environ, "COLLECTIVEX_OPERATOR_CONFIG": "/dev/null"},
+ )
def test_canonical_amd_stage_uses_config_not_world_writable_workspace(self) -> None:
common = ROOT / "runtime" / "common.sh"
From d3358c49b5864ae371bae70e709f8ecd613f9643 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 17:25:21 +0800
Subject: [PATCH 018/210] fix(collectivex): resolve canonical account staging
---
experimental/CollectiveX/README.md | 10 ++--
experimental/CollectiveX/runtime/common.sh | 50 ++++++++++++-------
.../tests/test_sampling_contract.py | 8 ++-
3 files changed, 44 insertions(+), 24 deletions(-)
diff --git a/experimental/CollectiveX/README.md b/experimental/CollectiveX/README.md
index 1969b88767..1836dba2ca 100644
--- a/experimental/CollectiveX/README.md
+++ b/experimental/CollectiveX/README.md
@@ -141,10 +141,12 @@ checkout or create a stage beneath image storage on AMD.
H100, H200, and B200 runners may omit `stage_dir`. Their isolated execution child is then created
under a runner-owned mode-0700 base in the validated operating-system account home, independent of
-the workflow's temporary `HOME`. The workflow still proves that base is visible from every allocated
-node before launch. The child is validated, marked, cross-node checked, and removed using the same
-contract. All other runners require the dedicated `stage_dir` above; no canonical run stages source
-beneath shared container storage.
+the workflow's temporary `HOME`. A symlinked account-home entry is resolved once to its canonical,
+runner-owned target; every created staging component must itself be non-symlinked and not writable by
+other users. The workflow still proves that base is visible from every allocated node before launch.
+The execution child is validated, marked, cross-node checked, and removed using the same contract.
+All other runners require the dedicated `stage_dir` above; no canonical run stages source beneath
+shared container storage.
Before import, each Docker Hub tag is resolved with bounded registry requests and must match its
pinned digest; digest-qualified overrides are rejected. Enroot imports use a fixed filesystem epoch
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 17011d5a4c..1c753ab198 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -1438,17 +1438,30 @@ import pwd
import stat
import sys
-home = Path(sys.argv[1] or pwd.getpwuid(os.getuid()).pw_dir)
+def reject(reason):
+ print(f"[collectivex] FATAL: implicit-stage-validation={reason}", file=sys.stderr)
+ raise SystemExit(1)
+
+try:
+ configured_home = Path(sys.argv[1] or pwd.getpwuid(os.getuid()).pw_dir)
+except (KeyError, OSError):
+ reject("account-home")
+if not configured_home.is_absolute():
+ reject("path-shape")
+home = Path(os.path.realpath(configured_home))
try:
- if not home.is_absolute() or Path(os.path.realpath(home)) != home:
- raise OSError
metadata = os.stat(home, follow_symlinks=False)
- if (
- not stat.S_ISDIR(metadata.st_mode)
- or metadata.st_uid != os.getuid()
- or stat.S_IMODE(metadata.st_mode) & (stat.S_IWGRP | stat.S_IWOTH)
- ):
- raise OSError
+except OSError:
+ reject("home-stat")
+if not stat.S_ISDIR(metadata.st_mode):
+ reject("home-type")
+if metadata.st_uid != os.getuid():
+ reject("home-owner")
+if stat.S_IMODE(metadata.st_mode) & stat.S_IWGRP:
+ reject("home-group-writable")
+if stat.S_IMODE(metadata.st_mode) & stat.S_IWOTH:
+ reject("home-world-writable")
+try:
current = home
for component in (".cache", "inferencex", "collectivex-stage"):
current /= component
@@ -1457,16 +1470,17 @@ try:
except FileExistsError:
pass
metadata = os.stat(current, follow_symlinks=False)
- if (
- not stat.S_ISDIR(metadata.st_mode)
- or metadata.st_uid != os.getuid()
- or Path(os.path.realpath(current)) != current
- or stat.S_IMODE(metadata.st_mode) & (stat.S_IWGRP | stat.S_IWOTH)
- ):
- raise OSError
+ if not stat.S_ISDIR(metadata.st_mode):
+ reject("child-type")
+ if metadata.st_uid != os.getuid():
+ reject("child-owner")
+ if Path(os.path.realpath(current)) != current:
+ reject("child-symlink")
+ if stat.S_IMODE(metadata.st_mode) & (stat.S_IWGRP | stat.S_IWOTH):
+ reject("child-writable")
os.chmod(current, 0o700)
-except (OSError, ValueError):
- raise SystemExit(1)
+except OSError:
+ reject("child-access")
print(current, end="")
PY
}
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index d7a67cf00e..9d3a82c559 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -2347,10 +2347,13 @@ def test_implicit_stage_base_is_private_and_non_symlinked(self) -> None:
safe_home = root / "safe-home"
unsafe_home = root / "unsafe-home"
linked_home = root / "linked-home"
+ linked_child_home = root / "linked-child-home"
safe_home.mkdir(mode=0o700)
unsafe_home.mkdir(mode=0o770)
unsafe_home.chmod(0o770)
linked_home.symlink_to(safe_home, target_is_directory=True)
+ linked_child_home.mkdir(mode=0o700)
+ (linked_child_home / ".cache").symlink_to(safe_home, target_is_directory=True)
command = r'''
set -euo pipefail
source "$1"
@@ -2358,13 +2361,14 @@ def test_implicit_stage_base_is_private_and_non_symlinked(self) -> None:
test "$base" = "$2/.cache/inferencex/collectivex-stage"
test "$(stat -c '%a' "$base" 2>/dev/null || stat -f '%Lp' "$base")" = 700
! cx_prepare_implicit_stage_base "$3"
- ! cx_prepare_implicit_stage_base "$4"
+ test "$(cx_prepare_implicit_stage_base "$4")" = "$2/.cache/inferencex/collectivex-stage"
+ ! cx_prepare_implicit_stage_base "$5"
'''
subprocess.run(
[
"bash", "-c", command, "_",
str(ROOT / "runtime" / "common.sh"), str(safe_home),
- str(unsafe_home), str(linked_home),
+ str(unsafe_home), str(linked_home), str(linked_child_home),
],
check=True,
env={**os.environ, "COLLECTIVEX_OPERATOR_CONFIG": "/dev/null"},
From 395eef6defea5de8bf49c622aa05cfe0c0144794 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 17:26:56 +0800
Subject: [PATCH 019/210] fix(collectivex): isolate account stage root
---
experimental/CollectiveX/README.md | 10 +++----
experimental/CollectiveX/runtime/common.sh | 30 +++++++++----------
.../tests/test_sampling_contract.py | 8 +++--
3 files changed, 24 insertions(+), 24 deletions(-)
diff --git a/experimental/CollectiveX/README.md b/experimental/CollectiveX/README.md
index 1836dba2ca..9dcdc405e9 100644
--- a/experimental/CollectiveX/README.md
+++ b/experimental/CollectiveX/README.md
@@ -142,11 +142,11 @@ checkout or create a stage beneath image storage on AMD.
H100, H200, and B200 runners may omit `stage_dir`. Their isolated execution child is then created
under a runner-owned mode-0700 base in the validated operating-system account home, independent of
the workflow's temporary `HOME`. A symlinked account-home entry is resolved once to its canonical,
-runner-owned target; every created staging component must itself be non-symlinked and not writable by
-other users. The workflow still proves that base is visible from every allocated node before launch.
-The execution child is validated, marked, cross-node checked, and removed using the same contract.
-All other runners require the dedicated `stage_dir` above; no canonical run stages source beneath
-shared container storage.
+runner-owned target; the single hidden staging base directly beneath it must itself be non-symlinked
+and not writable by other users. The workflow still proves that base is visible from every allocated
+node before launch. The execution child is validated, marked, cross-node checked, and removed using
+the same contract. All other runners require the dedicated `stage_dir` above; no canonical run
+stages source beneath shared container storage.
Before import, each Docker Hub tag is resolved with bounded registry requests and must match its
pinned digest; digest-qualified overrides are rejected. Enroot imports use a fixed filesystem epoch
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 1c753ab198..2daab8bcd6 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -1462,22 +1462,20 @@ if stat.S_IMODE(metadata.st_mode) & stat.S_IWGRP:
if stat.S_IMODE(metadata.st_mode) & stat.S_IWOTH:
reject("home-world-writable")
try:
- current = home
- for component in (".cache", "inferencex", "collectivex-stage"):
- current /= component
- try:
- os.mkdir(current, mode=0o700)
- except FileExistsError:
- pass
- metadata = os.stat(current, follow_symlinks=False)
- if not stat.S_ISDIR(metadata.st_mode):
- reject("child-type")
- if metadata.st_uid != os.getuid():
- reject("child-owner")
- if Path(os.path.realpath(current)) != current:
- reject("child-symlink")
- if stat.S_IMODE(metadata.st_mode) & (stat.S_IWGRP | stat.S_IWOTH):
- reject("child-writable")
+ current = home / ".inferencex-collectivex-stage"
+ try:
+ os.mkdir(current, mode=0o700)
+ except FileExistsError:
+ pass
+ metadata = os.stat(current, follow_symlinks=False)
+ if not stat.S_ISDIR(metadata.st_mode):
+ reject("child-type")
+ if metadata.st_uid != os.getuid():
+ reject("child-owner")
+ if Path(os.path.realpath(current)) != current:
+ reject("child-symlink")
+ if stat.S_IMODE(metadata.st_mode) & (stat.S_IWGRP | stat.S_IWOTH):
+ reject("child-writable")
os.chmod(current, 0o700)
except OSError:
reject("child-access")
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 9d3a82c559..f83e39cc43 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -2353,15 +2353,17 @@ def test_implicit_stage_base_is_private_and_non_symlinked(self) -> None:
unsafe_home.chmod(0o770)
linked_home.symlink_to(safe_home, target_is_directory=True)
linked_child_home.mkdir(mode=0o700)
- (linked_child_home / ".cache").symlink_to(safe_home, target_is_directory=True)
+ (linked_child_home / ".inferencex-collectivex-stage").symlink_to(
+ safe_home, target_is_directory=True
+ )
command = r'''
set -euo pipefail
source "$1"
base="$(cx_prepare_implicit_stage_base "$2")"
- test "$base" = "$2/.cache/inferencex/collectivex-stage"
+ test "$base" = "$2/.inferencex-collectivex-stage"
test "$(stat -c '%a' "$base" 2>/dev/null || stat -f '%Lp' "$base")" = 700
! cx_prepare_implicit_stage_base "$3"
- test "$(cx_prepare_implicit_stage_base "$4")" = "$2/.cache/inferencex/collectivex-stage"
+ test "$(cx_prepare_implicit_stage_base "$4")" = "$2/.inferencex-collectivex-stage"
! cx_prepare_implicit_stage_base "$5"
'''
subprocess.run(
From 862c8128280e87bed2163869dbd7f78fd8ddb9a5 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 17:31:34 +0800
Subject: [PATCH 020/210] fix(collectivex): tighten V1 decision contracts
---
experimental/CollectiveX/publisher.py | 9 +++--
experimental/CollectiveX/runtime/common.sh | 12 ++++++
experimental/CollectiveX/tests/ep_harness.py | 2 +-
.../CollectiveX/tests/test_publisher.py | 38 ++++++++++++++++++-
.../tests/test_sampling_contract.py | 17 +++++++++
5 files changed, 72 insertions(+), 6 deletions(-)
diff --git a/experimental/CollectiveX/publisher.py b/experimental/CollectiveX/publisher.py
index e64f52c124..2e57b76be6 100644
--- a/experimental/CollectiveX/publisher.py
+++ b/experimental/CollectiveX/publisher.py
@@ -3359,16 +3359,15 @@ def _p99_top_tie_ids(
)
baseline_id = ordered[0]["series_id"]
comparisons: dict[str, dict[str, Any]] = {}
- tie_end = 0
- for index, candidate in enumerate(ordered[1:], 1):
+ tie_ids = {baseline_id}
+ for candidate in ordered[1:]:
candidate_id = candidate["series_id"]
result = _hierarchical_p99_ratio(
baseline_id, candidate_id, token, internals, dataset_binding
)
comparisons[candidate_id] = result
if not result["baseline_wins"]:
- tie_end = index
- tie_ids = {member["series_id"] for member in ordered[:tie_end + 1]}
+ tie_ids.add(candidate_id)
internals[baseline_id].setdefault("decision_statistics", {})[
f"{cohort_id}:p99:{token}"
] = {
@@ -3541,6 +3540,8 @@ def build_decisions(
for cohort in cohorts:
if not cohort["eligibility"]["decision_grade"]:
continue
+ if cohort["kind"] == "precision-pair":
+ continue
members = [series_by_id[series_id] for series_id in cohort["series_ids"]]
tokens = sorted(set.intersection(*(
{point["tokens_per_rank"] for point in member["points"]} for member in members
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 2daab8bcd6..e1ff3b37a9 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -71,6 +71,18 @@ cx_fail_stage() {
diagnostic="network-or-timeout"
elif grep -aEqi 'salloc:|srun:.*(unable to create step|step creation|invalid partition|invalid account)|unable to create step|job allocation' "$log_path"; then
diagnostic="scheduler"
+ elif grep -aEqi 'ModuleNotFoundError|ImportError:' "$log_path"; then
+ diagnostic="python-import"
+ elif grep -aEqi 'AttributeError:|TypeError:.*(unexpected|argument|operand)|has no attribute' "$log_path"; then
+ diagnostic="backend-api"
+ elif grep -aEqi 'PrecisionError:|unsupported precision|precision profile.*(invalid|unsupported|differs)' "$log_path"; then
+ diagnostic="precision-contract"
+ elif grep -aEqi 'AssertionError:' "$log_path"; then
+ diagnostic="python-assertion"
+ elif grep -aEqi 'RuntimeError:' "$log_path"; then
+ diagnostic="python-runtime"
+ elif grep -aEqi 'Traceback \(most recent call last\)' "$log_path"; then
+ diagnostic="python-exception"
elif grep -aEqi 'SHARD done: [0-9]+/[0-9]+ case\(s\) failed|WARN: .* run failed rc=|completed with invalid semantic evidence' "$log_path"; then
diagnostic="benchmark-case-failure"
elif [ -s "$log_path" ]; then
diff --git a/experimental/CollectiveX/tests/ep_harness.py b/experimental/CollectiveX/tests/ep_harness.py
index af2ec98cae..13cebb517e 100644
--- a/experimental/CollectiveX/tests/ep_harness.py
+++ b/experimental/CollectiveX/tests/ep_harness.py
@@ -191,7 +191,7 @@ def add_common_args(ap: argparse.ArgumentParser) -> None:
"--qualification-index",
type=int,
choices=range(1, QUALIFICATION_RUNS + 1),
- default=os.environ.get("COLLECTIVEX_QUALIFICATION_INDEX", "1"),
+ default=os.environ.get("CX_QUALIFICATION_INDEX", "1"),
help="one-based qualification repeat used for deterministic measurement ordering",
)
# 32: B300/Blackwell needs ~30 untimed iters to reach steady-state GPU clocks +
diff --git a/experimental/CollectiveX/tests/test_publisher.py b/experimental/CollectiveX/tests/test_publisher.py
index abc43c6ec3..8b2846bc86 100644
--- a/experimental/CollectiveX/tests/test_publisher.py
+++ b/experimental/CollectiveX/tests/test_publisher.py
@@ -2737,6 +2737,38 @@ def test_p99_winner_requires_every_run_to_agree(self) -> None:
self.assertFalse(result["baseline_wins"])
self.assertTrue(result["tie"])
+ def test_p99_tie_set_does_not_include_decisively_beaten_middle_series(self) -> None:
+ fast, fast_internal = _series("tie-fast", "deepep", decision_grade=True)
+ middle, middle_internal = _series("tie-middle", "uccl", decision_grade=True)
+ uncertain, uncertain_internal = _series(
+ "tie-uncertain", "nccl-ep", decision_grade=True
+ )
+ for index, item in enumerate((fast, middle, uncertain), 1):
+ item["points"][0]["components"]["roundtrip"]["latency_us"]["p99"] = float(index)
+ outcomes = {
+ middle["series_id"]: {"baseline_wins": True},
+ uncertain["series_id"]: {"baseline_wins": False},
+ }
+ with mock.patch.object(
+ publisher,
+ "_hierarchical_p99_ratio",
+ side_effect=lambda _baseline, candidate, *_args: outcomes[candidate],
+ ):
+ tie_ids = publisher._p99_top_tie_ids(
+ [fast, middle, uncertain],
+ {
+ fast["series_id"]: fast_internal,
+ middle["series_id"]: middle_internal,
+ uncertain["series_id"]: uncertain_internal,
+ },
+ 8,
+ "d" * 64,
+ publisher._derived_id(
+ "cxcohort-v1-", {"fixture": "noncontiguous-tie"}
+ ),
+ )
+ self.assertEqual(tie_ids, {fast["series_id"], uncertain["series_id"]})
+
def test_precision_cohorts_isolate_axes_and_never_recommend(self) -> None:
profiles = (
identity.V1_CONTROL_PRECISION_PROFILE,
@@ -2786,7 +2818,7 @@ def test_precision_cohorts_isolate_axes_and_never_recommend(self) -> None:
and cohort["eligibility"]["decision_grade"]
for cohort in precision_cohorts
))
- self.assertEqual(len(rankings), 30)
+ self.assertEqual(len(rankings), 24)
self.assertEqual(len(sensitivities), 24)
self.assertEqual(recommendations, [])
pair = next(
@@ -2805,6 +2837,10 @@ def test_precision_cohorts_isolate_axes_and_never_recommend(self) -> None:
sensitivity["cohort_id"] == pair["cohort_id"]
for sensitivity in sensitivities
))
+ self.assertFalse(any(
+ ranking["cohort_id"] == pair["cohort_id"]
+ for ranking in rankings
+ ))
def test_private_trial_copy_is_component_extensible(self) -> None:
blocks = [[float(trial + iteration + 1) for iteration in range(8)]
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index f83e39cc43..471e00eccb 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -89,6 +89,17 @@ def test_identity_and_fixed_sampling_profile(self) -> None:
)
self.assertEqual((args.iters, args.trials, args.warmup), (8, 64, 32))
self.assertEqual(args.qualification_index, 1)
+ with mock.patch.dict(os.environ, {"CX_QUALIFICATION_INDEX": "3"}):
+ parser = argparse.ArgumentParser()
+ ep_harness.add_common_args(parser)
+ env_args = parser.parse_args(
+ [
+ "--runner", "test", "--topology-class", "test",
+ "--scope", "scale-up", "--scale-up-transport", "nvlink",
+ "--out", "result.json",
+ ]
+ )
+ self.assertEqual(env_args.qualification_index, 3)
for profile in ((8, 64, 32), (128, 4, 32), (8, 1, 4), (0, 64, 32)):
with self.subTest(profile=profile):
self.assertEqual(
@@ -2195,6 +2206,12 @@ def test_private_runtime_failure_signatures_override_case_footer(self) -> None:
"NVCC compilation failed": "jit-toolchain",
"CUDA out of memory": "accelerator-memory",
"torch rendezvous timed out": "network-or-timeout",
+ "ModuleNotFoundError: missing module": "python-import",
+ "AttributeError: backend has no attribute 'probe'": "backend-api",
+ "PrecisionError: unsupported precision profile": "precision-contract",
+ "AssertionError: probe invariant": "python-assertion",
+ "RuntimeError: probe execution failed": "python-runtime",
+ "Traceback (most recent call last):": "python-exception",
}
with tempfile.TemporaryDirectory() as temporary:
log = Path(temporary) / "runtime.log"
From b87b6f2d38272848be5dc7ececc1486a0d1b4c19 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 17:34:44 +0800
Subject: [PATCH 021/210] fix(collectivex): classify probe exceptions
---
experimental/CollectiveX/runtime/common.sh | 10 ++++++++++
.../CollectiveX/tests/test_sampling_contract.py | 5 +++++
2 files changed, 15 insertions(+)
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index e1ff3b37a9..6c0febc70e 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -81,6 +81,16 @@ cx_fail_stage() {
diagnostic="python-assertion"
elif grep -aEqi 'RuntimeError:' "$log_path"; then
diagnostic="python-runtime"
+ elif grep -aEqi 'ValueError:' "$log_path"; then
+ diagnostic="python-value"
+ elif grep -aEqi 'KeyError:' "$log_path"; then
+ diagnostic="python-key"
+ elif grep -aEqi '(FileNotFoundError|PermissionError|IsADirectoryError|NotADirectoryError|OSError):' "$log_path"; then
+ diagnostic="python-os"
+ elif grep -aEqi '(NotImplemented|System)Error:' "$log_path"; then
+ diagnostic="python-system"
+ elif grep -aEqi 'CalledProcessError:' "$log_path"; then
+ diagnostic="python-subprocess"
elif grep -aEqi 'Traceback \(most recent call last\)' "$log_path"; then
diagnostic="python-exception"
elif grep -aEqi 'SHARD done: [0-9]+/[0-9]+ case\(s\) failed|WARN: .* run failed rc=|completed with invalid semantic evidence' "$log_path"; then
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 471e00eccb..15c5af7171 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -2211,6 +2211,11 @@ def test_private_runtime_failure_signatures_override_case_footer(self) -> None:
"PrecisionError: unsupported precision profile": "precision-contract",
"AssertionError: probe invariant": "python-assertion",
"RuntimeError: probe execution failed": "python-runtime",
+ "ValueError: probe argument": "python-value",
+ "KeyError: 'probe-key'": "python-key",
+ "OSError: probe path": "python-os",
+ "NotImplementedError: probe API": "python-system",
+ "CalledProcessError: probe command": "python-subprocess",
"Traceback (most recent call last):": "python-exception",
}
with tempfile.TemporaryDirectory() as temporary:
From fd40dc4d547312c41d5c962fd4ba6b8802ae4123 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 17:39:21 +0800
Subject: [PATCH 022/210] fix(collectivex): classify probe error origins
---
experimental/CollectiveX/runtime/common.sh | 18 ++++++++++++++++++
.../tests/test_sampling_contract.py | 6 ++++++
2 files changed, 24 insertions(+)
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 6c0febc70e..d8cbeae4d2 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -81,6 +81,24 @@ cx_fail_stage() {
diagnostic="python-assertion"
elif grep -aEqi 'RuntimeError:' "$log_path"; then
diagnostic="python-runtime"
+ elif grep -aEqi 'ValueError:' "$log_path" \
+ && grep -aEqi 'probe_precision[.]py' "$log_path"; then
+ diagnostic="probe-manifest-value"
+ elif grep -aEqi 'ValueError:' "$log_path" \
+ && grep -aEqi 'ep_harness[.]py' "$log_path"; then
+ diagnostic="harness-value"
+ elif grep -aEqi 'ValueError:' "$log_path" \
+ && grep -aEqi 'workload[.]py|make_workloads[.]py' "$log_path"; then
+ diagnostic="workload-value"
+ elif grep -aEqi 'ValueError:' "$log_path" \
+ && grep -aEqi 'run_ep[.]py' "$log_path"; then
+ diagnostic="runner-value"
+ elif grep -aEqi 'ValueError:' "$log_path" \
+ && grep -aEqi 'ep_deepep[.]py' "$log_path"; then
+ diagnostic="deepep-adapter-value"
+ elif grep -aEqi 'ValueError:' "$log_path" \
+ && grep -aEqi '/(torch|numpy)/|site-packages/(torch|numpy)' "$log_path"; then
+ diagnostic="dependency-value"
elif grep -aEqi 'ValueError:' "$log_path"; then
diagnostic="python-value"
elif grep -aEqi 'KeyError:' "$log_path"; then
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 15c5af7171..6f38d98345 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -2211,6 +2211,12 @@ def test_private_runtime_failure_signatures_override_case_footer(self) -> None:
"PrecisionError: unsupported precision profile": "precision-contract",
"AssertionError: probe invariant": "python-assertion",
"RuntimeError: probe execution failed": "python-runtime",
+ "tests/probe_precision.py\nValueError: probe argument": "probe-manifest-value",
+ "tests/ep_harness.py\nValueError: harness argument": "harness-value",
+ "tests/workload.py\nValueError: workload argument": "workload-value",
+ "tests/run_ep.py\nValueError: runner argument": "runner-value",
+ "tests/ep_deepep.py\nValueError: adapter argument": "deepep-adapter-value",
+ "site-packages/torch/module.py\nValueError: dependency argument": "dependency-value",
"ValueError: probe argument": "python-value",
"KeyError: 'probe-key'": "python-key",
"OSError: probe path": "python-os",
From 3fe7391461d5461b8102f947f5ba1962056f832d Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 17:42:56 +0800
Subject: [PATCH 023/210] fix(collectivex): classify probe validation failures
---
experimental/CollectiveX/runtime/common.sh | 18 ++++++++++++++++++
.../tests/test_sampling_contract.py | 12 ++++++++++++
2 files changed, 30 insertions(+)
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index d8cbeae4d2..90c279e858 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -81,6 +81,24 @@ cx_fail_stage() {
diagnostic="python-assertion"
elif grep -aEqi 'RuntimeError:' "$log_path"; then
diagnostic="python-runtime"
+ elif grep -aEqi 'ValueError:.*(fields differ from collectivex[.]precision-probe|probe format, record type, schema, or contract differs)' "$log_path"; then
+ diagnostic="probe-schema-value"
+ elif grep -aEqi 'ValueError:.*probe target|ValueError:.*probe topology' "$log_path"; then
+ diagnostic="probe-target-value"
+ elif grep -aEqi 'ValueError:.*probe result|ValueError:.*statically promote the registry' "$log_path"; then
+ diagnostic="probe-result-value"
+ elif grep -aEqi 'ValueError:.*probe privacy' "$log_path"; then
+ diagnostic="probe-privacy-value"
+ elif grep -aEqi 'ValueError:.*probe API' "$log_path"; then
+ diagnostic="probe-api-value"
+ elif grep -aEqi 'ValueError:.*probe completion' "$log_path"; then
+ diagnostic="probe-completion-value"
+ elif grep -aEqi 'ValueError:.*probe (source SHA|image digest|backend provenance|backend component)' "$log_path"; then
+ diagnostic="probe-identity-value"
+ elif grep -aEqi 'ValueError:.*probe (precision correctness|scale shapes|precision .* (input|output|semantic|scales))' "$log_path"; then
+ diagnostic="probe-correctness-value"
+ elif grep -aEqi 'ValueError:.*probe transport' "$log_path"; then
+ diagnostic="probe-transport-value"
elif grep -aEqi 'ValueError:' "$log_path" \
&& grep -aEqi 'probe_precision[.]py' "$log_path"; then
diagnostic="probe-manifest-value"
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 6f38d98345..845bbb6602 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -2211,6 +2211,18 @@ def test_private_runtime_failure_signatures_override_case_footer(self) -> None:
"PrecisionError: unsupported precision profile": "precision-contract",
"AssertionError: probe invariant": "python-assertion",
"RuntimeError: probe execution failed": "python-runtime",
+ "ValueError: probe fields differ from collectivex.precision-probe.v1":
+ "probe-schema-value",
+ "ValueError: probe target is not a provisional native adapter cell":
+ "probe-target-value",
+ "ValueError: probe result reason is invalid": "probe-result-value",
+ "ValueError: probe privacy contract differs": "probe-privacy-value",
+ "ValueError: probe API calls are empty": "probe-api-value",
+ "ValueError: probe completion contract differs": "probe-completion-value",
+ "ValueError: probe image digest is invalid": "probe-identity-value",
+ "ValueError: probe precision correctness did not pass":
+ "probe-correctness-value",
+ "ValueError: probe transport fallback is present": "probe-transport-value",
"tests/probe_precision.py\nValueError: probe argument": "probe-manifest-value",
"tests/ep_harness.py\nValueError: harness argument": "harness-value",
"tests/workload.py\nValueError: workload argument": "workload-value",
From 612746584edb76c276610d49f026924221d9c9d9 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 17:46:52 +0800
Subject: [PATCH 024/210] fix(collectivex): validate native combine inputs
---
experimental/CollectiveX/runtime/common.sh | 8 +++++++
experimental/CollectiveX/tests/ep_harness.py | 2 ++
.../CollectiveX/tests/ep_precision.py | 24 +++----------------
.../tests/test_ep_precision_adapters.py | 4 ++++
.../tests/test_sampling_contract.py | 6 ++++-
5 files changed, 22 insertions(+), 22 deletions(-)
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 90c279e858..aab1a2ed92 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -95,6 +95,14 @@ cx_fail_stage() {
diagnostic="probe-completion-value"
elif grep -aEqi 'ValueError:.*probe (source SHA|image digest|backend provenance|backend component)' "$log_path"; then
diagnostic="probe-identity-value"
+ elif grep -aEqi 'ValueError:.*probe precision correctness did not pass' "$log_path"; then
+ diagnostic="probe-correctness-failed"
+ elif grep -aEqi 'ValueError:.*probe scale shapes are invalid' "$log_path"; then
+ diagnostic="probe-scale-shape"
+ elif grep -aEqi 'ValueError:.*probe precision .* is not finite' "$log_path"; then
+ diagnostic="probe-nonfinite"
+ elif grep -aEqi 'ValueError:.*probe precision .* shapes are empty|ValueError:.*probe precision .* shape is invalid' "$log_path"; then
+ diagnostic="probe-tensor-shape"
elif grep -aEqi 'ValueError:.*probe (precision correctness|scale shapes|precision .* (input|output|semantic|scales))' "$log_path"; then
diagnostic="probe-correctness-value"
elif grep -aEqi 'ValueError:.*probe transport' "$log_path"; then
diff --git a/experimental/CollectiveX/tests/ep_harness.py b/experimental/CollectiveX/tests/ep_harness.py
index 13cebb517e..af8d2b656a 100644
--- a/experimental/CollectiveX/tests/ep_harness.py
+++ b/experimental/CollectiveX/tests/ep_harness.py
@@ -911,6 +911,7 @@ def _run_expert_packed_oracle(
handle.oracle_packed_positions = view.packed_positions
problem.recv_tokens = receive_count
transformed = _expert_transform_expanded(torch, view.payload, view.expert_ids)
+ view.combine_input = transformed
combined = backend.combine_transformed(problem, handle, transformed)
torch.cuda.synchronize()
expected_combined = _expected_transformed_combine(torch, problem)
@@ -1111,6 +1112,7 @@ def _run_expert_oracle(
transformed = _expert_transform(
torch, view.payload, actual_ids, actual_weights, combine_weight_semantics
)
+ view.combine_input = transformed
combined = backend.combine_transformed(problem, handle, transformed)
torch.cuda.synchronize()
expected_combined = _expected_transformed_combine(torch, problem)
diff --git a/experimental/CollectiveX/tests/ep_precision.py b/experimental/CollectiveX/tests/ep_precision.py
index c4f19f54fe..cb40d25d55 100644
--- a/experimental/CollectiveX/tests/ep_precision.py
+++ b/experimental/CollectiveX/tests/ep_precision.py
@@ -297,28 +297,10 @@ def dequantize_expert_prefixes(
def _direct_cast_saturation(torch_module, profile: dict[str, Any], view) -> tuple[int, float]:
- """Count values clipped by MoRI's unscaled BF16-to-FP8 combine cast."""
- if not all(hasattr(view, field) for field in ("payload", "expert_ids", "weights")):
+ """Count values clipped in the exact native BF16-to-FP8 combine input."""
+ if not hasattr(view, "combine_input"):
return 0, 0.0
- expert_ids = view.expert_ids
- weights = view.weights
- if expert_ids.ndim != 2 or weights.shape != expert_ids.shape:
- return 0, 0.0
- valid = expert_ids >= 0
- expert = expert_ids.clamp(min=0).to(torch_module.int64)
- gate = weights.to(torch_module.float32).masked_fill(~valid, 0)
- scale = ((expert * 17 + 5) % 31 + 1).to(torch_module.float32) / 32
- offset_a = (((expert * 29 + 7) % 37) - 18).to(torch_module.float32) / 64
- offset_b = (((expert * 43 + 11) % 41) - 20).to(torch_module.float32) / 128
- columns = torch_module.arange(
- view.payload.shape[1], device=view.payload.device, dtype=torch_module.int64
- )
- pattern = (((columns * 13) % 17) - 8).to(torch_module.float32) / 8
- transformed = (
- view.payload.float() * (gate * scale).sum(dim=1, keepdim=True)
- + (gate * offset_a).sum(dim=1, keepdim=True)
- + (gate * offset_b).sum(dim=1, keepdim=True) * pattern.unsqueeze(0)
- )
+ transformed = view.combine_input.float()
dtype = _fp8_dtype(torch_module, profile["combine"])
fp8_max = float(torch_module.finfo(dtype).max)
count = int((transformed.abs() > fp8_max).sum().item())
diff --git a/experimental/CollectiveX/tests/test_ep_precision_adapters.py b/experimental/CollectiveX/tests/test_ep_precision_adapters.py
index a9655cdcdb..b2feba3ba2 100644
--- a/experimental/CollectiveX/tests/test_ep_precision_adapters.py
+++ b/experimental/CollectiveX/tests/test_ep_precision_adapters.py
@@ -121,9 +121,13 @@ def test_elastic_and_hybrid_constructors_enable_native_fp8(self):
def test_mori_wires_dispatch_scales_and_direct_cast_config(self):
source = (TESTS / "ep_mori.py").read_text(encoding="utf-8")
+ harness = (TESTS / "ep_harness.py").read_text(encoding="utf-8")
+ precision = (TESTS / "ep_precision.py").read_text(encoding="utf-8")
self.assertIn('"fp8_direct_cast" if self._direct_cast_combine', source)
self.assertIn("p.scales,", source)
self.assertIn("dispatch_scales=_scales", source)
+ self.assertEqual(harness.count("view.combine_input = transformed"), 2)
+ self.assertIn("transformed = view.combine_input.float()", precision)
if __name__ == "__main__":
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 845bbb6602..9afec028a3 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -2221,7 +2221,11 @@ def test_private_runtime_failure_signatures_override_case_footer(self) -> None:
"ValueError: probe completion contract differs": "probe-completion-value",
"ValueError: probe image digest is invalid": "probe-identity-value",
"ValueError: probe precision correctness did not pass":
- "probe-correctness-value",
+ "probe-correctness-failed",
+ "ValueError: probe scale shapes are invalid": "probe-scale-shape",
+ "ValueError: probe precision dispatch input is not finite": "probe-nonfinite",
+ "ValueError: probe precision combine output shape is invalid":
+ "probe-tensor-shape",
"ValueError: probe transport fallback is present": "probe-transport-value",
"tests/probe_precision.py\nValueError: probe argument": "probe-manifest-value",
"tests/ep_harness.py\nValueError: harness argument": "harness-value",
From 15d19a212710d56956c480ae74141d1deae6ca22 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 17:51:50 +0800
Subject: [PATCH 025/210] fix(collectivex): validate codec-specific correctness
---
AGENTS.md | 2 +-
experimental/CollectiveX/README.md | 6 +++
experimental/CollectiveX/contracts.py | 23 +++++---
experimental/CollectiveX/docs/methodology.md | 8 ++-
experimental/CollectiveX/identity.py | 26 ++++++++-
.../schemas/raw-case-v1.schema.json | 2 +-
.../schemas/terminal-outcome-v1.schema.json | 2 +-
experimental/CollectiveX/tests/ep_harness.py | 45 +++++++++-------
.../CollectiveX/tests/ep_precision.py | 1 +
.../CollectiveX/tests/probe_precision.py | 11 ++--
.../tests/test_precision_scheduling.py | 43 +++++++++++++++
.../tests/test_sampling_contract.py | 53 +++++++++++++++++++
12 files changed, 187 insertions(+), 35 deletions(-)
diff --git a/AGENTS.md b/AGENTS.md
index b49f547e2f..f1eac75137 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -2,7 +2,7 @@
Guidance for AI agents working with InferenceX.
-> **Temporary language override:** Until the user explicitly removes this override, ignore all other repository instructions that require Simplified Chinese or bilingual output. Work in English only: do not create or update `_zh.md` files, Chinese documentation, translations, or Chinese text in commits, pull requests, issues, reviews, or comments. Leave existing Chinese content untouched.
+> **Temporary language override (highest repository priority):** Until the user explicitly removes this override, ignore any instruction elsewhere in this repository that requires Simplified Chinese or bilingual output, including instructions in later sections, nested agent files, contribution guides, checklists, and templates. Work in English only: do not create or update `_zh.md` files, Chinese documentation, translations, or Chinese text in commits, pull requests, issues, reviews, or comments. Leave existing Chinese content untouched.
> **Mandatory reading: [`CONTRIBUTING.md`](CONTRIBUTING.md)** — read it before opening or reviewing any PR. It covers the full PR review flow, the CODEOWNER sign-off process, the `/reuse-sweep-run` merge path, post-merge responsibilities, and critical cluster rules (e.g. never leaving root-owned files on AMD runners).
diff --git a/experimental/CollectiveX/README.md b/experimental/CollectiveX/README.md
index 9dcdc405e9..2cccc58aa4 100644
--- a/experimental/CollectiveX/README.md
+++ b/experimental/CollectiveX/README.md
@@ -34,6 +34,12 @@ warmups before each measured component at every trial/point. Roundtrip is measur
iteration takes the cross-rank maximum before nearest-rank p50/p90/p95/p99, and roundtrip p99 is the
headline latency. A stdlib integer counter produces byte-identical routing and gate weights.
+Correctness is checked against the semantic value after the declared communication codec. The
+frozen combine gates are `rtol=0.05, atol=0.02` for BF16, `rtol=0.06, atol=0.03` for native
+logfmt10, and `rtol=0.08, atol=0.04` for native FP8 direct-cast combine. These are fail-closed
+publication thresholds, not estimates of codec error. FP8 direct-cast evidence also counts
+saturation on the exact transformed native combine input; any saturated value fails the point.
+
The BF16 planning baseline covers H100, H200, B200, B300, GB200, GB300, MI325X, and MI355X. It
requests
608 cases / 1,600 token points: 364 runnable cases / 940 points, emitted as 58 executable workflow
diff --git a/experimental/CollectiveX/contracts.py b/experimental/CollectiveX/contracts.py
index 04357da78c..49884bad1f 100644
--- a/experimental/CollectiveX/contracts.py
+++ b/experimental/CollectiveX/contracts.py
@@ -1453,9 +1453,16 @@ def _validate_component(
def _validate_oracle(
- value: Any, path: str, profile: dict[str, Any] | None = None
+ value: Any,
+ path: str,
+ profile: dict[str, Any] | None = None,
+ communication_precision: dict[str, Any] | None = None,
) -> dict[str, Any]:
profile = profile or identity.V1_NORMAL_CASE_PROFILE
+ communication_precision = communication_precision or identity.precision_profile(
+ identity.V1_CONTROL_PRECISION_PROFILE
+ )
+ tolerance = identity.combine_oracle_tolerances(communication_precision)
oracle = _keys(
value,
{"atol", "checks", "combine_weight_semantics", "contract", "dispatch_sha256",
@@ -1485,8 +1492,8 @@ def _validate_oracle(
)
if oracle["combine_weight_semantics"] != expected_weight_semantics:
raise ContractError(f"{path}.combine_weight_semantics differs from v1")
- _close(oracle["rtol"], 5e-2, f"{path}.rtol")
- _close(oracle["atol"], 2e-2, f"{path}.atol")
+ _close(oracle["rtol"], tolerance["rtol"], f"{path}.rtol")
+ _close(oracle["atol"], tolerance["atol"], f"{path}.atol")
for field in ("dispatch_sha256", "order_sha256"):
digest = oracle[field]
if digest is not None and (
@@ -1503,7 +1510,7 @@ def _validate_oracle(
expected_pass = (
all(checks.values())
and oracle["max_relative_error"] is not None
- and oracle["max_relative_error"] < 5e-2
+ and oracle["max_relative_error"] < tolerance["rtol"]
)
if oracle["passed"] != expected_pass:
raise ContractError(f"{path}.passed differs from its evidence")
@@ -1549,7 +1556,7 @@ def _validate_precision_evidence(
axis["encoded_payload_valid"]
and axis["dequantized_semantics"]
and (not expects_scales or (axis["scales_finite"] and axis["scales_positive"]))
- and saturation_count >= 0
+ and saturation_count == 0
)
if axis["passed"] != bool(expected_pass):
raise ContractError(f"{axis_path}.passed differs from its evidence")
@@ -1975,10 +1982,12 @@ def validate_raw_document(document: Any, samples_document: Any) -> dict[str, Any
if type(evidence["input_unchanged"]) is not bool or type(evidence["order_stable"]) is not bool:
raise ContractError(f"{evidence_path} stability fields must be boolean")
pre = _validate_oracle(
- evidence["pre_timing"], f"{evidence_path}.pre_timing", profile
+ evidence["pre_timing"], f"{evidence_path}.pre_timing", profile,
+ communication_precision,
)
post = _validate_oracle(
- evidence["post_timing"], f"{evidence_path}.post_timing", profile
+ evidence["post_timing"], f"{evidence_path}.post_timing", profile,
+ communication_precision,
)
if (
pre["receive_count"] != expected_payload_counts[evidence_rank]
diff --git a/experimental/CollectiveX/docs/methodology.md b/experimental/CollectiveX/docs/methodology.md
index 440f882a78..c5617f23e2 100644
--- a/experimental/CollectiveX/docs/methodology.md
+++ b/experimental/CollectiveX/docs/methodology.md
@@ -142,8 +142,12 @@ Normal-mode adapters use activation-only, unweighted rank-sum combine. The oracl
gate-weighted expert aggregate before combine, independently derives `sum(gate * expert(token))`,
and checks the dispatch metadata and transformed output. Low-latency adapters separately verify the
expert-packed source/expert assignment, native gate weights, and gate-weighted combined output. Both
-contracts check every element with recorded `rtol=0.05` and `atol=0.02`. Any failed rank or point
-makes the case ineligible.
+contracts compare against the semantic value after the declared communication codec. The frozen
+combine gates are `rtol=0.05, atol=0.02` for BF16, `rtol=0.06, atol=0.03` for native logfmt10, and
+`rtol=0.08, atol=0.04` for native FP8 direct-cast combine. These thresholds are correctness gates,
+not estimates of codec error. Direct-cast saturation is measured on the exact transformed native
+combine input, and the required saturation count is zero. Any failed rank or point makes the case
+ineligible.
Pre/post dispatch evidence is hashed in canonical source-token order. Native receive slots may be
assigned nondeterministically, so physical receive order is not treated as a correctness property.
diff --git a/experimental/CollectiveX/identity.py b/experimental/CollectiveX/identity.py
index 3b263cc9c3..9a0b300dbd 100644
--- a/experimental/CollectiveX/identity.py
+++ b/experimental/CollectiveX/identity.py
@@ -35,7 +35,7 @@
"eplb_reference_tokens_per_rank": 2048,
"mode": "normal",
"oracle_contract": "expert-specific-transform-v1",
- "oracle_tolerances": "rtol=0.05,atol=0.02",
+ "oracle_tolerances": "codec-specific-combine-v1",
"payload_unit": "token-rank",
"placement": "packed",
"percentile_method": "nearest-rank",
@@ -247,6 +247,30 @@ def _communication_axis(
},
}
+V1_COMBINE_ORACLE_TOLERANCES = {
+ "bf16": {"atol": 2e-2, "rtol": 5e-2},
+ "logfmt10": {"atol": 3e-2, "rtol": 6e-2},
+ "fp8-direct-cast": {"atol": 4e-2, "rtol": 8e-2},
+}
+
+
+def combine_oracle_tolerances(communication_precision: dict[str, Any]) -> dict[str, float]:
+ """Return the frozen combine-oracle gate for one exact native codec."""
+ combine = communication_precision["combine"]
+ communication_format = combine["communication_format"]
+ if communication_format == "bf16":
+ key = "bf16"
+ elif communication_format == "logfmt10":
+ key = "logfmt10"
+ elif (
+ communication_format in {"fp8-e4m3fn", "fp8-e4m3fnuz"}
+ and combine["quantization_origin"] == "backend-internal-direct-cast"
+ ):
+ key = "fp8-direct-cast"
+ else:
+ raise ValueError("precision profile has no frozen combine-oracle tolerance")
+ return dict(V1_COMBINE_ORACLE_TOLERANCES[key])
+
def case_profile(mode: str) -> dict[str, Any]:
"""Return the immutable measurement profile for one scheduled mode."""
diff --git a/experimental/CollectiveX/schemas/raw-case-v1.schema.json b/experimental/CollectiveX/schemas/raw-case-v1.schema.json
index d5c8a73a5f..4308fd46d6 100644
--- a/experimental/CollectiveX/schemas/raw-case-v1.schema.json
+++ b/experimental/CollectiveX/schemas/raw-case-v1.schema.json
@@ -214,7 +214,7 @@
"eplb_reference_tokens_per_rank": {"const":2048},
"mode": {"enum":["normal","low-latency"]},
"oracle_contract": {"enum":["expert-specific-transform-v1","expert-assignment-transform-v1"]},
- "oracle_tolerances": {"const":"rtol=0.05,atol=0.02"},
+ "oracle_tolerances": {"const":"codec-specific-combine-v1"},
"payload_unit": {"enum":["token-rank","token-expert"]},
"placement": {"const":"packed"},
"percentile_method": {"const":"nearest-rank"},
diff --git a/experimental/CollectiveX/schemas/terminal-outcome-v1.schema.json b/experimental/CollectiveX/schemas/terminal-outcome-v1.schema.json
index f4a9d99ca2..30d9bcebd1 100644
--- a/experimental/CollectiveX/schemas/terminal-outcome-v1.schema.json
+++ b/experimental/CollectiveX/schemas/terminal-outcome-v1.schema.json
@@ -231,7 +231,7 @@
"eplb_reference_tokens_per_rank":{"const":2048},
"mode":{"enum":["normal","low-latency"]},
"oracle_contract":{"enum":["expert-specific-transform-v1","expert-assignment-transform-v1"]},
- "oracle_tolerances":{"const":"rtol=0.05,atol=0.02"},
+ "oracle_tolerances":{"const":"codec-specific-combine-v1"},
"payload_unit":{"enum":["token-rank","token-expert"]},
"placement":{"const":"packed"},
"percentile_method":{"const":"nearest-rank"},
diff --git a/experimental/CollectiveX/tests/ep_harness.py b/experimental/CollectiveX/tests/ep_harness.py
index af8d2b656a..d07f924045 100644
--- a/experimental/CollectiveX/tests/ep_harness.py
+++ b/experimental/CollectiveX/tests/ep_harness.py
@@ -89,6 +89,11 @@
ORACLE_RTOL = 5e-2
ORACLE_ATOL = 2e-2
+
+def _oracle_tolerances(backend) -> tuple[float, float]:
+ tolerance = identity.combine_oracle_tolerances(backend.communication_precision)
+ return tolerance["atol"], tolerance["rtol"]
+
EPLB_REDUNDANT_EXPERTS = 32
EPLB_REFERENCE_TOKENS_PER_RANK = 2048
EPLB_PLANNER = "greedy-rank-major-v1"
@@ -647,15 +652,14 @@ def _precision_evidence(backend, problem, view, combined, expected_combined) ->
evidence = method(problem, view)
combine_axis = backend.communication_precision["combine"]
if combine_axis["communication_format"] != "bf16":
+ oracle_atol, oracle_rtol = _oracle_tolerances(backend)
if combined.shape == expected_combined.shape and combined.numel():
absolute = (combined.float() - expected_combined.float()).abs()
max_abs_error = float(absolute.max().item())
max_rel_error = max_abs_error / (
float(expected_combined.float().abs().max().item()) + 1e-6
)
- tolerance = ORACLE_ATOL + float(
- getattr(backend, "tolerance", ORACLE_RTOL)
- ) * expected_combined.float().abs()
+ tolerance = oracle_atol + oracle_rtol * expected_combined.float().abs()
semantics = bool((absolute <= tolerance).all().item())
elif combined.shape == expected_combined.shape:
max_abs_error = max_rel_error = 0.0
@@ -674,7 +678,10 @@ def _precision_evidence(backend, problem, view, combined, expected_combined) ->
and direction["scales_positive"] is not False
)
direction["passed"] = bool(
- direction["encoded_payload_valid"] and semantics and scale_ok
+ direction["encoded_payload_valid"]
+ and semantics
+ and scale_ok
+ and direction["saturation_count"] == 0
)
evidence["passed"] = bool(
evidence["dispatch"]["passed"] and direction["passed"]
@@ -758,6 +765,7 @@ def _run_expert_packed_oracle(
):
"""Verify an expert-packed dispatch and native gate-weighted combine."""
contract = LOW_LATENCY_ORACLE_CONTRACT
+ oracle_atol, oracle_rtol = _oracle_tolerances(backend)
handle = backend.dispatch(problem)
torch.cuda.synchronize()
try:
@@ -793,12 +801,12 @@ def _run_expert_packed_oracle(
backend, "combine_weight_semantics", "undeclared"
),
"receive_count": 0,
- "atol": ORACLE_ATOL,
+ "atol": oracle_atol,
"max_absolute_error": None,
"max_elementwise_relative_error": None,
"max_relative_error": None,
"max_weight_error": None,
- "rtol": ORACLE_RTOL,
+ "rtol": oracle_rtol,
"checks": {
"combine_values": False,
"counts": False,
@@ -922,10 +930,10 @@ def _run_expert_packed_oracle(
float(expected_combined.abs().max().item()) + 1e-6
)
max_elementwise_relative_error = float(
- (absolute_error / expected_combined.abs().clamp_min(ORACLE_ATOL)).max().item()
+ (absolute_error / expected_combined.abs().clamp_min(oracle_atol)).max().item()
)
combine_values_ok = bool(torch.allclose(
- combined.float(), expected_combined, rtol=ORACLE_RTOL, atol=ORACLE_ATOL
+ combined.float(), expected_combined, rtol=oracle_rtol, atol=oracle_atol
))
elif combined.shape == expected_combined.shape:
max_absolute_error = 0.0
@@ -937,7 +945,7 @@ def _run_expert_packed_oracle(
max_elementwise_relative_error = None
max_relative_error = None
combine_values_ok = False
- tolerance = float(getattr(backend, "tolerance", ORACLE_RTOL))
+ tolerance = oracle_rtol
precision = _precision_evidence(backend, problem, view, combined, expected_combined)
checks = {
"combine_values": combine_values_ok,
@@ -958,7 +966,7 @@ def _run_expert_packed_oracle(
and max_relative_error is not None
and max_relative_error < tolerance
),
- "atol": ORACLE_ATOL,
+ "atol": oracle_atol,
"combine_weight_semantics": backend.combine_weight_semantics,
"ordering_contract": ordering_contract,
"order_sha256": order_sha256,
@@ -968,7 +976,7 @@ def _run_expert_packed_oracle(
"max_elementwise_relative_error": max_elementwise_relative_error,
"max_relative_error": max_relative_error,
"max_weight_error": max_weight_error,
- "rtol": ORACLE_RTOL,
+ "rtol": oracle_rtol,
"checks": checks,
}
@@ -997,6 +1005,7 @@ def _run_expert_oracle(
experts_per_rank,
seed,
)
+ oracle_atol, oracle_rtol = _oracle_tolerances(backend)
handle = backend.dispatch(problem)
torch.cuda.synchronize()
try:
@@ -1021,12 +1030,12 @@ def _run_expert_oracle(
backend, "combine_weight_semantics", "undeclared"
),
"receive_count": 0,
- "atol": ORACLE_ATOL,
+ "atol": oracle_atol,
"max_absolute_error": None,
"max_elementwise_relative_error": None,
"max_relative_error": None,
"max_weight_error": None,
- "rtol": ORACLE_RTOL,
+ "rtol": oracle_rtol,
"checks": {
"combine_values": False,
"counts": False,
@@ -1123,10 +1132,10 @@ def _run_expert_oracle(
float(expected_combined.abs().max().item()) + 1e-6
)
max_elementwise_relative_error = float(
- (absolute_error / expected_combined.abs().clamp_min(ORACLE_ATOL)).max().item()
+ (absolute_error / expected_combined.abs().clamp_min(oracle_atol)).max().item()
)
combine_values_ok = bool(torch.allclose(
- combined.float(), expected_combined, rtol=ORACLE_RTOL, atol=ORACLE_ATOL
+ combined.float(), expected_combined, rtol=oracle_rtol, atol=oracle_atol
))
elif combined.shape == expected_combined.shape:
max_absolute_error = 0.0
@@ -1138,7 +1147,7 @@ def _run_expert_oracle(
max_elementwise_relative_error = None
max_relative_error = None
combine_values_ok = False
- tolerance = float(getattr(backend, "tolerance", 5e-2))
+ tolerance = oracle_rtol
precision = _precision_evidence(backend, problem, view, combined, expected_combined)
checks = {
"combine_values": combine_values_ok,
@@ -1159,7 +1168,7 @@ def _run_expert_oracle(
and max_relative_error is not None
and max_relative_error < tolerance
),
- "atol": ORACLE_ATOL,
+ "atol": oracle_atol,
"combine_weight_semantics": combine_weight_semantics,
"ordering_contract": ordering_contract,
"order_sha256": order_sha256,
@@ -1169,7 +1178,7 @@ def _run_expert_oracle(
"max_elementwise_relative_error": max_elementwise_relative_error,
"max_relative_error": max_relative_error,
"max_weight_error": max_weight_error,
- "rtol": ORACLE_RTOL,
+ "rtol": oracle_rtol,
"checks": checks,
}
diff --git a/experimental/CollectiveX/tests/ep_precision.py b/experimental/CollectiveX/tests/ep_precision.py
index cb40d25d55..01440b8f54 100644
--- a/experimental/CollectiveX/tests/ep_precision.py
+++ b/experimental/CollectiveX/tests/ep_precision.py
@@ -345,6 +345,7 @@ def precision_evidence(
count, rate = _direct_cast_saturation(torch_module, profile, view)
combine["saturation_count"] = count
combine["saturation_rate"] = rate
+ combine["passed"] = combine["passed"] and count == 0
return {
"profile_id": profile_id,
"dispatch": dispatch,
diff --git a/experimental/CollectiveX/tests/probe_precision.py b/experimental/CollectiveX/tests/probe_precision.py
index ec1afec2a1..8afdea1851 100644
--- a/experimental/CollectiveX/tests/probe_precision.py
+++ b/experimental/CollectiveX/tests/probe_precision.py
@@ -514,7 +514,9 @@ def _aggregate_tensor_summaries(records: list[dict[str, Any]]) -> dict[str, Any]
return {
"finite": all(record["finite"] for record in records),
"rank": ranks.pop(),
- "shapes": sorted({tuple(record["shape"]) for record in records}),
+ "shapes": [
+ list(shape) for shape in sorted({tuple(record["shape"]) for record in records})
+ ],
"storage_dtype": dtypes.pop(),
}
@@ -549,9 +551,10 @@ def _aggregate_scale_contracts(records: list[dict[str, Any]]) -> dict[str, Any]:
for field in ("finite", "positive"):
values = [record[field] for record in records]
result[field] = None if all(value is None for value in values) else all(value is True for value in values)
- result["runtime_shapes"] = sorted({
- tuple(record["runtime_shape"] or ()) for record in records
- }) if any(record["runtime_shape"] is not None for record in records) else []
+ result["runtime_shapes"] = [
+ list(shape)
+ for shape in sorted({tuple(record["runtime_shape"] or ()) for record in records})
+ ] if any(record["runtime_shape"] is not None for record in records) else []
return result
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
index 4441e820f0..8d289b1826 100644
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -108,6 +108,31 @@ def test_precision_probe_manifest_is_sanitized_and_runtime_evidence_is_required(
evidence=None,
)
+ def test_precision_probe_aggregates_shapes_as_manifest_arrays(self) -> None:
+ tensor = probe_precision._aggregate_tensor_summaries([
+ {"finite": True, "rank": 2, "shape": [8, 7168], "storage_dtype": "bfloat16"},
+ {"finite": True, "rank": 2, "shape": [16, 7168], "storage_dtype": "bfloat16"},
+ ])
+ self.assertEqual(tensor["shapes"], [[8, 7168], [16, 7168]])
+ self.assertTrue(all(type(shape) is list for shape in tensor["shapes"]))
+
+ scale = probe_precision._aggregate_scale_contracts([
+ {
+ "alignment": "hidden-block-128", "dtype": "f32", "finite": True,
+ "group_size": 128, "layout": "per-token-hidden-block",
+ "padding": "right-zero-pad-hidden-to-128", "positive": True,
+ "runtime_shape": [8, 56], "runtime_storage_dtype": "float32",
+ },
+ {
+ "alignment": "hidden-block-128", "dtype": "f32", "finite": True,
+ "group_size": 128, "layout": "per-token-hidden-block",
+ "padding": "right-zero-pad-hidden-to-128", "positive": True,
+ "runtime_shape": [16, 56], "runtime_storage_dtype": "float32",
+ },
+ ])
+ self.assertEqual(scale["runtime_shapes"], [[8, 56], [16, 56]])
+ self.assertTrue(all(type(shape) is list for shape in scale["runtime_shapes"]))
+
def test_precision_profiles_bind_exact_formats_and_timing_boundaries(self) -> None:
scheduled = set(identity.V1_NORMAL_PRECISION_PROFILE_IDS) | set(
identity.V1_LOW_LATENCY_PRECISION_PROFILE_IDS
@@ -158,6 +183,24 @@ def test_precision_profiles_bind_exact_formats_and_timing_boundaries(self) -> No
(logfmt["communication_format"], logfmt["scale_group_size"]),
("logfmt10", 64),
)
+ self.assertEqual(
+ identity.combine_oracle_tolerances(
+ identity.precision_profile(identity.V1_CONTROL_PRECISION_PROFILE)
+ ),
+ {"atol": 2e-2, "rtol": 5e-2},
+ )
+ self.assertEqual(
+ identity.combine_oracle_tolerances(
+ identity.precision_profile("d-bf16.c-logfmt10-dynamic64")
+ ),
+ {"atol": 3e-2, "rtol": 6e-2},
+ )
+ self.assertEqual(
+ identity.combine_oracle_tolerances(identity.precision_profile(
+ "d-bf16.c-fp8-e4m3fn-direct-cast-noscale"
+ )),
+ {"atol": 4e-2, "rtol": 8e-2},
+ )
base = {"mode": "normal"}
precision_case = {
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 9afec028a3..a5db5e2ee7 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -1841,6 +1841,58 @@ def test_oracle_pass_cannot_ignore_combined_value_failure(self) -> None:
with self.assertRaises(contracts.ContractError):
contracts._validate_oracle(tampered, "oracle")
+ for profile_id, expected in (
+ ("d-bf16.c-logfmt10-dynamic64", {"atol": 3e-2, "rtol": 6e-2}),
+ ("d-bf16.c-fp8-e4m3fn-direct-cast-noscale", {"atol": 4e-2, "rtol": 8e-2}),
+ ):
+ with self.subTest(profile_id=profile_id):
+ precision = identity.precision_profile(profile_id)
+ codec_oracle = copy.deepcopy(oracle)
+ codec_oracle.update(expected)
+ contracts._validate_oracle(
+ codec_oracle, "oracle", communication_precision=precision
+ )
+ codec_oracle["rtol"] = 5e-2
+ with self.assertRaisesRegex(contracts.ContractError, "rtol"):
+ contracts._validate_oracle(
+ codec_oracle, "oracle", communication_precision=precision
+ )
+
+ def test_precision_evidence_rejects_direct_cast_saturation(self) -> None:
+ profile_id = "d-bf16.c-fp8-e4m3fn-direct-cast-noscale"
+ communication_precision = identity.precision_profile(profile_id)
+ axis = {
+ "dequantized_semantics": True,
+ "encoded_payload_valid": True,
+ "max_abs_error": 0.0,
+ "max_rel_error": 0.0,
+ "passed": True,
+ "saturation_count": 0,
+ "saturation_rate": 0.0,
+ "scales_finite": None,
+ "scales_positive": None,
+ }
+ evidence = {
+ "combine": copy.deepcopy(axis),
+ "dispatch": copy.deepcopy(axis),
+ "passed": True,
+ "profile_id": profile_id,
+ }
+ contracts._validate_precision_evidence(
+ evidence, profile_id, communication_precision, "precision"
+ )
+ saturated = copy.deepcopy(evidence)
+ saturated["combine"].update({"saturation_count": 1, "saturation_rate": 0.01})
+ with self.assertRaisesRegex(contracts.ContractError, "passed differs"):
+ contracts._validate_precision_evidence(
+ saturated, profile_id, communication_precision, "precision"
+ )
+ saturated["combine"]["passed"] = False
+ saturated["passed"] = False
+ contracts._validate_precision_evidence(
+ saturated, profile_id, communication_precision, "precision"
+ )
+
def test_oracle_stability_canonicalizes_native_receive_order(self) -> None:
source = (HERE / "ep_harness.py").read_text()
begin = source.index("canonical_order = torch.argsort")
@@ -2658,6 +2710,7 @@ def test_stage_removes_its_execution_child_when_rsync_fails(self) -> None:
**os.environ,
"COLLECTIVEX_OPERATOR_CONFIG": "/dev/null",
"COLLECTIVEX_CANONICAL_GHA": "1",
+ "COLLECTIVEX_EXECUTION_ID": f"test-{root.name}",
"CX_STAGE_DIR": str(base),
"RSYNC_CALLED": str(sentinel),
},
From f15f04e75d0332e6b60d4ed67db793751a05c2c7 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 18:04:30 +0800
Subject: [PATCH 026/210] fix(collectivex): protect downloaded source archive
---
.github/workflows/collectivex-sweep.yml | 1 +
experimental/CollectiveX/tests/test_sampling_contract.py | 6 ++++++
2 files changed, 7 insertions(+)
diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml
index d66593256f..e4b38e615c 100644
--- a/.github/workflows/collectivex-sweep.yml
+++ b/.github/workflows/collectivex-sweep.yml
@@ -401,6 +401,7 @@ jobs:
destination="$CX_SOURCE_ROOT/experimental/CollectiveX"
seed_root="$destination/.cx_sources"
[ -f "$archive" ] && [ ! -e "$seed_root" ] && [ ! -L "$seed_root" ]
+ chmod 600 -- "$archive"
source "$destination/runtime/common.sh"
source_path="$(cx_backend_source_path "$seed_root" "$EXPECTED_BACKEND")"
source_basename="${source_path#"$seed_root/"}"
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index a5db5e2ee7..8d8da51f8d 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -1196,6 +1196,12 @@ def test_allocation_cleanup_fails_closed_when_scheduler_queries_fail(self) -> No
self.assertIn("CX_BACKEND_SOURCE_SEED_ROOT", workflow)
self.assertIn("steps.gen.outputs.source_backends", workflow)
self.assertIn('python3 "$destination/source_archive.py"', workflow)
+ install_source = workflow[workflow.index("- name: Install pinned backend source seed"):]
+ self.assertIn('chmod 600 -- "$archive"', install_source)
+ self.assertLess(
+ install_source.index('chmod 600 -- "$archive"'),
+ install_source.index('python3 "$destination/source_archive.py"'),
+ )
artifact_validation = workflow[workflow.index("- name: Validate shard artifact safety"):]
self.assertIn("steps.allocation_cleanup.outcome == 'success'", artifact_validation)
self.assertIn(
From df3528fd66a65d9084eb3a26fff1e89db8a1e376 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 18:07:35 +0800
Subject: [PATCH 027/210] fix(collectivex): freeze backend source validation
---
experimental/CollectiveX/runtime/common.sh | 5 +++--
experimental/CollectiveX/tests/test_sampling_contract.py | 4 +++-
2 files changed, 6 insertions(+), 3 deletions(-)
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index aab1a2ed92..91585019d6 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -1312,8 +1312,9 @@ cx_backend_source_is_valid() {
&& [ "$(cx_git_in_tree "$source" rev-parse HEAD 2>/dev/null)" = "$revision" ] \
&& [ "$(cx_git_in_tree "$source" rev-parse 'HEAD^{tree}' 2>/dev/null)" = "$tree" ] \
|| return 1
- status="$(cx_git_in_tree "$source" status --porcelain --untracked-files=all \
- --ignore-submodules=none 2>/dev/null)" || return 1
+ status="$(GIT_OPTIONAL_LOCKS=0 cx_git_in_tree "$source" \
+ status --porcelain --untracked-files=all --ignore-submodules=none \
+ 2>/dev/null)" || return 1
[ -z "$status" ] || return 1
ignored="$(cx_git_in_tree "$source" ls-files --others --ignored --exclude-standard \
2>/dev/null)" || return 1
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 8d8da51f8d..ac7a9b20dd 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -3193,7 +3193,9 @@ def test_backend_source_validation_rejects_status_errors_and_ignored_files(self)
case " $* " in
*' rev-parse HEAD '*) printf '%s\n' revision ;;
*' rev-parse HEAD^{tree} '*) printf '%s\n' tree ;;
- *' status --porcelain '*) [ "$mode" != status-error ] ;;
+ *' status --porcelain '*)
+ [ "${GIT_OPTIONAL_LOCKS:-}" = 0 ] && [ "$mode" != status-error ]
+ ;;
*' ls-files --others --ignored '*)
[ "$mode" != ignored ] || printf '%s\n' ignored.bin
;;
From f361c1ec570eeeeca7bbf4456b06ca6781577d4b Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 18:11:05 +0800
Subject: [PATCH 028/210] fix(collectivex): archive pinned source roots
---
.github/workflows/collectivex-sweep.yml | 15 ++++++++++++++-
.../CollectiveX/tests/test_sampling_contract.py | 7 +++++++
2 files changed, 21 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml
index e4b38e615c..bcfbca3f2c 100644
--- a/.github/workflows/collectivex-sweep.yml
+++ b/.github/workflows/collectivex-sweep.yml
@@ -171,12 +171,25 @@ jobs:
mkdir -p "$work/experimental/CollectiveX"
read -r -a backends <<< "$SOURCE_BACKENDS"
[ "${#backends[@]}" -gt 0 ]
+ source_root="$work/experimental/CollectiveX/.cx_sources"
+ members=()
for backend in "${backends[@]}"; do
cx_prepare_backend_source "$work" "$backend"
+ source_path="$(cx_backend_source_path "$source_root" "$backend")"
+ source_basename="${source_path#"$source_root/"}"
+ [ -n "$source_basename" ] \
+ && [ "$source_path" = "$source_root/$source_basename" ] \
+ && [[ "$source_basename" != */* ]]
+ members+=(".cx_sources/$source_basename")
done
cx_cleanup_private_logs 0
tar --sort=name --mtime='@1' --owner=0 --group=0 --numeric-owner \
- -C "$work/experimental/CollectiveX" -cf "$archive" .cx_sources
+ --no-recursion -C "$work/experimental/CollectiveX" \
+ -cf "$archive" .cx_sources
+ for member in "${members[@]}"; do
+ tar --sort=name --mtime='@1' --owner=0 --group=0 --numeric-owner \
+ -C "$work/experimental/CollectiveX" -rf "$archive" "$member"
+ done
sha256sum "$archive"
rm -rf -- "$work"
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index ac7a9b20dd..a92a8e8271 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -1195,6 +1195,13 @@ def test_allocation_cleanup_fails_closed_when_scheduler_queries_fail(self) -> No
self.assertIn("Install pinned backend source seed", workflow)
self.assertIn("CX_BACKEND_SOURCE_SEED_ROOT", workflow)
self.assertIn("steps.gen.outputs.source_backends", workflow)
+ prepare_start = workflow.index("- name: Prepare pinned backend source archive")
+ source_archive_step = workflow[
+ prepare_start:workflow.index("- uses: actions/upload-artifact", prepare_start)
+ ]
+ self.assertIn('--no-recursion -C "$work/experimental/CollectiveX"', source_archive_step)
+ self.assertIn('members+=(".cx_sources/$source_basename")', source_archive_step)
+ self.assertIn('-rf "$archive" "$member"', source_archive_step)
self.assertIn('python3 "$destination/source_archive.py"', workflow)
install_source = workflow[workflow.index("- name: Install pinned backend source seed"):]
self.assertIn('chmod 600 -- "$archive"', install_source)
From ce1f531b1b888b2fdbaf956497d28943717a94f3 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 18:41:52 +0800
Subject: [PATCH 029/210] fix(collectivex): align runner topology capabilities
---
experimental/CollectiveX/README.md | 17 +++++---
experimental/CollectiveX/capability.py | 27 +++++++++++-
experimental/CollectiveX/docs/methodology.md | 10 ++++-
experimental/CollectiveX/runtime/common.sh | 2 +-
.../tests/test_precision_scheduling.py | 41 +++++++++++++++----
.../tests/test_sampling_contract.py | 10 ++++-
6 files changed, 85 insertions(+), 22 deletions(-)
diff --git a/experimental/CollectiveX/README.md b/experimental/CollectiveX/README.md
index 2cccc58aa4..ea02c4f7dd 100644
--- a/experimental/CollectiveX/README.md
+++ b/experimental/CollectiveX/README.md
@@ -42,8 +42,8 @@ saturation on the exact transformed native combine input; any saturated value fa
The BF16 planning baseline covers H100, H200, B200, B300, GB200, GB300, MI325X, and MI355X. It
requests
-608 cases / 1,600 token points: 364 runnable cases / 940 points, emitted as 58 executable workflow
-shards/allocation cells, plus 244 explicit unsupported cases / 660 points. `sweep_matrix.py`
+608 cases / 1,600 token points: 338 runnable cases / 868 points, emitted as 54 executable workflow
+shards/allocation cells, plus 270 explicit unsupported cases / 732 points. `sweep_matrix.py`
materializes every token ladder and rejects missing, stale, malformed, or altered shard controls.
Shards are emitted round-robin by SKU so the bounded GHA matrix uses every runner pool early.
@@ -82,6 +82,10 @@ reports that its EP8 communicator lacks Device API symmetric-memory support; re-
requires an all-rank CUDA P2P/LSA-capable runtime. Other NVIDIA SKUs remain unvalidated until their
GPU outcomes pass the native correctness and publication gates.
+H100 EP16 is planned unsupported on the current runner pool because allocated compute nodes expose
+no active RDMA device. EP8 remains in scope, but requires a runner-owned shared `stage_dir` because
+the runner account home is not mounted on compute nodes.
+
Axes not implemented in this baseline include cached-layout `[cl]`, runtime-visible `[rv]`, FP8,
quantized combine,
extra routing distributions, activation profiles, uneven allocation, placement permutations, model
@@ -124,9 +128,10 @@ temporary copy before allocation. Required JSON fields are:
| SKU | Variables |
|---|---|
-| `h100-dgxc`, `b200-dgxc` | `partition`, `account`, `squash_dir` |
+| `h100-dgxc` | `partition`, `account`, `squash_dir`, `stage_dir` |
| `h200-dgxc` | `partition`, `squash_dir` |
-| `b300` | `partition`, `account`, `squash_dir`, `stage_dir` |
+| `b200-dgxc` | `partition`, `account`, `squash_dir` |
+| `b300` | `partition`, `account`, `squash_dir` |
| `gb200` | `partition`, `account`, ordered `storage_roots` |
| `gb300` | `partition`, `account`, `squash_dir`, `stage_dir`, `enroot_cache_path` |
| `mi325x`, `mi355x` | `partition`, `squash_dir`, `stage_dir` |
@@ -145,13 +150,13 @@ allocated node. Jobs create only a marked mode-0700 execution child, prove cross
visibility, and remove that exact child after allocation teardown; they never mount the runner
checkout or create a stage beneath image storage on AMD.
-H100, H200, and B200 runners may omit `stage_dir`. Their isolated execution child is then created
+H200, B200, and B300 runners may omit `stage_dir`. Their isolated execution child is then created
under a runner-owned mode-0700 base in the validated operating-system account home, independent of
the workflow's temporary `HOME`. A symlinked account-home entry is resolved once to its canonical,
runner-owned target; the single hidden staging base directly beneath it must itself be non-symlinked
and not writable by other users. The workflow still proves that base is visible from every allocated
node before launch. The execution child is validated, marked, cross-node checked, and removed using
-the same contract. All other runners require the dedicated `stage_dir` above; no canonical run
+the same contract. H100 and all other runners require the dedicated `stage_dir` above; no canonical run
stages source beneath shared container storage.
Before import, each Docker Hub tag is resolved with bounded registry requests and must match its
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
index 346e2ad142..f322f60bce 100644
--- a/experimental/CollectiveX/capability.py
+++ b/experimental/CollectiveX/capability.py
@@ -302,6 +302,17 @@ def _precision_rule(
),
}
+_H100_EP16_NO_RDMA_BASIS = "current-runner-has-no-active-rdma-device"
+PRECISION_CELL_OVERRIDES = {
+ (profile, rule["backend"], "h100-dgxc", 16, rule["mode"]): {
+ "basis": _H100_EP16_NO_RDMA_BASIS,
+ "disposition": "unsupported",
+ }
+ for profile, rules in PRECISION_CAPABILITIES.items()
+ for rule in rules
+ if "h100-dgxc" in rule["skus"] and 16 in rule["ep_degrees"]
+}
+
def runtime_identity_issues(
sku: str, *, vendor: str, arch: str, machine: str, device_name: str,
@@ -369,6 +380,8 @@ def _resolve_base(
topology = topology_for(sku, ep)
if topology is None or (nodes is not None and nodes != topology["nodes"]):
return False, f"{sku} does not register EP{ep} on {nodes} nodes"
+ if sku == "h100-dgxc" and ep == 16:
+ return False, _H100_EP16_NO_RDMA_BASIS
if routing not in {"uniform", "zipf"} or (eplb and routing != "zipf"):
return False, "v1 routing is uniform or zipf, with EPLB only on zipf"
if platform["vendor"] not in implementation["vendors"]:
@@ -401,14 +414,15 @@ def precision_targets(
if key in seen:
raise RuntimeError(f"duplicate precision capability target {key}")
seen.add(key)
+ override = PRECISION_CELL_OVERRIDES.get(key, rule)
targets.append({
"precision_profile": profile_name,
"backend": rule["backend"],
"sku": sku,
"ep": ep,
"mode": rule["mode"],
- "disposition": rule["disposition"],
- "basis": rule["basis"],
+ "disposition": override["disposition"],
+ "basis": override["basis"],
})
return targets
@@ -541,6 +555,15 @@ def _validate_precision_capabilities() -> None:
)
if empty:
raise RuntimeError(f"precision profiles have no native targets: {empty}")
+ declared_keys = {
+ (
+ target["precision_profile"], target["backend"], target["sku"],
+ target["ep"], target["mode"],
+ )
+ for target in precision_targets()
+ }
+ if not set(PRECISION_CELL_OVERRIDES) <= declared_keys:
+ raise RuntimeError("precision cell override has no declared native target")
for target in precision_targets():
if target["backend"] not in BACKENDS or target["sku"] not in PLATFORMS:
raise RuntimeError(f"unknown precision target: {target}")
diff --git a/experimental/CollectiveX/docs/methodology.md b/experimental/CollectiveX/docs/methodology.md
index c5617f23e2..8a9e6a202b 100644
--- a/experimental/CollectiveX/docs/methodology.md
+++ b/experimental/CollectiveX/docs/methodology.md
@@ -36,9 +36,9 @@ normal `layout-and-dispatch-v1` or low-latency `expert-packed-weighted-combine-v
- `ep-precision-normal-v1`: nonbaseline native dispatch/combine profiles at decode T=128 and prefill
T=512; BF16/BF16 endpoint controls are referenced rather than duplicated.
- `ep-precision-low-latency-v1`: nonbaseline native low-latency profiles at decode T=128.
-- BF16 planning baseline: 608 requested cases / 1,600 token points; 364 runnable cases / 940
+- BF16 planning baseline: 608 requested cases / 1,600 token points; 338 runnable cases / 868
points in
- 58 executable workflow shards/allocation cells; 244 unsupported cases / 660 points.
+ 54 executable workflow shards/allocation cells; 270 unsupported cases / 732 points.
| Systems | EP8 | EP16 |
|---|---|---|
@@ -60,6 +60,9 @@ declared unsupported combination in v1 because NCCL 2.30.4 reports no Device API
support for its EP8 communicator; that pool can return only after all-rank CUDA P2P/LSA support is
restored. This baseline omits `[cl]`, `[rv]`, quantization, alternate activation/routing profiles,
uneven allocation, placement permutations, model envelopes, and scaling.
+H100 EP16 is also planned unsupported on the current runner pool because allocated compute nodes
+expose no active RDMA device. H100 EP8 remains in scope with an operator-selected compute-visible
+shared stage; unlike the B300 runner account home, the H100 account home is not compute-visible.
FlashInfer is excluded from v1 after repeatable intermittent execution failures; those failures are
not converted into planned-unsupported coverage.
MoRI EP8 uses MI325X AsyncLL or MI355X IntraNode in normal mode. EP16 uses pinned InterNodeV1 over
@@ -243,6 +246,9 @@ including a safely identified partial claim. The same-run V2/Hybrid source archi
under fixed member and expanded-size bounds, and only the selected pinned root is extracted; a
symlink is accepted only when it is a relative leaf pointing to a regular member inside the same
backend root, followed by exact Git tree/submodule validation.
+H200, B200, and B300 may derive that private base beneath the validated operating-system account
+home when it is compute-visible. H100 requires an explicit shared `stage_dir`; the launcher still
+proves cross-node visibility before any benchmark starts.
## Artifact Validation And JIT Delivery
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 91585019d6..f8f4d5e27a 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -1598,7 +1598,7 @@ cx_lock_canonical_gha_env() {
|| cx_die "canonical CollectiveX execution requires shared container storage"
if [ -z "$trusted_stage_dir" ]; then
case "$runner" in
- h100-dgxc|h200-dgxc|b200-dgxc)
+ h100-dgxc|h200-dgxc|b200-dgxc|b300)
trusted_stage_dir="$(cx_prepare_implicit_stage_base)" \
|| cx_die "canonical CollectiveX execution cannot create an isolated stage directory"
;;
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
index 8d289b1826..c09d9cea84 100644
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -28,7 +28,7 @@ def test_precision_probe_inventory_is_exact_and_non_mutating(self) -> None:
item["precision_profile"],
)
self.assertEqual(targets, sorted(capability.provisional_precision_targets(), key=key))
- self.assertEqual(len(targets), 94)
+ self.assertEqual(len(targets), 85)
self.assertEqual(capability.PRECISION_CAPABILITIES, before)
self.assertEqual(
len({
@@ -217,8 +217,16 @@ def test_precision_profiles_bind_exact_formats_and_timing_boundaries(self) -> No
def test_capability_registry_uses_exact_native_targets(self) -> None:
targets = capability.precision_targets()
self.assertTrue(targets)
- self.assertTrue(all(item["disposition"] == "provisional" for item in targets))
- self.assertEqual(targets, capability.provisional_precision_targets())
+ self.assertEqual(
+ {item["disposition"] for item in targets}, {"provisional", "unsupported"}
+ )
+ self.assertEqual(
+ len(targets) - len(capability.provisional_precision_targets()), 9
+ )
+ self.assertTrue(all(
+ item["sku"] == "h100-dgxc" and item["ep"] == 16
+ for item in targets if item["disposition"] == "unsupported"
+ ))
keys = {
(
item["precision_profile"],
@@ -243,6 +251,7 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
(("mi355x", "mori", 16, "normal", direct), "not-applicable"),
(("mi325x", "mori", 8, "normal", fnuz_direct), "provisional"),
(("h200-dgxc", "deepep", 8, "low-latency", low_latency), "provisional"),
+ (("h100-dgxc", "deepep", 16, "low-latency", low_latency), "unsupported"),
(("h200-dgxc", "deepep-hybrid", 8, "low-latency", low_latency),
"not-applicable"),
)
@@ -268,6 +277,12 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
precision_profile=identity.V1_CONTROL_PRECISION_PROFILE,
)
self.assertEqual(control, "supported")
+ h100_ep16, reason = capability.resolve_disposition(
+ "h100-dgxc", "nccl-ep", ep=16, nodes=2,
+ precision_profile=identity.V1_CONTROL_PRECISION_PROFILE,
+ )
+ self.assertEqual(h100_ep16, "unsupported")
+ self.assertEqual(reason, capability._H100_EP16_NO_RDMA_BASIS)
def test_split_suites_are_provisional_and_do_not_duplicate_bf16(self) -> None:
suites = sweep_matrix._load("suites.yaml")
@@ -347,10 +362,6 @@ def load_config(name: str) -> dict[str, object]:
]["phases"])
for target in capability.precision_targets()
)
- unsupported_targets = [
- target for target in capability.precision_targets([normal_profile])
- if target["backend"] == "deepep"
- ]
with mock.patch.object(capability, "PRECISION_CAPABILITIES", promoted):
with self.assertRaisesRegex(SystemExit, "must track unresolved"):
sweep_matrix.validate_config_documents(suites, workloads)
@@ -358,6 +369,19 @@ def load_config(name: str) -> dict[str, object]:
matrix = sweep_matrix.validate_matrix_document(
sweep_matrix.resolve_matrix(suites=suite_names, backends="all")
)
+ expected_unsupported_cases = sum(
+ len(resolved_suites["suites"][
+ "ep-precision-normal-v1"
+ if target["mode"] == "normal"
+ else "ep-precision-low-latency-v1"
+ ]["phases"])
+ for target in capability.precision_targets()
+ if capability.resolve_disposition(
+ target["sku"], target["backend"], ep=target["ep"],
+ nodes=capability.topology_for(target["sku"], target["ep"])["nodes"],
+ mode=target["mode"], precision_profile=target["precision_profile"],
+ )[0] != "supported"
+ )
unsupported = [
item for item in matrix["requested_cases"]
@@ -365,8 +389,7 @@ def load_config(name: str) -> dict[str, object]:
]
self.assertEqual(
len(unsupported),
- len(unsupported_targets)
- * len(resolved_suites["suites"]["ep-precision-normal-v1"]["phases"]),
+ expected_unsupported_cases,
)
self.assertTrue(all(
item["reason"] == "precision-profile-unsupported" for item in unsupported
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index a92a8e8271..5dc4cbad58 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -208,7 +208,7 @@ def test_matrix_cases_and_shards_are_identity_bound(self) -> None:
sum(len(item["case"]["ladder"].split()) for item in runnable_cases),
sum(len(item["case"]["ladder"].split()) for item in unsupported_cases),
),
- (58, 608, 364, 244, 1600, 940, 660),
+ (54, 608, 338, 270, 1600, 868, 732),
)
expected_topologies = {}
for sku, product in (
@@ -263,7 +263,7 @@ def test_matrix_cases_and_shards_are_identity_bound(self) -> None:
{shard["n"] for shard in matrix["include"]}, {6, 7}
)
self.assertEqual(
- sum(shard["n"] == 7 for shard in matrix["include"]), 16
+ sum(shard["n"] == 7 for shard in matrix["include"]), 14
)
ll_cases = [
item for item in matrix["requested_cases"]
@@ -2401,6 +2401,12 @@ def test_canonical_gha_environment_is_locked_but_manual_overrides_survive(self)
cx_lock_canonical_gha_env b200-dgxc
test "$CX_STAGE_DIR" = "$TEST_IMPLICIT_STAGE"
+ export COLLECTIVEX_OPERATOR_CONFIG_LOADED=$$
+ export CX_SHARD_SKU=b300 CX_NODES=1 CX_GPUS_PER_NODE=8
+ unset CX_STAGE_DIR
+ cx_lock_canonical_gha_env b300
+ test "$CX_STAGE_DIR" = "$TEST_IMPLICIT_STAGE"
+
export CX_STAGE_DIR=/shared/gb-stage
export CX_SHARD_SKU=gb300 CX_NODES=2 CX_GPUS_PER_NODE=4
export CX_IMAGE=untrusted CX_NGPUS=1 CX_MORI_KERNEL_TYPE=untrusted
From 5e96e43eeaa455470d96f79df959fcd38e12b85a Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 18:42:42 +0800
Subject: [PATCH 030/210] fix(collectivex): align shared stage schema
---
experimental/CollectiveX/runtime/common.sh | 10 ++++------
.../CollectiveX/tests/test_deepep_v2_contract.py | 9 ++++++++-
2 files changed, 12 insertions(+), 7 deletions(-)
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index f8f4d5e27a..bae2bd1133 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -235,22 +235,20 @@ NETWORK_FIELDS = {
"socket_ifname", "rdma_devices", "ib_gid_index", "rdma_service_level",
}
REQUIRED = {
- "h100-dgxc": {"partition", "account", "squash_dir"},
+ "h100-dgxc": {"partition", "account", "squash_dir", "stage_dir"},
"h200-dgxc": {"partition", "squash_dir"},
"b200-dgxc": {"partition", "account", "squash_dir"},
- "b300": {
- "partition", "account", "squash_dir", "stage_dir",
- },
+ "b300": {"partition", "account", "squash_dir"},
"gb200": {"partition", "account", "storage_roots"},
"gb300": {"partition", "account", "squash_dir", "stage_dir", "enroot_cache_path"},
"mi325x": {"partition", "squash_dir", "stage_dir"},
"mi355x": {"partition", "squash_dir", "stage_dir"},
}
ALLOWED = {
- "h100-dgxc": REQUIRED["h100-dgxc"] | {"exclude_nodes", "stage_dir"} | NETWORK_FIELDS,
+ "h100-dgxc": REQUIRED["h100-dgxc"] | {"exclude_nodes"} | NETWORK_FIELDS,
"h200-dgxc": REQUIRED["h200-dgxc"] | {"account", "exclude_nodes", "stage_dir"} | NETWORK_FIELDS,
"b200-dgxc": REQUIRED["b200-dgxc"] | {"exclude_nodes", "stage_dir"} | NETWORK_FIELDS,
- "b300": REQUIRED["b300"] | {"exclude_nodes"} | NETWORK_FIELDS,
+ "b300": REQUIRED["b300"] | {"exclude_nodes", "stage_dir"} | NETWORK_FIELDS,
"gb200": REQUIRED["gb200"] | NETWORK_FIELDS,
"gb300": REQUIRED["gb300"] | NETWORK_FIELDS,
"mi325x": REQUIRED["mi325x"] | {"exclude_nodes", "nodelist", "stage_dir", "lock_dir"} | NETWORK_FIELDS,
diff --git a/experimental/CollectiveX/tests/test_deepep_v2_contract.py b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
index 994a17e3f7..7a63bdd2a6 100644
--- a/experimental/CollectiveX/tests/test_deepep_v2_contract.py
+++ b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
@@ -2061,9 +2061,15 @@ def test_operator_config_is_strict_per_runner_json(self) -> None:
del missing_rdma["runners"]["mi355x"]["rdma_devices"]
missing_amd_stage = json.loads(json.dumps(document))
del missing_amd_stage["runners"]["mi325x"]["stage_dir"]
+ missing_h100_stage = json.loads(json.dumps(document))
+ del missing_h100_stage["runners"]["h100-dgxc"]["stage_dir"]
+ missing_b300_stage = json.loads(json.dumps(document))
+ del missing_b300_stage["runners"]["b300"]["stage_dir"]
missing_nvidia_account = json.loads(json.dumps(document))
del missing_nvidia_account["runners"]["h100-dgxc"]["account"]
- for invalid in (rejected, boolean_version, missing_nvidia_account):
+ for invalid in (
+ rejected, boolean_version, missing_h100_stage, missing_nvidia_account,
+ ):
config.write_text(json.dumps(invalid))
config.chmod(0o600)
result = subprocess.run(
@@ -2083,6 +2089,7 @@ def test_operator_config_is_strict_per_runner_json(self) -> None:
(missing_socket, "h100-dgxc"),
(missing_rdma, "h100-dgxc"),
(missing_amd_stage, "h100-dgxc"),
+ (missing_b300_stage, "b300"),
):
config.write_text(json.dumps(valid))
config.chmod(0o600)
From 91a08ab236fbc0578094fe780011189b0b64f085 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 18:43:47 +0800
Subject: [PATCH 031/210] fix(collectivex): stage H100 on shared storage
---
experimental/CollectiveX/README.md | 11 ++++++-----
experimental/CollectiveX/docs/methodology.md | 10 ++++++----
experimental/CollectiveX/runtime/common.sh | 10 +++++++---
.../CollectiveX/tests/test_deepep_v2_contract.py | 5 ++---
.../CollectiveX/tests/test_sampling_contract.py | 13 ++++++++++++-
5 files changed, 33 insertions(+), 16 deletions(-)
diff --git a/experimental/CollectiveX/README.md b/experimental/CollectiveX/README.md
index ea02c4f7dd..21ea9c1d0a 100644
--- a/experimental/CollectiveX/README.md
+++ b/experimental/CollectiveX/README.md
@@ -83,8 +83,8 @@ requires an all-rank CUDA P2P/LSA-capable runtime. Other NVIDIA SKUs remain unva
GPU outcomes pass the native correctness and publication gates.
H100 EP16 is planned unsupported on the current runner pool because allocated compute nodes expose
-no active RDMA device. EP8 remains in scope, but requires a runner-owned shared `stage_dir` because
-the runner account home is not mounted on compute nodes.
+no active RDMA device. EP8 remains in scope and derives its private stage beside the existing shared
+container directory because the runner account home is not mounted on compute nodes.
Axes not implemented in this baseline include cached-layout `[cl]`, runtime-visible `[rv]`, FP8,
quantized combine,
@@ -128,7 +128,7 @@ temporary copy before allocation. Required JSON fields are:
| SKU | Variables |
|---|---|
-| `h100-dgxc` | `partition`, `account`, `squash_dir`, `stage_dir` |
+| `h100-dgxc` | `partition`, `account`, `squash_dir` |
| `h200-dgxc` | `partition`, `squash_dir` |
| `b200-dgxc` | `partition`, `account`, `squash_dir` |
| `b300` | `partition`, `account`, `squash_dir` |
@@ -156,8 +156,9 @@ the workflow's temporary `HOME`. A symlinked account-home entry is resolved once
runner-owned target; the single hidden staging base directly beneath it must itself be non-symlinked
and not writable by other users. The workflow still proves that base is visible from every allocated
node before launch. The execution child is validated, marked, cross-node checked, and removed using
-the same contract. H100 and all other runners require the dedicated `stage_dir` above; no canonical run
-stages source beneath shared container storage.
+the same contract. H100 may also omit `stage_dir`; its private base is created beside, never beneath,
+the configured shared container directory so it is compute-visible. All other runners require the
+dedicated `stage_dir` above; no canonical run stages source beneath shared container storage.
Before import, each Docker Hub tag is resolved with bounded registry requests and must match its
pinned digest; digest-qualified overrides are rejected. Enroot imports use a fixed filesystem epoch
diff --git a/experimental/CollectiveX/docs/methodology.md b/experimental/CollectiveX/docs/methodology.md
index 8a9e6a202b..54e88f1a92 100644
--- a/experimental/CollectiveX/docs/methodology.md
+++ b/experimental/CollectiveX/docs/methodology.md
@@ -61,8 +61,9 @@ support for its EP8 communicator; that pool can return only after all-rank CUDA
restored. This baseline omits `[cl]`, `[rv]`, quantization, alternate activation/routing profiles,
uneven allocation, placement permutations, model envelopes, and scaling.
H100 EP16 is also planned unsupported on the current runner pool because allocated compute nodes
-expose no active RDMA device. H100 EP8 remains in scope with an operator-selected compute-visible
-shared stage; unlike the B300 runner account home, the H100 account home is not compute-visible.
+expose no active RDMA device. H100 EP8 remains in scope with a private stage beside its configured
+shared container directory; unlike the B300 runner account home, the H100 account home is not
+compute-visible.
FlashInfer is excluded from v1 after repeatable intermittent execution failures; those failures are
not converted into planned-unsupported coverage.
MoRI EP8 uses MI325X AsyncLL or MI355X IntraNode in normal mode. EP16 uses pinned InterNodeV1 over
@@ -247,8 +248,9 @@ under fixed member and expanded-size bounds, and only the selected pinned root i
symlink is accepted only when it is a relative leaf pointing to a regular member inside the same
backend root, followed by exact Git tree/submodule validation.
H200, B200, and B300 may derive that private base beneath the validated operating-system account
-home when it is compute-visible. H100 requires an explicit shared `stage_dir`; the launcher still
-proves cross-node visibility before any benchmark starts.
+home when it is compute-visible. H100 instead derives a sibling of its shared container directory,
+never a child of image storage. The launcher still proves cross-node visibility before any benchmark
+starts.
## Artifact Validation And JIT Delivery
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index bae2bd1133..c3928f55c9 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -235,7 +235,7 @@ NETWORK_FIELDS = {
"socket_ifname", "rdma_devices", "ib_gid_index", "rdma_service_level",
}
REQUIRED = {
- "h100-dgxc": {"partition", "account", "squash_dir", "stage_dir"},
+ "h100-dgxc": {"partition", "account", "squash_dir"},
"h200-dgxc": {"partition", "squash_dir"},
"b200-dgxc": {"partition", "account", "squash_dir"},
"b300": {"partition", "account", "squash_dir"},
@@ -245,7 +245,7 @@ REQUIRED = {
"mi355x": {"partition", "squash_dir", "stage_dir"},
}
ALLOWED = {
- "h100-dgxc": REQUIRED["h100-dgxc"] | {"exclude_nodes"} | NETWORK_FIELDS,
+ "h100-dgxc": REQUIRED["h100-dgxc"] | {"exclude_nodes", "stage_dir"} | NETWORK_FIELDS,
"h200-dgxc": REQUIRED["h200-dgxc"] | {"account", "exclude_nodes", "stage_dir"} | NETWORK_FIELDS,
"b200-dgxc": REQUIRED["b200-dgxc"] | {"exclude_nodes", "stage_dir"} | NETWORK_FIELDS,
"b300": REQUIRED["b300"] | {"exclude_nodes", "stage_dir"} | NETWORK_FIELDS,
@@ -1596,7 +1596,11 @@ cx_lock_canonical_gha_env() {
|| cx_die "canonical CollectiveX execution requires shared container storage"
if [ -z "$trusted_stage_dir" ]; then
case "$runner" in
- h100-dgxc|h200-dgxc|b200-dgxc|b300)
+ h100-dgxc)
+ trusted_stage_dir="$(cx_prepare_implicit_stage_base "${CX_SQUASH_DIR%/*}")" \
+ || cx_die "canonical CollectiveX execution cannot create an isolated shared stage directory"
+ ;;
+ h200-dgxc|b200-dgxc|b300)
trusted_stage_dir="$(cx_prepare_implicit_stage_base)" \
|| cx_die "canonical CollectiveX execution cannot create an isolated stage directory"
;;
diff --git a/experimental/CollectiveX/tests/test_deepep_v2_contract.py b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
index 7a63bdd2a6..28861a0361 100644
--- a/experimental/CollectiveX/tests/test_deepep_v2_contract.py
+++ b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
@@ -2067,9 +2067,7 @@ def test_operator_config_is_strict_per_runner_json(self) -> None:
del missing_b300_stage["runners"]["b300"]["stage_dir"]
missing_nvidia_account = json.loads(json.dumps(document))
del missing_nvidia_account["runners"]["h100-dgxc"]["account"]
- for invalid in (
- rejected, boolean_version, missing_h100_stage, missing_nvidia_account,
- ):
+ for invalid in (rejected, boolean_version, missing_nvidia_account):
config.write_text(json.dumps(invalid))
config.chmod(0o600)
result = subprocess.run(
@@ -2089,6 +2087,7 @@ def test_operator_config_is_strict_per_runner_json(self) -> None:
(missing_socket, "h100-dgxc"),
(missing_rdma, "h100-dgxc"),
(missing_amd_stage, "h100-dgxc"),
+ (missing_h100_stage, "h100-dgxc"),
(missing_b300_stage, "b300"),
):
config.write_text(json.dumps(valid))
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 5dc4cbad58..48b5126abb 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -2394,7 +2394,17 @@ def test_canonical_gha_environment_is_locked_but_manual_overrides_survive(self)
test -z "${CX_PREPARED_BACKEND_CACHE+x}${CX_BACKEND_SOURCE_ROOT+x}"
test -z "${CX_DRYRUN+x}"
- cx_prepare_implicit_stage_base() { printf '%s' "$TEST_IMPLICIT_STAGE"; }
+ cx_prepare_implicit_stage_base() {
+ if [ -n "${1:-}" ]; then test "$1" = "$TEST_SHARED_PARENT"; fi
+ printf '%s' "$TEST_IMPLICIT_STAGE"
+ }
+ export COLLECTIVEX_OPERATOR_CONFIG_LOADED=$$
+ export CX_SHARD_SKU=h100-dgxc CX_NODES=1 CX_GPUS_PER_NODE=8
+ export CX_SQUASH_DIR="$TEST_SHARED_PARENT/containers"
+ unset CX_STAGE_DIR
+ cx_lock_canonical_gha_env h100-dgxc
+ test "$CX_STAGE_DIR" = "$TEST_IMPLICIT_STAGE"
+
export COLLECTIVEX_OPERATOR_CONFIG_LOADED=$$
export CX_SHARD_SKU=b200-dgxc CX_NODES=1 CX_GPUS_PER_NODE=8
unset CX_STAGE_DIR
@@ -2450,6 +2460,7 @@ def test_canonical_gha_environment_is_locked_but_manual_overrides_survive(self)
**os.environ,
"HOME": str(home),
"TEST_IMPLICIT_STAGE": str(implicit_stage),
+ "TEST_SHARED_PARENT": str(root),
"COLLECTIVEX_OPERATOR_CONFIG": "/dev/null",
"GITHUB_WORKSPACE": str(workspace),
},
From eab3aa1c4373ce1e8165aee5dfd6728c32598851 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 18:46:19 +0800
Subject: [PATCH 032/210] fix(collectivex): ignore stale B300 stage config
---
experimental/CollectiveX/README.md | 2 ++
experimental/CollectiveX/docs/methodology.md | 2 ++
experimental/CollectiveX/runtime/common.sh | 3 +++
experimental/CollectiveX/tests/test_sampling_contract.py | 2 +-
4 files changed, 8 insertions(+), 1 deletion(-)
diff --git a/experimental/CollectiveX/README.md b/experimental/CollectiveX/README.md
index 21ea9c1d0a..74611f1715 100644
--- a/experimental/CollectiveX/README.md
+++ b/experimental/CollectiveX/README.md
@@ -159,6 +159,8 @@ node before launch. The execution child is validated, marked, cross-node checked
the same contract. H100 may also omit `stage_dir`; its private base is created beside, never beneath,
the configured shared container directory so it is compute-visible. All other runners require the
dedicated `stage_dir` above; no canonical run stages source beneath shared container storage.
+Canonical B300 execution ignores any legacy configured `stage_dir` and always uses the validated
+compute-visible account-home base.
Before import, each Docker Hub tag is resolved with bounded registry requests and must match its
pinned digest; digest-qualified overrides are rejected. Enroot imports use a fixed filesystem epoch
diff --git a/experimental/CollectiveX/docs/methodology.md b/experimental/CollectiveX/docs/methodology.md
index 54e88f1a92..883b1f66ea 100644
--- a/experimental/CollectiveX/docs/methodology.md
+++ b/experimental/CollectiveX/docs/methodology.md
@@ -251,6 +251,8 @@ H200, B200, and B300 may derive that private base beneath the validated operatin
home when it is compute-visible. H100 instead derives a sibling of its shared container directory,
never a child of image storage. The launcher still proves cross-node visibility before any benchmark
starts.
+Canonical B300 execution ignores the legacy operator `stage_dir` field and always derives the base
+from the validated shared account home.
## Artifact Validation And JIT Delivery
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index c3928f55c9..421c64cf3b 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -1575,6 +1575,9 @@ cx_lock_canonical_gha_env() {
trusted_rdma_service_level="${CX_RDMA_SERVICE_LEVEL:-}"
trusted_audit_salt="${CX_AUDIT_SALT:-}"
fi
+ # The legacy B300 operator row contains a root-owned stage path. B300's
+ # compute-visible account home is the canonical source for its private base.
+ [ "$runner" != b300 ] || trusted_stage_dir=""
unset CX_NCCL_HOME CX_MASTER_PORT CX_MORI_KERNEL_TYPE CX_LOCK_DIR CX_STAGE_DIR
unset MASTER_ADDR MASTER_PORT RANK WORLD_SIZE LOCAL_RANK LOCAL_WORLD_SIZE
unset CX_SOCKET_IFNAME CX_RDMA_DEVICES CX_IB_GID_INDEX CX_RDMA_SERVICE_LEVEL
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 48b5126abb..117654a295 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -2413,7 +2413,7 @@ def test_canonical_gha_environment_is_locked_but_manual_overrides_survive(self)
export COLLECTIVEX_OPERATOR_CONFIG_LOADED=$$
export CX_SHARD_SKU=b300 CX_NODES=1 CX_GPUS_PER_NODE=8
- unset CX_STAGE_DIR
+ export CX_STAGE_DIR=/legacy/root-owned-stage
cx_lock_canonical_gha_env b300
test "$CX_STAGE_DIR" = "$TEST_IMPLICIT_STAGE"
From 4aa236a084f215f09c3d1c963ceeade582abc876 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 18:48:56 +0800
Subject: [PATCH 033/210] feat(collectivex): record validated precision cells
---
experimental/CollectiveX/README.md | 4 +--
experimental/CollectiveX/capability.py | 35 +++++++++++++++++++
experimental/CollectiveX/docs/methodology.md | 2 ++
.../tests/test_precision_scheduling.py | 27 +++++++-------
4 files changed, 54 insertions(+), 14 deletions(-)
diff --git a/experimental/CollectiveX/README.md b/experimental/CollectiveX/README.md
index 74611f1715..f131924ab8 100644
--- a/experimental/CollectiveX/README.md
+++ b/experimental/CollectiveX/README.md
@@ -13,8 +13,8 @@ combine, and paired roundtrip latency across EP libraries and accelerator system
> rankings, recommendations, or regression baselines.
> Development status: the V1 precision, point-level publication, branch-only delivery, and frontend
-> contracts are implemented. Native precision capability probes are still resolving provisional
-> cells, so the BF16 counts below remain a planning baseline and V1 counts/digests are not frozen.
+> contracts are implemented. Validated B200 EP8, H200 EP8, and GB200 EP8/EP16 probe outcomes are
+> recorded; 57 native precision cells remain provisional, so V1 counts/digests are not frozen.
## Implemented Pre-V1 Execution Profile
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
index f322f60bce..b892338656 100644
--- a/experimental/CollectiveX/capability.py
+++ b/experimental/CollectiveX/capability.py
@@ -313,6 +313,41 @@ def _precision_rule(
if "h100-dgxc" in rule["skus"] and 16 in rule["ep_degrees"]
}
+_VALIDATED_NATIVE_PROBE_CELLS = (
+ # run, SKU, EP, backend, mode, profile, disposition, result
+ ("28737315879", "b200-dgxc", 8, "deepep", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed"),
+ ("28737315879", "b200-dgxc", 8, "deepep", "low-latency", _LL_FP8_PROFILE, "supported", "native-probe-passed"),
+ ("28737315879", "b200-dgxc", 8, "deepep", "low-latency", _LL_LOGFMT_PROFILE, "supported", "native-probe-passed"),
+ ("28737315879", "b200-dgxc", 8, "deepep", "low-latency", _LL_FP8_LOGFMT_PROFILE, "supported", "native-probe-passed"),
+ ("28737315879", "b200-dgxc", 8, "deepep-hybrid", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "native-operation-failed"),
+ ("28737315879", "b200-dgxc", 8, "deepep-v2", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "backend-construction-failed"),
+ ("28737422303", "h200-dgxc", 8, "deepep", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed"),
+ ("28737422303", "h200-dgxc", 8, "deepep", "low-latency", _LL_FP8_PROFILE, "supported", "native-probe-passed"),
+ ("28737422303", "h200-dgxc", 8, "deepep", "low-latency", _LL_LOGFMT_PROFILE, "supported", "native-probe-passed"),
+ ("28737422303", "h200-dgxc", 8, "deepep", "low-latency", _LL_FP8_LOGFMT_PROFILE, "supported", "native-probe-passed"),
+ ("28737422303", "h200-dgxc", 8, "deepep-hybrid", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "native-operation-failed"),
+ ("28737422303", "h200-dgxc", 8, "deepep-v2", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "backend-construction-failed"),
+ *(("28737422303", "h200-dgxc", 8, "uccl", mode, profile, "supported", "native-probe-passed")
+ for profile, mode in ((_NORMAL_E4M3FN_PROFILE, "normal"), (_LL_FP8_PROFILE, "low-latency"),
+ (_LL_LOGFMT_PROFILE, "low-latency"), (_LL_FP8_LOGFMT_PROFILE, "low-latency"))),
+ *(("28737422902", "gb200", ep, "deepep", mode, profile, "supported", "native-probe-passed")
+ for ep in (8, 16)
+ for profile, mode in ((_NORMAL_E4M3FN_PROFILE, "normal"), (_LL_FP8_PROFILE, "low-latency"),
+ (_LL_LOGFMT_PROFILE, "low-latency"), (_LL_FP8_LOGFMT_PROFILE, "low-latency"))),
+ *(("28737422902", "gb200", ep, "deepep-v2", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed")
+ for ep in (8, 16)),
+ *(("28737422902", "gb200", ep, "deepep-hybrid", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "native-operation-failed")
+ for ep in (8, 16)),
+)
+PRECISION_CELL_OVERRIDES.update({
+ (profile, backend, sku, ep, mode): {
+ "basis": f"native-probe-v1-run-{run_id}-{result}",
+ "disposition": disposition,
+ }
+ for run_id, sku, ep, backend, mode, profile, disposition, result
+ in _VALIDATED_NATIVE_PROBE_CELLS
+})
+
def runtime_identity_issues(
sku: str, *, vendor: str, arch: str, machine: str, device_name: str,
diff --git a/experimental/CollectiveX/docs/methodology.md b/experimental/CollectiveX/docs/methodology.md
index 883b1f66ea..535ab87360 100644
--- a/experimental/CollectiveX/docs/methodology.md
+++ b/experimental/CollectiveX/docs/methodology.md
@@ -10,6 +10,8 @@ This document describes the implemented BF16 baseline. It is not yet the V1 qual
Before any V1-tagged run, this English document must match the implemented precision, measurement,
publication, and frontend contracts; counts and digests remain unfrozen. Chinese documentation
synchronization is explicitly deferred for the V1 implementation phase.
+Validated B200 EP8, H200 EP8, and GB200 EP8/EP16 native probe outcomes are recorded in the
+capability registry; 57 native precision cells remain provisional.
## Product Boundary
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
index c09d9cea84..142f47dfb6 100644
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -28,7 +28,7 @@ def test_precision_probe_inventory_is_exact_and_non_mutating(self) -> None:
item["precision_profile"],
)
self.assertEqual(targets, sorted(capability.provisional_precision_targets(), key=key))
- self.assertEqual(len(targets), 85)
+ self.assertEqual(len(targets), 57)
self.assertEqual(capability.PRECISION_CAPABILITIES, before)
self.assertEqual(
len({
@@ -53,10 +53,10 @@ def test_precision_probe_selects_only_one_exact_provisional_cell(self) -> None:
)
def test_precision_probe_workflow_plan_binds_exact_controls(self) -> None:
- plan = probe_precision.workflow_plan(backend="deepep", only_sku="b200-dgxc")
+ plan = probe_precision.workflow_plan(backend="deepep", only_sku="b300")
self.assertTrue(plan["include"])
self.assertTrue(all(
- row["backend"] == "deepep" and row["sku"] == "b200-dgxc"
+ row["backend"] == "deepep" and row["sku"] == "b300"
for row in plan["include"]
))
row = plan["include"][0]
@@ -78,7 +78,7 @@ def test_precision_probe_workflow_plan_binds_exact_controls(self) -> None:
with self.assertRaisesRegex(ValueError, "select no provisional"):
probe_precision.workflow_plan(backend="mori", only_sku="b200-dgxc")
ep8 = probe_precision.workflow_plan(
- backend="deepep", only_sku="b200-dgxc", max_nodes=1,
+ backend="deepep", only_sku="b300", max_nodes=1,
)
self.assertTrue(ep8["include"])
self.assertEqual({row["ep"] for row in ep8["include"]}, {8})
@@ -218,15 +218,18 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
targets = capability.precision_targets()
self.assertTrue(targets)
self.assertEqual(
- {item["disposition"] for item in targets}, {"provisional", "unsupported"}
+ {item["disposition"] for item in targets},
+ {"provisional", "supported", "unsupported"},
)
self.assertEqual(
- len(targets) - len(capability.provisional_precision_targets()), 9
+ len(targets) - len(capability.provisional_precision_targets()), 37
+ )
+ self.assertEqual(
+ sum(item["disposition"] == "supported" for item in targets), 22
+ )
+ self.assertEqual(
+ sum(item["disposition"] == "unsupported" for item in targets), 15
)
- self.assertTrue(all(
- item["sku"] == "h100-dgxc" and item["ep"] == 16
- for item in targets if item["disposition"] == "unsupported"
- ))
keys = {
(
item["precision_profile"],
@@ -244,13 +247,13 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
fnuz_direct = "d-bf16.c-fp8-e4m3fnuz-direct-cast-noscale"
low_latency = "d-bf16.c-logfmt10-dynamic64"
cases = (
- (("h200-dgxc", "deepep-v2", 8, "normal", normal), "provisional"),
+ (("h200-dgxc", "deepep-v2", 8, "normal", normal), "unsupported"),
(("h100-dgxc", "deepep-v2", 8, "normal", normal), "not-applicable"),
(("h200-dgxc", "nccl-ep", 8, "normal", normal), "not-applicable"),
(("mi355x", "mori", 8, "normal", direct), "provisional"),
(("mi355x", "mori", 16, "normal", direct), "not-applicable"),
(("mi325x", "mori", 8, "normal", fnuz_direct), "provisional"),
- (("h200-dgxc", "deepep", 8, "low-latency", low_latency), "provisional"),
+ (("h200-dgxc", "deepep", 8, "low-latency", low_latency), "supported"),
(("h100-dgxc", "deepep", 16, "low-latency", low_latency), "unsupported"),
(("h200-dgxc", "deepep-hybrid", 8, "low-latency", low_latency),
"not-applicable"),
From 99b08a647789ae3874ee04aaab78485c801b4f0e Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 18:53:33 +0800
Subject: [PATCH 034/210] fix(collectivex): isolate B300 staging by execution
---
experimental/CollectiveX/README.md | 5 ++-
experimental/CollectiveX/docs/methodology.md | 5 ++-
experimental/CollectiveX/runtime/common.sh | 39 +++++++++++++++----
.../tests/test_sampling_contract.py | 6 +++
4 files changed, 46 insertions(+), 9 deletions(-)
diff --git a/experimental/CollectiveX/README.md b/experimental/CollectiveX/README.md
index f131924ab8..45246df14d 100644
--- a/experimental/CollectiveX/README.md
+++ b/experimental/CollectiveX/README.md
@@ -160,7 +160,10 @@ the same contract. H100 may also omit `stage_dir`; its private base is created b
the configured shared container directory so it is compute-visible. All other runners require the
dedicated `stage_dir` above; no canonical run stages source beneath shared container storage.
Canonical B300 execution ignores any legacy configured `stage_dir` and always uses the validated
-compute-visible account-home base.
+compute-visible account-home base. Its UID-mapped Actions shell may accept that exact base when its
+owner matches the private parent owner; this exception is not available to configured stages or
+other runners. A hashed execution-ID suffix isolates parallel B300 workers without exposing private
+runner identity.
Before import, each Docker Hub tag is resolved with bounded registry requests and must match its
pinned digest; digest-qualified overrides are rejected. Enroot imports use a fixed filesystem epoch
diff --git a/experimental/CollectiveX/docs/methodology.md b/experimental/CollectiveX/docs/methodology.md
index 535ab87360..f224fdcf5e 100644
--- a/experimental/CollectiveX/docs/methodology.md
+++ b/experimental/CollectiveX/docs/methodology.md
@@ -254,7 +254,10 @@ home when it is compute-visible. H100 instead derives a sibling of its shared co
never a child of image storage. The launcher still proves cross-node visibility before any benchmark
starts.
Canonical B300 execution ignores the legacy operator `stage_dir` field and always derives the base
-from the validated shared account home.
+from the validated shared account home. Its UID-mapped Actions shell may accept that exact base when
+its owner matches the private parent owner; explicit stages and all other runners retain the strict
+effective-UID ownership rule. A hashed execution-ID suffix isolates parallel B300 workers without
+exposing private runner identity.
## Artifact Validation And JIT Delivery
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 421c64cf3b..c9f87f4c66 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -1496,7 +1496,8 @@ cx_materialize_backend_source() {
}
cx_prepare_implicit_stage_base() {
- python3 - "${1:-}" <<'PY'
+ python3 - "${1:-}" "${2:-}" <<'PY'
+import hashlib
import os
from pathlib import Path
import pwd
@@ -1526,8 +1527,13 @@ if stat.S_IMODE(metadata.st_mode) & stat.S_IWGRP:
reject("home-group-writable")
if stat.S_IMODE(metadata.st_mode) & stat.S_IWOTH:
reject("home-world-writable")
+home_owner = metadata.st_uid
try:
- current = home / ".inferencex-collectivex-stage"
+ isolation_key = sys.argv[2]
+ suffix = ""
+ if isolation_key:
+ suffix = "-" + hashlib.sha256(isolation_key.encode("utf-8")).hexdigest()[:16]
+ current = home / f".inferencex-collectivex-stage{suffix}"
try:
os.mkdir(current, mode=0o700)
except FileExistsError:
@@ -1535,7 +1541,7 @@ try:
metadata = os.stat(current, follow_symlinks=False)
if not stat.S_ISDIR(metadata.st_mode):
reject("child-type")
- if metadata.st_uid != os.getuid():
+ if metadata.st_uid not in {os.getuid(), home_owner}:
reject("child-owner")
if Path(os.path.realpath(current)) != current:
reject("child-symlink")
@@ -1579,6 +1585,7 @@ cx_lock_canonical_gha_env() {
# compute-visible account home is the canonical source for its private base.
[ "$runner" != b300 ] || trusted_stage_dir=""
unset CX_NCCL_HOME CX_MASTER_PORT CX_MORI_KERNEL_TYPE CX_LOCK_DIR CX_STAGE_DIR
+ unset CX_STAGE_PARENT_OWNER_OK
unset MASTER_ADDR MASTER_PORT RANK WORLD_SIZE LOCAL_RANK LOCAL_WORLD_SIZE
unset CX_SOCKET_IFNAME CX_RDMA_DEVICES CX_IB_GID_INDEX CX_RDMA_SERVICE_LEVEL
unset CX_AUDIT_SALT
@@ -1603,7 +1610,12 @@ cx_lock_canonical_gha_env() {
trusted_stage_dir="$(cx_prepare_implicit_stage_base "${CX_SQUASH_DIR%/*}")" \
|| cx_die "canonical CollectiveX execution cannot create an isolated shared stage directory"
;;
- h200-dgxc|b200-dgxc|b300)
+ b300)
+ trusted_stage_dir="$(cx_prepare_implicit_stage_base "" \
+ "${COLLECTIVEX_EXECUTION_ID:-${GITHUB_RUN_ID:-}}")" \
+ || cx_die "canonical CollectiveX execution cannot create an isolated stage directory"
+ ;;
+ h200-dgxc|b200-dgxc)
trusted_stage_dir="$(cx_prepare_implicit_stage_base)" \
|| cx_die "canonical CollectiveX execution cannot create an isolated stage directory"
;;
@@ -1612,6 +1624,9 @@ cx_lock_canonical_gha_env() {
fi
[[ "$trusted_audit_salt" =~ ^[0-9a-f]{64}$ ]] \
|| cx_die "canonical CollectiveX execution requires a private audit salt"
+ if [ "$runner" = b300 ]; then
+ CX_STAGE_PARENT_OWNER_OK=1
+ fi
case "$runner" in
h100-dgxc|h200-dgxc|b200-dgxc|b300)
@@ -1935,12 +1950,13 @@ cx_stage_path() {
stage_path="${stage_base%/}/job_$safe_tag"
fi
python3 - "$repo_root" "$stage_base" "$stage_path" \
- "${CX_JOB_ROOT:-}" "${GITHUB_WORKSPACE:-}" <<'PY'
+ "${CX_JOB_ROOT:-}" "${GITHUB_WORKSPACE:-}" \
+ "${CX_STAGE_PARENT_OWNER_OK:-0}" <<'PY'
import os
import stat
import sys
-repo, base, child, job_root, workspace = sys.argv[1:]
+repo, base, child, job_root, workspace, allow_parent_owner = sys.argv[1:]
def reject(reason):
print(f"[collectivex] FATAL: stage-base-validation={reason}", file=sys.stderr)
raise SystemExit(1)
@@ -1970,7 +1986,16 @@ try:
if not stat.S_ISDIR(metadata.st_mode):
reject("not-directory")
if metadata.st_uid != os.getuid():
- reject("owner")
+ if allow_parent_owner != "1":
+ reject("owner")
+ parent = os.path.dirname(base.rstrip("/"))
+ parent_metadata = os.stat(parent, follow_symlinks=False)
+ if (
+ not stat.S_ISDIR(parent_metadata.st_mode)
+ or parent_metadata.st_uid != metadata.st_uid
+ or stat.S_IMODE(parent_metadata.st_mode) & (stat.S_IWGRP | stat.S_IWOTH)
+ ):
+ reject("parent-owner")
if stat.S_IMODE(metadata.st_mode) & stat.S_IWGRP:
reject("group-writable")
if stat.S_IMODE(metadata.st_mode) & stat.S_IWOTH:
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 117654a295..5a69ee401a 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -2396,6 +2396,7 @@ def test_canonical_gha_environment_is_locked_but_manual_overrides_survive(self)
cx_prepare_implicit_stage_base() {
if [ -n "${1:-}" ]; then test "$1" = "$TEST_SHARED_PARENT"; fi
+ if [ -n "${2:-}" ]; then test "$2" = 123; fi
printf '%s' "$TEST_IMPLICIT_STAGE"
}
export COLLECTIVEX_OPERATOR_CONFIG_LOADED=$$
@@ -2416,6 +2417,7 @@ def test_canonical_gha_environment_is_locked_but_manual_overrides_survive(self)
export CX_STAGE_DIR=/legacy/root-owned-stage
cx_lock_canonical_gha_env b300
test "$CX_STAGE_DIR" = "$TEST_IMPLICIT_STAGE"
+ test "$CX_STAGE_PARENT_OWNER_OK" = 1
export CX_STAGE_DIR=/shared/gb-stage
export CX_SHARD_SKU=gb300 CX_NODES=2 CX_GPUS_PER_NODE=4
@@ -2427,6 +2429,7 @@ def test_canonical_gha_environment_is_locked_but_manual_overrides_survive(self)
test "$CX_NGPUS:$CX_SEED:$CX_RUN_TIMEOUT" = 8:67:900
test "$CX_NCCL_HOME:$CX_MASTER_PORT" = /usr:29551
test "$CX_STAGE_DIR" = /shared/gb-stage
+ test -z "${CX_STAGE_PARENT_OWNER_OK+x}"
test -z "${CX_MORI_KERNEL_TYPE+x}${MORI_ENABLE_SDMA+x}"
export COLLECTIVEX_OPERATOR_CONFIG_LOADED=$$
@@ -2490,6 +2493,9 @@ def test_implicit_stage_base_is_private_and_non_symlinked(self) -> None:
test "$(stat -c '%a' "$base" 2>/dev/null || stat -f '%Lp' "$base")" = 700
! cx_prepare_implicit_stage_base "$3"
test "$(cx_prepare_implicit_stage_base "$4")" = "$2/.inferencex-collectivex-stage"
+ isolated="$(cx_prepare_implicit_stage_base "$2" runner-a)"
+ test "$isolated" != "$2/.inferencex-collectivex-stage"
+ case "$isolated" in "$2"/.inferencex-collectivex-stage-*) ;; *) exit 1 ;; esac
! cx_prepare_implicit_stage_base "$5"
'''
subprocess.run(
From 7d0dd462b33fc9a0d05848d264749279729183bc Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 18:55:37 +0800
Subject: [PATCH 035/210] fix(collectivex): handle B300 NFS UID mapping
---
experimental/CollectiveX/README.md | 3 ++-
experimental/CollectiveX/docs/methodology.md | 3 ++-
experimental/CollectiveX/runtime/common.sh | 8 ++++++--
3 files changed, 10 insertions(+), 4 deletions(-)
diff --git a/experimental/CollectiveX/README.md b/experimental/CollectiveX/README.md
index 45246df14d..4d0080ba28 100644
--- a/experimental/CollectiveX/README.md
+++ b/experimental/CollectiveX/README.md
@@ -163,7 +163,8 @@ Canonical B300 execution ignores any legacy configured `stage_dir` and always us
compute-visible account-home base. Its UID-mapped Actions shell may accept that exact base when its
owner matches the private parent owner; this exception is not available to configured stages or
other runners. A hashed execution-ID suffix isolates parallel B300 workers without exposing private
-runner identity.
+runner identity. On this NFS export, a newly created hashed base may realize as UID 0; only that
+creation path is accepted, while a pre-existing root-owned base is rejected.
Before import, each Docker Hub tag is resolved with bounded registry requests and must match its
pinned digest; digest-qualified overrides are rejected. Enroot imports use a fixed filesystem epoch
diff --git a/experimental/CollectiveX/docs/methodology.md b/experimental/CollectiveX/docs/methodology.md
index f224fdcf5e..111225f353 100644
--- a/experimental/CollectiveX/docs/methodology.md
+++ b/experimental/CollectiveX/docs/methodology.md
@@ -257,7 +257,8 @@ Canonical B300 execution ignores the legacy operator `stage_dir` field and alway
from the validated shared account home. Its UID-mapped Actions shell may accept that exact base when
its owner matches the private parent owner; explicit stages and all other runners retain the strict
effective-UID ownership rule. A hashed execution-ID suffix isolates parallel B300 workers without
-exposing private runner identity.
+exposing private runner identity. The current NFS export may realize a newly created hashed base as
+UID 0; only that creation path is accepted, while a pre-existing root-owned base is rejected.
## Artifact Validation And JIT Delivery
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index c9f87f4c66..d895c03714 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -1534,14 +1534,18 @@ try:
if isolation_key:
suffix = "-" + hashlib.sha256(isolation_key.encode("utf-8")).hexdigest()[:16]
current = home / f".inferencex-collectivex-stage{suffix}"
+ created = False
try:
os.mkdir(current, mode=0o700)
+ created = True
except FileExistsError:
pass
metadata = os.stat(current, follow_symlinks=False)
if not stat.S_ISDIR(metadata.st_mode):
reject("child-type")
- if metadata.st_uid not in {os.getuid(), home_owner}:
+ if metadata.st_uid not in {os.getuid(), home_owner} and not (
+ isolation_key and created and metadata.st_uid == 0
+ ):
reject("child-owner")
if Path(os.path.realpath(current)) != current:
reject("child-symlink")
@@ -1992,7 +1996,7 @@ try:
parent_metadata = os.stat(parent, follow_symlinks=False)
if (
not stat.S_ISDIR(parent_metadata.st_mode)
- or parent_metadata.st_uid != metadata.st_uid
+ or metadata.st_uid not in {parent_metadata.st_uid, 0}
or stat.S_IMODE(parent_metadata.st_mode) & (stat.S_IWGRP | stat.S_IWOTH)
):
reject("parent-owner")
From 929047c703e59bade66b20919c7b31796a8f2559 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 18:57:20 +0800
Subject: [PATCH 036/210] fix(collectivex): isolate GB300 staging
---
experimental/CollectiveX/README.md | 4 +++-
experimental/CollectiveX/docs/methodology.md | 2 ++
experimental/CollectiveX/runtime/common.sh | 11 ++++++++---
.../CollectiveX/tests/test_deepep_v2_contract.py | 3 +++
.../CollectiveX/tests/test_sampling_contract.py | 5 +++--
5 files changed, 19 insertions(+), 6 deletions(-)
diff --git a/experimental/CollectiveX/README.md b/experimental/CollectiveX/README.md
index 4d0080ba28..037490c158 100644
--- a/experimental/CollectiveX/README.md
+++ b/experimental/CollectiveX/README.md
@@ -133,7 +133,7 @@ temporary copy before allocation. Required JSON fields are:
| `b200-dgxc` | `partition`, `account`, `squash_dir` |
| `b300` | `partition`, `account`, `squash_dir` |
| `gb200` | `partition`, `account`, ordered `storage_roots` |
-| `gb300` | `partition`, `account`, `squash_dir`, `stage_dir`, `enroot_cache_path` |
+| `gb300` | `partition`, `account`, `squash_dir`, `enroot_cache_path` |
| `mi325x`, `mi355x` | `partition`, `squash_dir`, `stage_dir` |
Every selected non-MNNVL EP16 placement additionally requires `socket_ifname` and `rdma_devices`
@@ -165,6 +165,8 @@ owner matches the private parent owner; this exception is not available to confi
other runners. A hashed execution-ID suffix isolates parallel B300 workers without exposing private
runner identity. On this NFS export, a newly created hashed base may realize as UID 0; only that
creation path is accepted, while a pre-existing root-owned base is rejected.
+Canonical GB300 execution likewise ignores its legacy group-writable `stage_dir` and derives an
+execution-hashed private base beneath the validated compute-visible account home.
Before import, each Docker Hub tag is resolved with bounded registry requests and must match its
pinned digest; digest-qualified overrides are rejected. Enroot imports use a fixed filesystem epoch
diff --git a/experimental/CollectiveX/docs/methodology.md b/experimental/CollectiveX/docs/methodology.md
index 111225f353..faf8b41f94 100644
--- a/experimental/CollectiveX/docs/methodology.md
+++ b/experimental/CollectiveX/docs/methodology.md
@@ -259,6 +259,8 @@ its owner matches the private parent owner; explicit stages and all other runner
effective-UID ownership rule. A hashed execution-ID suffix isolates parallel B300 workers without
exposing private runner identity. The current NFS export may realize a newly created hashed base as
UID 0; only that creation path is accepted, while a pre-existing root-owned base is rejected.
+Canonical GB300 execution likewise ignores its legacy group-writable `stage_dir` and derives an
+execution-hashed private base beneath the validated compute-visible account home.
## Artifact Validation And JIT Delivery
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index d895c03714..1d29370d57 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -240,7 +240,7 @@ REQUIRED = {
"b200-dgxc": {"partition", "account", "squash_dir"},
"b300": {"partition", "account", "squash_dir"},
"gb200": {"partition", "account", "storage_roots"},
- "gb300": {"partition", "account", "squash_dir", "stage_dir", "enroot_cache_path"},
+ "gb300": {"partition", "account", "squash_dir", "enroot_cache_path"},
"mi325x": {"partition", "squash_dir", "stage_dir"},
"mi355x": {"partition", "squash_dir", "stage_dir"},
}
@@ -250,7 +250,7 @@ ALLOWED = {
"b200-dgxc": REQUIRED["b200-dgxc"] | {"exclude_nodes", "stage_dir"} | NETWORK_FIELDS,
"b300": REQUIRED["b300"] | {"exclude_nodes", "stage_dir"} | NETWORK_FIELDS,
"gb200": REQUIRED["gb200"] | NETWORK_FIELDS,
- "gb300": REQUIRED["gb300"] | NETWORK_FIELDS,
+ "gb300": REQUIRED["gb300"] | {"stage_dir"} | NETWORK_FIELDS,
"mi325x": REQUIRED["mi325x"] | {"exclude_nodes", "nodelist", "stage_dir", "lock_dir"} | NETWORK_FIELDS,
"mi355x": REQUIRED["mi355x"] | {"exclude_nodes", "nodelist", "stage_dir", "lock_dir"} | NETWORK_FIELDS,
}
@@ -1587,7 +1587,7 @@ cx_lock_canonical_gha_env() {
fi
# The legacy B300 operator row contains a root-owned stage path. B300's
# compute-visible account home is the canonical source for its private base.
- [ "$runner" != b300 ] || trusted_stage_dir=""
+ case "$runner" in b300|gb300) trusted_stage_dir="" ;; esac
unset CX_NCCL_HOME CX_MASTER_PORT CX_MORI_KERNEL_TYPE CX_LOCK_DIR CX_STAGE_DIR
unset CX_STAGE_PARENT_OWNER_OK
unset MASTER_ADDR MASTER_PORT RANK WORLD_SIZE LOCAL_RANK LOCAL_WORLD_SIZE
@@ -1619,6 +1619,11 @@ cx_lock_canonical_gha_env() {
"${COLLECTIVEX_EXECUTION_ID:-${GITHUB_RUN_ID:-}}")" \
|| cx_die "canonical CollectiveX execution cannot create an isolated stage directory"
;;
+ gb300)
+ trusted_stage_dir="$(cx_prepare_implicit_stage_base "" \
+ "${COLLECTIVEX_EXECUTION_ID:-${GITHUB_RUN_ID:-}}")" \
+ || cx_die "canonical CollectiveX execution cannot create an isolated stage directory"
+ ;;
h200-dgxc|b200-dgxc)
trusted_stage_dir="$(cx_prepare_implicit_stage_base)" \
|| cx_die "canonical CollectiveX execution cannot create an isolated stage directory"
diff --git a/experimental/CollectiveX/tests/test_deepep_v2_contract.py b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
index 28861a0361..f74e97c1d3 100644
--- a/experimental/CollectiveX/tests/test_deepep_v2_contract.py
+++ b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
@@ -2065,6 +2065,8 @@ def test_operator_config_is_strict_per_runner_json(self) -> None:
del missing_h100_stage["runners"]["h100-dgxc"]["stage_dir"]
missing_b300_stage = json.loads(json.dumps(document))
del missing_b300_stage["runners"]["b300"]["stage_dir"]
+ missing_gb300_stage = json.loads(json.dumps(document))
+ del missing_gb300_stage["runners"]["gb300"]["stage_dir"]
missing_nvidia_account = json.loads(json.dumps(document))
del missing_nvidia_account["runners"]["h100-dgxc"]["account"]
for invalid in (rejected, boolean_version, missing_nvidia_account):
@@ -2089,6 +2091,7 @@ def test_operator_config_is_strict_per_runner_json(self) -> None:
(missing_amd_stage, "h100-dgxc"),
(missing_h100_stage, "h100-dgxc"),
(missing_b300_stage, "b300"),
+ (missing_gb300_stage, "gb300"),
):
config.write_text(json.dumps(valid))
config.chmod(0o600)
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 5a69ee401a..b343714dd1 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -2419,7 +2419,8 @@ def test_canonical_gha_environment_is_locked_but_manual_overrides_survive(self)
test "$CX_STAGE_DIR" = "$TEST_IMPLICIT_STAGE"
test "$CX_STAGE_PARENT_OWNER_OK" = 1
- export CX_STAGE_DIR=/shared/gb-stage
+ export COLLECTIVEX_OPERATOR_CONFIG_LOADED=$$
+ export CX_STAGE_DIR=/legacy/group-writable-stage
export CX_SHARD_SKU=gb300 CX_NODES=2 CX_GPUS_PER_NODE=4
export CX_IMAGE=untrusted CX_NGPUS=1 CX_MORI_KERNEL_TYPE=untrusted
export MORI_ENABLE_SDMA=0 CX_NCCL_HOME=/untrusted CX_MASTER_PORT=1
@@ -2428,7 +2429,7 @@ def test_canonical_gha_environment_is_locked_but_manual_overrides_survive(self)
test "$CX_IMAGE_DIGEST" = "$CX_IMAGE_MULTIARCH_DIGEST"
test "$CX_NGPUS:$CX_SEED:$CX_RUN_TIMEOUT" = 8:67:900
test "$CX_NCCL_HOME:$CX_MASTER_PORT" = /usr:29551
- test "$CX_STAGE_DIR" = /shared/gb-stage
+ test "$CX_STAGE_DIR" = "$TEST_IMPLICIT_STAGE"
test -z "${CX_STAGE_PARENT_OWNER_OK+x}"
test -z "${CX_MORI_KERNEL_TYPE+x}${MORI_ENABLE_SDMA+x}"
From ba5831f32f7a23f295e4bed9b295394b62d57a0e Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 18:59:10 +0800
Subject: [PATCH 037/210] feat(collectivex): record completed H100 precision
cells
---
experimental/CollectiveX/README.md | 3 ++-
experimental/CollectiveX/capability.py | 7 +++++++
experimental/CollectiveX/docs/methodology.md | 3 ++-
.../CollectiveX/tests/test_precision_scheduling.py | 8 ++++----
4 files changed, 15 insertions(+), 6 deletions(-)
diff --git a/experimental/CollectiveX/README.md b/experimental/CollectiveX/README.md
index 037490c158..33fe811c1d 100644
--- a/experimental/CollectiveX/README.md
+++ b/experimental/CollectiveX/README.md
@@ -14,7 +14,8 @@ combine, and paired roundtrip latency across EP libraries and accelerator system
> Development status: the V1 precision, point-level publication, branch-only delivery, and frontend
> contracts are implemented. Validated B200 EP8, H200 EP8, and GB200 EP8/EP16 probe outcomes are
-> recorded; 57 native precision cells remain provisional, so V1 counts/digests are not frozen.
+> recorded, together with seven completed H100 EP8 outcomes; 50 native precision cells remain
+> provisional, so V1 counts/digests are not frozen.
## Implemented Pre-V1 Execution Profile
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
index b892338656..f02a4dab41 100644
--- a/experimental/CollectiveX/capability.py
+++ b/experimental/CollectiveX/capability.py
@@ -338,6 +338,13 @@ def _precision_rule(
for ep in (8, 16)),
*(("28737422902", "gb200", ep, "deepep-hybrid", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "native-operation-failed")
for ep in (8, 16)),
+ ("28738113606", "h100-dgxc", 8, "deepep", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed"),
+ ("28738113606", "h100-dgxc", 8, "deepep", "low-latency", _LL_FP8_PROFILE, "supported", "native-probe-passed"),
+ ("28738113606", "h100-dgxc", 8, "deepep", "low-latency", _LL_LOGFMT_PROFILE, "supported", "native-probe-passed"),
+ ("28738113606", "h100-dgxc", 8, "deepep", "low-latency", _LL_FP8_LOGFMT_PROFILE, "supported", "native-probe-passed"),
+ ("28738113606", "h100-dgxc", 8, "deepep-hybrid", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "native-operation-failed"),
+ ("28738113606", "h100-dgxc", 8, "uccl", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed"),
+ ("28738113606", "h100-dgxc", 8, "uccl", "low-latency", _LL_LOGFMT_PROFILE, "supported", "native-probe-passed"),
)
PRECISION_CELL_OVERRIDES.update({
(profile, backend, sku, ep, mode): {
diff --git a/experimental/CollectiveX/docs/methodology.md b/experimental/CollectiveX/docs/methodology.md
index faf8b41f94..03a4a9c1aa 100644
--- a/experimental/CollectiveX/docs/methodology.md
+++ b/experimental/CollectiveX/docs/methodology.md
@@ -11,7 +11,8 @@ Before any V1-tagged run, this English document must match the implemented preci
publication, and frontend contracts; counts and digests remain unfrozen. Chinese documentation
synchronization is explicitly deferred for the V1 implementation phase.
Validated B200 EP8, H200 EP8, and GB200 EP8/EP16 native probe outcomes are recorded in the
-capability registry; 57 native precision cells remain provisional.
+capability registry, together with seven completed H100 EP8 outcomes; 50 native precision cells
+remain provisional.
## Product Boundary
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
index 142f47dfb6..a8b0aac40e 100644
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -28,7 +28,7 @@ def test_precision_probe_inventory_is_exact_and_non_mutating(self) -> None:
item["precision_profile"],
)
self.assertEqual(targets, sorted(capability.provisional_precision_targets(), key=key))
- self.assertEqual(len(targets), 57)
+ self.assertEqual(len(targets), 50)
self.assertEqual(capability.PRECISION_CAPABILITIES, before)
self.assertEqual(
len({
@@ -222,13 +222,13 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
{"provisional", "supported", "unsupported"},
)
self.assertEqual(
- len(targets) - len(capability.provisional_precision_targets()), 37
+ len(targets) - len(capability.provisional_precision_targets()), 44
)
self.assertEqual(
- sum(item["disposition"] == "supported" for item in targets), 22
+ sum(item["disposition"] == "supported" for item in targets), 28
)
self.assertEqual(
- sum(item["disposition"] == "unsupported" for item in targets), 15
+ sum(item["disposition"] == "unsupported" for item in targets), 16
)
keys = {
(
From 8e021c95f6b92fc8146d6f24750f63a61af01279 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 19:00:23 +0800
Subject: [PATCH 038/210] test(collectivex): update frozen matrix baseline
---
experimental/CollectiveX/tests/test_publisher.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/experimental/CollectiveX/tests/test_publisher.py b/experimental/CollectiveX/tests/test_publisher.py
index 8b2846bc86..b4fd0ffb30 100644
--- a/experimental/CollectiveX/tests/test_publisher.py
+++ b/experimental/CollectiveX/tests/test_publisher.py
@@ -2024,7 +2024,7 @@ def test_workflow_matrix_and_catalog_digests_do_not_drift(self) -> None:
if item["disposition"] == "unsupported"
),
),
- (58, 608, 364, 244, 1600, 940, 660),
+ (54, 608, 338, 270, 1600, 868, 732),
)
library: dict[tuple, set[str]] = {}
system: dict[tuple, set[str]] = {}
From 328d30e12705af38d9661796f77782c1559d43e0 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 19:08:41 +0800
Subject: [PATCH 039/210] fix(collectivex): clean UID-mapped B300 stages
---
experimental/CollectiveX/runtime/common.sh | 21 ++++++++++++++++---
.../tests/test_sampling_contract.py | 9 ++++++++
2 files changed, 27 insertions(+), 3 deletions(-)
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 1d29370d57..4ac3af226a 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -2095,7 +2095,7 @@ cx_cleanup_stage() {
cx_log "ERROR: refusing to remove an unrecognized stage directory"
return 1
fi
- if ! python3 - "$mount_src" "$tag" <<'PY'
+ if ! python3 - "$mount_src" "$tag" "${CX_STAGE_PARENT_OWNER_OK:-0}" <<'PY'
import os
from pathlib import Path
import stat
@@ -2103,12 +2103,27 @@ import sys
root = Path(sys.argv[1])
expected = f"collectivex-stage-v1\n{sys.argv[2]}\n"
+allow_uid_mapped_root = sys.argv[3] == "1"
try:
metadata = os.stat(root, follow_symlinks=False)
marker = root / ".collectivex-stage-v1"
+ owner_ok = metadata.st_uid == os.getuid()
+ if not owner_ok and allow_uid_mapped_root and metadata.st_uid == 0:
+ base = root.parent
+ private_parent = base.parent
+ base_metadata = os.stat(base, follow_symlinks=False)
+ parent_metadata = os.stat(private_parent, follow_symlinks=False)
+ owner_ok = (
+ stat.S_ISDIR(base_metadata.st_mode)
+ and base_metadata.st_uid == 0
+ and stat.S_IMODE(base_metadata.st_mode) == 0o700
+ and stat.S_ISDIR(parent_metadata.st_mode)
+ and parent_metadata.st_uid != 0
+ and not stat.S_IMODE(parent_metadata.st_mode) & (stat.S_IWGRP | stat.S_IWOTH)
+ )
if (
not stat.S_ISDIR(metadata.st_mode)
- or metadata.st_uid != os.getuid()
+ or not owner_ok
or (stat.S_IMODE(metadata.st_mode) & 0o777) != 0o700
):
raise OSError
@@ -2117,7 +2132,7 @@ try:
marker_metadata = os.stat(marker, follow_symlinks=False)
if (
not stat.S_ISREG(marker_metadata.st_mode)
- or marker_metadata.st_uid != os.getuid()
+ or marker_metadata.st_uid != metadata.st_uid
or stat.S_IMODE(marker_metadata.st_mode) != 0o600
):
raise OSError
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index b343714dd1..78419a0422 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -856,6 +856,15 @@ def test_launchers_preserve_platform_specific_runtime_requirements(self) -> None
self.assertIn('distribution.read_text("direct_url.json")', runtime)
self.assertIn("6548e9c504a12b2471af4b7f4d9546321210a57a456b5dc55bd4a8dad0f932ac", runtime)
self.assertIn("2671cff7baf8c2c214ff4bac721af875d513130670bec57601998bd1aae82882", runtime)
+ stage_cleanup = common[
+ common.index("cx_cleanup_stage()"):
+ common.index("cx_has_result_doc()")
+ ]
+ self.assertIn('allow_uid_mapped_root = sys.argv[3] == "1"', stage_cleanup)
+ self.assertIn("base_metadata.st_uid == 0", stage_cleanup)
+ self.assertIn("stat.S_IMODE(base_metadata.st_mode) == 0o700", stage_cleanup)
+ self.assertIn("parent_metadata.st_uid != 0", stage_cleanup)
+ self.assertIn("stat.S_IWGRP | stat.S_IWOTH", stage_cleanup)
def test_deferred_backend_provenance_resolves_before_measurement(self) -> None:
harness = (ROOT / "tests" / "ep_harness.py").read_text()
From b08a19036c0ffc98f52d6bba9d8f46c0fa16cdce Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 19:11:23 +0800
Subject: [PATCH 040/210] feat(collectivex): resolve H100 UCCL fused probes
---
experimental/CollectiveX/capability.py | 2 ++
experimental/CollectiveX/tests/test_precision_scheduling.py | 6 +++---
2 files changed, 5 insertions(+), 3 deletions(-)
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
index f02a4dab41..15b3afcd6f 100644
--- a/experimental/CollectiveX/capability.py
+++ b/experimental/CollectiveX/capability.py
@@ -345,6 +345,8 @@ def _precision_rule(
("28738113606", "h100-dgxc", 8, "deepep-hybrid", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "native-operation-failed"),
("28738113606", "h100-dgxc", 8, "uccl", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed"),
("28738113606", "h100-dgxc", 8, "uccl", "low-latency", _LL_LOGFMT_PROFILE, "supported", "native-probe-passed"),
+ ("28738113606", "h100-dgxc", 8, "uccl", "low-latency", _LL_FP8_PROFILE, "unsupported", "native-operation-timeout"),
+ ("28738113606", "h100-dgxc", 8, "uccl", "low-latency", _LL_FP8_LOGFMT_PROFILE, "unsupported", "native-operation-timeout"),
)
PRECISION_CELL_OVERRIDES.update({
(profile, backend, sku, ep, mode): {
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
index a8b0aac40e..45ec571147 100644
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -28,7 +28,7 @@ def test_precision_probe_inventory_is_exact_and_non_mutating(self) -> None:
item["precision_profile"],
)
self.assertEqual(targets, sorted(capability.provisional_precision_targets(), key=key))
- self.assertEqual(len(targets), 50)
+ self.assertEqual(len(targets), 48)
self.assertEqual(capability.PRECISION_CAPABILITIES, before)
self.assertEqual(
len({
@@ -222,13 +222,13 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
{"provisional", "supported", "unsupported"},
)
self.assertEqual(
- len(targets) - len(capability.provisional_precision_targets()), 44
+ len(targets) - len(capability.provisional_precision_targets()), 46
)
self.assertEqual(
sum(item["disposition"] == "supported" for item in targets), 28
)
self.assertEqual(
- sum(item["disposition"] == "unsupported" for item in targets), 16
+ sum(item["disposition"] == "unsupported" for item in targets), 18
)
keys = {
(
From 9ceb93de19c066c41039e6d68ebc56e9cc223d34 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 19:13:24 +0800
Subject: [PATCH 041/210] fix(collectivex): use NVLink for B300 EP8 NVSHMEM
---
experimental/CollectiveX/runtime/common.sh | 8 +++++++-
experimental/CollectiveX/tests/test_sampling_contract.py | 8 +++++++-
2 files changed, 14 insertions(+), 2 deletions(-)
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 4ac3af226a..511a187e79 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -532,7 +532,7 @@ PY
# Explicit Slurm export boundary. Operator config, runner credentials, HOME,
# workspace paths, and unrelated service secrets never enter the container.
cx_container_exports() {
- printf '%s' 'COLLECTIVEX_SOURCE_SHA,COLLECTIVEX_ARTIFACT_NAME,COLLECTIVEX_EXECUTION_ID,COLLECTIVEX_CONTROL_SHA256,COLLECTIVEX_IMAGE,COLLECTIVEX_IMAGE_DIGEST,COLLECTIVEX_IMAGE_DIGEST_VERIFIED,COLLECTIVEX_SQUASH_SHA256,GITHUB_REF_NAME,GITHUB_REF,GITHUB_REPOSITORY,GITHUB_JOB,GITHUB_RUN_ID,GITHUB_RUN_ATTEMPT,GITHUB_SHA,CX_RUNNER,CX_BENCH,CX_NODES,CX_GPUS_PER_NODE,CX_SCALE_UP_DOMAIN,CX_SHARD_FILE,CX_SHARD_SKU,CX_PRECISION_PROBE,CX_NGPUS,CX_TS,CX_TOPO,CX_SCOPE,CX_TRANSPORT,CX_SCALE_UP_TRANSPORT,CX_SCALE_OUT_TRANSPORT,CX_MODE,CX_PHASE,CX_ROUTING,CX_EPLB,CX_CASE_ID,CX_SUITE,CX_WORKLOAD_NAME,CX_REQUIRED_PUBLICATION,CX_PRECISION_PROFILE,CX_QUALIFICATION_INDEX,CX_HIDDEN,CX_TOPK,CX_EXPERTS,CX_TOKENS_LADDER,CX_CANONICAL,CX_ITERS,CX_TRIALS,CX_WARMUP,CX_SAMPLES_PER_POINT,CX_WARMUP_SEMANTICS,CX_SEED,CX_RUN_TIMEOUT,CX_NCCL_HOME,CX_ALLOW_MNNVL,CX_ATTEMPT_ID,CX_RUNTIME_MARKER,CX_MORI_KERNEL_TYPE,CX_WORKLOAD_DIR,CX_BACKEND_CACHE_ROOT,CX_BACKEND_CACHE_SENTINEL_SHA256,CX_BACKEND_SOURCE_ROOT,CX_AUDIT_SALT,CX_SOCKET_IFNAME,CX_RDMA_DEVICES,CX_IB_GID_INDEX,CX_RDMA_SERVICE_LEVEL,MASTER_ADDR,MASTER_PORT,RANK,WORLD_SIZE,LOCAL_RANK,LOCAL_WORLD_SIZE,NCCL_NET,NCCL_SOCKET_IFNAME,GLOO_SOCKET_IFNAME,NCCL_IB_HCA,NCCL_IB_GID_INDEX,NCCL_IB_SL,NVSHMEM_HCA_LIST,NVSHMEM_IB_GID_INDEX,NVSHMEM_IB_SL,NVSHMEM_IB_ENABLE_IBGDA,NVSHMEM_IBGDA_NIC_HANDLER,EP_NIC_NAME,EP_OVERRIDE_RDMA_SL,UCCL_SOCKET_IFNAME,UCCL_IB_GID_INDEX,UCCL_IB_SL,MORI_RDMA_DEVICES,HYBRID_EP_MULTINODE,USE_NIXL,RDMA_CORE_HOME,DEEPEP_HYBRID_BUILD_MODE,NCCL_CUMEM_ENABLE,NCCL_MNNVL_ENABLE,MC_FORCE_MNNVL,MORI_DISABLE_AUTO_XGMI,MORI_ENABLE_SDMA,MORI_APP_LOG_LEVEL,MORI_SHMEM_LOG_LEVEL,MORI_IO_LOG_LEVEL'
+ printf '%s' 'COLLECTIVEX_SOURCE_SHA,COLLECTIVEX_ARTIFACT_NAME,COLLECTIVEX_EXECUTION_ID,COLLECTIVEX_CONTROL_SHA256,COLLECTIVEX_IMAGE,COLLECTIVEX_IMAGE_DIGEST,COLLECTIVEX_IMAGE_DIGEST_VERIFIED,COLLECTIVEX_SQUASH_SHA256,GITHUB_REF_NAME,GITHUB_REF,GITHUB_REPOSITORY,GITHUB_JOB,GITHUB_RUN_ID,GITHUB_RUN_ATTEMPT,GITHUB_SHA,CX_RUNNER,CX_BENCH,CX_NODES,CX_GPUS_PER_NODE,CX_SCALE_UP_DOMAIN,CX_SHARD_FILE,CX_SHARD_SKU,CX_PRECISION_PROBE,CX_NGPUS,CX_TS,CX_TOPO,CX_SCOPE,CX_TRANSPORT,CX_SCALE_UP_TRANSPORT,CX_SCALE_OUT_TRANSPORT,CX_MODE,CX_PHASE,CX_ROUTING,CX_EPLB,CX_CASE_ID,CX_SUITE,CX_WORKLOAD_NAME,CX_REQUIRED_PUBLICATION,CX_PRECISION_PROFILE,CX_QUALIFICATION_INDEX,CX_HIDDEN,CX_TOPK,CX_EXPERTS,CX_TOKENS_LADDER,CX_CANONICAL,CX_ITERS,CX_TRIALS,CX_WARMUP,CX_SAMPLES_PER_POINT,CX_WARMUP_SEMANTICS,CX_SEED,CX_RUN_TIMEOUT,CX_NCCL_HOME,CX_ALLOW_MNNVL,CX_ATTEMPT_ID,CX_RUNTIME_MARKER,CX_MORI_KERNEL_TYPE,CX_WORKLOAD_DIR,CX_BACKEND_CACHE_ROOT,CX_BACKEND_CACHE_SENTINEL_SHA256,CX_BACKEND_SOURCE_ROOT,CX_AUDIT_SALT,CX_SOCKET_IFNAME,CX_RDMA_DEVICES,CX_IB_GID_INDEX,CX_RDMA_SERVICE_LEVEL,MASTER_ADDR,MASTER_PORT,RANK,WORLD_SIZE,LOCAL_RANK,LOCAL_WORLD_SIZE,NCCL_NET,NCCL_SOCKET_IFNAME,GLOO_SOCKET_IFNAME,NCCL_IB_HCA,NCCL_IB_GID_INDEX,NCCL_IB_SL,NVSHMEM_DISABLE_IB,NVSHMEM_HCA_LIST,NVSHMEM_IB_GID_INDEX,NVSHMEM_IB_SL,NVSHMEM_IB_ENABLE_IBGDA,NVSHMEM_IBGDA_NIC_HANDLER,EP_NIC_NAME,EP_OVERRIDE_RDMA_SL,UCCL_SOCKET_IFNAME,UCCL_IB_GID_INDEX,UCCL_IB_SL,MORI_RDMA_DEVICES,HYBRID_EP_MULTINODE,USE_NIXL,RDMA_CORE_HOME,DEEPEP_HYBRID_BUILD_MODE,NCCL_CUMEM_ENABLE,NCCL_MNNVL_ENABLE,MC_FORCE_MNNVL,MORI_DISABLE_AUTO_XGMI,MORI_ENABLE_SDMA,MORI_APP_LOG_LEVEL,MORI_SHMEM_LOG_LEVEL,MORI_IO_LOG_LEVEL'
printf '%s' ',MORI_COMMIT'
}
@@ -617,10 +617,14 @@ cx_apply_network_profile() {
[[ "$nodes" =~ ^[1-9][0-9]*$ ]] || cx_die "invalid network placement"
unset NCCL_NET NCCL_SOCKET_IFNAME GLOO_SOCKET_IFNAME NCCL_IB_HCA
unset NCCL_IB_GID_INDEX NCCL_IB_SL
+ unset NVSHMEM_DISABLE_IB
unset NVSHMEM_HCA_LIST NVSHMEM_IB_GID_INDEX NVSHMEM_IB_SL
unset NVSHMEM_IB_ENABLE_IBGDA NVSHMEM_IBGDA_NIC_HANDLER
unset EP_NIC_NAME EP_OVERRIDE_RDMA_SL
unset UCCL_SOCKET_IFNAME UCCL_IB_GID_INDEX UCCL_IB_SL MORI_RDMA_DEVICES
+ if [ "${CX_SHARD_SKU:-}" = b300 ] && [ "$nodes" = 1 ]; then
+ export NVSHMEM_DISABLE_IB=1
+ fi
[ "$nodes" -gt 1 ] && [ "$transport" != mnnvl ] || return 0
[ -n "${CX_SOCKET_IFNAME:-}" ] && [ -n "${CX_RDMA_DEVICES:-}" ] \
|| cx_die "multi-node execution requires private socket and RDMA selectors"
@@ -1595,6 +1599,7 @@ cx_lock_canonical_gha_env() {
unset CX_AUDIT_SALT
unset NCCL_NET NCCL_SOCKET_IFNAME GLOO_SOCKET_IFNAME NCCL_IB_HCA
unset NCCL_IB_GID_INDEX NCCL_IB_SL
+ unset NVSHMEM_DISABLE_IB
unset NVSHMEM_HCA_LIST NVSHMEM_IB_GID_INDEX NVSHMEM_IB_SL
unset NVSHMEM_IB_ENABLE_IBGDA NVSHMEM_IBGDA_NIC_HANDLER
unset EP_NIC_NAME EP_OVERRIDE_RDMA_SL
@@ -1635,6 +1640,7 @@ cx_lock_canonical_gha_env() {
|| cx_die "canonical CollectiveX execution requires a private audit salt"
if [ "$runner" = b300 ]; then
CX_STAGE_PARENT_OWNER_OK=1
+ [ "${CX_NODES:-}" != 1 ] || export NVSHMEM_DISABLE_IB=1
fi
case "$runner" in
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 78419a0422..4011e9d123 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -597,8 +597,12 @@ def test_scaleout_network_profile_is_explicit_and_allowlisted(self) -> None:
! (export CX_SOCKET_IFNAME=eth0; unset CX_RDMA_DEVICES; cx_apply_network_profile 2 nvlink-rdma)
export CX_SOCKET_IFNAME=ib0 CX_RDMA_DEVICES=mlx5_0:1,mlx5_1:1
export NCCL_NET=Socket NCCL_IB_HCA=stale NVSHMEM_HCA_LIST=stale
+ export CX_SHARD_SKU=b300
cx_apply_network_profile 1 nvlink
- test -z "${NCCL_NET+x}${NCCL_IB_HCA+x}${NVSHMEM_HCA_LIST+x}"
+ test "$NVSHMEM_DISABLE_IB" = 1
+ unset CX_SHARD_SKU
+ cx_apply_network_profile 1 nvlink
+ test -z "${NCCL_NET+x}${NCCL_IB_HCA+x}${NVSHMEM_DISABLE_IB+x}${NVSHMEM_HCA_LIST+x}"
cx_apply_network_profile 4 mnnvl
test -z "${NCCL_NET+x}${NCCL_IB_HCA+x}${NVSHMEM_HCA_LIST+x}"
export CX_IB_GID_INDEX=3 CX_RDMA_SERVICE_LEVEL=2
@@ -2427,6 +2431,7 @@ def test_canonical_gha_environment_is_locked_but_manual_overrides_survive(self)
cx_lock_canonical_gha_env b300
test "$CX_STAGE_DIR" = "$TEST_IMPLICIT_STAGE"
test "$CX_STAGE_PARENT_OWNER_OK" = 1
+ test "$NVSHMEM_DISABLE_IB" = 1
export COLLECTIVEX_OPERATOR_CONFIG_LOADED=$$
export CX_STAGE_DIR=/legacy/group-writable-stage
@@ -2440,6 +2445,7 @@ def test_canonical_gha_environment_is_locked_but_manual_overrides_survive(self)
test "$CX_NCCL_HOME:$CX_MASTER_PORT" = /usr:29551
test "$CX_STAGE_DIR" = "$TEST_IMPLICIT_STAGE"
test -z "${CX_STAGE_PARENT_OWNER_OK+x}"
+ test -z "${NVSHMEM_DISABLE_IB+x}"
test -z "${CX_MORI_KERNEL_TYPE+x}${MORI_ENABLE_SDMA+x}"
export COLLECTIVEX_OPERATOR_CONFIG_LOADED=$$
From 7e26390b5c4ac0a6e1c2a509f55eb9981465aa20 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 19:14:42 +0800
Subject: [PATCH 042/210] feat(collectivex): record completed GB300 precision
cells
---
experimental/CollectiveX/capability.py | 5 +++++
.../CollectiveX/tests/test_precision_scheduling.py | 8 ++++----
2 files changed, 9 insertions(+), 4 deletions(-)
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
index 15b3afcd6f..f54e7390ab 100644
--- a/experimental/CollectiveX/capability.py
+++ b/experimental/CollectiveX/capability.py
@@ -347,6 +347,11 @@ def _precision_rule(
("28738113606", "h100-dgxc", 8, "uccl", "low-latency", _LL_LOGFMT_PROFILE, "supported", "native-probe-passed"),
("28738113606", "h100-dgxc", 8, "uccl", "low-latency", _LL_FP8_PROFILE, "unsupported", "native-operation-timeout"),
("28738113606", "h100-dgxc", 8, "uccl", "low-latency", _LL_FP8_LOGFMT_PROFILE, "unsupported", "native-operation-timeout"),
+ *(("28738445591", "gb300", ep, "deepep", mode, profile, "supported", "native-probe-passed")
+ for ep in (8, 16)
+ for profile, mode in ((_NORMAL_E4M3FN_PROFILE, "normal"), (_LL_FP8_PROFILE, "low-latency"),
+ (_LL_LOGFMT_PROFILE, "low-latency"), (_LL_FP8_LOGFMT_PROFILE, "low-latency"))),
+ ("28738445591", "gb300", 8, "deepep-hybrid", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "native-operation-failed"),
)
PRECISION_CELL_OVERRIDES.update({
(profile, backend, sku, ep, mode): {
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
index 45ec571147..c0703e6078 100644
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -28,7 +28,7 @@ def test_precision_probe_inventory_is_exact_and_non_mutating(self) -> None:
item["precision_profile"],
)
self.assertEqual(targets, sorted(capability.provisional_precision_targets(), key=key))
- self.assertEqual(len(targets), 48)
+ self.assertEqual(len(targets), 39)
self.assertEqual(capability.PRECISION_CAPABILITIES, before)
self.assertEqual(
len({
@@ -222,13 +222,13 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
{"provisional", "supported", "unsupported"},
)
self.assertEqual(
- len(targets) - len(capability.provisional_precision_targets()), 46
+ len(targets) - len(capability.provisional_precision_targets()), 55
)
self.assertEqual(
- sum(item["disposition"] == "supported" for item in targets), 28
+ sum(item["disposition"] == "supported" for item in targets), 36
)
self.assertEqual(
- sum(item["disposition"] == "unsupported" for item in targets), 18
+ sum(item["disposition"] == "unsupported" for item in targets), 19
)
keys = {
(
From 39776a835f242e84e73549302c79c99726625771 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 19:15:12 +0800
Subject: [PATCH 043/210] feat(collectivex): record completed B300 precision
cells
---
experimental/CollectiveX/capability.py | 2 ++
.../CollectiveX/tests/test_precision_scheduling.py | 8 ++++----
2 files changed, 6 insertions(+), 4 deletions(-)
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
index f54e7390ab..f1b7cee427 100644
--- a/experimental/CollectiveX/capability.py
+++ b/experimental/CollectiveX/capability.py
@@ -352,6 +352,8 @@ def _precision_rule(
for profile, mode in ((_NORMAL_E4M3FN_PROFILE, "normal"), (_LL_FP8_PROFILE, "low-latency"),
(_LL_LOGFMT_PROFILE, "low-latency"), (_LL_FP8_LOGFMT_PROFILE, "low-latency"))),
("28738445591", "gb300", 8, "deepep-hybrid", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "native-operation-failed"),
+ ("28738738793", "b300", 8, "deepep", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed"),
+ ("28738738793", "b300", 8, "deepep-hybrid", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "native-operation-failed"),
)
PRECISION_CELL_OVERRIDES.update({
(profile, backend, sku, ep, mode): {
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
index c0703e6078..08e3a7d073 100644
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -28,7 +28,7 @@ def test_precision_probe_inventory_is_exact_and_non_mutating(self) -> None:
item["precision_profile"],
)
self.assertEqual(targets, sorted(capability.provisional_precision_targets(), key=key))
- self.assertEqual(len(targets), 39)
+ self.assertEqual(len(targets), 37)
self.assertEqual(capability.PRECISION_CAPABILITIES, before)
self.assertEqual(
len({
@@ -222,13 +222,13 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
{"provisional", "supported", "unsupported"},
)
self.assertEqual(
- len(targets) - len(capability.provisional_precision_targets()), 55
+ len(targets) - len(capability.provisional_precision_targets()), 57
)
self.assertEqual(
- sum(item["disposition"] == "supported" for item in targets), 36
+ sum(item["disposition"] == "supported" for item in targets), 37
)
self.assertEqual(
- sum(item["disposition"] == "unsupported" for item in targets), 19
+ sum(item["disposition"] == "unsupported" for item in targets), 20
)
keys = {
(
From a617860fd656334c686c304a8f76b7b59a553d0c Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 19:18:09 +0800
Subject: [PATCH 044/210] feat(collectivex): complete GB300 precision inventory
---
experimental/CollectiveX/capability.py | 5 ++++-
.../CollectiveX/tests/test_precision_scheduling.py | 8 ++++----
2 files changed, 8 insertions(+), 5 deletions(-)
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
index f1b7cee427..ebd3a6a352 100644
--- a/experimental/CollectiveX/capability.py
+++ b/experimental/CollectiveX/capability.py
@@ -351,7 +351,10 @@ def _precision_rule(
for ep in (8, 16)
for profile, mode in ((_NORMAL_E4M3FN_PROFILE, "normal"), (_LL_FP8_PROFILE, "low-latency"),
(_LL_LOGFMT_PROFILE, "low-latency"), (_LL_FP8_LOGFMT_PROFILE, "low-latency"))),
- ("28738445591", "gb300", 8, "deepep-hybrid", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "native-operation-failed"),
+ *(("28738445591", "gb300", ep, "deepep-v2", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed")
+ for ep in (8, 16)),
+ *(("28738445591", "gb300", ep, "deepep-hybrid", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "native-operation-failed")
+ for ep in (8, 16)),
("28738738793", "b300", 8, "deepep", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed"),
("28738738793", "b300", 8, "deepep-hybrid", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "native-operation-failed"),
)
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
index 08e3a7d073..265b611c0b 100644
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -28,7 +28,7 @@ def test_precision_probe_inventory_is_exact_and_non_mutating(self) -> None:
item["precision_profile"],
)
self.assertEqual(targets, sorted(capability.provisional_precision_targets(), key=key))
- self.assertEqual(len(targets), 37)
+ self.assertEqual(len(targets), 34)
self.assertEqual(capability.PRECISION_CAPABILITIES, before)
self.assertEqual(
len({
@@ -222,13 +222,13 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
{"provisional", "supported", "unsupported"},
)
self.assertEqual(
- len(targets) - len(capability.provisional_precision_targets()), 57
+ len(targets) - len(capability.provisional_precision_targets()), 60
)
self.assertEqual(
- sum(item["disposition"] == "supported" for item in targets), 37
+ sum(item["disposition"] == "supported" for item in targets), 39
)
self.assertEqual(
- sum(item["disposition"] == "unsupported" for item in targets), 20
+ sum(item["disposition"] == "unsupported" for item in targets), 21
)
keys = {
(
From 26e4cce87ce2854c47f9b02780e9ec863a2f02cc Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 19:21:18 +0800
Subject: [PATCH 045/210] chore(collectivex): expose private V2 version
mismatch
---
experimental/CollectiveX/runtime/run_in_container.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/experimental/CollectiveX/runtime/run_in_container.sh b/experimental/CollectiveX/runtime/run_in_container.sh
index eeb8f632f9..050dcc7765 100644
--- a/experimental/CollectiveX/runtime/run_in_container.sh
+++ b/experimental/CollectiveX/runtime/run_in_container.sh
@@ -407,7 +407,7 @@ assert metadata.version("numpy") == "2.2.6"
import deep_ep
assert deep_ep.__version__ == "2.0.0", deep_ep.__version__
-assert metadata.version("deep_ep") == "2.0.0+fa8a9b1"
+assert metadata.version("deep_ep") == "2.0.0+fa8a9b1", metadata.version("deep_ep")
assert inspect.isclass(deep_ep.ElasticBuffer)
assert deep_ep.ElasticBuffer.__name__ == "ElasticBuffer"
assert os.environ.get("EP_SUPPRESS_NCCL_CHECK") is None
From a52211fc9cae67723e089b95013b40f86bf45ec5 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 19:24:14 +0800
Subject: [PATCH 046/210] feat(collectivex): add private network config overlay
---
.github/workflows/collectivex-sweep.yml | 50 +++++++++++++++++++
.../tests/test_sampling_contract.py | 4 ++
2 files changed, 54 insertions(+)
diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml
index bcfbca3f2c..768837fa30 100644
--- a/.github/workflows/collectivex-sweep.yml
+++ b/.github/workflows/collectivex-sweep.yml
@@ -451,11 +451,61 @@ jobs:
id: sweep_shard
env:
COLLECTIVEX_OPERATOR_CONFIG_CONTENT: ${{ secrets.COLLECTIVEX_OPERATOR_CONFIG_V1 }}
+ COLLECTIVEX_NETWORK_CONFIG_CONTENT: ${{ secrets.COLLECTIVEX_NETWORK_CONFIG_V1 }}
COLLECTIVEX_OPERATOR_AUDIT_SALT: ${{ secrets.COLLECTIVEX_AUDIT_SALT_V1 }}
COLLECTIVEX_OPERATOR_CONFIG_REQUIRED: '1'
run: |
set -euo pipefail
umask 077
+ operator_config="$CX_JOB_ROOT/operator-config.json"
+ python3 - "$operator_config" <<'PY'
+ import json
+ import os
+ from pathlib import Path
+ import sys
+
+ def pairs(items):
+ result = {}
+ for key, value in items:
+ if key in result:
+ raise ValueError("duplicate configuration key")
+ result[key] = value
+ return result
+
+ def load(name):
+ return json.loads(
+ os.environ[name], object_pairs_hook=pairs,
+ parse_constant=lambda _: (_ for _ in ()).throw(ValueError()),
+ )
+
+ base = load("COLLECTIVEX_OPERATOR_CONFIG_CONTENT")
+ overlay_raw = os.environ.get("COLLECTIVEX_NETWORK_CONFIG_CONTENT", "")
+ if overlay_raw:
+ overlay = load("COLLECTIVEX_NETWORK_CONFIG_CONTENT")
+ network_fields = {
+ "socket_ifname", "rdma_devices", "ib_gid_index", "rdma_service_level",
+ }
+ if set(overlay) != {"schema_version", "runners"} or overlay["schema_version"] != 1:
+ raise ValueError("invalid network overlay envelope")
+ if not isinstance(overlay["runners"], dict) or not overlay["runners"]:
+ raise ValueError("invalid network overlay runners")
+ for runner, fields in overlay["runners"].items():
+ if runner not in base.get("runners", {}) or not isinstance(fields, dict) or not fields:
+ raise ValueError("invalid network overlay runner")
+ if set(fields) - network_fields:
+ raise ValueError("network overlay contains a non-network field")
+ base["runners"][runner].update(fields)
+ payload = json.dumps(base, sort_keys=True, separators=(",", ":")) + "\n"
+ if len(payload.encode()) > 65536:
+ raise ValueError("merged operator configuration is too large")
+ path = Path(sys.argv[1])
+ descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
+ with os.fdopen(descriptor, "w", encoding="utf-8") as stream:
+ stream.write(payload)
+ PY
+ unset COLLECTIVEX_OPERATOR_CONFIG_CONTENT COLLECTIVEX_NETWORK_CONFIG_CONTENT
+ export COLLECTIVEX_OPERATOR_CONFIG="$operator_config"
+ export COLLECTIVEX_OPERATOR_CONFIG_EPHEMERAL=1
: > "$CX_JOB_ROOT/cleanup-unsafe"
rm -f -- "$CX_JOB_ROOT/cleanup-safe"
cd "$CX_SOURCE_ROOT" 2>/dev/null \
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 4011e9d123..cb621e80ec 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -1208,6 +1208,10 @@ def test_allocation_cleanup_fails_closed_when_scheduler_queries_fail(self) -> No
self.assertIn("Install pinned backend source seed", workflow)
self.assertIn("CX_BACKEND_SOURCE_SEED_ROOT", workflow)
self.assertIn("steps.gen.outputs.source_backends", workflow)
+ self.assertIn("secrets.COLLECTIVEX_NETWORK_CONFIG_V1", workflow)
+ self.assertIn("network_fields = {", workflow)
+ self.assertIn("set(fields) - network_fields", workflow)
+ self.assertIn("COLLECTIVEX_OPERATOR_CONFIG_EPHEMERAL=1", workflow)
prepare_start = workflow.index("- name: Prepare pinned backend source archive")
source_archive_step = workflow[
prepare_start:workflow.index("- uses: actions/upload-artifact", prepare_start)
From 0bfe3dd7c8e8b2f40d0a05610750d7db9870587d Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 19:27:35 +0800
Subject: [PATCH 047/210] fix(collectivex): load merged runner config
---
.github/workflows/collectivex-sweep.yml | 1 +
experimental/CollectiveX/tests/test_sampling_contract.py | 4 ++++
2 files changed, 5 insertions(+)
diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml
index 768837fa30..5aaccc6132 100644
--- a/.github/workflows/collectivex-sweep.yml
+++ b/.github/workflows/collectivex-sweep.yml
@@ -504,6 +504,7 @@ jobs:
stream.write(payload)
PY
unset COLLECTIVEX_OPERATOR_CONFIG_CONTENT COLLECTIVEX_NETWORK_CONFIG_CONTENT
+ unset COLLECTIVEX_OPERATOR_CONFIG_REQUIRED
export COLLECTIVEX_OPERATOR_CONFIG="$operator_config"
export COLLECTIVEX_OPERATOR_CONFIG_EPHEMERAL=1
: > "$CX_JOB_ROOT/cleanup-unsafe"
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index cb621e80ec..df36c32ca9 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -1212,6 +1212,10 @@ def test_allocation_cleanup_fails_closed_when_scheduler_queries_fail(self) -> No
self.assertIn("network_fields = {", workflow)
self.assertIn("set(fields) - network_fields", workflow)
self.assertIn("COLLECTIVEX_OPERATOR_CONFIG_EPHEMERAL=1", workflow)
+ self.assertLess(
+ workflow.index("unset COLLECTIVEX_OPERATOR_CONFIG_REQUIRED"),
+ workflow.index('export COLLECTIVEX_OPERATOR_CONFIG="$operator_config"'),
+ )
prepare_start = workflow.index("- name: Prepare pinned backend source archive")
source_archive_step = workflow[
prepare_start:workflow.index("- uses: actions/upload-artifact", prepare_start)
From 58160de68ed382c20a3dbf47b4118e8537fb9e1c Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 19:31:26 +0800
Subject: [PATCH 048/210] fix(collectivex): bind B300 low latency RDMA
---
experimental/CollectiveX/runtime/common.sh | 74 +++++++++++++------
.../tests/test_sampling_contract.py | 24 +++++-
2 files changed, 73 insertions(+), 25 deletions(-)
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 511a187e79..44b92a373f 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -613,6 +613,7 @@ cx_require_single_node() {
# never addresses; the rendezvous hostname is derived from the allocation.
cx_apply_network_profile() {
local nodes="$1" transport="$2" selector rdma_name rdma_names="" ep_nic=""
+ local scaleout=0 single_node_rdma=0
local -a selectors
[[ "$nodes" =~ ^[1-9][0-9]*$ ]] || cx_die "invalid network placement"
unset NCCL_NET NCCL_SOCKET_IFNAME GLOO_SOCKET_IFNAME NCCL_IB_HCA
@@ -622,13 +623,22 @@ cx_apply_network_profile() {
unset NVSHMEM_IB_ENABLE_IBGDA NVSHMEM_IBGDA_NIC_HANDLER
unset EP_NIC_NAME EP_OVERRIDE_RDMA_SL
unset UCCL_SOCKET_IFNAME UCCL_IB_GID_INDEX UCCL_IB_SL MORI_RDMA_DEVICES
- if [ "${CX_SHARD_SKU:-}" = b300 ] && [ "$nodes" = 1 ]; then
+ if [ "$nodes" -gt 1 ] && [ "$transport" != mnnvl ]; then
+ scaleout=1
+ elif [ "${CX_SHARD_SKU:-}" = b300 ] && [ "$nodes" = 1 ] \
+ && [ "${CX_BENCH:-}" = deepep ] && [ "${CX_MODE:-}" = low-latency ]; then
+ # DeepEP V1 low-latency kernels use pure RDMA even for EP8 within one
+ # NVLink domain. Keep NCCL on NVLink, but bind NVSHMEM to the private HCA.
+ single_node_rdma=1
+ elif [ "${CX_SHARD_SKU:-}" = b300 ] && [ "$nodes" = 1 ]; then
export NVSHMEM_DISABLE_IB=1
fi
- [ "$nodes" -gt 1 ] && [ "$transport" != mnnvl ] || return 0
- [ -n "${CX_SOCKET_IFNAME:-}" ] && [ -n "${CX_RDMA_DEVICES:-}" ] \
- || cx_die "multi-node execution requires private socket and RDMA selectors"
- if [ -n "${CX_SOCKET_IFNAME:-}" ]; then
+ [ "$scaleout" = 1 ] || [ "$single_node_rdma" = 1 ] || return 0
+ [ -n "${CX_RDMA_DEVICES:-}" ] \
+ || cx_die "RDMA execution requires a private device selector"
+ if [ "$scaleout" = 1 ]; then
+ [ -n "${CX_SOCKET_IFNAME:-}" ] \
+ || cx_die "multi-node execution requires a private socket selector"
[[ "$CX_SOCKET_IFNAME" =~ ^[A-Za-z][A-Za-z0-9_.-]{0,31}(,[A-Za-z][A-Za-z0-9_.-]{0,31})*$ ]] \
|| cx_die "invalid private socket interface selector"
export NCCL_SOCKET_IFNAME="$CX_SOCKET_IFNAME" GLOO_SOCKET_IFNAME="$CX_SOCKET_IFNAME"
@@ -643,22 +653,28 @@ cx_apply_network_profile() {
rdma_names="${rdma_names}${rdma_names:+,}${rdma_name}"
[ -n "$ep_nic" ] || ep_nic="$rdma_name"
done
- export NCCL_NET=IB NCCL_IB_HCA="=$CX_RDMA_DEVICES"
export NVSHMEM_HCA_LIST="$CX_RDMA_DEVICES"
- export MORI_RDMA_DEVICES="$rdma_names" EP_NIC_NAME="$ep_nic"
+ if [ "$scaleout" = 1 ]; then
+ export NCCL_NET=IB NCCL_IB_HCA="=$CX_RDMA_DEVICES"
+ export MORI_RDMA_DEVICES="$rdma_names" EP_NIC_NAME="$ep_nic"
+ fi
fi
if [ -n "${CX_IB_GID_INDEX:-}" ]; then
[[ "$CX_IB_GID_INDEX" =~ ^[0-9]+$ ]] && [ "$CX_IB_GID_INDEX" -le 255 ] \
|| cx_die "invalid private IB GID index"
- export NCCL_IB_GID_INDEX="$CX_IB_GID_INDEX" NVSHMEM_IB_GID_INDEX="$CX_IB_GID_INDEX"
- export UCCL_IB_GID_INDEX="$CX_IB_GID_INDEX"
+ export NVSHMEM_IB_GID_INDEX="$CX_IB_GID_INDEX"
+ if [ "$scaleout" = 1 ]; then
+ export NCCL_IB_GID_INDEX="$CX_IB_GID_INDEX" UCCL_IB_GID_INDEX="$CX_IB_GID_INDEX"
+ fi
fi
if [ -n "${CX_RDMA_SERVICE_LEVEL:-}" ]; then
[[ "$CX_RDMA_SERVICE_LEVEL" =~ ^[0-9]+$ ]] && [ "$CX_RDMA_SERVICE_LEVEL" -le 15 ] \
|| cx_die "invalid private RDMA service level"
- export NCCL_IB_SL="$CX_RDMA_SERVICE_LEVEL" NVSHMEM_IB_SL="$CX_RDMA_SERVICE_LEVEL"
- export UCCL_IB_SL="$CX_RDMA_SERVICE_LEVEL"
- export EP_OVERRIDE_RDMA_SL="$CX_RDMA_SERVICE_LEVEL"
+ export NVSHMEM_IB_SL="$CX_RDMA_SERVICE_LEVEL"
+ if [ "$scaleout" = 1 ]; then
+ export NCCL_IB_SL="$CX_RDMA_SERVICE_LEVEL" UCCL_IB_SL="$CX_RDMA_SERVICE_LEVEL"
+ export EP_OVERRIDE_RDMA_SL="$CX_RDMA_SERVICE_LEVEL"
+ fi
fi
export NVSHMEM_IB_ENABLE_IBGDA=1 NVSHMEM_IBGDA_NIC_HANDLER=gpu
}
@@ -667,29 +683,40 @@ cx_apply_network_profile() {
# node before image import or backend initialization. Selector values and node
# diagnostics stay in the runner-private log.
cx_validate_network_profile_on_job() {
- local job_id="$1" nodes="$2" transport="$3" log rc=0
- [ "$nodes" -gt 1 ] && [ "$transport" != mnnvl ] || return 0
+ local job_id="$1" nodes="$2" transport="$3" log rc=0 scaleout=0
+ local single_node_rdma=0
+ if [ "$nodes" -gt 1 ] && [ "$transport" != mnnvl ]; then
+ scaleout=1
+ elif [ "${CX_SHARD_SKU:-}" = b300 ] && [ "$nodes" = 1 ] \
+ && [ "${CX_BENCH:-}" = deepep ] && [ "${CX_MODE:-}" = low-latency ]; then
+ single_node_rdma=1
+ fi
+ [ "$scaleout" = 1 ] || [ "$single_node_rdma" = 1 ] || return 0
[[ "$job_id" =~ ^[1-9][0-9]*$ && "$nodes" =~ ^[1-9][0-9]*$ ]] \
|| return 1
- [ -n "${CX_SOCKET_IFNAME:-}" ] && [ -n "${CX_RDMA_DEVICES:-}" ] \
- || return 1
+ [ -n "${CX_RDMA_DEVICES:-}" ] || return 1
+ [ "$scaleout" = 0 ] || [ -n "${CX_SOCKET_IFNAME:-}" ] || return 1
log="$(cx_private_log_path network-profile)" || return 1
srun --jobid="$job_id" --nodes="$nodes" --ntasks="$nodes" --ntasks-per-node=1 \
--chdir=/tmp --input=all \
--export="$(cx_host_exports),CX_SOCKET_IFNAME,CX_RDMA_DEVICES,CX_IB_GID_INDEX" \
bash -s > "$log" 2>&1 <<'BASH' || rc=$?
set -euo pipefail
-[[ "$CX_SOCKET_IFNAME" =~ ^[A-Za-z][A-Za-z0-9_.-]{0,31}(,[A-Za-z][A-Za-z0-9_.-]{0,31})*$ ]]
+if [ -n "${CX_SOCKET_IFNAME:-}" ]; then
+ [[ "$CX_SOCKET_IFNAME" =~ ^[A-Za-z][A-Za-z0-9_.-]{0,31}(,[A-Za-z][A-Za-z0-9_.-]{0,31})*$ ]]
+fi
[[ "$CX_RDMA_DEVICES" =~ ^[A-Za-z][A-Za-z0-9_.-]{0,31}(:[1-9][0-9]*)?(,[A-Za-z][A-Za-z0-9_.-]{0,31}(:[1-9][0-9]*)?)*$ ]]
if [ -n "${CX_IB_GID_INDEX:-}" ]; then
[[ "$CX_IB_GID_INDEX" =~ ^[0-9]+$ ]] && [ "$CX_IB_GID_INDEX" -le 255 ]
fi
-IFS=, read -r -a interfaces <<< "$CX_SOCKET_IFNAME"
-for interface in "${interfaces[@]}"; do
- [ -d "/sys/class/net/$interface" ]
- state="$(cat "/sys/class/net/$interface/operstate")"
- [ "$state" = up ] || [ "$state" = unknown ]
-done
+if [ -n "${CX_SOCKET_IFNAME:-}" ]; then
+ IFS=, read -r -a interfaces <<< "$CX_SOCKET_IFNAME"
+ for interface in "${interfaces[@]}"; do
+ [ -d "/sys/class/net/$interface" ]
+ state="$(cat "/sys/class/net/$interface/operstate")"
+ [ "$state" = up ] || [ "$state" = unknown ]
+ done
+fi
check_port() {
local port_path="$1" state gid
[ -d "$port_path" ] || return 1
@@ -1640,7 +1667,6 @@ cx_lock_canonical_gha_env() {
|| cx_die "canonical CollectiveX execution requires a private audit salt"
if [ "$runner" = b300 ]; then
CX_STAGE_PARENT_OWNER_OK=1
- [ "${CX_NODES:-}" != 1 ] || export NVSHMEM_DISABLE_IB=1
fi
case "$runner" in
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index df36c32ca9..f67d80d129 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -600,6 +600,12 @@ def test_scaleout_network_profile_is_explicit_and_allowlisted(self) -> None:
export CX_SHARD_SKU=b300
cx_apply_network_profile 1 nvlink
test "$NVSHMEM_DISABLE_IB" = 1
+ export CX_BENCH=deepep CX_MODE=low-latency CX_IB_GID_INDEX=3
+ cx_apply_network_profile 1 nvlink
+ test -z "${NVSHMEM_DISABLE_IB+x}${NCCL_NET+x}${NCCL_IB_HCA+x}"
+ test "$NVSHMEM_HCA_LIST:$NVSHMEM_IB_GID_INDEX" = mlx5_0:1,mlx5_1:1:3
+ test "$NVSHMEM_IB_ENABLE_IBGDA:$NVSHMEM_IBGDA_NIC_HANDLER" = 1:gpu
+ unset CX_BENCH CX_MODE CX_IB_GID_INDEX
unset CX_SHARD_SKU
cx_apply_network_profile 1 nvlink
test -z "${NCCL_NET+x}${NCCL_IB_HCA+x}${NVSHMEM_DISABLE_IB+x}${NVSHMEM_HCA_LIST+x}"
@@ -686,6 +692,22 @@ def test_network_profile_validation_is_private_and_all_node(self) -> None:
)
self.assertFalse(arguments.exists())
+ single_node_command = (
+ 'source "$1"; export COLLECTIVEX_EXECUTION_ID="network-test-$$"; '
+ "trap 'cx_cleanup_private_logs 0' EXIT; "
+ "export CX_SHARD_SKU=b300 CX_BENCH=deepep CX_MODE=low-latency; "
+ 'cx_validate_network_profile_on_job 42 1 nvlink'
+ )
+ single_node = subprocess.run(
+ ["bash", "-c", single_node_command, "_", str(ROOT / "runtime" / "common.sh")],
+ text=True,
+ capture_output=True,
+ env={**environment, "CX_SOCKET_IFNAME": ""},
+ )
+ self.assertEqual(single_node.returncode, 0, single_node.stderr)
+ self.assertIn("--nodes=1", arguments.read_text())
+ self.assertIn("--ntasks=1", arguments.read_text())
+
def test_allocation_preflight_proves_shared_write_visibility(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary).resolve()
@@ -2439,7 +2461,7 @@ def test_canonical_gha_environment_is_locked_but_manual_overrides_survive(self)
cx_lock_canonical_gha_env b300
test "$CX_STAGE_DIR" = "$TEST_IMPLICIT_STAGE"
test "$CX_STAGE_PARENT_OWNER_OK" = 1
- test "$NVSHMEM_DISABLE_IB" = 1
+ test -z "${NVSHMEM_DISABLE_IB+x}"
export COLLECTIVEX_OPERATOR_CONFIG_LOADED=$$
export CX_STAGE_DIR=/legacy/group-writable-stage
From 1ed28ac35921f1815dca6d58556c23ab73741ad6 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 19:33:54 +0800
Subject: [PATCH 049/210] fix(collectivex): accept pinned V2 local builds
---
experimental/CollectiveX/runtime/run_in_container.sh | 2 +-
experimental/CollectiveX/tests/ep_deepep_v2.py | 9 +++++++--
experimental/CollectiveX/tests/test_sampling_contract.py | 6 ++++++
3 files changed, 14 insertions(+), 3 deletions(-)
diff --git a/experimental/CollectiveX/runtime/run_in_container.sh b/experimental/CollectiveX/runtime/run_in_container.sh
index 050dcc7765..4f12d9095f 100644
--- a/experimental/CollectiveX/runtime/run_in_container.sh
+++ b/experimental/CollectiveX/runtime/run_in_container.sh
@@ -407,7 +407,7 @@ assert metadata.version("numpy") == "2.2.6"
import deep_ep
assert deep_ep.__version__ == "2.0.0", deep_ep.__version__
-assert metadata.version("deep_ep") == "2.0.0+fa8a9b1", metadata.version("deep_ep")
+assert metadata.version("deep_ep") in {"2.0.0+fa8a9b1", "2.0.0+local"}, metadata.version("deep_ep")
assert inspect.isclass(deep_ep.ElasticBuffer)
assert deep_ep.ElasticBuffer.__name__ == "ElasticBuffer"
assert os.environ.get("EP_SUPPRESS_NCCL_CHECK") is None
diff --git a/experimental/CollectiveX/tests/ep_deepep_v2.py b/experimental/CollectiveX/tests/ep_deepep_v2.py
index c6c30ad276..f9572a53cd 100644
--- a/experimental/CollectiveX/tests/ep_deepep_v2.py
+++ b/experimental/CollectiveX/tests/ep_deepep_v2.py
@@ -34,7 +34,7 @@
DEEPEP_V2_TREE = "29809e75c5874e6609dac4804e7b651d5226959f"
DEEPEP_V2_FMT_COMMIT = "a4c7e17133ee9cb6a2f45545f6e974dd3c393efa"
DEEPEP_V2_VERSION = "2.0.0"
-DEEPEP_V2_DISTRIBUTION = "2.0.0+fa8a9b1"
+DEEPEP_V2_DISTRIBUTIONS = frozenset({"2.0.0+fa8a9b1", "2.0.0+local"})
DEEPEP_V2_JIT_RANDOM_SEED = "collectivex-deepep-v2-fa8a9b1"
TORCH_VERSION = "2.10.0+cu130"
NCCL_VERSION = "2.30.4"
@@ -277,7 +277,6 @@ def _require_runtime() -> tuple[str, str]:
}
required = {
"deep_ep": DEEPEP_V2_VERSION,
- "deep_ep distribution": DEEPEP_V2_DISTRIBUTION,
"torch": TORCH_VERSION,
"nvidia-nccl-cu13": NCCL_VERSION,
"nvidia-nvshmem-cu12": NVSHMEM_VERSION,
@@ -287,6 +286,12 @@ def _require_runtime() -> tuple[str, str]:
for name, value in required.items()
if actual[name] != value
)
+ if actual["deep_ep distribution"] not in DEEPEP_V2_DISTRIBUTIONS:
+ mismatches.append(
+ "deep_ep distribution="
+ f"{actual['deep_ep distribution']!r}, expected one of "
+ f"{sorted(DEEPEP_V2_DISTRIBUTIONS)!r}"
+ )
if not inspect.isclass(ElasticBuffer) or ElasticBuffer.__name__ != "ElasticBuffer":
mismatches.append("deep_ep.ElasticBuffer is absent")
if os.environ.get("EP_SUPPRESS_NCCL_CHECK"):
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index f67d80d129..6420413e30 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -626,6 +626,12 @@ def test_scaleout_network_profile_is_explicit_and_allowlisted(self) -> None:
env={**os.environ, "COLLECTIVEX_OPERATOR_CONFIG": "/dev/null"},
)
+ v2_adapter = (ROOT / "tests" / "ep_deepep_v2.py").read_text()
+ container_runtime = (ROOT / "runtime" / "run_in_container.sh").read_text()
+ for version in ("2.0.0+fa8a9b1", "2.0.0+local"):
+ self.assertIn(version, v2_adapter)
+ self.assertIn(version, container_runtime)
+
def test_network_profile_validation_is_private_and_all_node(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary).resolve()
From 2cb27a7547c259126246c9f8a12089bd973faa07 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 19:38:37 +0800
Subject: [PATCH 050/210] fix(collectivex): normalize exact HCA selectors
---
experimental/CollectiveX/runtime/common.sh | 7 ++++++-
experimental/CollectiveX/runtime/run_in_container.sh | 4 ++--
experimental/CollectiveX/tests/test_sampling_contract.py | 4 ++++
3 files changed, 12 insertions(+), 3 deletions(-)
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 44b92a373f..85ab4e9c70 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -608,6 +608,11 @@ cx_require_single_node() {
[ "${CX_NODES:-1}" = "1" ] || cx_die "$1 supports one-node EP only"
}
+cx_nccl_hca_device_name() {
+ local selector="${1#=}"
+ printf '%s' "${selector%%:*}"
+}
+
# Convert private, runner-local network selectors into the public library
# variables needed inside the container. Values are interface/HCA identifiers,
# never addresses; the rendezvous hostname is derived from the allocation.
@@ -950,7 +955,7 @@ cx_cancel_job() {
local job_id="$1" active delay
[[ "$job_id" =~ ^[0-9]+$ ]] || return 1
scancel "$job_id" >/dev/null 2>&1 || true
- for delay in 1 2 4 8 16 32; do
+ for delay in 1 2 4 8 16 32 64; do
if ! active="$(squeue -h -j "$job_id" -o %A 2>/dev/null)"; then
sleep "$delay"
continue
diff --git a/experimental/CollectiveX/runtime/run_in_container.sh b/experimental/CollectiveX/runtime/run_in_container.sh
index 4f12d9095f..ed5d066878 100644
--- a/experimental/CollectiveX/runtime/run_in_container.sh
+++ b/experimental/CollectiveX/runtime/run_in_container.sh
@@ -677,7 +677,7 @@ cx_configure_deepep_hybrid_build() {
done
IFS=, read -r -a devices <<< "$NCCL_IB_HCA"
for device in "${devices[@]}"; do
- rdma_name="${device%%:*}"
+ rdma_name="$(cx_nccl_hca_device_name "$device")"
[ -d "/sys/class/infiniband/$rdma_name" ] \
|| { cx_log "ERROR: configured hybrid-ep RDMA device is absent"; return 1; }
done
@@ -924,7 +924,7 @@ cx_probe_scaleout_network() {
done
IFS=, read -r -a devices <<< "$NCCL_IB_HCA"
for device in "${devices[@]}"; do
- rdma_name="${device%%:*}"
+ rdma_name="$(cx_nccl_hca_device_name "$device")"
[ -d "/sys/class/infiniband/$rdma_name" ] \
|| { cx_log "ERROR: configured scale-out RDMA device is absent"; return 1; }
done
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 6420413e30..965738a2fe 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -593,6 +593,8 @@ def test_scaleout_network_profile_is_explicit_and_allowlisted(self) -> None:
command = r'''
set -euo pipefail
source "$1"
+ test "$(cx_nccl_hca_device_name '=mlx5_0:1')" = mlx5_0
+ test "$(cx_nccl_hca_device_name 'mlx5_1')" = mlx5_1
! (unset CX_SOCKET_IFNAME CX_RDMA_DEVICES; cx_apply_network_profile 2 nvlink-rdma)
! (export CX_SOCKET_IFNAME=eth0; unset CX_RDMA_DEVICES; cx_apply_network_profile 2 nvlink-rdma)
export CX_SOCKET_IFNAME=ib0 CX_RDMA_DEVICES=mlx5_0:1,mlx5_1:1
@@ -1205,6 +1207,8 @@ def test_salloc_query_failure_and_interruption_remain_cleanup_unsafe(self) -> No
self.assertTrue(interrupted["cleanup_unsafe"])
def test_allocation_cleanup_fails_closed_when_scheduler_queries_fail(self) -> None:
+ common = (ROOT / "runtime" / "common.sh").read_text()
+ self.assertIn("for delay in 1 2 4 8 16 32 64; do", common)
with tempfile.TemporaryDirectory() as temporary:
directory = Path(temporary)
for name, body in {
From eae76f72767956add33fd792a1820d8717e3ffae Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 19:40:30 +0800
Subject: [PATCH 051/210] fix(collectivex): enable pinned NVSHMEM HCA mapping
---
experimental/CollectiveX/runtime/common.sh | 7 ++++---
experimental/CollectiveX/tests/test_sampling_contract.py | 2 ++
2 files changed, 6 insertions(+), 3 deletions(-)
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 85ab4e9c70..71961a0f69 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -532,7 +532,7 @@ PY
# Explicit Slurm export boundary. Operator config, runner credentials, HOME,
# workspace paths, and unrelated service secrets never enter the container.
cx_container_exports() {
- printf '%s' 'COLLECTIVEX_SOURCE_SHA,COLLECTIVEX_ARTIFACT_NAME,COLLECTIVEX_EXECUTION_ID,COLLECTIVEX_CONTROL_SHA256,COLLECTIVEX_IMAGE,COLLECTIVEX_IMAGE_DIGEST,COLLECTIVEX_IMAGE_DIGEST_VERIFIED,COLLECTIVEX_SQUASH_SHA256,GITHUB_REF_NAME,GITHUB_REF,GITHUB_REPOSITORY,GITHUB_JOB,GITHUB_RUN_ID,GITHUB_RUN_ATTEMPT,GITHUB_SHA,CX_RUNNER,CX_BENCH,CX_NODES,CX_GPUS_PER_NODE,CX_SCALE_UP_DOMAIN,CX_SHARD_FILE,CX_SHARD_SKU,CX_PRECISION_PROBE,CX_NGPUS,CX_TS,CX_TOPO,CX_SCOPE,CX_TRANSPORT,CX_SCALE_UP_TRANSPORT,CX_SCALE_OUT_TRANSPORT,CX_MODE,CX_PHASE,CX_ROUTING,CX_EPLB,CX_CASE_ID,CX_SUITE,CX_WORKLOAD_NAME,CX_REQUIRED_PUBLICATION,CX_PRECISION_PROFILE,CX_QUALIFICATION_INDEX,CX_HIDDEN,CX_TOPK,CX_EXPERTS,CX_TOKENS_LADDER,CX_CANONICAL,CX_ITERS,CX_TRIALS,CX_WARMUP,CX_SAMPLES_PER_POINT,CX_WARMUP_SEMANTICS,CX_SEED,CX_RUN_TIMEOUT,CX_NCCL_HOME,CX_ALLOW_MNNVL,CX_ATTEMPT_ID,CX_RUNTIME_MARKER,CX_MORI_KERNEL_TYPE,CX_WORKLOAD_DIR,CX_BACKEND_CACHE_ROOT,CX_BACKEND_CACHE_SENTINEL_SHA256,CX_BACKEND_SOURCE_ROOT,CX_AUDIT_SALT,CX_SOCKET_IFNAME,CX_RDMA_DEVICES,CX_IB_GID_INDEX,CX_RDMA_SERVICE_LEVEL,MASTER_ADDR,MASTER_PORT,RANK,WORLD_SIZE,LOCAL_RANK,LOCAL_WORLD_SIZE,NCCL_NET,NCCL_SOCKET_IFNAME,GLOO_SOCKET_IFNAME,NCCL_IB_HCA,NCCL_IB_GID_INDEX,NCCL_IB_SL,NVSHMEM_DISABLE_IB,NVSHMEM_HCA_LIST,NVSHMEM_IB_GID_INDEX,NVSHMEM_IB_SL,NVSHMEM_IB_ENABLE_IBGDA,NVSHMEM_IBGDA_NIC_HANDLER,EP_NIC_NAME,EP_OVERRIDE_RDMA_SL,UCCL_SOCKET_IFNAME,UCCL_IB_GID_INDEX,UCCL_IB_SL,MORI_RDMA_DEVICES,HYBRID_EP_MULTINODE,USE_NIXL,RDMA_CORE_HOME,DEEPEP_HYBRID_BUILD_MODE,NCCL_CUMEM_ENABLE,NCCL_MNNVL_ENABLE,MC_FORCE_MNNVL,MORI_DISABLE_AUTO_XGMI,MORI_ENABLE_SDMA,MORI_APP_LOG_LEVEL,MORI_SHMEM_LOG_LEVEL,MORI_IO_LOG_LEVEL'
+ printf '%s' 'COLLECTIVEX_SOURCE_SHA,COLLECTIVEX_ARTIFACT_NAME,COLLECTIVEX_EXECUTION_ID,COLLECTIVEX_CONTROL_SHA256,COLLECTIVEX_IMAGE,COLLECTIVEX_IMAGE_DIGEST,COLLECTIVEX_IMAGE_DIGEST_VERIFIED,COLLECTIVEX_SQUASH_SHA256,GITHUB_REF_NAME,GITHUB_REF,GITHUB_REPOSITORY,GITHUB_JOB,GITHUB_RUN_ID,GITHUB_RUN_ATTEMPT,GITHUB_SHA,CX_RUNNER,CX_BENCH,CX_NODES,CX_GPUS_PER_NODE,CX_SCALE_UP_DOMAIN,CX_SHARD_FILE,CX_SHARD_SKU,CX_PRECISION_PROBE,CX_NGPUS,CX_TS,CX_TOPO,CX_SCOPE,CX_TRANSPORT,CX_SCALE_UP_TRANSPORT,CX_SCALE_OUT_TRANSPORT,CX_MODE,CX_PHASE,CX_ROUTING,CX_EPLB,CX_CASE_ID,CX_SUITE,CX_WORKLOAD_NAME,CX_REQUIRED_PUBLICATION,CX_PRECISION_PROFILE,CX_QUALIFICATION_INDEX,CX_HIDDEN,CX_TOPK,CX_EXPERTS,CX_TOKENS_LADDER,CX_CANONICAL,CX_ITERS,CX_TRIALS,CX_WARMUP,CX_SAMPLES_PER_POINT,CX_WARMUP_SEMANTICS,CX_SEED,CX_RUN_TIMEOUT,CX_NCCL_HOME,CX_ALLOW_MNNVL,CX_ATTEMPT_ID,CX_RUNTIME_MARKER,CX_MORI_KERNEL_TYPE,CX_WORKLOAD_DIR,CX_BACKEND_CACHE_ROOT,CX_BACKEND_CACHE_SENTINEL_SHA256,CX_BACKEND_SOURCE_ROOT,CX_AUDIT_SALT,CX_SOCKET_IFNAME,CX_RDMA_DEVICES,CX_IB_GID_INDEX,CX_RDMA_SERVICE_LEVEL,MASTER_ADDR,MASTER_PORT,RANK,WORLD_SIZE,LOCAL_RANK,LOCAL_WORLD_SIZE,NCCL_NET,NCCL_SOCKET_IFNAME,GLOO_SOCKET_IFNAME,NCCL_IB_HCA,NCCL_IB_GID_INDEX,NCCL_IB_SL,NVSHMEM_DISABLE_IB,NVSHMEM_ENABLE_NIC_PE_MAPPING,NVSHMEM_HCA_LIST,NVSHMEM_IB_GID_INDEX,NVSHMEM_IB_SL,NVSHMEM_IB_ENABLE_IBGDA,NVSHMEM_IBGDA_NIC_HANDLER,EP_NIC_NAME,EP_OVERRIDE_RDMA_SL,UCCL_SOCKET_IFNAME,UCCL_IB_GID_INDEX,UCCL_IB_SL,MORI_RDMA_DEVICES,HYBRID_EP_MULTINODE,USE_NIXL,RDMA_CORE_HOME,DEEPEP_HYBRID_BUILD_MODE,NCCL_CUMEM_ENABLE,NCCL_MNNVL_ENABLE,MC_FORCE_MNNVL,MORI_DISABLE_AUTO_XGMI,MORI_ENABLE_SDMA,MORI_APP_LOG_LEVEL,MORI_SHMEM_LOG_LEVEL,MORI_IO_LOG_LEVEL'
printf '%s' ',MORI_COMMIT'
}
@@ -623,7 +623,7 @@ cx_apply_network_profile() {
[[ "$nodes" =~ ^[1-9][0-9]*$ ]] || cx_die "invalid network placement"
unset NCCL_NET NCCL_SOCKET_IFNAME GLOO_SOCKET_IFNAME NCCL_IB_HCA
unset NCCL_IB_GID_INDEX NCCL_IB_SL
- unset NVSHMEM_DISABLE_IB
+ unset NVSHMEM_DISABLE_IB NVSHMEM_ENABLE_NIC_PE_MAPPING
unset NVSHMEM_HCA_LIST NVSHMEM_IB_GID_INDEX NVSHMEM_IB_SL
unset NVSHMEM_IB_ENABLE_IBGDA NVSHMEM_IBGDA_NIC_HANDLER
unset EP_NIC_NAME EP_OVERRIDE_RDMA_SL
@@ -659,6 +659,7 @@ cx_apply_network_profile() {
[ -n "$ep_nic" ] || ep_nic="$rdma_name"
done
export NVSHMEM_HCA_LIST="$CX_RDMA_DEVICES"
+ export NVSHMEM_ENABLE_NIC_PE_MAPPING=1
if [ "$scaleout" = 1 ]; then
export NCCL_NET=IB NCCL_IB_HCA="=$CX_RDMA_DEVICES"
export MORI_RDMA_DEVICES="$rdma_names" EP_NIC_NAME="$ep_nic"
@@ -1631,7 +1632,7 @@ cx_lock_canonical_gha_env() {
unset CX_AUDIT_SALT
unset NCCL_NET NCCL_SOCKET_IFNAME GLOO_SOCKET_IFNAME NCCL_IB_HCA
unset NCCL_IB_GID_INDEX NCCL_IB_SL
- unset NVSHMEM_DISABLE_IB
+ unset NVSHMEM_DISABLE_IB NVSHMEM_ENABLE_NIC_PE_MAPPING
unset NVSHMEM_HCA_LIST NVSHMEM_IB_GID_INDEX NVSHMEM_IB_SL
unset NVSHMEM_IB_ENABLE_IBGDA NVSHMEM_IBGDA_NIC_HANDLER
unset EP_NIC_NAME EP_OVERRIDE_RDMA_SL
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 965738a2fe..c289150cc0 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -606,6 +606,7 @@ def test_scaleout_network_profile_is_explicit_and_allowlisted(self) -> None:
cx_apply_network_profile 1 nvlink
test -z "${NVSHMEM_DISABLE_IB+x}${NCCL_NET+x}${NCCL_IB_HCA+x}"
test "$NVSHMEM_HCA_LIST:$NVSHMEM_IB_GID_INDEX" = mlx5_0:1,mlx5_1:1:3
+ test "$NVSHMEM_ENABLE_NIC_PE_MAPPING" = 1
test "$NVSHMEM_IB_ENABLE_IBGDA:$NVSHMEM_IBGDA_NIC_HANDLER" = 1:gpu
unset CX_BENCH CX_MODE CX_IB_GID_INDEX
unset CX_SHARD_SKU
@@ -618,6 +619,7 @@ def test_scaleout_network_profile_is_explicit_and_allowlisted(self) -> None:
test "$NCCL_SOCKET_IFNAME:$GLOO_SOCKET_IFNAME:$UCCL_SOCKET_IFNAME" = ib0:ib0:ib0
test "$NCCL_NET:$NCCL_IB_HCA" = 'IB:=mlx5_0:1,mlx5_1:1'
test "$NVSHMEM_HCA_LIST" = mlx5_0:1,mlx5_1:1
+ test "$NVSHMEM_ENABLE_NIC_PE_MAPPING" = 1
test "$MORI_RDMA_DEVICES:$EP_NIC_NAME" = mlx5_0,mlx5_1:mlx5_0
test "$NCCL_IB_GID_INDEX:$NCCL_IB_SL" = 3:2
test "$NVSHMEM_IB_ENABLE_IBGDA:$NVSHMEM_IBGDA_NIC_HANDLER" = 1:gpu
From 7fb656621663693b4f625ee58c264109a0c91fa1 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 20:00:55 +0800
Subject: [PATCH 052/210] fix(collectivex): repair V2 identity and AMD staging
---
experimental/CollectiveX/README.md | 10 ++--
experimental/CollectiveX/capability.py | 5 +-
experimental/CollectiveX/contracts.py | 5 +-
experimental/CollectiveX/docs/methodology.md | 3 +-
experimental/CollectiveX/runtime/common.sh | 50 +++++++++++++++++--
.../CollectiveX/runtime/run_in_container.sh | 15 +++---
.../schemas/raw-case-v1.schema.json | 4 ++
.../CollectiveX/tests/ep_deepep_v2.py | 8 ++-
.../tests/test_deepep_v2_contract.py | 21 ++++++--
.../tests/test_precision_scheduling.py | 8 +--
.../tests/test_sampling_contract.py | 34 +++++++++++++
11 files changed, 138 insertions(+), 25 deletions(-)
diff --git a/experimental/CollectiveX/README.md b/experimental/CollectiveX/README.md
index 33fe811c1d..11acdad9bf 100644
--- a/experimental/CollectiveX/README.md
+++ b/experimental/CollectiveX/README.md
@@ -60,7 +60,7 @@ scale-up domain.
| Backend | Current scope |
|---|---|
| DeepEP V1 | Image-pinned `deep_ep.Buffer`: normal and native low-latency APIs; upstream v1.2.1 on x86 and the image's GB fork on arm64 |
-| DeepEP V2 | PR #605 `ElasticBuffer` plus #630: LSA for scale-up and GIN for x86 EP16 scale-out; source/SASS-bound reproducible JIT |
+| DeepEP V2 | PR #605 `ElasticBuffer` plus exact upstream #630 and #640 fixes: LSA for scale-up and GIN for x86 EP16 scale-out; source/SASS-bound reproducible JIT |
| DeepEP Hybrid | Pinned `HybridEPBuffer`: x86 EP16 multi-domain RDMA/DOCA; GB EP8/EP16 in one MNNVL communication domain |
| UCCL | Pinned 0.1.1 wheel and wrapper with normal and native low-latency APIs on Hopper; Blackwell is explicitly unsupported |
| NCCL/RCCL A2A | Portable rank-deduplicated payload plus expert/routing-metadata reference |
@@ -71,9 +71,11 @@ misreported as a platform capability limitation and can return after a stable pi
DeepEP V2 means the `ElasticBuffer` implementation introduced by
[DeepEP PR #605](https://github.com/deepseek-ai/DeepEP/pull/605), not a newer legacy `Buffer` build.
-The pinned source is the minimal upstream [PR #630](https://github.com/deepseek-ai/DeepEP/pull/630)
-follow-up: its parent is the #605 merge tree and its only source change fixes pure scale-up
-initialization when GIN is unavailable. Scale-up cases request NCCL Device API LSA and fail closed
+The pinned source is the [PR #630](https://github.com/deepseek-ai/DeepEP/pull/630) head, whose parent
+is the #605 merge tree, plus the exact one-line library matcher from upstream
+[PR #640](https://github.com/deepseek-ai/DeepEP/pull/640). The first fixes pure scale-up
+initialization when GIN is unavailable; the second prevents NCCL shared-memory mappings from being
+misclassified as duplicate NCCL libraries. Scale-up cases request NCCL Device API LSA and fail closed
unless the realized LSA team covers the full EP world. x86 EP16 scale-out cases instead require the
hybrid path with GIN, two logical scale-out domains represented by two physical RDMA ranks, and eight
scale-up ranks per domain; GB EP16 remains MNNVL scale-up and therefore uses LSA. The isolated build
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
index ebd3a6a352..2e0af58cf8 100644
--- a/experimental/CollectiveX/capability.py
+++ b/experimental/CollectiveX/capability.py
@@ -142,7 +142,7 @@ def _platform(
"deepep-v2": {
"vendors": {"nvidia"},
"implementation": "deep_ep.ElasticBuffer",
- "source": "deepseek-ai/DeepEP#605+#630",
+ "source": "deepseek-ai/DeepEP#605+#630+#640",
"commit": DEEPEP_V2_COMMIT,
"communication_backend": "nccl-device-lsa",
"torch": "2.10.0+cu130",
@@ -357,6 +357,9 @@ def _precision_rule(
for ep in (8, 16)),
("28738738793", "b300", 8, "deepep", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed"),
("28738738793", "b300", 8, "deepep-hybrid", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "native-operation-failed"),
+ ("28739555164", "h200-dgxc", 16, "deepep", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed"),
+ ("28739555164", "h200-dgxc", 16, "deepep", "low-latency", _LL_FP8_LOGFMT_PROFILE, "supported", "native-probe-passed"),
+ ("28739555164", "h200-dgxc", 16, "deepep-hybrid", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "native-operation-failed"),
)
PRECISION_CELL_OVERRIDES.update({
(profile, backend, sku, ep, mode): {
diff --git a/experimental/CollectiveX/contracts.py b/experimental/CollectiveX/contracts.py
index 49884bad1f..b196a4fcbf 100644
--- a/experimental/CollectiveX/contracts.py
+++ b/experimental/CollectiveX/contracts.py
@@ -89,6 +89,8 @@
"deepep_tree": "29809e75c5874e6609dac4804e7b651d5226959f",
"deepep_pr": 605,
"deepep_fix_pr": 630,
+ "deepep_nccl_check_fix_pr": 640,
+ "deepep_nccl_check_commit": "93d0564188f7a0a6288c6e316484861b0efa042e",
"fmt_commit": "a4c7e17133ee9cb6a2f45545f6e974dd3c393efa",
"torch_version": "2.10.0+cu130",
"nccl_package_version": "2.30.4",
@@ -130,7 +132,8 @@
"api", "api_signature_sha256", "backend", "backend_lineage", "block_num",
"block_num_floored", "block_num_target", "branch", "collective_library",
"combine_dtype", "combine_warps", "communication_backend", "cuda_version",
- "deepep_commit", "deepep_distribution_version", "deepep_fix_pr", "deepep_pr", "deepep_tree",
+ "deepep_commit", "deepep_distribution_version", "deepep_fix_pr",
+ "deepep_nccl_check_commit", "deepep_nccl_check_fix_pr", "deepep_pr", "deepep_tree",
"deepep_version", "deterministic", "device_cus",
"device_sms", "dispatch_dtype", "dispatch_warps", "enable_sdma", "fmt_commit",
"gin_enabled",
diff --git a/experimental/CollectiveX/docs/methodology.md b/experimental/CollectiveX/docs/methodology.md
index 03a4a9c1aa..2a8ae75352 100644
--- a/experimental/CollectiveX/docs/methodology.md
+++ b/experimental/CollectiveX/docs/methodology.md
@@ -53,7 +53,8 @@ Physical host count does not define scope. Both GB cells remain inside one 72-GP
domain.
Unsupported combinations are terminal outcomes, not silently skipped coverage. DeepEP V2 is the
-`ElasticBuffer` introduced by PR #605, pinned with upstream PR #630's minimal pure-scale-up fix.
+`ElasticBuffer` introduced by PR #605, pinned with upstream PR #630's minimal pure-scale-up fix and
+the exact upstream PR #640 library matcher that excludes NCCL shared-memory mappings.
Scale-up cases request NCCL Device API LSA and fail closed unless the realized LSA team covers the
full EP world. x86 EP16 scale-out uses the hybrid path with GIN and requires two logical scale-out
domains represented by two physical RDMA ranks, with eight scale-up ranks per domain. GB EP16
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 71961a0f69..c3b27a4f55 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -11,6 +11,10 @@ CX_SQUASH_SOURCE_DATE_EPOCH=1
CX_DEEPEP_V2_COMMIT="fa8a9b16898204afd347c663b89e65ef87dc6ce6" # pragma: allowlist secret
CX_DEEPEP_V2_TREE="29809e75c5874e6609dac4804e7b651d5226959f" # pragma: allowlist secret
CX_DEEPEP_V2_FMT_COMMIT="a4c7e17133ee9cb6a2f45545f6e974dd3c393efa" # pragma: allowlist secret
+# Consumed by run_in_container.sh after this helper is sourced.
+# shellcheck disable=SC2034
+CX_DEEPEP_V2_NCCL_CHECK_COMMIT="93d0564188f7a0a6288c6e316484861b0efa042e" # pragma: allowlist secret
+CX_DEEPEP_V2_INIT_SHA256="f090bbacc38c7d5dba29f07fd38a918eb820b6adac6f76903fe56273060e4870"
CX_DEEPEP_HYBRID_COMMIT="e0a5b1d9848ab3e7b4a67842bf06f067bfac67f8" # pragma: allowlist secret
CX_DEEPEP_HYBRID_TREE="d77aeab7f1bb52b615666fe178d26ced41fae08e" # pragma: allowlist secret
CX_DEEPEP_HYBRID_NCCL_COMMIT="1e0c869c39bb33f1034cb9920bd2a8a8406f04a3" # pragma: allowlist secret
@@ -241,8 +245,8 @@ REQUIRED = {
"b300": {"partition", "account", "squash_dir"},
"gb200": {"partition", "account", "storage_roots"},
"gb300": {"partition", "account", "squash_dir", "enroot_cache_path"},
- "mi325x": {"partition", "squash_dir", "stage_dir"},
- "mi355x": {"partition", "squash_dir", "stage_dir"},
+ "mi325x": {"partition", "squash_dir"},
+ "mi355x": {"partition", "squash_dir"},
}
ALLOWED = {
"h100-dgxc": REQUIRED["h100-dgxc"] | {"exclude_nodes", "stage_dir"} | NETWORK_FIELDS,
@@ -1350,7 +1354,14 @@ cx_backend_source_is_valid() {
status="$(GIT_OPTIONAL_LOCKS=0 cx_git_in_tree "$source" \
status --porcelain --untracked-files=all --ignore-submodules=none \
2>/dev/null)" || return 1
- [ -z "$status" ] || return 1
+ if [ "$backend" = deepep-v2 ]; then
+ [ "$status" = " M deep_ep/__init__.py" ] \
+ && [ "$(sha256sum "$source/deep_ep/__init__.py" | awk '{print $1}')" = \
+ "$CX_DEEPEP_V2_INIT_SHA256" ] \
+ || return 1
+ else
+ [ -z "$status" ] || return 1
+ fi
ignored="$(cx_git_in_tree "$source" ls-files --others --ignored --exclude-standard \
2>/dev/null)" || return 1
[ -z "$ignored" ] || return 1
@@ -1361,6 +1372,26 @@ cx_backend_source_is_valid() {
|| [ "$(cx_git_in_tree "$source/third-party/nccl" rev-parse HEAD 2>/dev/null)" = "$nccl" ]
}
+cx_apply_deepep_v2_nccl_check_fix() {
+ local source="$1"
+ python3 - "$source/deep_ep/__init__.py" "$CX_DEEPEP_V2_INIT_SHA256" <<'PY'
+import hashlib
+from pathlib import Path
+import sys
+
+path = Path(sys.argv[1])
+expected = sys.argv[2]
+old = "for so in [line.strip().split(' ')[-1] for line in f if 'nccl' in line]:"
+new = "for so in [line.strip().split(' ')[-1] for line in f if 'libnccl' in line]:"
+payload = path.read_text(encoding="utf-8")
+if payload.count(old) != 1 or new in payload:
+ raise SystemExit(1)
+path.write_text(payload.replace(old, new), encoding="utf-8")
+if hashlib.sha256(path.read_bytes()).hexdigest() != expected:
+ raise SystemExit(1)
+PY
+}
+
cx_extension_pair_sha256() {
python3 - "$1" "$2" "$3" <<'PY'
import hashlib
@@ -1488,6 +1519,13 @@ _cx_prepare_backend_source() {
rm -rf -- "$temporary"
return 1
fi
+ if [ "$backend" = deepep-v2 ]; then
+ CX_BACKEND_SOURCE_STEP="upstream NCCL check fix"
+ cx_apply_deepep_v2_nccl_check_fix "$temporary" || {
+ rm -rf -- "$temporary"
+ return 1
+ }
+ fi
CX_BACKEND_SOURCE_STEP="source publication validation"
if ! cx_backend_source_is_valid "$backend" "$temporary" \
|| ! mv -- "$temporary" "$source"; then
@@ -1666,6 +1704,12 @@ cx_lock_canonical_gha_env() {
trusted_stage_dir="$(cx_prepare_implicit_stage_base)" \
|| cx_die "canonical CollectiveX execution cannot create an isolated stage directory"
;;
+ mi325x|mi355x)
+ # AMD Slurm jobs already consume the configured squash directory from
+ # compute-visible shared storage. Use it as the isolated staging base
+ # when the private row does not provide a separate location.
+ trusted_stage_dir="$CX_SQUASH_DIR"
+ ;;
*) cx_die "canonical CollectiveX execution requires a configured shared stage directory" ;;
esac
fi
diff --git a/experimental/CollectiveX/runtime/run_in_container.sh b/experimental/CollectiveX/runtime/run_in_container.sh
index ed5d066878..69b37ef1d2 100644
--- a/experimental/CollectiveX/runtime/run_in_container.sh
+++ b/experimental/CollectiveX/runtime/run_in_container.sh
@@ -311,8 +311,8 @@ PY
}
# DeepEP V2 is PR #605's ElasticBuffer implementation with upstream PR #630's pure scale-up
-# initialization fix. Canonical launchers stage the pinned source and mount a private cluster-local
-# build cache at /cx-cache.
+# initialization fix and PR #640's exact libnccl mapping check. Canonical launchers stage the
+# pinned source and mount a private cluster-local build cache at /cx-cache.
cx_deepep_v2_root() {
local arch cpu base identity key image_digest
arch="$(cx_cuda_arch)" || return 1
@@ -325,7 +325,7 @@ cx_deepep_v2_root() {
|| return 1
# Bump the recipe generation whenever the build procedure changes. Benchmark-only
# source revisions must reuse the same immutable environment instead of leaking GBs.
- identity="deepep-v2-cache-v2|$cpu|sm${arch/./}|image=$image_digest|recipe=aot-persistent-nvshmem-active-cuda-maxjobs16-v2|$CX_DEEPEP_V2_COMMIT|$CX_DEEPEP_V2_TREE|$CX_DEEPEP_V2_FMT_COMMIT|pip=26.1.2|setuptools=82.0.1|wheel=0.47.0|ninja=1.13.0|numpy=2.2.6|torch=2.10.0+cu130|nccl=2.30.4|nvshmem=3.3.9|max-jobs=16"
+ identity="deepep-v2-cache-v3|$cpu|sm${arch/./}|image=$image_digest|recipe=aot-persistent-nvshmem-active-cuda-maxjobs16-v3|$CX_DEEPEP_V2_COMMIT|$CX_DEEPEP_V2_TREE|$CX_DEEPEP_V2_FMT_COMMIT|$CX_DEEPEP_V2_NCCL_CHECK_COMMIT|pip=26.1.2|setuptools=82.0.1|wheel=0.47.0|ninja=1.13.0|numpy=2.2.6|torch=2.10.0+cu130|nccl=2.30.4|nvshmem=3.3.9|max-jobs=16"
key="$(printf '%s' "$identity" | sha256sum | awk '{print $1}')"
[[ "$key" =~ ^[0-9a-f]{64}$ ]] || return 1
printf '%s/deepep-v2-%s' "$base" "$key"
@@ -356,11 +356,13 @@ cx_activate_deepep_v2() {
# from seeding a later run; all ranks and cases in this shard still share one cold build.
export EP_JIT_CACHE_DIR="$stage_root/.cx_backend/deepep-v2-jit"
export EP_REUSE_NCCL_COMM=1
- export DEEPEP_V2_PR=605 DEEPEP_V2_FIX_PR=630
+ export DEEPEP_V2_PR=605 DEEPEP_V2_FIX_PR=630 DEEPEP_V2_NCCL_CHECK_FIX_PR=640
DEEPEP_V2_COMMIT="$CX_DEEPEP_V2_COMMIT"
DEEPEP_V2_TREE="$CX_DEEPEP_V2_TREE"
DEEPEP_V2_FMT_COMMIT="$CX_DEEPEP_V2_FMT_COMMIT"
+ DEEPEP_V2_NCCL_CHECK_COMMIT="$CX_DEEPEP_V2_NCCL_CHECK_COMMIT"
export DEEPEP_V2_COMMIT DEEPEP_V2_TREE DEEPEP_V2_FMT_COMMIT
+ export DEEPEP_V2_NCCL_CHECK_COMMIT
[ ! -L "$stage_root/.cx_backend" ] && [ ! -L "$EP_JIT_CACHE_DIR" ] \
|| { cx_log "ERROR: DeepEP V2 JIT cache path is unsafe"; return 1; }
if ! mkdir -p "$EP_JIT_CACHE_DIR" \
@@ -571,7 +573,7 @@ cx_build_deepep_v2() {
lock_path="${root}.lock"
command -v flock >/dev/null || { cx_log "ERROR: flock is required for DeepEP V2"; return 1; }
mkdir -p "${root%/*}" || return 1
- cx_log "DeepEP V2: preparing PR #605 implementation with upstream PR #630 fix ($revision)"
+ cx_log "DeepEP V2: preparing PR #605 with upstream PR #630 and #640 fixes ($revision)"
if ! (
[ ! -L "$lock_path" ] \
|| { cx_log "ERROR: DeepEP V2 cache lock is unsafe"; exit 1; }
@@ -888,7 +890,8 @@ cx_persist_backend_env() {
NVSHMEM_DIR DEEPEP_COMMIT DEEPEP_TREE
EP_NCCL_ROOT_DIR EP_NVSHMEM_ROOT_DIR EP_JIT_CACHE_DIR EP_REUSE_NCCL_COMM
EP_JIT_DUMP_SASS
- DEEPEP_V2_PR DEEPEP_V2_FIX_PR DEEPEP_V2_COMMIT DEEPEP_V2_TREE DEEPEP_V2_FMT_COMMIT
+ DEEPEP_V2_PR DEEPEP_V2_FIX_PR DEEPEP_V2_NCCL_CHECK_FIX_PR DEEPEP_V2_COMMIT
+ DEEPEP_V2_TREE DEEPEP_V2_FMT_COMMIT DEEPEP_V2_NCCL_CHECK_COMMIT
DEEPEP_V2_JIT_RANDOM_SEED
HYBRID_EP_MULTINODE USE_NIXL RDMA_CORE_HOME DEEPEP_HYBRID_BUILD_MODE
UCCL_COMMIT UCCL_WRAPPER_COMMIT CX_UCCL_WRAPPER)
diff --git a/experimental/CollectiveX/schemas/raw-case-v1.schema.json b/experimental/CollectiveX/schemas/raw-case-v1.schema.json
index 4308fd46d6..6b12afc21b 100644
--- a/experimental/CollectiveX/schemas/raw-case-v1.schema.json
+++ b/experimental/CollectiveX/schemas/raw-case-v1.schema.json
@@ -650,6 +650,8 @@
"allow_hybrid_mode": {"type":"boolean"},
"communication_backend": {"enum":["nccl-device-lsa","nccl-gin"]},
"deepep_fix_pr": {"const":630},
+ "deepep_nccl_check_commit": {"const":"93d0564188f7a0a6288c6e316484861b0efa042e"},
+ "deepep_nccl_check_fix_pr": {"const":640},
"deepep_pr": {"const":605},
"deterministic": {"type": "boolean"},
"gin_enabled": {"type":"boolean"},
@@ -728,6 +730,8 @@
"deepep_commit",
"deepep_distribution_version",
"deepep_fix_pr",
+ "deepep_nccl_check_commit",
+ "deepep_nccl_check_fix_pr",
"deepep_pr",
"deepep_tree",
"deepep_version",
diff --git a/experimental/CollectiveX/tests/ep_deepep_v2.py b/experimental/CollectiveX/tests/ep_deepep_v2.py
index f9572a53cd..e3da3f6090 100644
--- a/experimental/CollectiveX/tests/ep_deepep_v2.py
+++ b/experimental/CollectiveX/tests/ep_deepep_v2.py
@@ -1,5 +1,5 @@
#!/usr/bin/env python3
-"""DeepEP PR #605 adapter with PR #630's pure scale-up initialization fix."""
+"""DeepEP PR #605 adapter with the exact upstream PR #630 and #640 fixes."""
from __future__ import annotations
@@ -30,6 +30,8 @@
DEEPEP_V2_PR = 605
DEEPEP_V2_FIX_PR = 630
+DEEPEP_V2_NCCL_CHECK_FIX_PR = 640
+DEEPEP_V2_NCCL_CHECK_COMMIT = "93d0564188f7a0a6288c6e316484861b0efa042e"
DEEPEP_V2_COMMIT = "fa8a9b16898204afd347c663b89e65ef87dc6ce6"
DEEPEP_V2_TREE = "29809e75c5874e6609dac4804e7b651d5226959f"
DEEPEP_V2_FMT_COMMIT = "a4c7e17133ee9cb6a2f45545f6e974dd3c393efa"
@@ -254,9 +256,11 @@ def _require_runtime() -> tuple[str, str]:
expected = {
"DEEPEP_V2_PR": str(DEEPEP_V2_PR),
"DEEPEP_V2_FIX_PR": str(DEEPEP_V2_FIX_PR),
+ "DEEPEP_V2_NCCL_CHECK_FIX_PR": str(DEEPEP_V2_NCCL_CHECK_FIX_PR),
"DEEPEP_V2_COMMIT": DEEPEP_V2_COMMIT,
"DEEPEP_V2_TREE": DEEPEP_V2_TREE,
"DEEPEP_V2_FMT_COMMIT": DEEPEP_V2_FMT_COMMIT,
+ "DEEPEP_V2_NCCL_CHECK_COMMIT": DEEPEP_V2_NCCL_CHECK_COMMIT,
"DEEPEP_V2_JIT_RANDOM_SEED": DEEPEP_V2_JIT_RANDOM_SEED,
"EP_JIT_DUMP_SASS": "1",
}
@@ -423,6 +427,8 @@ def __init__(self, args, rank, world_size, local_rank, device):
"deepep_tree": DEEPEP_V2_TREE,
"deepep_pr": DEEPEP_V2_PR,
"deepep_fix_pr": DEEPEP_V2_FIX_PR,
+ "deepep_nccl_check_fix_pr": DEEPEP_V2_NCCL_CHECK_FIX_PR,
+ "deepep_nccl_check_commit": DEEPEP_V2_NCCL_CHECK_COMMIT,
"fmt_commit": DEEPEP_V2_FMT_COMMIT,
"api": "deep_ep.ElasticBuffer",
"api_signature_sha256": _api_sha256(),
diff --git a/experimental/CollectiveX/tests/test_deepep_v2_contract.py b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
index f74e97c1d3..c8eded7452 100644
--- a/experimental/CollectiveX/tests/test_deepep_v2_contract.py
+++ b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
@@ -33,6 +33,7 @@
COMMIT = "fa8a9b16898204afd347c663b89e65ef87dc6ce6"
TREE = "29809e75c5874e6609dac4804e7b651d5226959f"
FMT_COMMIT = "a4c7e17133ee9cb6a2f45545f6e974dd3c393efa"
+NCCL_CHECK_COMMIT = "93d0564188f7a0a6288c6e316484861b0efa042e"
def deepep_v2_jit_provenance() -> list[dict[str, str]]:
@@ -127,7 +128,7 @@ def test_capability_is_explicit_for_every_sku(self) -> None:
(backend["implementation"], backend["commit"], backend["torch"], backend["nccl"]),
("deep_ep.ElasticBuffer", COMMIT, "2.10.0+cu130", "2.30.4"),
)
- self.assertEqual(backend["source"], "deepseek-ai/DeepEP#605+#630")
+ self.assertEqual(backend["source"], "deepseek-ai/DeepEP#605+#630+#640")
self.assertEqual(backend["communication_backend"], "nccl-device-lsa")
self.assertEqual(set(backend["sku_capabilities"]), set(capability.PLATFORMS))
for sku, platform in capability.PLATFORMS.items():
@@ -165,6 +166,8 @@ def test_adapter_ast_pins_elastic_api_and_weight_semantics(self) -> None:
self.assertEqual(constants["DEEPEP_V2_FMT_COMMIT"], FMT_COMMIT)
self.assertEqual(constants["DEEPEP_V2_PR"], 605)
self.assertEqual(constants["DEEPEP_V2_FIX_PR"], 630)
+ self.assertEqual(constants["DEEPEP_V2_NCCL_CHECK_FIX_PR"], 640)
+ self.assertEqual(constants["DEEPEP_V2_NCCL_CHECK_COMMIT"], NCCL_CHECK_COMMIT)
self.assertEqual(
constants["DEEPEP_V2_JIT_RANDOM_SEED"],
"collectivex-deepep-v2-fa8a9b1",
@@ -346,6 +349,13 @@ def test_adapter_ast_pins_elastic_api_and_weight_semantics(self) -> None:
self.assertEqual(provenance["properties"]["gin_enabled"], {"type": "boolean"})
self.assertEqual(provenance["properties"]["deepep_pr"], {"const": 605})
self.assertEqual(provenance["properties"]["deepep_fix_pr"], {"const": 630})
+ self.assertEqual(
+ provenance["properties"]["deepep_nccl_check_fix_pr"], {"const": 640}
+ )
+ self.assertEqual(
+ provenance["properties"]["deepep_nccl_check_commit"],
+ {"const": NCCL_CHECK_COMMIT},
+ )
self.assertEqual(
provenance["properties"]["communication_backend"],
{"enum": ["nccl-device-lsa", "nccl-gin"]},
@@ -1444,12 +1454,12 @@ def test_runtime_pins_uccl_wheel_and_hybrid_source_tree(self) -> None:
]
self.assertIn('cccl="${CX_CUDA_CCCL:-}"', jit)
self.assertNotIn("/usr/local/cuda*", jit)
- self.assertIn("deepep-v2-cache-v2|$cpu|sm${arch/./}", runtime)
+ self.assertIn("deepep-v2-cache-v3|$cpu|sm${arch/./}", runtime)
self.assertNotIn("deepep-v2-cache-v1|", runtime)
self.assertIn('base="${CX_BACKEND_CACHE_ROOT:-}"', runtime)
self.assertNotIn("${CX_BACKEND_CACHE_ROOT:-$PWD/.cx_backend}", runtime)
self.assertIn(
- "recipe=aot-persistent-nvshmem-active-cuda-maxjobs16-v2", runtime
+ "recipe=aot-persistent-nvshmem-active-cuda-maxjobs16-v3", runtime
)
self.assertNotIn("recipe=aot-source-date-epoch-arch-maxjobs16-v1", runtime)
self.assertNotIn("recipe=$source_sha", runtime)
@@ -2092,6 +2102,7 @@ def test_operator_config_is_strict_per_runner_json(self) -> None:
(missing_h100_stage, "h100-dgxc"),
(missing_b300_stage, "b300"),
(missing_gb300_stage, "gb300"),
+ (missing_amd_stage, "mi325x"),
):
config.write_text(json.dumps(valid))
config.chmod(0o600)
@@ -2139,7 +2150,9 @@ def test_operator_config_is_strict_per_runner_json(self) -> None:
"COLLECTIVEX_OPERATOR_CONFIG": str(config),
},
)
- self.assertNotEqual(selected_missing_stage.returncode, 0)
+ self.assertEqual(
+ selected_missing_stage.returncode, 0, selected_missing_stage.stderr
+ )
config.write_text(json.dumps(document))
config.chmod(0o644)
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
index 265b611c0b..a2d54d4e85 100644
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -28,7 +28,7 @@ def test_precision_probe_inventory_is_exact_and_non_mutating(self) -> None:
item["precision_profile"],
)
self.assertEqual(targets, sorted(capability.provisional_precision_targets(), key=key))
- self.assertEqual(len(targets), 34)
+ self.assertEqual(len(targets), 31)
self.assertEqual(capability.PRECISION_CAPABILITIES, before)
self.assertEqual(
len({
@@ -222,13 +222,13 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
{"provisional", "supported", "unsupported"},
)
self.assertEqual(
- len(targets) - len(capability.provisional_precision_targets()), 60
+ len(targets) - len(capability.provisional_precision_targets()), 63
)
self.assertEqual(
- sum(item["disposition"] == "supported" for item in targets), 39
+ sum(item["disposition"] == "supported" for item in targets), 41
)
self.assertEqual(
- sum(item["disposition"] == "unsupported" for item in targets), 21
+ sum(item["disposition"] == "unsupported" for item in targets), 22
)
keys = {
(
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index c289150cc0..65444d4f61 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -2499,6 +2499,14 @@ def test_canonical_gha_environment_is_locked_but_manual_overrides_survive(self)
test "$CX_STAGE_DIR" = /validated/amd-stage
test "$MORI_COMMIT" = "$CX_MORI_COMMIT_MI355"
+ export COLLECTIVEX_OPERATOR_CONFIG_LOADED=$$
+ export CX_SHARD_SKU=mi355x CX_NODES=1 CX_GPUS_PER_NODE=8
+ export CX_SQUASH_DIR=/validated/amd-shared
+ export CX_AUDIT_SALT="$(printf 'a%.0s' {1..64})"
+ unset CX_STAGE_DIR
+ cx_lock_canonical_gha_env mi355x
+ test "$CX_STAGE_DIR" = /validated/amd-shared
+
unset COLLECTIVEX_CANONICAL_GHA
unset COLLECTIVEX_OPERATOR_CONFIG_LOADED
CX_IMAGE=manual CX_IMAGE_DIGEST=manual CX_NGPUS=3
@@ -3302,6 +3310,32 @@ def test_backend_source_validation_rejects_status_errors_and_ignored_files(self)
check=True,
)
+ def test_deepep_v2_applies_only_the_pinned_upstream_nccl_check_fix(self) -> None:
+ common = ROOT / "runtime" / "common.sh"
+ with tempfile.TemporaryDirectory() as temporary:
+ source = Path(temporary)
+ init = source / "deep_ep" / "__init__.py"
+ init.parent.mkdir()
+ init.write_text(
+ "for so in [line.strip().split(' ')[-1] for line in f if 'nccl' in line]:\n"
+ )
+ command = r'''
+ set -euo pipefail
+ source "$1"
+ expected_file="$2/expected"
+ sed "s/if 'nccl' in line/if 'libnccl' in line/" \
+ "$2/deep_ep/__init__.py" > "$expected_file"
+ CX_DEEPEP_V2_INIT_SHA256="$(sha256sum "$expected_file" | awk '{print $1}')"
+ rm -f "$expected_file"
+ cx_apply_deepep_v2_nccl_check_fix "$2"
+ grep -Fq "if 'libnccl' in line" "$2/deep_ep/__init__.py"
+ ! cx_apply_deepep_v2_nccl_check_fix "$2"
+ '''
+ subprocess.run(
+ ["bash", "-c", command, "_", str(common), temporary],
+ check=True,
+ )
+
def test_backend_source_root_normalizes_inherited_special_mode(self) -> None:
common = ROOT / "runtime" / "common.sh"
with tempfile.TemporaryDirectory() as temporary:
From ab729d9311450a4528b308d2783744ac287a59cb Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 20:04:02 +0800
Subject: [PATCH 053/210] fix(collectivex): stabilize multinode startup
---
experimental/CollectiveX/runtime/common.sh | 8 ++++---
.../CollectiveX/runtime/run_in_container.sh | 15 ++++++++----
.../tests/test_deepep_v2_contract.py | 3 +++
.../tests/test_sampling_contract.py | 24 +++++++++++++++++++
4 files changed, 42 insertions(+), 8 deletions(-)
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index c3b27a4f55..770a737158 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -766,10 +766,12 @@ BASH
}
cx_resolve_slurm_rendezvous() {
- local job_id="$1" nodes master_addr master_port
+ local job_id="$1" master_addr master_port
[[ "$job_id" =~ ^[1-9][0-9]*$ ]] || cx_die "invalid rendezvous allocation"
- nodes="$(squeue -j "$job_id" -h -o %N 2>/dev/null)"
- master_addr="$(scontrol show hostnames "$nodes" 2>/dev/null | head -n1)"
+ # Expanded Slurm nodelists may be sorted differently from task placement.
+ # Query relative node zero directly so MASTER_ADDR always hosts global rank 0.
+ master_addr="$(srun --jobid="$job_id" --nodes=1 --ntasks=1 --relative=0 \
+ --chdir=/tmp --export="$(cx_host_exports)" hostname -s 2>/dev/null | head -n1)"
master_port="${CX_MASTER_PORT:-29551}"
[[ "$master_addr" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] \
|| cx_die "could not resolve the allocated primary node"
diff --git a/experimental/CollectiveX/runtime/run_in_container.sh b/experimental/CollectiveX/runtime/run_in_container.sh
index 69b37ef1d2..5edbd9e9f5 100644
--- a/experimental/CollectiveX/runtime/run_in_container.sh
+++ b/experimental/CollectiveX/runtime/run_in_container.sh
@@ -831,7 +831,7 @@ cx_build_uccl() {
python3 -c "import torch; from uccl_deepep import Buffer" 2>/dev/null || return 1
return 0
fi
- local version="0.1.1" tag="v0.1.1"
+ local version="0.1.1" tag="v0.1.1" node_id wrapper_root
local wheel_sha256="390c1320918972206546e44d79b132988f2818ec07e23afcd0595f7183916cec"
cx_log "UCCL EP: installing uccl==$version + cu12 runtime shim"
export PIP_BREAK_SYSTEM_PACKAGES=1
@@ -860,16 +860,21 @@ cx_build_uccl() {
# (uccl_deepep) so it doesn't shadow the container's real deep_ep. Its Buffer(group, num_nvl_bytes,
# ...) takes a torch ProcessGroup (matching DeepEP + ep_uccl.py's calls) and runs the full
# proxy/IPC-handle/runtime.sync bootstrap that the low-level uccl.ep.Buffer(rank,num_ranks) lacks.
- rm -rf /tmp/uccl_src /tmp/uccl_deepep_pkg
+ case "${SLURM_NODEID:-0}" in ""|*[!0-9]*) return 1 ;; esac
+ node_id="${SLURM_NODEID:-0}"
+ wrapper_root="$PWD/.cx_backend/uccl-wrapper-node-$node_id"
+ case "$wrapper_root" in /ix/experimental/CollectiveX/.cx_backend/uccl-wrapper-node-*) ;; *) return 1 ;; esac
+ rm -rf /tmp/uccl_src "$wrapper_root"
# Pin the wrapper to the SAME tag as the installed wheel (pkg-0.1.1 -> v0.1.1): the wrapper's
# dispatch calls into uccl.ep (get_rdma_buffer etc.), so a main-branch wrapper vs a 0.1.1 wheel
# mismatches signatures. Match them.
if git clone --depth 1 --branch "$tag" https://github.com/uccl-project/uccl /tmp/uccl_src >&2 2>&1 \
&& [ "$(git -C /tmp/uccl_src rev-parse HEAD)" = "73ee4f12ba71717d6de34ba06806e1baaabe3f42" ] \
&& [ -d /tmp/uccl_src/ep/deep_ep_wrapper/deep_ep ]; then
- mkdir -p /tmp/uccl_deepep_pkg/uccl_deepep
- cp /tmp/uccl_src/ep/deep_ep_wrapper/deep_ep/*.py /tmp/uccl_deepep_pkg/uccl_deepep/ 2>/dev/null
- export PYTHONPATH="/tmp/uccl_deepep_pkg:${PYTHONPATH:-}"
+ mkdir -p "$wrapper_root/uccl_deepep"
+ chmod 700 "$PWD/.cx_backend" "$wrapper_root" "$wrapper_root/uccl_deepep"
+ cp /tmp/uccl_src/ep/deep_ep_wrapper/deep_ep/*.py "$wrapper_root/uccl_deepep/" 2>/dev/null
+ export PYTHONPATH="$wrapper_root:${PYTHONPATH:-}"
python3 -c "import torch; from uccl_deepep import Buffer; print('uccl_deepep wrapper ready')" >&2 \
|| { cx_log "ERROR: uccl_deepep wrapper import failed"; return 1; }
export CX_UCCL_WRAPPER=1
diff --git a/experimental/CollectiveX/tests/test_deepep_v2_contract.py b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
index c8eded7452..bf8e3cad0a 100644
--- a/experimental/CollectiveX/tests/test_deepep_v2_contract.py
+++ b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
@@ -1407,6 +1407,9 @@ def test_runtime_pins_uccl_wheel_and_hybrid_source_tree(self) -> None:
runtime,
)
self.assertIn("--require-hashes", runtime)
+ self.assertIn('$PWD/.cx_backend/uccl-wrapper-node-$node_id', runtime)
+ self.assertNotIn("/tmp/uccl_deepep_pkg", runtime)
+ self.assertIn('export PYTHONPATH="$wrapper_root:${PYTHONPATH:-}"', runtime)
self.assertIn("d77aeab7f1bb52b615666fe178d26ced41fae08e", common)
self.assertIn("HEAD^{tree}", runtime)
self.assertIn("$PWD/.cx_backend/deepep-hybrid-", runtime)
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 65444d4f61..ad4d6c5740 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -2536,6 +2536,30 @@ def test_canonical_gha_environment_is_locked_but_manual_overrides_survive(self)
)
self.assertEqual(list(workspace.iterdir()), [])
+ def test_slurm_rendezvous_uses_relative_node_zero(self) -> None:
+ common = ROOT / "runtime" / "common.sh"
+ command = r'''
+ set -euo pipefail
+ source "$1"
+ cx_host_exports() { printf '%s' HOME,PATH,USER; }
+ squeue() { return 91; }
+ scontrol() { return 92; }
+ srun() {
+ test "$*" = "--jobid=123 --nodes=1 --ntasks=1 --relative=0 --chdir=/tmp --export=HOME,PATH,USER hostname -s" \
+ || return 93
+ printf '%s\n' rank-zero-node
+ }
+ cx_resolve_slurm_rendezvous 123
+ test "$MASTER_ADDR:$MASTER_PORT" = rank-zero-node:29551
+ CX_MASTER_PORT=30444
+ cx_resolve_slurm_rendezvous 123
+ test "$MASTER_ADDR:$MASTER_PORT" = rank-zero-node:30444
+ '''
+ subprocess.run(
+ ["bash", "-c", command, "_", str(common)],
+ check=True,
+ )
+
def test_implicit_stage_base_is_private_and_non_symlinked(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary).resolve()
From 2b77bdd4440995b7c14172e86c477f7b1be53d75 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 20:12:33 +0800
Subject: [PATCH 054/210] fix(collectivex): distinguish native IB from RoCE
---
experimental/CollectiveX/README.md | 5 +++
experimental/CollectiveX/capability.py | 3 ++
experimental/CollectiveX/runtime/common.sh | 43 +++++++++++++++++--
.../tests/test_precision_scheduling.py | 6 +--
.../tests/test_sampling_contract.py | 30 +++++++++----
5 files changed, 72 insertions(+), 15 deletions(-)
diff --git a/experimental/CollectiveX/README.md b/experimental/CollectiveX/README.md
index 11acdad9bf..1284b7f347 100644
--- a/experimental/CollectiveX/README.md
+++ b/experimental/CollectiveX/README.md
@@ -147,6 +147,11 @@ all configured interfaces and active HCA ports exist before backend setup. Scale
clear these overrides. Scale-out NCCL/RCCL is pinned to `IB` with exact-match HCA selectors so a
socket fallback fails instead of being mislabeled as RDMA.
+`ib_gid_index` is applied only when every selected HCA port reports an Ethernet link layer, where it
+selects the operator-approved RoCE GID. Native InfiniBand profiles retain explicit HCA and service
+level pinning but leave the RoCE-only GID override unset so NVSHMEM/NCCL can use the native LID path.
+Mixed Ethernet and InfiniBand HCA lists are rejected.
+
`stage_dir` is a pre-existing, runner-owned, non-symlinked base outside the checkout and workflow
workspace. It is not group- or world-writable and is visible at the same path on the runner and every
allocated node. Jobs create only a marked mode-0700 execution child, prove cross-node read/write
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
index 2e0af58cf8..9f179ad9aa 100644
--- a/experimental/CollectiveX/capability.py
+++ b/experimental/CollectiveX/capability.py
@@ -360,6 +360,9 @@ def _precision_rule(
("28739555164", "h200-dgxc", 16, "deepep", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed"),
("28739555164", "h200-dgxc", 16, "deepep", "low-latency", _LL_FP8_LOGFMT_PROFILE, "supported", "native-probe-passed"),
("28739555164", "h200-dgxc", 16, "deepep-hybrid", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "native-operation-failed"),
+ ("28740154697", "h200-dgxc", 16, "deepep", "low-latency", _LL_FP8_PROFILE, "supported", "native-probe-passed"),
+ ("28740154697", "h200-dgxc", 16, "deepep", "low-latency", _LL_LOGFMT_PROFILE, "supported", "native-probe-passed"),
+ ("28740154697", "h200-dgxc", 16, "uccl", "low-latency", _LL_LOGFMT_PROFILE, "supported", "native-probe-passed"),
)
PRECISION_CELL_OVERRIDES.update({
(profile, backend, sku, ep, mode): {
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 770a737158..021a6b7fbb 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -617,11 +617,43 @@ cx_nccl_hca_device_name() {
printf '%s' "${selector%%:*}"
}
+cx_rdma_profile_link_layer() {
+ local sysfs_root="$1" selector device configured_port port_path layer profile=""
+ local -a selectors port_paths
+ IFS=, read -r -a selectors <<< "$CX_RDMA_DEVICES"
+ for selector in "${selectors[@]}"; do
+ device="${selector%%:*}"
+ configured_port=""
+ [ "$selector" = "$device" ] || configured_port="${selector#*:}"
+ if [ -n "$configured_port" ]; then
+ port_paths=("$sysfs_root/$device/ports/$configured_port")
+ else
+ port_paths=("$sysfs_root/$device/ports"/*)
+ fi
+ [ "${#port_paths[@]}" -gt 0 ] || cx_die "RDMA selector has no visible ports"
+ for port_path in "${port_paths[@]}"; do
+ [ -r "$port_path/link_layer" ] || cx_die "RDMA link layer is unavailable"
+ layer="$(< "$port_path/link_layer")"
+ case "$layer" in
+ Ethernet) layer=roce ;;
+ InfiniBand) layer=infiniband ;;
+ *) cx_die "unsupported RDMA link layer" ;;
+ esac
+ [ -z "$profile" ] || [ "$profile" = "$layer" ] \
+ || cx_die "mixed RDMA link layers are unsupported"
+ profile="$layer"
+ done
+ done
+ [ -n "$profile" ] || cx_die "RDMA link layer is unavailable"
+ printf '%s' "$profile"
+}
+
# Convert private, runner-local network selectors into the public library
# variables needed inside the container. Values are interface/HCA identifiers,
# never addresses; the rendezvous hostname is derived from the allocation.
cx_apply_network_profile() {
- local nodes="$1" transport="$2" selector rdma_name rdma_names="" ep_nic=""
+ local nodes="$1" transport="$2" rdma_sysfs_root="${3:-/sys/class/infiniband}"
+ local selector rdma_name rdma_names="" ep_nic="" rdma_link_layer=""
local scaleout=0 single_node_rdma=0
local -a selectors
[[ "$nodes" =~ ^[1-9][0-9]*$ ]] || cx_die "invalid network placement"
@@ -672,9 +704,12 @@ cx_apply_network_profile() {
if [ -n "${CX_IB_GID_INDEX:-}" ]; then
[[ "$CX_IB_GID_INDEX" =~ ^[0-9]+$ ]] && [ "$CX_IB_GID_INDEX" -le 255 ] \
|| cx_die "invalid private IB GID index"
- export NVSHMEM_IB_GID_INDEX="$CX_IB_GID_INDEX"
- if [ "$scaleout" = 1 ]; then
- export NCCL_IB_GID_INDEX="$CX_IB_GID_INDEX" UCCL_IB_GID_INDEX="$CX_IB_GID_INDEX"
+ rdma_link_layer="$(cx_rdma_profile_link_layer "$rdma_sysfs_root")"
+ if [ "$rdma_link_layer" = roce ]; then
+ export NVSHMEM_IB_GID_INDEX="$CX_IB_GID_INDEX"
+ if [ "$scaleout" = 1 ]; then
+ export NCCL_IB_GID_INDEX="$CX_IB_GID_INDEX" UCCL_IB_GID_INDEX="$CX_IB_GID_INDEX"
+ fi
fi
fi
if [ -n "${CX_RDMA_SERVICE_LEVEL:-}" ]; then
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
index a2d54d4e85..bc401db471 100644
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -28,7 +28,7 @@ def test_precision_probe_inventory_is_exact_and_non_mutating(self) -> None:
item["precision_profile"],
)
self.assertEqual(targets, sorted(capability.provisional_precision_targets(), key=key))
- self.assertEqual(len(targets), 31)
+ self.assertEqual(len(targets), 28)
self.assertEqual(capability.PRECISION_CAPABILITIES, before)
self.assertEqual(
len({
@@ -222,10 +222,10 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
{"provisional", "supported", "unsupported"},
)
self.assertEqual(
- len(targets) - len(capability.provisional_precision_targets()), 63
+ len(targets) - len(capability.provisional_precision_targets()), 66
)
self.assertEqual(
- sum(item["disposition"] == "supported" for item in targets), 41
+ sum(item["disposition"] == "supported" for item in targets), 44
)
self.assertEqual(
sum(item["disposition"] == "unsupported" for item in targets), 22
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index ad4d6c5740..949f4d2c94 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -590,7 +590,13 @@ def invoke(
self.assertIn("validation-missing-required-account", rejected.stderr)
def test_scaleout_network_profile_is_explicit_and_allowlisted(self) -> None:
- command = r'''
+ with tempfile.TemporaryDirectory() as temporary:
+ sysfs = Path(temporary)
+ for device in ("mlx5_0", "mlx5_1"):
+ port = sysfs / device / "ports" / "1"
+ port.mkdir(parents=True)
+ (port / "link_layer").write_text("Ethernet\n")
+ command = r'''
set -euo pipefail
source "$1"
test "$(cx_nccl_hca_device_name '=mlx5_0:1')" = mlx5_0
@@ -603,7 +609,7 @@ def test_scaleout_network_profile_is_explicit_and_allowlisted(self) -> None:
cx_apply_network_profile 1 nvlink
test "$NVSHMEM_DISABLE_IB" = 1
export CX_BENCH=deepep CX_MODE=low-latency CX_IB_GID_INDEX=3
- cx_apply_network_profile 1 nvlink
+ cx_apply_network_profile 1 nvlink "$2"
test -z "${NVSHMEM_DISABLE_IB+x}${NCCL_NET+x}${NCCL_IB_HCA+x}"
test "$NVSHMEM_HCA_LIST:$NVSHMEM_IB_GID_INDEX" = mlx5_0:1,mlx5_1:1:3
test "$NVSHMEM_ENABLE_NIC_PE_MAPPING" = 1
@@ -615,7 +621,7 @@ def test_scaleout_network_profile_is_explicit_and_allowlisted(self) -> None:
cx_apply_network_profile 4 mnnvl
test -z "${NCCL_NET+x}${NCCL_IB_HCA+x}${NVSHMEM_HCA_LIST+x}"
export CX_IB_GID_INDEX=3 CX_RDMA_SERVICE_LEVEL=2
- cx_apply_network_profile 2 nvlink-rdma
+ cx_apply_network_profile 2 nvlink-rdma "$2"
test "$NCCL_SOCKET_IFNAME:$GLOO_SOCKET_IFNAME:$UCCL_SOCKET_IFNAME" = ib0:ib0:ib0
test "$NCCL_NET:$NCCL_IB_HCA" = 'IB:=mlx5_0:1,mlx5_1:1'
test "$NVSHMEM_HCA_LIST" = mlx5_0:1,mlx5_1:1
@@ -623,12 +629,20 @@ def test_scaleout_network_profile_is_explicit_and_allowlisted(self) -> None:
test "$MORI_RDMA_DEVICES:$EP_NIC_NAME" = mlx5_0,mlx5_1:mlx5_0
test "$NCCL_IB_GID_INDEX:$NCCL_IB_SL" = 3:2
test "$NVSHMEM_IB_ENABLE_IBGDA:$NVSHMEM_IBGDA_NIC_HANDLER" = 1:gpu
+ printf 'InfiniBand\n' > "$2/mlx5_0/ports/1/link_layer"
+ printf 'InfiniBand\n' > "$2/mlx5_1/ports/1/link_layer"
+ cx_apply_network_profile 2 nvlink-rdma "$2"
+ test -z "${NVSHMEM_IB_GID_INDEX+x}${NCCL_IB_GID_INDEX+x}${UCCL_IB_GID_INDEX+x}"
+ test "$NVSHMEM_IB_SL:$NCCL_IB_SL:$UCCL_IB_SL" = 2:2:2
'''
- subprocess.run(
- ["bash", "-c", command, "_", str(ROOT / "runtime" / "common.sh")],
- check=True,
- env={**os.environ, "COLLECTIVEX_OPERATOR_CONFIG": "/dev/null"},
- )
+ subprocess.run(
+ [
+ "bash", "-c", command, "_",
+ str(ROOT / "runtime" / "common.sh"), str(sysfs),
+ ],
+ check=True,
+ env={**os.environ, "COLLECTIVEX_OPERATOR_CONFIG": "/dev/null"},
+ )
v2_adapter = (ROOT / "tests" / "ep_deepep_v2.py").read_text()
container_runtime = (ROOT / "runtime" / "run_in_container.sh").read_text()
From 70c9ac1116b23946dcd7a8344dc0237a265aab92 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 20:18:49 +0800
Subject: [PATCH 055/210] fix(collectivex): use runner-owned AMD staging
---
experimental/CollectiveX/README.md | 5 +++++
experimental/CollectiveX/capability.py | 7 +++++++
experimental/CollectiveX/runtime/common.sh | 20 +++++++++++++++----
.../tests/test_precision_scheduling.py | 10 +++++-----
.../tests/test_sampling_contract.py | 3 ++-
5 files changed, 35 insertions(+), 10 deletions(-)
diff --git a/experimental/CollectiveX/README.md b/experimental/CollectiveX/README.md
index 1284b7f347..37f22e7a2f 100644
--- a/experimental/CollectiveX/README.md
+++ b/experimental/CollectiveX/README.md
@@ -158,6 +158,11 @@ allocated node. Jobs create only a marked mode-0700 execution child, prove cross
visibility, and remove that exact child after allocation teardown; they never mount the runner
checkout or create a stage beneath image storage on AMD.
+When an AMD operator row omits `stage_dir`, the self-hosted runner derives a private base beside its
+standard `_work` directory on the shared runner filesystem. The root-owned squash cache is never
+used as a repository stage. The derived parent must be runner-owned, non-symlinked, and not writable
+by group or world before any benchmark source is copied.
+
H200, B200, and B300 runners may omit `stage_dir`. Their isolated execution child is then created
under a runner-owned mode-0700 base in the validated operating-system account home, independent of
the workflow's temporary `HOME`. A symlinked account-home entry is resolved once to its canonical,
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
index 9f179ad9aa..b098dd0e4b 100644
--- a/experimental/CollectiveX/capability.py
+++ b/experimental/CollectiveX/capability.py
@@ -363,6 +363,13 @@ def _precision_rule(
("28740154697", "h200-dgxc", 16, "deepep", "low-latency", _LL_FP8_PROFILE, "supported", "native-probe-passed"),
("28740154697", "h200-dgxc", 16, "deepep", "low-latency", _LL_LOGFMT_PROFILE, "supported", "native-probe-passed"),
("28740154697", "h200-dgxc", 16, "uccl", "low-latency", _LL_LOGFMT_PROFILE, "supported", "native-probe-passed"),
+ ("28740074613", "b300", 8, "deepep-v2", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed"),
+ ("28740154697", "h200-dgxc", 16, "uccl", "low-latency", _LL_FP8_PROFILE, "supported", "native-probe-passed"),
+ ("28740154697", "h200-dgxc", 16, "uccl", "low-latency", _LL_FP8_LOGFMT_PROFILE, "supported", "native-probe-passed"),
+ ("28740154697", "h200-dgxc", 16, "uccl", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed"),
+ *(('28740381789', 'b300', 8, 'deepep', 'low-latency', profile,
+ 'unsupported', 'backend-construction-failed')
+ for profile in (_LL_FP8_PROFILE, _LL_LOGFMT_PROFILE, _LL_FP8_LOGFMT_PROFILE)),
)
PRECISION_CELL_OVERRIDES.update({
(profile, backend, sku, ep, mode): {
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 021a6b7fbb..d70f46b3a1 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -1670,6 +1670,17 @@ print(current, end="")
PY
}
+cx_prepare_runner_shared_stage_base() {
+ local runner_temp="${RUNNER_TEMP:-}" runner_root
+ case "$runner_temp" in
+ /*/_work/_temp) runner_root="${runner_temp%/_work/_temp}" ;;
+ *) cx_die "canonical AMD execution requires a standard shared runner temp" ;;
+ esac
+ [ -n "$runner_root" ] && [ "$runner_root" != "$runner_temp" ] \
+ || cx_die "canonical AMD execution requires a shared runner root"
+ cx_prepare_implicit_stage_base "$runner_root"
+}
+
cx_lock_canonical_gha_env() {
local runner="$1" expected_nodes expected_gpn expected_world trusted_lock_dir=""
local trusted_stage_dir=""
@@ -1742,10 +1753,11 @@ cx_lock_canonical_gha_env() {
|| cx_die "canonical CollectiveX execution cannot create an isolated stage directory"
;;
mi325x|mi355x)
- # AMD Slurm jobs already consume the configured squash directory from
- # compute-visible shared storage. Use it as the isolated staging base
- # when the private row does not provide a separate location.
- trusted_stage_dir="$CX_SQUASH_DIR"
+ # AMD self-hosted runners and compute nodes share the runner filesystem,
+ # while the image cache may be root-owned. Derive a runner-owned base
+ # outside _work instead of weakening stage ownership validation.
+ trusted_stage_dir="$(cx_prepare_runner_shared_stage_base)" \
+ || cx_die "canonical AMD execution cannot create an isolated shared stage directory"
;;
*) cx_die "canonical CollectiveX execution requires a configured shared stage directory" ;;
esac
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
index bc401db471..c60102b1d2 100644
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -28,7 +28,7 @@ def test_precision_probe_inventory_is_exact_and_non_mutating(self) -> None:
item["precision_profile"],
)
self.assertEqual(targets, sorted(capability.provisional_precision_targets(), key=key))
- self.assertEqual(len(targets), 28)
+ self.assertEqual(len(targets), 21)
self.assertEqual(capability.PRECISION_CAPABILITIES, before)
self.assertEqual(
len({
@@ -78,7 +78,7 @@ def test_precision_probe_workflow_plan_binds_exact_controls(self) -> None:
with self.assertRaisesRegex(ValueError, "select no provisional"):
probe_precision.workflow_plan(backend="mori", only_sku="b200-dgxc")
ep8 = probe_precision.workflow_plan(
- backend="deepep", only_sku="b300", max_nodes=1,
+ backend="mori", only_sku="mi355x", max_nodes=1,
)
self.assertTrue(ep8["include"])
self.assertEqual({row["ep"] for row in ep8["include"]}, {8})
@@ -222,13 +222,13 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
{"provisional", "supported", "unsupported"},
)
self.assertEqual(
- len(targets) - len(capability.provisional_precision_targets()), 66
+ len(targets) - len(capability.provisional_precision_targets()), 73
)
self.assertEqual(
- sum(item["disposition"] == "supported" for item in targets), 44
+ sum(item["disposition"] == "supported" for item in targets), 48
)
self.assertEqual(
- sum(item["disposition"] == "unsupported" for item in targets), 22
+ sum(item["disposition"] == "unsupported" for item in targets), 25
)
keys = {
(
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 949f4d2c94..602a073cf6 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -2516,10 +2516,11 @@ def test_canonical_gha_environment_is_locked_but_manual_overrides_survive(self)
export COLLECTIVEX_OPERATOR_CONFIG_LOADED=$$
export CX_SHARD_SKU=mi355x CX_NODES=1 CX_GPUS_PER_NODE=8
export CX_SQUASH_DIR=/validated/amd-shared
+ export RUNNER_TEMP="$TEST_SHARED_PARENT/_work/_temp"
export CX_AUDIT_SALT="$(printf 'a%.0s' {1..64})"
unset CX_STAGE_DIR
cx_lock_canonical_gha_env mi355x
- test "$CX_STAGE_DIR" = /validated/amd-shared
+ test "$CX_STAGE_DIR" = "$TEST_IMPLICIT_STAGE"
unset COLLECTIVEX_CANONICAL_GHA
unset COLLECTIVEX_OPERATOR_CONFIG_LOADED
From 05252758435f9c2e9462890727b44647195666e0 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 20:25:29 +0800
Subject: [PATCH 056/210] fix(collectivex): validate RDMA on allocated nodes
---
experimental/CollectiveX/capability.py | 1 +
experimental/CollectiveX/runtime/common.sh | 87 ++++++++---------
.../tests/test_precision_scheduling.py | 6 +-
.../tests/test_sampling_contract.py | 93 ++++++++++++++-----
4 files changed, 120 insertions(+), 67 deletions(-)
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
index b098dd0e4b..d3a50ac13c 100644
--- a/experimental/CollectiveX/capability.py
+++ b/experimental/CollectiveX/capability.py
@@ -367,6 +367,7 @@ def _precision_rule(
("28740154697", "h200-dgxc", 16, "uccl", "low-latency", _LL_FP8_PROFILE, "supported", "native-probe-passed"),
("28740154697", "h200-dgxc", 16, "uccl", "low-latency", _LL_FP8_LOGFMT_PROFILE, "supported", "native-probe-passed"),
("28740154697", "h200-dgxc", 16, "uccl", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed"),
+ ("28740154697", "h200-dgxc", 16, "deepep-v2", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "backend-setup-timeout"),
*(('28740381789', 'b300', 8, 'deepep', 'low-latency', profile,
'unsupported', 'backend-construction-failed')
for profile in (_LL_FP8_PROFILE, _LL_LOGFMT_PROFILE, _LL_FP8_LOGFMT_PROFILE)),
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index d70f46b3a1..bc6f3fcc1a 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -617,43 +617,29 @@ cx_nccl_hca_device_name() {
printf '%s' "${selector%%:*}"
}
-cx_rdma_profile_link_layer() {
- local sysfs_root="$1" selector device configured_port port_path layer profile=""
- local -a selectors port_paths
- IFS=, read -r -a selectors <<< "$CX_RDMA_DEVICES"
- for selector in "${selectors[@]}"; do
- device="${selector%%:*}"
- configured_port=""
- [ "$selector" = "$device" ] || configured_port="${selector#*:}"
- if [ -n "$configured_port" ]; then
- port_paths=("$sysfs_root/$device/ports/$configured_port")
- else
- port_paths=("$sysfs_root/$device/ports"/*)
- fi
- [ "${#port_paths[@]}" -gt 0 ] || cx_die "RDMA selector has no visible ports"
- for port_path in "${port_paths[@]}"; do
- [ -r "$port_path/link_layer" ] || cx_die "RDMA link layer is unavailable"
- layer="$(< "$port_path/link_layer")"
- case "$layer" in
- Ethernet) layer=roce ;;
- InfiniBand) layer=infiniband ;;
- *) cx_die "unsupported RDMA link layer" ;;
- esac
- [ -z "$profile" ] || [ "$profile" = "$layer" ] \
- || cx_die "mixed RDMA link layers are unsupported"
- profile="$layer"
- done
- done
- [ -n "$profile" ] || cx_die "RDMA link layer is unavailable"
- printf '%s' "$profile"
+cx_export_gid_index_for_link_layer() {
+ local link_layer="$1" scaleout="$2"
+ unset NVSHMEM_IB_GID_INDEX NCCL_IB_GID_INDEX UCCL_IB_GID_INDEX
+ [ -n "${CX_IB_GID_INDEX:-}" ] || return 0
+ case "$link_layer" in
+ roce)
+ export NVSHMEM_IB_GID_INDEX="$CX_IB_GID_INDEX"
+ if [ "$scaleout" = 1 ]; then
+ export NCCL_IB_GID_INDEX="$CX_IB_GID_INDEX"
+ export UCCL_IB_GID_INDEX="$CX_IB_GID_INDEX"
+ fi
+ ;;
+ infiniband) ;;
+ *) cx_die "unsupported RDMA link layer" ;;
+ esac
}
# Convert private, runner-local network selectors into the public library
# variables needed inside the container. Values are interface/HCA identifiers,
# never addresses; the rendezvous hostname is derived from the allocation.
cx_apply_network_profile() {
- local nodes="$1" transport="$2" rdma_sysfs_root="${3:-/sys/class/infiniband}"
- local selector rdma_name rdma_names="" ep_nic="" rdma_link_layer=""
+ local nodes="$1" transport="$2"
+ local selector rdma_name rdma_names="" ep_nic=""
local scaleout=0 single_node_rdma=0
local -a selectors
[[ "$nodes" =~ ^[1-9][0-9]*$ ]] || cx_die "invalid network placement"
@@ -704,13 +690,6 @@ cx_apply_network_profile() {
if [ -n "${CX_IB_GID_INDEX:-}" ]; then
[[ "$CX_IB_GID_INDEX" =~ ^[0-9]+$ ]] && [ "$CX_IB_GID_INDEX" -le 255 ] \
|| cx_die "invalid private IB GID index"
- rdma_link_layer="$(cx_rdma_profile_link_layer "$rdma_sysfs_root")"
- if [ "$rdma_link_layer" = roce ]; then
- export NVSHMEM_IB_GID_INDEX="$CX_IB_GID_INDEX"
- if [ "$scaleout" = 1 ]; then
- export NCCL_IB_GID_INDEX="$CX_IB_GID_INDEX" UCCL_IB_GID_INDEX="$CX_IB_GID_INDEX"
- fi
- fi
fi
if [ -n "${CX_RDMA_SERVICE_LEVEL:-}" ]; then
[[ "$CX_RDMA_SERVICE_LEVEL" =~ ^[0-9]+$ ]] && [ "$CX_RDMA_SERVICE_LEVEL" -le 15 ] \
@@ -728,7 +707,7 @@ cx_apply_network_profile() {
# node before image import or backend initialization. Selector values and node
# diagnostics stay in the runner-private log.
cx_validate_network_profile_on_job() {
- local job_id="$1" nodes="$2" transport="$3" log rc=0 scaleout=0
+ local job_id="$1" nodes="$2" transport="$3" log rc=0 scaleout=0 marker_count link_layer
local single_node_rdma=0
if [ "$nodes" -gt 1 ] && [ "$transport" != mnnvl ]; then
scaleout=1
@@ -763,7 +742,7 @@ if [ -n "${CX_SOCKET_IFNAME:-}" ]; then
done
fi
check_port() {
- local port_path="$1" state gid
+ local port_path="$1" state gid link_layer
[ -d "$port_path" ] || return 1
read -r state _ < "$port_path/state"
[ "$state" = 4: ] || return 1
@@ -772,7 +751,17 @@ check_port() {
gid="$(tr -d ':0[:space:]' < "$port_path/gids/$CX_IB_GID_INDEX")"
[ -n "$gid" ] || return 1
fi
+ [ -r "$port_path/link_layer" ] || return 1
+ link_layer="$(< "$port_path/link_layer")"
+ case "$link_layer" in
+ Ethernet) link_layer=roce ;;
+ InfiniBand) link_layer=infiniband ;;
+ *) return 1 ;;
+ esac
+ [ -z "${profile:-}" ] || [ "$profile" = "$link_layer" ] || return 1
+ profile="$link_layer"
}
+profile=""
IFS=, read -r -a devices <<< "$CX_RDMA_DEVICES"
for selector in "${devices[@]}"; do
device="${selector%%:*}"
@@ -787,17 +776,28 @@ for selector in "${devices[@]}"; do
for port_path in "$ports"/*; do
if check_port "$port_path"; then
active=1
- break
fi
done
[ "$active" = 1 ]
fi
done
+[ -n "$profile" ]
+printf '[collectivex-private] rdma-link-layer=%s\n' "$profile"
BASH
if [ "$rc" != 0 ]; then
cx_fail_stage setup "$log" || true
return "$rc"
fi
+ link_layer="$(
+ sed -nE 's/^\[collectivex-private\] rdma-link-layer=(roce|infiniband)$/\1/p' "$log" \
+ | sort -u
+ )"
+ marker_count="$(grep -Ec '^\[collectivex-private\] rdma-link-layer=(roce|infiniband)$' "$log")"
+ case "$marker_count:$link_layer" in
+ "$nodes":roce|"$nodes":infiniband) ;;
+ *) cx_fail_stage setup "$log" || true; return 1 ;;
+ esac
+ cx_export_gid_index_for_link_layer "$link_layer" "$scaleout"
}
cx_resolve_slurm_rendezvous() {
@@ -2729,7 +2729,7 @@ PY
cx_launcher_cleanup() {
local rc="$1" stage_root="${MOUNT_SRC:-}" source_root out_dir allocation_stopped=1
source_root="${stage_root:-${REPO_ROOT:-}}"
- trap - EXIT
+ trap - EXIT HUP INT TERM
if [ -n "${COLLECTIVEX_EPHEMERAL_CONFIG_PATH:-}" ]; then
rm -f -- "$COLLECTIVEX_EPHEMERAL_CONFIG_PATH" >/dev/null 2>&1 || true
unset COLLECTIVEX_EPHEMERAL_CONFIG_PATH
@@ -2775,4 +2775,7 @@ cx_launcher_cleanup() {
cx_install_launcher_fail_safe() {
CX_ALLOCATION_UNCERTAIN=0
trap 'cx_launcher_cleanup "$?"' EXIT
+ trap 'cx_launcher_cleanup 129' HUP
+ trap 'cx_launcher_cleanup 130' INT
+ trap 'cx_launcher_cleanup 143' TERM
}
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
index c60102b1d2..32df3de2e6 100644
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -28,7 +28,7 @@ def test_precision_probe_inventory_is_exact_and_non_mutating(self) -> None:
item["precision_profile"],
)
self.assertEqual(targets, sorted(capability.provisional_precision_targets(), key=key))
- self.assertEqual(len(targets), 21)
+ self.assertEqual(len(targets), 20)
self.assertEqual(capability.PRECISION_CAPABILITIES, before)
self.assertEqual(
len({
@@ -222,13 +222,13 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
{"provisional", "supported", "unsupported"},
)
self.assertEqual(
- len(targets) - len(capability.provisional_precision_targets()), 73
+ len(targets) - len(capability.provisional_precision_targets()), 74
)
self.assertEqual(
sum(item["disposition"] == "supported" for item in targets), 48
)
self.assertEqual(
- sum(item["disposition"] == "unsupported" for item in targets), 25
+ sum(item["disposition"] == "unsupported" for item in targets), 26
)
keys = {
(
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 602a073cf6..27d804246b 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -590,13 +590,7 @@ def invoke(
self.assertIn("validation-missing-required-account", rejected.stderr)
def test_scaleout_network_profile_is_explicit_and_allowlisted(self) -> None:
- with tempfile.TemporaryDirectory() as temporary:
- sysfs = Path(temporary)
- for device in ("mlx5_0", "mlx5_1"):
- port = sysfs / device / "ports" / "1"
- port.mkdir(parents=True)
- (port / "link_layer").write_text("Ethernet\n")
- command = r'''
+ command = r'''
set -euo pipefail
source "$1"
test "$(cx_nccl_hca_device_name '=mlx5_0:1')" = mlx5_0
@@ -609,8 +603,10 @@ def test_scaleout_network_profile_is_explicit_and_allowlisted(self) -> None:
cx_apply_network_profile 1 nvlink
test "$NVSHMEM_DISABLE_IB" = 1
export CX_BENCH=deepep CX_MODE=low-latency CX_IB_GID_INDEX=3
- cx_apply_network_profile 1 nvlink "$2"
+ cx_apply_network_profile 1 nvlink
test -z "${NVSHMEM_DISABLE_IB+x}${NCCL_NET+x}${NCCL_IB_HCA+x}"
+ test -z "${NVSHMEM_IB_GID_INDEX+x}"
+ cx_export_gid_index_for_link_layer roce 0
test "$NVSHMEM_HCA_LIST:$NVSHMEM_IB_GID_INDEX" = mlx5_0:1,mlx5_1:1:3
test "$NVSHMEM_ENABLE_NIC_PE_MAPPING" = 1
test "$NVSHMEM_IB_ENABLE_IBGDA:$NVSHMEM_IBGDA_NIC_HANDLER" = 1:gpu
@@ -621,28 +617,26 @@ def test_scaleout_network_profile_is_explicit_and_allowlisted(self) -> None:
cx_apply_network_profile 4 mnnvl
test -z "${NCCL_NET+x}${NCCL_IB_HCA+x}${NVSHMEM_HCA_LIST+x}"
export CX_IB_GID_INDEX=3 CX_RDMA_SERVICE_LEVEL=2
- cx_apply_network_profile 2 nvlink-rdma "$2"
+ cx_apply_network_profile 2 nvlink-rdma
test "$NCCL_SOCKET_IFNAME:$GLOO_SOCKET_IFNAME:$UCCL_SOCKET_IFNAME" = ib0:ib0:ib0
test "$NCCL_NET:$NCCL_IB_HCA" = 'IB:=mlx5_0:1,mlx5_1:1'
test "$NVSHMEM_HCA_LIST" = mlx5_0:1,mlx5_1:1
test "$NVSHMEM_ENABLE_NIC_PE_MAPPING" = 1
test "$MORI_RDMA_DEVICES:$EP_NIC_NAME" = mlx5_0,mlx5_1:mlx5_0
+ test -z "${NCCL_IB_GID_INDEX+x}${NVSHMEM_IB_GID_INDEX+x}${UCCL_IB_GID_INDEX+x}"
+ cx_export_gid_index_for_link_layer roce 1
test "$NCCL_IB_GID_INDEX:$NCCL_IB_SL" = 3:2
test "$NVSHMEM_IB_ENABLE_IBGDA:$NVSHMEM_IBGDA_NIC_HANDLER" = 1:gpu
- printf 'InfiniBand\n' > "$2/mlx5_0/ports/1/link_layer"
- printf 'InfiniBand\n' > "$2/mlx5_1/ports/1/link_layer"
- cx_apply_network_profile 2 nvlink-rdma "$2"
+ cx_apply_network_profile 2 nvlink-rdma
+ cx_export_gid_index_for_link_layer infiniband 1
test -z "${NVSHMEM_IB_GID_INDEX+x}${NCCL_IB_GID_INDEX+x}${UCCL_IB_GID_INDEX+x}"
test "$NVSHMEM_IB_SL:$NCCL_IB_SL:$UCCL_IB_SL" = 2:2:2
'''
- subprocess.run(
- [
- "bash", "-c", command, "_",
- str(ROOT / "runtime" / "common.sh"), str(sysfs),
- ],
- check=True,
- env={**os.environ, "COLLECTIVEX_OPERATOR_CONFIG": "/dev/null"},
- )
+ subprocess.run(
+ ["bash", "-c", command, "_", str(ROOT / "runtime" / "common.sh")],
+ check=True,
+ env={**os.environ, "COLLECTIVEX_OPERATOR_CONFIG": "/dev/null"},
+ )
v2_adapter = (ROOT / "tests" / "ep_deepep_v2.py").read_text()
container_runtime = (ROOT / "runtime" / "run_in_container.sh").read_text()
@@ -660,6 +654,15 @@ def test_network_profile_validation_is_private_and_all_node(self) -> None:
"#!/usr/bin/env bash\n"
"printf '%s\\n' \"$@\" > \"$CAPTURE_ARGS\"\n"
"cat > \"$CAPTURE_SCRIPT\"\n"
+ "tasks=1\n"
+ "for arg in \"$@\"; do\n"
+ " case \"$arg\" in --ntasks=*) tasks=\"${arg#--ntasks=}\" ;; esac\n"
+ "done\n"
+ "if [ \"${SRUN_RC:-0}\" = 0 ]; then\n"
+ " for ((task=0; task None:
self.assertIn("--ntasks=2", invoked)
self.assertIn("--input=all", invoked)
self.assertIn("CX_SOCKET_IFNAME,CX_RDMA_DEVICES,CX_IB_GID_INDEX", invoked)
- self.assertIn('/sys/class/infiniband/$device/ports', script.read_text())
+ validation_script = script.read_text()
+ self.assertIn('/sys/class/infiniband/$device/ports', validation_script)
+ self.assertIn('rdma-link-layer=%s', validation_script)
self.assertNotIn("privateif0", result.stdout + result.stderr)
self.assertNotIn("privatehca0", result.stdout + result.stderr)
@@ -1246,7 +1251,51 @@ def test_allocation_cleanup_fails_closed_when_scheduler_queries_fail(self) -> No
env={**os.environ, "PATH": f"{directory}:{os.environ['PATH']}"},
)
self.assertNotEqual(result.returncode, 0)
- self.assertIn("did not terminate", result.stderr)
+ self.assertIn("did not terminate", result.stderr)
+
+ def test_launcher_signal_traps_cancel_the_allocation(self) -> None:
+ prefix = f"inferencex-collectivex-{os.getpid()}-1-"
+ with tempfile.TemporaryDirectory(prefix=prefix, dir="/tmp") as temporary:
+ root = Path(temporary)
+ binary = root / "bin"
+ repo = root / "repo"
+ calls = root / "scancel-calls"
+ binary.mkdir()
+ repo.mkdir()
+ for name, body in {
+ "scancel": 'printf "%s\\n" "$*" >> "$CX_TEST_SCANCEL_CALLS"',
+ "squeue": "exit 0",
+ }.items():
+ path = binary / name
+ path.write_text(f"#!/usr/bin/env bash\n{body}\n")
+ path.chmod(0o700)
+ command = r'''
+ set -euo pipefail
+ source "$1"
+ export JOB_ID=5151 CX_JOB_ROOT="$2" REPO_ROOT="$2/repo" MOUNT_SRC="$2/repo"
+ export COLLECTIVEX_CANONICAL_GHA=1
+ cx_write_cleanup_guard() {
+ rm -f -- "$CX_JOB_ROOT/cleanup-safe" "$CX_JOB_ROOT/cleanup-unsafe"
+ : > "$CX_JOB_ROOT/cleanup-$1"
+ }
+ cx_install_launcher_fail_safe
+ kill -TERM $$
+ exit 99
+ '''
+ result = subprocess.run(
+ ["bash", "-c", command, "_", str(ROOT / "runtime" / "common.sh"), str(root)],
+ text=True,
+ capture_output=True,
+ env={
+ **os.environ,
+ "PATH": f"{binary}:{os.environ['PATH']}",
+ "CX_TEST_SCANCEL_CALLS": str(calls),
+ },
+ )
+ self.assertEqual(result.returncode, 143, result.stderr)
+ self.assertEqual(calls.read_text().splitlines(), ["5151"])
+ self.assertTrue((root / "cleanup-safe").is_file())
+ self.assertFalse((root / "cleanup-unsafe").exists())
workflow = (ROOT.parent.parent / ".github" / "workflows" / "collectivex-sweep.yml").read_text()
self.assertIn("cleanup-unsafe", workflow)
From 43d91c4e16baf59c493cb01ec1a72e430a6425f5 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 20:54:32 +0800
Subject: [PATCH 057/210] test(collectivex): record terminal EP probe outcomes
---
experimental/CollectiveX/capability.py | 7 +++++++
.../CollectiveX/tests/test_precision_scheduling.py | 10 +++++-----
2 files changed, 12 insertions(+), 5 deletions(-)
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
index d3a50ac13c..cdfc2ecbe8 100644
--- a/experimental/CollectiveX/capability.py
+++ b/experimental/CollectiveX/capability.py
@@ -368,6 +368,13 @@ def _precision_rule(
("28740154697", "h200-dgxc", 16, "uccl", "low-latency", _LL_FP8_LOGFMT_PROFILE, "supported", "native-probe-passed"),
("28740154697", "h200-dgxc", 16, "uccl", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed"),
("28740154697", "h200-dgxc", 16, "deepep-v2", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "backend-setup-timeout"),
+ ("28740533382", "mi355x", 8, "mori", "normal", _MORI_E4M3FN_BOTH_PROFILE, "unsupported", "backend-construction-failed"),
+ ("28740712155", "b300", 16, "deepep", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "backend-setup-timeout"),
+ ("28740712155", "b300", 16, "deepep-v2", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "backend-construction-failed"),
+ ("28740712155", "b300", 16, "deepep-hybrid", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "native-operation-failed"),
+ *(("28740712155", "b300", 16, "deepep", "low-latency", profile,
+ "unsupported", "backend-setup-timeout")
+ for profile in (_LL_FP8_PROFILE, _LL_LOGFMT_PROFILE, _LL_FP8_LOGFMT_PROFILE)),
*(('28740381789', 'b300', 8, 'deepep', 'low-latency', profile,
'unsupported', 'backend-construction-failed')
for profile in (_LL_FP8_PROFILE, _LL_LOGFMT_PROFILE, _LL_FP8_LOGFMT_PROFILE)),
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
index 32df3de2e6..6e0be14b7e 100644
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -28,7 +28,7 @@ def test_precision_probe_inventory_is_exact_and_non_mutating(self) -> None:
item["precision_profile"],
)
self.assertEqual(targets, sorted(capability.provisional_precision_targets(), key=key))
- self.assertEqual(len(targets), 20)
+ self.assertEqual(len(targets), 13)
self.assertEqual(capability.PRECISION_CAPABILITIES, before)
self.assertEqual(
len({
@@ -53,10 +53,10 @@ def test_precision_probe_selects_only_one_exact_provisional_cell(self) -> None:
)
def test_precision_probe_workflow_plan_binds_exact_controls(self) -> None:
- plan = probe_precision.workflow_plan(backend="deepep", only_sku="b300")
+ plan = probe_precision.workflow_plan(backend="deepep", only_sku="b200-dgxc")
self.assertTrue(plan["include"])
self.assertTrue(all(
- row["backend"] == "deepep" and row["sku"] == "b300"
+ row["backend"] == "deepep" and row["sku"] == "b200-dgxc"
for row in plan["include"]
))
row = plan["include"][0]
@@ -222,13 +222,13 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
{"provisional", "supported", "unsupported"},
)
self.assertEqual(
- len(targets) - len(capability.provisional_precision_targets()), 74
+ len(targets) - len(capability.provisional_precision_targets()), 81
)
self.assertEqual(
sum(item["disposition"] == "supported" for item in targets), 48
)
self.assertEqual(
- sum(item["disposition"] == "unsupported" for item in targets), 26
+ sum(item["disposition"] == "unsupported" for item in targets), 33
)
keys = {
(
From 9ac3a646e16a9d230e57a2640ef3f4d5c24d0477 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 21:09:13 +0800
Subject: [PATCH 058/210] fix(collectivex): diagnose NCCL transport guards
---
experimental/CollectiveX/tests/ep_nccl.py | 6 ++++--
experimental/CollectiveX/tests/test_deepep_v2_contract.py | 3 +++
2 files changed, 7 insertions(+), 2 deletions(-)
diff --git a/experimental/CollectiveX/tests/ep_nccl.py b/experimental/CollectiveX/tests/ep_nccl.py
index d440f8ecf9..c30f0c6d72 100644
--- a/experimental/CollectiveX/tests/ep_nccl.py
+++ b/experimental/CollectiveX/tests/ep_nccl.py
@@ -67,12 +67,14 @@ def __init__(self, args, rank, world_size, local_rank, device):
_library, _version = _runtime_collective(args, torch)
if args.scale_out_transport:
hcas = os.environ.get("NCCL_IB_HCA", "")
- if os.environ.get("NCCL_NET") != "IB" or not re.fullmatch(
+ if os.environ.get("NCCL_NET") != "IB":
+ raise RuntimeError("scale-out collective network mode is not IB")
+ if not re.fullmatch(
r"=[A-Za-z][A-Za-z0-9_.-]{0,31}(?::[1-9][0-9]*)?"
r"(?:,[A-Za-z][A-Za-z0-9_.-]{0,31}(?::[1-9][0-9]*)?)*",
hcas,
):
- raise RuntimeError("scale-out collective transport is not pinned to RDMA")
+ raise RuntimeError("scale-out collective HCA allowlist is invalid")
self.kernel_generation = contracts.collective_kernel_generation(_library)
self.backend_provenance = {
"backend": f"{_library}-all2all",
diff --git a/experimental/CollectiveX/tests/test_deepep_v2_contract.py b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
index bf8e3cad0a..50e454e1de 100644
--- a/experimental/CollectiveX/tests/test_deepep_v2_contract.py
+++ b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
@@ -1489,6 +1489,9 @@ def test_runtime_pins_uccl_wheel_and_hybrid_source_tree(self) -> None:
self.assertIn('EP_NVSHMEM_ROOT_DIR="$NVSHMEM_DIR"', runtime)
reference = (HERE / "ep_nccl.py").read_text()
self.assertIn("self.kernel_generation = contracts.collective_kernel_generation", reference)
+ self.assertIn("scale-out collective network mode is not IB", reference)
+ self.assertIn("scale-out collective HCA allowlist is invalid", reference)
+ self.assertNotIn("scale-out collective transport is not pinned to RDMA", reference)
def test_deepep_v2_cache_recovers_from_an_unpublished_partial_build(self) -> None:
runtime = ROOT / "runtime" / "run_in_container.sh"
From fbfb77df4b22c6c6dce62398d38dc3e4e88b9436 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 21:15:53 +0800
Subject: [PATCH 059/210] fix(collectivex): classify HCA selector shape
---
experimental/CollectiveX/tests/ep_nccl.py | 13 ++++++++++++-
.../CollectiveX/tests/test_deepep_v2_contract.py | 1 +
2 files changed, 13 insertions(+), 1 deletion(-)
diff --git a/experimental/CollectiveX/tests/ep_nccl.py b/experimental/CollectiveX/tests/ep_nccl.py
index c30f0c6d72..efd4d89b7a 100644
--- a/experimental/CollectiveX/tests/ep_nccl.py
+++ b/experimental/CollectiveX/tests/ep_nccl.py
@@ -74,7 +74,18 @@ def __init__(self, args, rank, world_size, local_rank, device):
r"(?:,[A-Za-z][A-Za-z0-9_.-]{0,31}(?::[1-9][0-9]*)?)*",
hcas,
):
- raise RuntimeError("scale-out collective HCA allowlist is invalid")
+ entries = hcas.removeprefix("=").split(",") if hcas else []
+ shaped = all(
+ re.fullmatch(
+ r"[A-Za-z][A-Za-z0-9_.-]{0,31}(?::[1-9][0-9]*)?", entry
+ )
+ for entry in entries
+ )
+ raise RuntimeError(
+ "scale-out collective HCA allowlist is invalid "
+ f"(present={bool(hcas)}, exact={hcas.startswith('=')}, "
+ f"entries={len(entries)}, shaped={shaped})"
+ )
self.kernel_generation = contracts.collective_kernel_generation(_library)
self.backend_provenance = {
"backend": f"{_library}-all2all",
diff --git a/experimental/CollectiveX/tests/test_deepep_v2_contract.py b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
index 50e454e1de..780e0f86d8 100644
--- a/experimental/CollectiveX/tests/test_deepep_v2_contract.py
+++ b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
@@ -1491,6 +1491,7 @@ def test_runtime_pins_uccl_wheel_and_hybrid_source_tree(self) -> None:
self.assertIn("self.kernel_generation = contracts.collective_kernel_generation", reference)
self.assertIn("scale-out collective network mode is not IB", reference)
self.assertIn("scale-out collective HCA allowlist is invalid", reference)
+ self.assertIn('entries={len(entries)}, shaped={shaped}', reference)
self.assertNotIn("scale-out collective transport is not pinned to RDMA", reference)
def test_deepep_v2_cache_recovers_from_an_unpublished_partial_build(self) -> None:
From 582451d057054480a8d4fa4477594762cf8f3b6f Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 21:21:37 +0800
Subject: [PATCH 060/210] fix(collectivex): restore exact HCA selection
---
experimental/CollectiveX/README.md | 6 +++---
experimental/CollectiveX/capability.py | 1 +
experimental/CollectiveX/docs/methodology.md | 6 +++---
experimental/CollectiveX/runtime/common.sh | 14 ++++++++++++++
.../CollectiveX/runtime/run_in_container.sh | 1 +
.../CollectiveX/tests/test_precision_scheduling.py | 8 ++++----
.../CollectiveX/tests/test_sampling_contract.py | 4 ++++
7 files changed, 30 insertions(+), 10 deletions(-)
diff --git a/experimental/CollectiveX/README.md b/experimental/CollectiveX/README.md
index 37f22e7a2f..16b89f495f 100644
--- a/experimental/CollectiveX/README.md
+++ b/experimental/CollectiveX/README.md
@@ -13,9 +13,9 @@ combine, and paired roundtrip latency across EP libraries and accelerator system
> rankings, recommendations, or regression baselines.
> Development status: the V1 precision, point-level publication, branch-only delivery, and frontend
-> contracts are implemented. Validated B200 EP8, H200 EP8, and GB200 EP8/EP16 probe outcomes are
-> recorded, together with seven completed H100 EP8 outcomes; 50 native precision cells remain
-> provisional, so V1 counts/digests are not frozen.
+> contracts are implemented. H100, H200, GB200, and GB300 are terminal; B200 and B300 EP8 are
+> terminal, and B300 EP16 has terminal native outcomes. Twelve B200/MI325X/MI355X precision cells
+> remain provisional, so V1 counts and digests are not frozen.
## Implemented Pre-V1 Execution Profile
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
index cdfc2ecbe8..3f105d7864 100644
--- a/experimental/CollectiveX/capability.py
+++ b/experimental/CollectiveX/capability.py
@@ -369,6 +369,7 @@ def _precision_rule(
("28740154697", "h200-dgxc", 16, "uccl", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed"),
("28740154697", "h200-dgxc", 16, "deepep-v2", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "backend-setup-timeout"),
("28740533382", "mi355x", 8, "mori", "normal", _MORI_E4M3FN_BOTH_PROFILE, "unsupported", "backend-construction-failed"),
+ ("28740533382", "mi355x", 8, "mori", "normal", _MORI_E4M3FN_DIRECT_PROFILE, "unsupported", "backend-construction-failed"),
("28740712155", "b300", 16, "deepep", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "backend-setup-timeout"),
("28740712155", "b300", 16, "deepep-v2", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "backend-construction-failed"),
("28740712155", "b300", 16, "deepep-hybrid", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "native-operation-failed"),
diff --git a/experimental/CollectiveX/docs/methodology.md b/experimental/CollectiveX/docs/methodology.md
index 2a8ae75352..6368c92faf 100644
--- a/experimental/CollectiveX/docs/methodology.md
+++ b/experimental/CollectiveX/docs/methodology.md
@@ -10,9 +10,9 @@ This document describes the implemented BF16 baseline. It is not yet the V1 qual
Before any V1-tagged run, this English document must match the implemented precision, measurement,
publication, and frontend contracts; counts and digests remain unfrozen. Chinese documentation
synchronization is explicitly deferred for the V1 implementation phase.
-Validated B200 EP8, H200 EP8, and GB200 EP8/EP16 native probe outcomes are recorded in the
-capability registry, together with seven completed H100 EP8 outcomes; 50 native precision cells
-remain provisional.
+H100, H200, GB200, and GB300 precision cells are terminal. B200 and B300 EP8 are terminal, and
+B300 EP16 has terminal native outcomes. Twelve B200/MI325X/MI355X precision cells remain
+provisional.
## Product Boundary
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index bc6f3fcc1a..a288147464 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -703,6 +703,20 @@ cx_apply_network_profile() {
export NVSHMEM_IB_ENABLE_IBGDA=1 NVSHMEM_IBGDA_NIC_HANDLER=gpu
}
+# Slurm may remove NCCL's leading exact-match marker while propagating an
+# inherited environment. Reconstruct it from the validated private selector at
+# the container boundary instead of accepting a prefix-matched HCA list.
+cx_restore_exact_hca_selector() {
+ if [ "${CX_NODES:-1}" -le 1 ] || [ "${CX_TRANSPORT:-}" = mnnvl ]; then
+ return 0
+ fi
+ [ -n "${CX_RDMA_DEVICES:-}" ] \
+ || { cx_log "ERROR: scale-out RDMA selector is unavailable"; return 1; }
+ [[ "$CX_RDMA_DEVICES" =~ ^[A-Za-z][A-Za-z0-9_.-]{0,31}(:[1-9][0-9]*)?(,[A-Za-z][A-Za-z0-9_.-]{0,31}(:[1-9][0-9]*)?)*$ ]] \
+ || { cx_log "ERROR: invalid scale-out RDMA selector"; return 1; }
+ export NCCL_IB_HCA="=$CX_RDMA_DEVICES"
+}
+
# Prove that the operator-pinned scale-out fabric exists on every allocated
# node before image import or backend initialization. Selector values and node
# diagnostics stay in the runner-private log.
diff --git a/experimental/CollectiveX/runtime/run_in_container.sh b/experimental/CollectiveX/runtime/run_in_container.sh
index 5edbd9e9f5..e1dd5e4e92 100644
--- a/experimental/CollectiveX/runtime/run_in_container.sh
+++ b/experimental/CollectiveX/runtime/run_in_container.sh
@@ -923,6 +923,7 @@ cx_probe_scaleout_network() {
if [ "${CX_NODES:-1}" -le 1 ] || [ "${CX_TRANSPORT:-}" = mnnvl ]; then
return 0
fi
+ cx_restore_exact_hca_selector || return 1
[ -n "${GLOO_SOCKET_IFNAME:-}" ] && [ -n "${NCCL_IB_HCA:-}" ] \
|| { cx_log "ERROR: scale-out network selectors are unavailable"; return 1; }
IFS=, read -r -a interfaces <<< "$GLOO_SOCKET_IFNAME"
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
index 6e0be14b7e..2b79d8fd0d 100644
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -28,7 +28,7 @@ def test_precision_probe_inventory_is_exact_and_non_mutating(self) -> None:
item["precision_profile"],
)
self.assertEqual(targets, sorted(capability.provisional_precision_targets(), key=key))
- self.assertEqual(len(targets), 13)
+ self.assertEqual(len(targets), 12)
self.assertEqual(capability.PRECISION_CAPABILITIES, before)
self.assertEqual(
len({
@@ -222,13 +222,13 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
{"provisional", "supported", "unsupported"},
)
self.assertEqual(
- len(targets) - len(capability.provisional_precision_targets()), 81
+ len(targets) - len(capability.provisional_precision_targets()), 82
)
self.assertEqual(
sum(item["disposition"] == "supported" for item in targets), 48
)
self.assertEqual(
- sum(item["disposition"] == "unsupported" for item in targets), 33
+ sum(item["disposition"] == "unsupported" for item in targets), 34
)
keys = {
(
@@ -250,7 +250,7 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
(("h200-dgxc", "deepep-v2", 8, "normal", normal), "unsupported"),
(("h100-dgxc", "deepep-v2", 8, "normal", normal), "not-applicable"),
(("h200-dgxc", "nccl-ep", 8, "normal", normal), "not-applicable"),
- (("mi355x", "mori", 8, "normal", direct), "provisional"),
+ (("mi355x", "mori", 8, "normal", direct), "unsupported"),
(("mi355x", "mori", 16, "normal", direct), "not-applicable"),
(("mi325x", "mori", 8, "normal", fnuz_direct), "provisional"),
(("h200-dgxc", "deepep", 8, "low-latency", low_latency), "supported"),
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 27d804246b..93a22bd6a7 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -620,6 +620,10 @@ def test_scaleout_network_profile_is_explicit_and_allowlisted(self) -> None:
cx_apply_network_profile 2 nvlink-rdma
test "$NCCL_SOCKET_IFNAME:$GLOO_SOCKET_IFNAME:$UCCL_SOCKET_IFNAME" = ib0:ib0:ib0
test "$NCCL_NET:$NCCL_IB_HCA" = 'IB:=mlx5_0:1,mlx5_1:1'
+ NCCL_IB_HCA=mlx5_0:1,mlx5_1:1
+ export NCCL_IB_HCA CX_NODES=2 CX_TRANSPORT=nvlink-rdma
+ cx_restore_exact_hca_selector
+ test "$NCCL_IB_HCA" = '=mlx5_0:1,mlx5_1:1'
test "$NVSHMEM_HCA_LIST" = mlx5_0:1,mlx5_1:1
test "$NVSHMEM_ENABLE_NIC_PE_MAPPING" = 1
test "$MORI_RDMA_DEVICES:$EP_NIC_NAME" = mlx5_0,mlx5_1:mlx5_0
From 50b559dd6bc108acd9e66a9be7ea64c12afacd18 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 21:27:07 +0800
Subject: [PATCH 061/210] fix(collectivex): normalize HCA selection per rank
---
experimental/CollectiveX/runtime/common.sh | 4 ++++
experimental/CollectiveX/tests/test_sampling_contract.py | 4 ++++
2 files changed, 8 insertions(+)
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index a288147464..aea6ba891c 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -838,6 +838,10 @@ case "${SLURM_PROCID:-}:${SLURM_NTASKS:-}:${SLURM_LOCALID:-}:${SLURM_NODEID:-}"
esac
[ "$SLURM_NTASKS" = "$CX_NGPUS" ] || exit 67
[ "$SLURM_LOCALID" -lt "$CX_GPUS_PER_NODE" ] || exit 67
+if [ "${CX_NODES:-1}" -gt 1 ] && [ "${CX_TRANSPORT:-}" != mnnvl ]; then
+ . /ix/experimental/CollectiveX/runtime/common.sh || exit 68
+ cx_restore_exact_hca_selector || exit 68
+fi
export RANK="$SLURM_PROCID" WORLD_SIZE="$SLURM_NTASKS"
export LOCAL_RANK="$SLURM_LOCALID" LOCAL_WORLD_SIZE="$CX_GPUS_PER_NODE"
case "${CX_PRECISION_PROBE:-0}" in
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 93a22bd6a7..5353085029 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -624,6 +624,10 @@ def test_scaleout_network_profile_is_explicit_and_allowlisted(self) -> None:
export NCCL_IB_HCA CX_NODES=2 CX_TRANSPORT=nvlink-rdma
cx_restore_exact_hca_selector
test "$NCCL_IB_HCA" = '=mlx5_0:1,mlx5_1:1'
+ wrapper="$(cx_slurm_rank_wrapper)"
+ bash -n <<< "$wrapper"
+ grep -Fq '. /ix/experimental/CollectiveX/runtime/common.sh' <<< "$wrapper"
+ grep -Fq 'cx_restore_exact_hca_selector || exit 68' <<< "$wrapper"
test "$NVSHMEM_HCA_LIST" = mlx5_0:1,mlx5_1:1
test "$NVSHMEM_ENABLE_NIC_PE_MAPPING" = 1
test "$MORI_RDMA_DEVICES:$EP_NIC_NAME" = mlx5_0,mlx5_1:mlx5_0
From e9a87c3c2e43bfd99245fb710e156bfdc3fc2262 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 21:36:23 +0800
Subject: [PATCH 062/210] fix(collectivex): reconcile cancelled allocations
---
.github/workflows/collectivex-sweep.yml | 8 +++
experimental/CollectiveX/runtime/common.sh | 67 +++++++++++++++++++
.../tests/test_sampling_contract.py | 53 +++++++++++++++
3 files changed, 128 insertions(+)
diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml
index 5aaccc6132..9ea2f50f7f 100644
--- a/.github/workflows/collectivex-sweep.yml
+++ b/.github/workflows/collectivex-sweep.yml
@@ -512,6 +512,14 @@ jobs:
cd "$CX_SOURCE_ROOT" 2>/dev/null \
|| { echo "CollectiveX source is unavailable" >&2; exit 1; }
bash "experimental/CollectiveX/launchers/launch_${{ matrix.launcher }}.sh"
+ - name: Reconcile allocation cleanup
+ if: ${{ always() && steps.source.outputs.prepared == 'true' }}
+ run: |
+ set -euo pipefail
+ cd "$CX_SOURCE_ROOT" 2>/dev/null \
+ || { echo "CollectiveX source is unavailable" >&2; exit 1; }
+ source experimental/CollectiveX/runtime/common.sh
+ cx_reconcile_recorded_allocation "$CX_JOB_ROOT"
- name: Confirm allocation cleanup
id: allocation_cleanup
if: ${{ always() && steps.source.outputs.prepared == 'true' }}
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index aea6ba891c..13def11b5a 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -1001,6 +1001,7 @@ cx_salloc_jobid() {
return 1
fi
[ -n "$JOB_ID" ] || cx_reconcile_salloc_jobid "$job_name" || true
+ [ -z "$JOB_ID" ] || cx_record_allocation_jobid "$JOB_ID" || true
cx_fail_stage scheduler-allocation "$log"
return 1
fi
@@ -1009,6 +1010,36 @@ cx_salloc_jobid() {
cx_fail_stage scheduler-allocation "$log"
return 1
fi
+ cx_record_allocation_jobid "$JOB_ID" || return 1
+}
+
+cx_record_allocation_jobid() {
+ local job_id="$1" root="${CX_JOB_ROOT:-}" path temporary
+ [[ "$job_id" =~ ^[1-9][0-9]*$ ]] || return 1
+ [ -n "$root" ] || return 0
+ [[ "$root" =~ ^/tmp/inferencex-collectivex-[0-9]+-[0-9]+-[A-Za-z0-9._-]+$ ]] \
+ && [ -d "$root" ] && [ ! -L "$root" ] \
+ && [ "$(stat -c '%u:%a' "$root" 2>/dev/null)" = "$(id -u):700" ] || return 1
+ path="$root/allocation-job-id"
+ temporary="$(mktemp "$root/.allocation-job-id.XXXXXX")" || return 1
+ chmod 600 "$temporary" || { rm -f -- "$temporary"; return 1; }
+ printf '%s\n' "$job_id" > "$temporary" \
+ || { rm -f -- "$temporary"; return 1; }
+ mv -f -- "$temporary" "$path" || { rm -f -- "$temporary"; return 1; }
+}
+
+cx_clear_allocation_jobid() {
+ local root="${CX_JOB_ROOT:-}" path
+ [ -n "$root" ] || return 0
+ [[ "$root" =~ ^/tmp/inferencex-collectivex-[0-9]+-[0-9]+-[A-Za-z0-9._-]+$ ]] \
+ && [ -d "$root" ] && [ ! -L "$root" ] \
+ && [ "$(stat -c '%u:%a' "$root" 2>/dev/null)" = "$(id -u):700" ] || return 1
+ path="$root/allocation-job-id"
+ [ ! -e "$path" ] || {
+ [ -f "$path" ] && [ ! -L "$path" ] \
+ && [ "$(stat -c '%u:%a' "$path" 2>/dev/null)" = "$(id -u):600" ] || return 1
+ rm -f -- "$path"
+ }
}
cx_cancel_job() {
@@ -1042,6 +1073,39 @@ cx_write_cleanup_guard() {
esac
}
+# A workflow cancellation may kill a foreground Slurm step before Bash can run
+# the launcher trap. Reconcile the mode-0600 allocation record from an always()
+# workflow step before isolated source cleanup is allowed.
+cx_reconcile_recorded_allocation() {
+ local root="$1" path job_id
+ [[ "$root" =~ ^/tmp/inferencex-collectivex-[0-9]+-[0-9]+-[A-Za-z0-9._-]+$ ]] \
+ && [ -d "$root" ] && [ ! -L "$root" ] \
+ && [ "$(stat -c '%u:%a' "$root" 2>/dev/null)" = "$(id -u):700" ] || return 1
+ export CX_JOB_ROOT="$root"
+ path="$root/allocation-job-id"
+ if [ ! -e "$path" ]; then
+ [ -f "$root/cleanup-safe" ] && [ ! -e "$root/cleanup-unsafe" ]
+ return
+ fi
+ [ -f "$path" ] && [ ! -L "$path" ] \
+ && [ "$(stat -c '%u:%a' "$path" 2>/dev/null)" = "$(id -u):600" ] \
+ || { cx_write_cleanup_guard unsafe || true; return 1; }
+ IFS= read -r job_id < "$path" || {
+ cx_write_cleanup_guard unsafe || true
+ return 1
+ }
+ [[ "$job_id" =~ ^[1-9][0-9]*$ ]] || {
+ cx_write_cleanup_guard unsafe || true
+ return 1
+ }
+ if cx_cancel_job "$job_id" && cx_clear_allocation_jobid; then
+ cx_write_cleanup_guard safe
+ return
+ fi
+ cx_write_cleanup_guard unsafe || true
+ return 1
+}
+
# Single multi-arch container for ALL NVIDIA SKUs: tag `v0.5.11-cu130` is an OCI
# image index covering linux/amd64 (B200) + linux/arm64 (GB200); enroot import
# pulls the matching arch. (cu130 = CUDA 13, system nccl.h in /usr/include, torch 2.9.x.)
@@ -2756,6 +2820,9 @@ cx_launcher_cleanup() {
if ! cx_cancel_job "$JOB_ID"; then
allocation_stopped=0
[ "$rc" != 0 ] || rc=1
+ elif ! cx_clear_allocation_jobid; then
+ allocation_stopped=0
+ [ "$rc" != 0 ] || rc=1
fi
elif [ "${CX_ALLOCATION_UNCERTAIN:-0}" = 1 ]; then
allocation_stopped=0
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 5353085029..96cc585bc7 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -1108,6 +1108,7 @@ def _run_salloc_scenario(
if [ "$3" = cleanup ]; then
export CX_JOB_ROOT="$2" REPO_ROOT="$2/repo" MOUNT_SRC="$2/repo"
export COLLECTIVEX_CANONICAL_GHA=1
+ cx_clear_allocation_jobid() { rm -f -- "$CX_JOB_ROOT/allocation-job-id"; }
cx_write_cleanup_guard() {
rm -f -- "$CX_JOB_ROOT/cleanup-safe" "$CX_JOB_ROOT/cleanup-unsafe"
: > "$CX_JOB_ROOT/cleanup-$1"
@@ -1282,6 +1283,7 @@ def test_launcher_signal_traps_cancel_the_allocation(self) -> None:
source "$1"
export JOB_ID=5151 CX_JOB_ROOT="$2" REPO_ROOT="$2/repo" MOUNT_SRC="$2/repo"
export COLLECTIVEX_CANONICAL_GHA=1
+ cx_clear_allocation_jobid() { rm -f -- "$CX_JOB_ROOT/allocation-job-id"; }
cx_write_cleanup_guard() {
rm -f -- "$CX_JOB_ROOT/cleanup-safe" "$CX_JOB_ROOT/cleanup-unsafe"
: > "$CX_JOB_ROOT/cleanup-$1"
@@ -1308,7 +1310,12 @@ def test_launcher_signal_traps_cancel_the_allocation(self) -> None:
workflow = (ROOT.parent.parent / ".github" / "workflows" / "collectivex-sweep.yml").read_text()
self.assertIn("cleanup-unsafe", workflow)
self.assertIn("cleanup-safe", workflow)
+ self.assertIn("Reconcile allocation cleanup", workflow)
self.assertIn("Confirm allocation cleanup", workflow)
+ self.assertLess(
+ workflow.index("Reconcile allocation cleanup"),
+ workflow.index("Confirm allocation cleanup"),
+ )
self.assertIn("Prepare pinned backend source archive", workflow)
self.assertIn("Install pinned backend source seed", workflow)
self.assertIn("CX_BACKEND_SOURCE_SEED_ROOT", workflow)
@@ -1321,6 +1328,52 @@ def test_launcher_signal_traps_cancel_the_allocation(self) -> None:
workflow.index("unset COLLECTIVEX_OPERATOR_CONFIG_REQUIRED"),
workflow.index('export COLLECTIVEX_OPERATOR_CONFIG="$operator_config"'),
)
+
+ def test_workflow_reconciles_a_recorded_allocation_after_launcher_loss(self) -> None:
+ workflow = (
+ ROOT.parent.parent / ".github" / "workflows" / "collectivex-sweep.yml"
+ ).read_text()
+ prefix = f"inferencex-collectivex-{os.getpid()}-1-"
+ with tempfile.TemporaryDirectory(prefix=prefix, dir="/tmp") as temporary:
+ root = Path(temporary)
+ binary = root / "bin"
+ calls = root / "scancel-calls"
+ binary.mkdir()
+ for name, body in {
+ "scancel": 'printf "%s\\n" "$*" >> "$CX_TEST_SCANCEL_CALLS"',
+ "squeue": "exit 0",
+ "stat": (
+ 'case "$3" in */allocation-job-id) mode=600 ;; *) mode=700 ;; esac\n'
+ 'printf "%s:%s\\n" "$CX_TEST_UID" "$mode"'
+ ),
+ }.items():
+ path = binary / name
+ path.write_text(f"#!/usr/bin/env bash\n{body}\n")
+ path.chmod(0o700)
+ command = r'''
+ set -euo pipefail
+ source "$1"
+ export CX_JOB_ROOT="$2"
+ : > "$CX_JOB_ROOT/cleanup-unsafe"
+ cx_record_allocation_jobid 6262
+ cx_reconcile_recorded_allocation "$CX_JOB_ROOT"
+ '''
+ result = subprocess.run(
+ ["bash", "-c", command, "_", str(ROOT / "runtime" / "common.sh"), str(root)],
+ text=True,
+ capture_output=True,
+ env={
+ **os.environ,
+ "PATH": f"{binary}:{os.environ['PATH']}",
+ "CX_TEST_SCANCEL_CALLS": str(calls),
+ "CX_TEST_UID": str(os.getuid()),
+ },
+ )
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertEqual(calls.read_text().splitlines(), ["6262"])
+ self.assertTrue((root / "cleanup-safe").is_file())
+ self.assertFalse((root / "cleanup-unsafe").exists())
+ self.assertFalse((root / "allocation-job-id").exists())
prepare_start = workflow.index("- name: Prepare pinned backend source archive")
source_archive_step = workflow[
prepare_start:workflow.index("- uses: actions/upload-artifact", prepare_start)
From 8e7410f47428d7e6c5c0ac79474dcb87e840c97a Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 21:38:45 +0800
Subject: [PATCH 063/210] chore(collectivex): classify probe failure stages
---
experimental/CollectiveX/tests/probe_precision.py | 13 ++++++++++++-
.../CollectiveX/tests/test_precision_scheduling.py | 10 ++++++++++
2 files changed, 22 insertions(+), 1 deletion(-)
diff --git a/experimental/CollectiveX/tests/probe_precision.py b/experimental/CollectiveX/tests/probe_precision.py
index 8afdea1851..fd672d3644 100644
--- a/experimental/CollectiveX/tests/probe_precision.py
+++ b/experimental/CollectiveX/tests/probe_precision.py
@@ -1034,8 +1034,8 @@ def run_target(target: dict[str, Any], output: Path) -> int:
os.environ.setdefault("MASTER_PORT", "12355")
torch.cuda.set_device(local_rank)
device = torch.device(f"cuda:{local_rank}")
- _init_distributed(torch, dist, target["backend"], device, rank, world_size)
backend = None
+ diagnostic_stage = "distributed-init"
topology = capability.topology_for(target["sku"], target["ep"])
topology_record = _topology_record(topology, False) if topology is not None else {
"gpus_per_node": 1, "nodes": target["ep"], "placement_valid": False,
@@ -1044,6 +1044,8 @@ def run_target(target: dict[str, Any], output: Path) -> int:
"topology_class": "unknown", "transport": "unknown", "world_size": target["ep"],
}
try:
+ _init_distributed(torch, dist, target["backend"], device, rank, world_size)
+ diagnostic_stage = "runtime-context"
topology, placement, fingerprint = _runtime_context(
torch, dist, target, device, local_rank
)
@@ -1057,6 +1059,7 @@ def run_target(target: dict[str, Any], output: Path) -> int:
except Exception:
construction = {"ok": False, "reason": "backend-construction-failed"}
gathered: list[Any] = [None] * world_size
+ diagnostic_stage = "construction-consensus"
dist.all_gather_object(gathered, construction)
if not all(record.get("ok") is True for record in gathered):
manifest = build_manifest(
@@ -1073,6 +1076,7 @@ def run_target(target: dict[str, Any], output: Path) -> int:
except Exception:
local = {"ok": False, "reason": "native-operation-failed"}
gathered = [None] * world_size
+ diagnostic_stage = "operation-consensus"
dist.all_gather_object(gathered, local)
if not all(record.get("ok") is True for record in gathered):
reasons = {record.get("reason") for record in gathered}
@@ -1082,6 +1086,7 @@ def run_target(target: dict[str, Any], output: Path) -> int:
reason=reason, runtime_executed=True, evidence=None,
)
else:
+ diagnostic_stage = "evidence-aggregation"
evidence = _aggregate_local([record["evidence"] for record in gathered])
manifest = build_manifest(
target=target, topology=topology_record, disposition="supported",
@@ -1092,6 +1097,12 @@ def run_target(target: dict[str, Any], output: Path) -> int:
target=target, topology=topology_record, disposition="unsupported",
reason=exc.reason, runtime_executed=False, evidence=None,
)
+ except Exception:
+ print(
+ f"[collectivex] precision-probe-diagnostic={diagnostic_stage}-exception",
+ file=sys.stderr,
+ )
+ raise
if rank == 0:
_write_atomic(output, manifest)
print(json.dumps(manifest, allow_nan=False, sort_keys=True, separators=(",", ":")))
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
index 2b79d8fd0d..dce0414a2c 100644
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -418,6 +418,16 @@ def load_config(name: str) -> dict[str, object]:
self.assertTrue(direct_cases)
self.assertEqual({item["case"]["ep"] for item in direct_cases}, {8})
+ def test_probe_runtime_diagnostics_are_closed_stage_codes(self) -> None:
+ probe = (ROOT / "tests" / "probe_precision.py").read_text()
+ for stage in (
+ "distributed-init", "runtime-context", "construction-consensus",
+ "operation-consensus", "evidence-aggregation",
+ ):
+ self.assertIn(f'"{stage}"', probe)
+ self.assertIn("precision-probe-diagnostic={diagnostic_stage}-exception", probe)
+ self.assertNotIn("precision-probe-diagnostic={exc}", probe)
+
if __name__ == "__main__":
unittest.main()
From f6a0f61b3f94144ead91196db0efd8509c6be78e Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 21:57:15 +0800
Subject: [PATCH 064/210] fix(collectivex): apply case-aware network profiles
---
.../CollectiveX/launchers/launch_gb-nv.sh | 1 +
.../CollectiveX/launchers/launch_mi-amds.sh | 3 +-
.../launchers/launch_single-slurm.sh | 3 +-
experimental/CollectiveX/runtime/common.sh | 46 ++++++++++++++++++-
.../CollectiveX/runtime/run_in_container.sh | 3 ++
.../tests/test_sampling_contract.py | 42 +++++++++++++++++
6 files changed, 95 insertions(+), 3 deletions(-)
diff --git a/experimental/CollectiveX/launchers/launch_gb-nv.sh b/experimental/CollectiveX/launchers/launch_gb-nv.sh
index 21aae4c13c..16371719ee 100644
--- a/experimental/CollectiveX/launchers/launch_gb-nv.sh
+++ b/experimental/CollectiveX/launchers/launch_gb-nv.sh
@@ -48,6 +48,7 @@ case "$CX_BENCH" in
*) cx_die "unsupported $PRODUCT EP backend: $CX_BENCH" ;;
esac
cx_validate_shard_control "$CX_DIR"
+cx_load_network_control_mode "$CX_DIR" || cx_die "cannot resolve network control mode"
cx_require_vars CX_PARTITION CX_ACCOUNT CX_SQUASH_DIR CX_STAGE_DIR
[ "$PRODUCT" != gb300 ] || cx_require_vars CX_ENROOT_CACHE_PATH
PARTITION="$CX_PARTITION"; ACCOUNT="$CX_ACCOUNT"; SQUASH_DIR="$CX_SQUASH_DIR"
diff --git a/experimental/CollectiveX/launchers/launch_mi-amds.sh b/experimental/CollectiveX/launchers/launch_mi-amds.sh
index f66f820f54..162c22adf5 100644
--- a/experimental/CollectiveX/launchers/launch_mi-amds.sh
+++ b/experimental/CollectiveX/launchers/launch_mi-amds.sh
@@ -73,8 +73,9 @@ else
unset CX_SCALE_OUT_TRANSPORT
fi
export CX_RUN_TIMEOUT="${CX_RUN_TIMEOUT:-1800}"
-cx_apply_network_profile "$NODES" "$CX_TRANSPORT"
cx_validate_shard_control "$CX_DIR"
+cx_load_network_control_mode "$CX_DIR" || cx_die "cannot resolve network control mode"
+cx_apply_network_profile "$NODES" "$CX_TRANSPORT"
cx_require_vars CX_PARTITION CX_SQUASH_DIR CX_STAGE_DIR
PARTITION="$CX_PARTITION"; SQUASH_DIR="$CX_SQUASH_DIR"
diff --git a/experimental/CollectiveX/launchers/launch_single-slurm.sh b/experimental/CollectiveX/launchers/launch_single-slurm.sh
index eade8fb751..2ebacecf33 100644
--- a/experimental/CollectiveX/launchers/launch_single-slurm.sh
+++ b/experimental/CollectiveX/launchers/launch_single-slurm.sh
@@ -67,8 +67,9 @@ else
unset CX_SCALE_OUT_TRANSPORT
fi
export CX_NCCL_HOME="${CX_NCCL_HOME:-/usr}" NCCL_CUMEM_ENABLE=1
-cx_apply_network_profile "$NODES" "$CX_TRANSPORT"
cx_validate_shard_control "$CX_DIR"
+cx_load_network_control_mode "$CX_DIR" || cx_die "cannot resolve network control mode"
+cx_apply_network_profile "$NODES" "$CX_TRANSPORT"
cx_require_vars CX_PARTITION CX_SQUASH_DIR
[ "$REQUIRE_ACCOUNT" = 0 ] || cx_require_vars CX_ACCOUNT
[ "$RUNNER" != b300 ] || cx_require_vars CX_STAGE_DIR
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 13def11b5a..9c0d491e65 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -536,7 +536,7 @@ PY
# Explicit Slurm export boundary. Operator config, runner credentials, HOME,
# workspace paths, and unrelated service secrets never enter the container.
cx_container_exports() {
- printf '%s' 'COLLECTIVEX_SOURCE_SHA,COLLECTIVEX_ARTIFACT_NAME,COLLECTIVEX_EXECUTION_ID,COLLECTIVEX_CONTROL_SHA256,COLLECTIVEX_IMAGE,COLLECTIVEX_IMAGE_DIGEST,COLLECTIVEX_IMAGE_DIGEST_VERIFIED,COLLECTIVEX_SQUASH_SHA256,GITHUB_REF_NAME,GITHUB_REF,GITHUB_REPOSITORY,GITHUB_JOB,GITHUB_RUN_ID,GITHUB_RUN_ATTEMPT,GITHUB_SHA,CX_RUNNER,CX_BENCH,CX_NODES,CX_GPUS_PER_NODE,CX_SCALE_UP_DOMAIN,CX_SHARD_FILE,CX_SHARD_SKU,CX_PRECISION_PROBE,CX_NGPUS,CX_TS,CX_TOPO,CX_SCOPE,CX_TRANSPORT,CX_SCALE_UP_TRANSPORT,CX_SCALE_OUT_TRANSPORT,CX_MODE,CX_PHASE,CX_ROUTING,CX_EPLB,CX_CASE_ID,CX_SUITE,CX_WORKLOAD_NAME,CX_REQUIRED_PUBLICATION,CX_PRECISION_PROFILE,CX_QUALIFICATION_INDEX,CX_HIDDEN,CX_TOPK,CX_EXPERTS,CX_TOKENS_LADDER,CX_CANONICAL,CX_ITERS,CX_TRIALS,CX_WARMUP,CX_SAMPLES_PER_POINT,CX_WARMUP_SEMANTICS,CX_SEED,CX_RUN_TIMEOUT,CX_NCCL_HOME,CX_ALLOW_MNNVL,CX_ATTEMPT_ID,CX_RUNTIME_MARKER,CX_MORI_KERNEL_TYPE,CX_WORKLOAD_DIR,CX_BACKEND_CACHE_ROOT,CX_BACKEND_CACHE_SENTINEL_SHA256,CX_BACKEND_SOURCE_ROOT,CX_AUDIT_SALT,CX_SOCKET_IFNAME,CX_RDMA_DEVICES,CX_IB_GID_INDEX,CX_RDMA_SERVICE_LEVEL,MASTER_ADDR,MASTER_PORT,RANK,WORLD_SIZE,LOCAL_RANK,LOCAL_WORLD_SIZE,NCCL_NET,NCCL_SOCKET_IFNAME,GLOO_SOCKET_IFNAME,NCCL_IB_HCA,NCCL_IB_GID_INDEX,NCCL_IB_SL,NVSHMEM_DISABLE_IB,NVSHMEM_ENABLE_NIC_PE_MAPPING,NVSHMEM_HCA_LIST,NVSHMEM_IB_GID_INDEX,NVSHMEM_IB_SL,NVSHMEM_IB_ENABLE_IBGDA,NVSHMEM_IBGDA_NIC_HANDLER,EP_NIC_NAME,EP_OVERRIDE_RDMA_SL,UCCL_SOCKET_IFNAME,UCCL_IB_GID_INDEX,UCCL_IB_SL,MORI_RDMA_DEVICES,HYBRID_EP_MULTINODE,USE_NIXL,RDMA_CORE_HOME,DEEPEP_HYBRID_BUILD_MODE,NCCL_CUMEM_ENABLE,NCCL_MNNVL_ENABLE,MC_FORCE_MNNVL,MORI_DISABLE_AUTO_XGMI,MORI_ENABLE_SDMA,MORI_APP_LOG_LEVEL,MORI_SHMEM_LOG_LEVEL,MORI_IO_LOG_LEVEL'
+ printf '%s' 'COLLECTIVEX_SOURCE_SHA,COLLECTIVEX_ARTIFACT_NAME,COLLECTIVEX_EXECUTION_ID,COLLECTIVEX_CONTROL_SHA256,COLLECTIVEX_IMAGE,COLLECTIVEX_IMAGE_DIGEST,COLLECTIVEX_IMAGE_DIGEST_VERIFIED,COLLECTIVEX_SQUASH_SHA256,GITHUB_REF_NAME,GITHUB_REF,GITHUB_REPOSITORY,GITHUB_JOB,GITHUB_RUN_ID,GITHUB_RUN_ATTEMPT,GITHUB_SHA,CX_RUNNER,CX_BENCH,CX_NODES,CX_GPUS_PER_NODE,CX_SCALE_UP_DOMAIN,CX_SHARD_FILE,CX_SHARD_SKU,CX_PRECISION_PROBE,CX_NGPUS,CX_TS,CX_TOPO,CX_SCOPE,CX_TRANSPORT,CX_SCALE_UP_TRANSPORT,CX_SCALE_OUT_TRANSPORT,CX_MODE,CX_PHASE,CX_ROUTING,CX_EPLB,CX_CASE_ID,CX_SUITE,CX_WORKLOAD_NAME,CX_REQUIRED_PUBLICATION,CX_PRECISION_PROFILE,CX_QUALIFICATION_INDEX,CX_HIDDEN,CX_TOPK,CX_EXPERTS,CX_TOKENS_LADDER,CX_CANONICAL,CX_ITERS,CX_TRIALS,CX_WARMUP,CX_SAMPLES_PER_POINT,CX_WARMUP_SEMANTICS,CX_SEED,CX_RUN_TIMEOUT,CX_NCCL_HOME,CX_ALLOW_MNNVL,CX_ATTEMPT_ID,CX_RUNTIME_MARKER,CX_MORI_KERNEL_TYPE,CX_WORKLOAD_DIR,CX_BACKEND_CACHE_ROOT,CX_BACKEND_CACHE_SENTINEL_SHA256,CX_BACKEND_SOURCE_ROOT,CX_AUDIT_SALT,CX_SOCKET_IFNAME,CX_RDMA_DEVICES,CX_IB_GID_INDEX,CX_RDMA_SERVICE_LEVEL,CX_RDMA_LINK_LAYER,MASTER_ADDR,MASTER_PORT,RANK,WORLD_SIZE,LOCAL_RANK,LOCAL_WORLD_SIZE,NCCL_NET,NCCL_SOCKET_IFNAME,GLOO_SOCKET_IFNAME,NCCL_IB_HCA,NCCL_IB_GID_INDEX,NCCL_IB_SL,NVSHMEM_DISABLE_IB,NVSHMEM_ENABLE_NIC_PE_MAPPING,NVSHMEM_HCA_LIST,NVSHMEM_IB_GID_INDEX,NVSHMEM_IB_SL,NVSHMEM_IB_ENABLE_IBGDA,NVSHMEM_IBGDA_NIC_HANDLER,EP_NIC_NAME,EP_OVERRIDE_RDMA_SL,UCCL_SOCKET_IFNAME,UCCL_IB_GID_INDEX,UCCL_IB_SL,MORI_RDMA_DEVICES,HYBRID_EP_MULTINODE,USE_NIXL,RDMA_CORE_HOME,DEEPEP_HYBRID_BUILD_MODE,NCCL_CUMEM_ENABLE,NCCL_MNNVL_ENABLE,MC_FORCE_MNNVL,MORI_DISABLE_AUTO_XGMI,MORI_ENABLE_SDMA,MORI_APP_LOG_LEVEL,MORI_SHMEM_LOG_LEVEL,MORI_IO_LOG_LEVEL'
printf '%s' ',MORI_COMMIT'
}
@@ -648,6 +648,7 @@ cx_apply_network_profile() {
unset NVSHMEM_DISABLE_IB NVSHMEM_ENABLE_NIC_PE_MAPPING
unset NVSHMEM_HCA_LIST NVSHMEM_IB_GID_INDEX NVSHMEM_IB_SL
unset NVSHMEM_IB_ENABLE_IBGDA NVSHMEM_IBGDA_NIC_HANDLER
+ unset NVSHMEM_HCA_PE_MAPPING NVSHMEM_REMOTE_TRANSPORT
unset EP_NIC_NAME EP_OVERRIDE_RDMA_SL
unset UCCL_SOCKET_IFNAME UCCL_IB_GID_INDEX UCCL_IB_SL MORI_RDMA_DEVICES
if [ "$nodes" -gt 1 ] && [ "$transport" != mnnvl ]; then
@@ -701,6 +702,13 @@ cx_apply_network_profile() {
fi
fi
export NVSHMEM_IB_ENABLE_IBGDA=1 NVSHMEM_IBGDA_NIC_HANDLER=gpu
+ if [ -n "${CX_RDMA_LINK_LAYER:-}" ]; then
+ case "$CX_RDMA_LINK_LAYER" in
+ roce|infiniband) ;;
+ *) cx_die "invalid validated RDMA link layer" ;;
+ esac
+ cx_export_gid_index_for_link_layer "$CX_RDMA_LINK_LAYER" "$scaleout"
+ fi
}
# Slurm may remove NCCL's leading exact-match marker while propagating an
@@ -811,6 +819,7 @@ BASH
"$nodes":roce|"$nodes":infiniband) ;;
*) cx_fail_stage setup "$log" || true; return 1 ;;
esac
+ export CX_RDMA_LINK_LAYER="$link_layer"
cx_export_gid_index_for_link_layer "$link_layer" "$scaleout"
}
@@ -907,6 +916,39 @@ print("|".join(map(str, values)))
PY
}
+# Load only the case mode needed to choose the allocation/network profile. A
+# consolidated shard may contain normal and low-latency cases; selecting the
+# strictest mode here makes allocation preflight prove every required device.
+# The in-container dispatcher reapplies the profile for each individual case.
+cx_load_network_control_mode() {
+ local cx_root="$1" shard="${CX_SHARD_FILE:-}" path fields mode
+ [ -n "$shard" ] || return 0
+ path="$shard"
+ [ -f "$path" ] || path="${cx_root%/}/$shard"
+ [ -f "$path" ] || return 1
+ if [ "${CX_PRECISION_PROBE:-0}" = 1 ]; then
+ fields="$(cx_precision_probe_control_fields "$cx_root")" || return 1
+ IFS='|' read -r _ _ _ _ mode _ <<< "$fields"
+ else
+ mode="$(python3 - "$path" <<'PY'
+import json
+import sys
+
+with open(sys.argv[1]) as stream:
+ cases = json.load(stream)["cases"]
+modes = {case["mode"] for case in cases}
+if not modes or modes - {"normal", "low-latency"}:
+ raise SystemExit("invalid shard mode")
+print("low-latency" if "low-latency" in modes else "normal")
+PY
+)" || return 1
+ fi
+ case "$mode" in
+ normal|low-latency) export CX_MODE="$mode" ;;
+ *) return 1 ;;
+ esac
+}
+
cx_apply_timing_profile() {
[ -n "${CX_TIMING:-}" ] || return 0
local iters trials warmup extra
@@ -1803,6 +1845,7 @@ cx_lock_canonical_gha_env() {
unset NVSHMEM_DISABLE_IB NVSHMEM_ENABLE_NIC_PE_MAPPING
unset NVSHMEM_HCA_LIST NVSHMEM_IB_GID_INDEX NVSHMEM_IB_SL
unset NVSHMEM_IB_ENABLE_IBGDA NVSHMEM_IBGDA_NIC_HANDLER
+ unset NVSHMEM_HCA_PE_MAPPING NVSHMEM_REMOTE_TRANSPORT
unset EP_NIC_NAME EP_OVERRIDE_RDMA_SL
unset UCCL_SOCKET_IFNAME UCCL_IB_GID_INDEX UCCL_IB_SL MORI_RDMA_DEVICES
unset HYBRID_EP_MULTINODE USE_NIXL RDMA_CORE_HOME DEEPEP_HYBRID_BUILD_MODE
@@ -2712,6 +2755,7 @@ PY
export CX_ITERS="$case_iters" CX_TRIALS="$case_trials" CX_WARMUP="$case_warmup"
export CX_SAMPLES_PER_POINT="$((case_iters * case_trials))"
export CX_WARMUP_SEMANTICS="full-roundtrip-before-each-component-trial-point-v1"
+ cx_apply_network_profile "$NODES" "$transport"
cx_log "EP${NGPUS}[$ci] id=${case_id:-manual} $mode/$ph $CX_BENCH"
if [ "$ep" != "$NGPUS" ] || [ "$nodes" != "$NODES" ] || [ "$gpn" != "$GPN" ] \
|| [ "$domain" != "$SCALE_UP_DOMAIN" ]; then
diff --git a/experimental/CollectiveX/runtime/run_in_container.sh b/experimental/CollectiveX/runtime/run_in_container.sh
index e1dd5e4e92..b48a4d3d53 100644
--- a/experimental/CollectiveX/runtime/run_in_container.sh
+++ b/experimental/CollectiveX/runtime/run_in_container.sh
@@ -1019,6 +1019,8 @@ run_precision_probe() {
rc=0
cx_validate_shard_control "$PWD"
+cx_load_network_control_mode "$PWD" || cx_die "cannot resolve network control mode"
+cx_apply_network_profile "${CX_NODES:-1}" "${CX_TRANSPORT:-}"
# Build-only mode: rack launchers run the shared backend preparation hook once per
# node inside a persistent named container, then direct rank processes reuse it.
if [ -n "${CX_BUILD_ONLY:-}" ]; then
@@ -1089,6 +1091,7 @@ print("\n".join(lines))
PY
)"
eval "$_exports"
+ cx_apply_network_profile "$CX_NODES" "$CX_TRANSPORT"
# Each case has its OWN routing/dims -> its own canonical workload manifest. cx_stage_canonical
# short-circuits when CX_WORKLOAD_DIR is already set, so without this unset the first case's
# staged dir is reused for the rest and run_ep.py can't find the later cases' manifests
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 96cc585bc7..b3e5373a40 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -599,9 +599,11 @@ def test_scaleout_network_profile_is_explicit_and_allowlisted(self) -> None:
! (export CX_SOCKET_IFNAME=eth0; unset CX_RDMA_DEVICES; cx_apply_network_profile 2 nvlink-rdma)
export CX_SOCKET_IFNAME=ib0 CX_RDMA_DEVICES=mlx5_0:1,mlx5_1:1
export NCCL_NET=Socket NCCL_IB_HCA=stale NVSHMEM_HCA_LIST=stale
+ export NVSHMEM_HCA_PE_MAPPING=stale NVSHMEM_REMOTE_TRANSPORT=stale
export CX_SHARD_SKU=b300
cx_apply_network_profile 1 nvlink
test "$NVSHMEM_DISABLE_IB" = 1
+ test -z "${NVSHMEM_HCA_PE_MAPPING+x}${NVSHMEM_REMOTE_TRANSPORT+x}"
export CX_BENCH=deepep CX_MODE=low-latency CX_IB_GID_INDEX=3
cx_apply_network_profile 1 nvlink
test -z "${NVSHMEM_DISABLE_IB+x}${NCCL_NET+x}${NCCL_IB_HCA+x}"
@@ -639,6 +641,9 @@ def test_scaleout_network_profile_is_explicit_and_allowlisted(self) -> None:
cx_export_gid_index_for_link_layer infiniband 1
test -z "${NVSHMEM_IB_GID_INDEX+x}${NCCL_IB_GID_INDEX+x}${UCCL_IB_GID_INDEX+x}"
test "$NVSHMEM_IB_SL:$NCCL_IB_SL:$UCCL_IB_SL" = 2:2:2
+ export CX_RDMA_LINK_LAYER=roce
+ cx_apply_network_profile 2 nvlink-rdma
+ test "$NVSHMEM_IB_GID_INDEX:$NCCL_IB_GID_INDEX:$UCCL_IB_GID_INDEX" = 3:3:3
'''
subprocess.run(
["bash", "-c", command, "_", str(ROOT / "runtime" / "common.sh")],
@@ -652,6 +657,43 @@ def test_scaleout_network_profile_is_explicit_and_allowlisted(self) -> None:
self.assertIn(version, v2_adapter)
self.assertIn(version, container_runtime)
+ def test_network_mode_is_loaded_from_validated_control(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary)
+ mixed = root / "mixed.json"
+ mixed.write_text(json.dumps({"cases": [
+ {"mode": "normal"}, {"mode": "low-latency"},
+ ]}))
+ normal = root / "normal.json"
+ normal.write_text(json.dumps({"cases": [{"mode": "normal"}]}))
+ probe = root / "probe.json"
+ probe.write_text(json.dumps({
+ "id": "probe-id",
+ "target": {
+ "backend": "deepep", "sku": "b300", "ep": 8,
+ "mode": "low-latency", "precision_profile": "profile",
+ },
+ }))
+ command = r'''
+ set -euo pipefail
+ source "$1"
+ export CX_SHARD_FILE="$2" CX_PRECISION_PROBE="$3"
+ cx_load_network_control_mode "$4"
+ printf '%s' "$CX_MODE"
+ '''
+ for path, precision, expected in (
+ (mixed, "0", "low-latency"),
+ (normal, "0", "normal"),
+ (probe, "1", "low-latency"),
+ ):
+ result = subprocess.run(
+ ["bash", "-c", command, "_", str(ROOT / "runtime" / "common.sh"),
+ str(path), precision, str(ROOT)],
+ text=True, capture_output=True, check=True,
+ env={**os.environ, "COLLECTIVEX_OPERATOR_CONFIG": "/dev/null"},
+ )
+ self.assertEqual(result.stdout, expected)
+
def test_network_profile_validation_is_private_and_all_node(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary).resolve()
From 188e2b105ab2dc8d2391597e286bcf29a3d4e171 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 21:59:56 +0800
Subject: [PATCH 065/210] fix(collectivex): reopen invalid B300 LL probes
---
experimental/CollectiveX/capability.py | 3 ---
.../CollectiveX/tests/test_precision_scheduling.py | 7 ++++---
2 files changed, 4 insertions(+), 6 deletions(-)
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
index 3f105d7864..0242a4d1cc 100644
--- a/experimental/CollectiveX/capability.py
+++ b/experimental/CollectiveX/capability.py
@@ -376,9 +376,6 @@ def _precision_rule(
*(("28740712155", "b300", 16, "deepep", "low-latency", profile,
"unsupported", "backend-setup-timeout")
for profile in (_LL_FP8_PROFILE, _LL_LOGFMT_PROFILE, _LL_FP8_LOGFMT_PROFILE)),
- *(('28740381789', 'b300', 8, 'deepep', 'low-latency', profile,
- 'unsupported', 'backend-construction-failed')
- for profile in (_LL_FP8_PROFILE, _LL_LOGFMT_PROFILE, _LL_FP8_LOGFMT_PROFILE)),
)
PRECISION_CELL_OVERRIDES.update({
(profile, backend, sku, ep, mode): {
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
index dce0414a2c..4325f89ed7 100644
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -28,7 +28,7 @@ def test_precision_probe_inventory_is_exact_and_non_mutating(self) -> None:
item["precision_profile"],
)
self.assertEqual(targets, sorted(capability.provisional_precision_targets(), key=key))
- self.assertEqual(len(targets), 12)
+ self.assertEqual(len(targets), 15)
self.assertEqual(capability.PRECISION_CAPABILITIES, before)
self.assertEqual(
len({
@@ -222,14 +222,15 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
{"provisional", "supported", "unsupported"},
)
self.assertEqual(
- len(targets) - len(capability.provisional_precision_targets()), 82
+ len(targets) - len(capability.provisional_precision_targets()), 79
)
self.assertEqual(
sum(item["disposition"] == "supported" for item in targets), 48
)
self.assertEqual(
- sum(item["disposition"] == "unsupported" for item in targets), 34
+ sum(item["disposition"] == "unsupported" for item in targets), 31
)
+ self.assertEqual(len(capability.provisional_precision_targets()), 15)
keys = {
(
item["precision_profile"],
From c19f7f38d8c8e028a0674f66341303ab6530992f Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 22:00:53 +0800
Subject: [PATCH 066/210] fix(collectivex): reopen invalid B300 EP16 probes
---
experimental/CollectiveX/capability.py | 6 ------
.../CollectiveX/tests/test_precision_scheduling.py | 8 ++++----
2 files changed, 4 insertions(+), 10 deletions(-)
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
index 0242a4d1cc..871e420f4b 100644
--- a/experimental/CollectiveX/capability.py
+++ b/experimental/CollectiveX/capability.py
@@ -370,12 +370,6 @@ def _precision_rule(
("28740154697", "h200-dgxc", 16, "deepep-v2", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "backend-setup-timeout"),
("28740533382", "mi355x", 8, "mori", "normal", _MORI_E4M3FN_BOTH_PROFILE, "unsupported", "backend-construction-failed"),
("28740533382", "mi355x", 8, "mori", "normal", _MORI_E4M3FN_DIRECT_PROFILE, "unsupported", "backend-construction-failed"),
- ("28740712155", "b300", 16, "deepep", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "backend-setup-timeout"),
- ("28740712155", "b300", 16, "deepep-v2", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "backend-construction-failed"),
- ("28740712155", "b300", 16, "deepep-hybrid", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "native-operation-failed"),
- *(("28740712155", "b300", 16, "deepep", "low-latency", profile,
- "unsupported", "backend-setup-timeout")
- for profile in (_LL_FP8_PROFILE, _LL_LOGFMT_PROFILE, _LL_FP8_LOGFMT_PROFILE)),
)
PRECISION_CELL_OVERRIDES.update({
(profile, backend, sku, ep, mode): {
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
index 4325f89ed7..59b8a064ca 100644
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -28,7 +28,7 @@ def test_precision_probe_inventory_is_exact_and_non_mutating(self) -> None:
item["precision_profile"],
)
self.assertEqual(targets, sorted(capability.provisional_precision_targets(), key=key))
- self.assertEqual(len(targets), 15)
+ self.assertEqual(len(targets), 21)
self.assertEqual(capability.PRECISION_CAPABILITIES, before)
self.assertEqual(
len({
@@ -222,15 +222,15 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
{"provisional", "supported", "unsupported"},
)
self.assertEqual(
- len(targets) - len(capability.provisional_precision_targets()), 79
+ len(targets) - len(capability.provisional_precision_targets()), 73
)
self.assertEqual(
sum(item["disposition"] == "supported" for item in targets), 48
)
self.assertEqual(
- sum(item["disposition"] == "unsupported" for item in targets), 31
+ sum(item["disposition"] == "unsupported" for item in targets), 25
)
- self.assertEqual(len(capability.provisional_precision_targets()), 15)
+ self.assertEqual(len(capability.provisional_precision_targets()), 21)
keys = {
(
item["precision_profile"],
From 55f6a7ce70fc1580fbcd2b3f4106b00ca00f0a2c Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 22:05:23 +0800
Subject: [PATCH 067/210] test(collectivex): record B300 LL support
---
experimental/CollectiveX/capability.py | 3 +++
.../CollectiveX/tests/test_precision_scheduling.py | 8 ++++----
2 files changed, 7 insertions(+), 4 deletions(-)
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
index 871e420f4b..25ad036a54 100644
--- a/experimental/CollectiveX/capability.py
+++ b/experimental/CollectiveX/capability.py
@@ -370,6 +370,9 @@ def _precision_rule(
("28740154697", "h200-dgxc", 16, "deepep-v2", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "backend-setup-timeout"),
("28740533382", "mi355x", 8, "mori", "normal", _MORI_E4M3FN_BOTH_PROFILE, "unsupported", "backend-construction-failed"),
("28740533382", "mi355x", 8, "mori", "normal", _MORI_E4M3FN_DIRECT_PROFILE, "unsupported", "backend-construction-failed"),
+ *(("28743235213", "b300", 8, "deepep", "low-latency", profile,
+ "supported", "native-probe-passed")
+ for profile in (_LL_FP8_PROFILE, _LL_LOGFMT_PROFILE, _LL_FP8_LOGFMT_PROFILE)),
)
PRECISION_CELL_OVERRIDES.update({
(profile, backend, sku, ep, mode): {
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
index 59b8a064ca..2af87722a0 100644
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -28,7 +28,7 @@ def test_precision_probe_inventory_is_exact_and_non_mutating(self) -> None:
item["precision_profile"],
)
self.assertEqual(targets, sorted(capability.provisional_precision_targets(), key=key))
- self.assertEqual(len(targets), 21)
+ self.assertEqual(len(targets), 18)
self.assertEqual(capability.PRECISION_CAPABILITIES, before)
self.assertEqual(
len({
@@ -222,15 +222,15 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
{"provisional", "supported", "unsupported"},
)
self.assertEqual(
- len(targets) - len(capability.provisional_precision_targets()), 73
+ len(targets) - len(capability.provisional_precision_targets()), 76
)
self.assertEqual(
- sum(item["disposition"] == "supported" for item in targets), 48
+ sum(item["disposition"] == "supported" for item in targets), 51
)
self.assertEqual(
sum(item["disposition"] == "unsupported" for item in targets), 25
)
- self.assertEqual(len(capability.provisional_precision_targets()), 21)
+ self.assertEqual(len(capability.provisional_precision_targets()), 18)
keys = {
(
item["precision_profile"],
From 3f41d529647ffec0038eb28cc8447704fbc1d3b4 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 22:15:10 +0800
Subject: [PATCH 068/210] fix(collectivex): support private B200 placement
---
.github/workflows/collectivex-sweep.yml | 22 ++++++++++++++-----
.../launchers/launch_single-slurm.sh | 1 +
experimental/CollectiveX/runtime/common.sh | 10 ++++++---
.../tests/test_deepep_v2_contract.py | 3 ++-
.../tests/test_sampling_contract.py | 8 +++++--
5 files changed, 32 insertions(+), 12 deletions(-)
diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml
index 9ea2f50f7f..5696998d08 100644
--- a/.github/workflows/collectivex-sweep.yml
+++ b/.github/workflows/collectivex-sweep.yml
@@ -452,6 +452,7 @@ jobs:
env:
COLLECTIVEX_OPERATOR_CONFIG_CONTENT: ${{ secrets.COLLECTIVEX_OPERATOR_CONFIG_V1 }}
COLLECTIVEX_NETWORK_CONFIG_CONTENT: ${{ secrets.COLLECTIVEX_NETWORK_CONFIG_V1 }}
+ COLLECTIVEX_B200_CONFIG_CONTENT: ${{ secrets.COLLECTIVEX_B200_CONFIG_V1 }}
COLLECTIVEX_OPERATOR_AUDIT_SALT: ${{ secrets.COLLECTIVEX_AUDIT_SALT_V1 }}
COLLECTIVEX_OPERATOR_CONFIG_REQUIRED: '1'
run: |
@@ -479,21 +480,29 @@ jobs:
)
base = load("COLLECTIVEX_OPERATOR_CONFIG_CONTENT")
- overlay_raw = os.environ.get("COLLECTIVEX_NETWORK_CONFIG_CONTENT", "")
- if overlay_raw:
- overlay = load("COLLECTIVEX_NETWORK_CONFIG_CONTENT")
- network_fields = {
+ for overlay_name in (
+ "COLLECTIVEX_NETWORK_CONFIG_CONTENT", "COLLECTIVEX_B200_CONFIG_CONTENT",
+ ):
+ overlay_raw = os.environ.get(overlay_name, "")
+ if not overlay_raw:
+ continue
+ overlay = load(overlay_name)
+ private_fields = {
"socket_ifname", "rdma_devices", "ib_gid_index", "rdma_service_level",
}
if set(overlay) != {"schema_version", "runners"} or overlay["schema_version"] != 1:
raise ValueError("invalid network overlay envelope")
if not isinstance(overlay["runners"], dict) or not overlay["runners"]:
raise ValueError("invalid network overlay runners")
+ if overlay_name == "COLLECTIVEX_B200_CONFIG_CONTENT":
+ if set(overlay["runners"]) != {"b200-dgxc"}:
+ raise ValueError("invalid B200 overlay runner")
+ private_fields.add("qos")
for runner, fields in overlay["runners"].items():
if runner not in base.get("runners", {}) or not isinstance(fields, dict) or not fields:
raise ValueError("invalid network overlay runner")
- if set(fields) - network_fields:
- raise ValueError("network overlay contains a non-network field")
+ if set(fields) - private_fields:
+ raise ValueError("private overlay contains a disallowed field")
base["runners"][runner].update(fields)
payload = json.dumps(base, sort_keys=True, separators=(",", ":")) + "\n"
if len(payload.encode()) > 65536:
@@ -504,6 +513,7 @@ jobs:
stream.write(payload)
PY
unset COLLECTIVEX_OPERATOR_CONFIG_CONTENT COLLECTIVEX_NETWORK_CONFIG_CONTENT
+ unset COLLECTIVEX_B200_CONFIG_CONTENT
unset COLLECTIVEX_OPERATOR_CONFIG_REQUIRED
export COLLECTIVEX_OPERATOR_CONFIG="$operator_config"
export COLLECTIVEX_OPERATOR_CONFIG_EPHEMERAL=1
diff --git a/experimental/CollectiveX/launchers/launch_single-slurm.sh b/experimental/CollectiveX/launchers/launch_single-slurm.sh
index 2ebacecf33..21284e5f5b 100644
--- a/experimental/CollectiveX/launchers/launch_single-slurm.sh
+++ b/experimental/CollectiveX/launchers/launch_single-slurm.sh
@@ -103,6 +103,7 @@ allocation=(--partition="$CX_PARTITION" --nodes="$NODES" --gres=gpu:"$GPN" --exc
--time="$TIME_MIN" "${ALLOC_EXTRA[@]}")
[ "$NODES" = 1 ] || allocation+=(--ntasks-per-node="$GPN")
[ -z "${CX_ACCOUNT:-}" ] || allocation+=(--account="$CX_ACCOUNT")
+[ -z "${CX_QOS:-}" ] || allocation+=(--qos="$CX_QOS")
[ -z "${CX_EXCLUDE_NODES:-}" ] || allocation+=(--exclude="$CX_EXCLUDE_NODES")
cx_salloc_jobid "${allocation[@]}"
[ -n "$JOB_ID" ] || cx_die "could not resolve allocated JOB_ID from salloc"
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 9c0d491e65..5c8ee87d3c 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -163,7 +163,7 @@ cx_load_operator_config() {
local audit_salt_override validation_code
audit_salt_override="${COLLECTIVEX_OPERATOR_AUDIT_SALT:-}"
unset COLLECTIVEX_OPERATOR_AUDIT_SALT
- unset CX_PARTITION CX_ACCOUNT CX_SQUASH_DIR CX_STAGE_DIR CX_ENROOT_CACHE_PATH
+ unset CX_PARTITION CX_ACCOUNT CX_QOS CX_SQUASH_DIR CX_STAGE_DIR CX_ENROOT_CACHE_PATH
unset ENROOT_CACHE_PATH
unset CX_EXCLUDE_NODES CX_NODELIST CX_LOCK_DIR CX_MASTER_PORT
unset CX_SOCKET_IFNAME CX_RDMA_DEVICES CX_IB_GID_INDEX CX_RDMA_SERVICE_LEVEL
@@ -224,6 +224,7 @@ RUNNERS = {
FIELDS = {
"partition": "CX_PARTITION",
"account": "CX_ACCOUNT",
+ "qos": "CX_QOS",
"squash_dir": "CX_SQUASH_DIR",
"stage_dir": "CX_STAGE_DIR",
"enroot_cache_path": "CX_ENROOT_CACHE_PATH",
@@ -251,7 +252,7 @@ REQUIRED = {
ALLOWED = {
"h100-dgxc": REQUIRED["h100-dgxc"] | {"exclude_nodes", "stage_dir"} | NETWORK_FIELDS,
"h200-dgxc": REQUIRED["h200-dgxc"] | {"account", "exclude_nodes", "stage_dir"} | NETWORK_FIELDS,
- "b200-dgxc": REQUIRED["b200-dgxc"] | {"exclude_nodes", "stage_dir"} | NETWORK_FIELDS,
+ "b200-dgxc": REQUIRED["b200-dgxc"] | {"exclude_nodes", "stage_dir", "qos"} | NETWORK_FIELDS,
"b300": REQUIRED["b300"] | {"exclude_nodes", "stage_dir"} | NETWORK_FIELDS,
"gb200": REQUIRED["gb200"] | NETWORK_FIELDS,
"gb300": REQUIRED["gb300"] | {"stage_dir"} | NETWORK_FIELDS,
@@ -1808,6 +1809,7 @@ cx_prepare_runner_shared_stage_base() {
cx_lock_canonical_gha_env() {
local runner="$1" expected_nodes expected_gpn expected_world trusted_lock_dir=""
local trusted_stage_dir=""
+ local trusted_qos=""
local trusted_socket_ifname="" trusted_rdma_devices=""
local trusted_ib_gid_index="" trusted_rdma_service_level=""
local trusted_audit_salt=""
@@ -1826,6 +1828,7 @@ cx_lock_canonical_gha_env() {
if [ "${COLLECTIVEX_OPERATOR_CONFIG_LOADED:-}" = "$$" ]; then
trusted_lock_dir="${CX_LOCK_DIR:-}"
trusted_stage_dir="${CX_STAGE_DIR:-}"
+ trusted_qos="${CX_QOS:-}"
trusted_socket_ifname="${CX_SOCKET_IFNAME:-}"
trusted_rdma_devices="${CX_RDMA_DEVICES:-}"
trusted_ib_gid_index="${CX_IB_GID_INDEX:-}"
@@ -1835,7 +1838,7 @@ cx_lock_canonical_gha_env() {
# The legacy B300 operator row contains a root-owned stage path. B300's
# compute-visible account home is the canonical source for its private base.
case "$runner" in b300|gb300) trusted_stage_dir="" ;; esac
- unset CX_NCCL_HOME CX_MASTER_PORT CX_MORI_KERNEL_TYPE CX_LOCK_DIR CX_STAGE_DIR
+ unset CX_NCCL_HOME CX_MASTER_PORT CX_MORI_KERNEL_TYPE CX_LOCK_DIR CX_STAGE_DIR CX_QOS
unset CX_STAGE_PARENT_OWNER_OK
unset MASTER_ADDR MASTER_PORT RANK WORLD_SIZE LOCAL_RANK LOCAL_WORLD_SIZE
unset CX_SOCKET_IFNAME CX_RDMA_DEVICES CX_IB_GID_INDEX CX_RDMA_SERVICE_LEVEL
@@ -1948,6 +1951,7 @@ cx_lock_canonical_gha_env() {
mi325x:?*|mi355x:?*) export CX_LOCK_DIR="$trusted_lock_dir" ;;
esac
CX_STAGE_DIR="$trusted_stage_dir"
+ [ -z "$trusted_qos" ] || export CX_QOS="$trusted_qos"
[ -z "$trusted_socket_ifname" ] \
|| export CX_SOCKET_IFNAME="$trusted_socket_ifname"
[ -z "$trusted_rdma_devices" ] \
diff --git a/experimental/CollectiveX/tests/test_deepep_v2_contract.py b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
index 780e0f86d8..d34dd21e31 100644
--- a/experimental/CollectiveX/tests/test_deepep_v2_contract.py
+++ b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
@@ -94,7 +94,7 @@ def operator_config(root: Path) -> dict[str, object]:
"partition": "test", "squash_dir": path, "stage_dir": path, **network,
},
"b200-dgxc": {
- "partition": "test", "account": "test", "squash_dir": path,
+ "partition": "test", "account": "test", "qos": "test-qos", "squash_dir": path,
"stage_dir": path, **network,
},
"b300": {
@@ -1987,6 +1987,7 @@ def test_operator_config_is_strict_per_runner_json(self) -> None:
'source "$1"; export COLLECTIVEX_EXECUTION_ID="operator-config-$$"; '
"trap 'cx_cleanup_private_logs 0' EXIT; cx_load_operator_config; "
'test "$CX_PARTITION" = test; '
+ '[ "$CX_RUNNER" != b200-dgxc ] || test "$CX_QOS" = test-qos; '
'test -z "${COLLECTIVEX_OPERATOR_CONFIG_CONTENT+x}"; '
'test -z "${ENROOT_CACHE_PATH+x}"'
)
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index b3e5373a40..5b9644ea0f 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -1363,8 +1363,12 @@ def test_launcher_signal_traps_cancel_the_allocation(self) -> None:
self.assertIn("CX_BACKEND_SOURCE_SEED_ROOT", workflow)
self.assertIn("steps.gen.outputs.source_backends", workflow)
self.assertIn("secrets.COLLECTIVEX_NETWORK_CONFIG_V1", workflow)
- self.assertIn("network_fields = {", workflow)
- self.assertIn("set(fields) - network_fields", workflow)
+ self.assertIn("secrets.COLLECTIVEX_B200_CONFIG_V1", workflow)
+ self.assertIn('set(overlay["runners"]) != {"b200-dgxc"}', workflow)
+ self.assertIn("set(fields) - private_fields", workflow)
+ self.assertIn('[ -z "${CX_QOS:-}" ] || allocation+=(--qos="$CX_QOS")', (
+ ROOT / "launchers" / "launch_single-slurm.sh"
+ ).read_text())
self.assertIn("COLLECTIVEX_OPERATOR_CONFIG_EPHEMERAL=1", workflow)
self.assertLess(
workflow.index("unset COLLECTIVEX_OPERATOR_CONFIG_REQUIRED"),
From 34ad173d609cf3913ab653fa2d93bf5f89ae73fc Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 22:38:26 +0800
Subject: [PATCH 069/210] fix(collectivex): diagnose distributed probe stalls
---
experimental/CollectiveX/runtime/common.sh | 5 +++++
.../CollectiveX/tests/probe_precision.py | 22 +++++++++++++++----
.../tests/test_sampling_contract.py | 2 ++
3 files changed, 25 insertions(+), 4 deletions(-)
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 5c8ee87d3c..c23a435dab 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -852,6 +852,7 @@ if [ "${CX_NODES:-1}" -gt 1 ] && [ "${CX_TRANSPORT:-}" != mnnvl ]; then
. /ix/experimental/CollectiveX/runtime/common.sh || exit 68
cx_restore_exact_hca_selector || exit 68
fi
+cx_write_runtime_stage execution || exit 68
export RANK="$SLURM_PROCID" WORLD_SIZE="$SLURM_NTASKS"
export LOCAL_RANK="$SLURM_LOCALID" LOCAL_WORLD_SIZE="$CX_GPUS_PER_NODE"
case "${CX_PRECISION_PROBE:-0}" in
@@ -2678,6 +2679,10 @@ cx_run_distributed_shard() {
"$runtime_log" 2>&1
run_rc=$?
set -e
+ if [ "$run_rc" = 124 ] || [ "$run_rc" = 137 ]; then
+ printf '[collectivex] precision probe timed out rc=%s limit=%ss\n' \
+ "$run_rc" "${CX_RUN_TIMEOUT:-900}" >> "$runtime_log"
+ fi
if [ "$run_rc" != 0 ] || ! python3 "$CX_DIR/tests/probe_precision.py" \
--validate-manifest "$expected_out" >/dev/null 2>&1; then
[ "$run_rc" != 0 ] || run_rc=1
diff --git a/experimental/CollectiveX/tests/probe_precision.py b/experimental/CollectiveX/tests/probe_precision.py
index fd672d3644..12e1973b44 100644
--- a/experimental/CollectiveX/tests/probe_precision.py
+++ b/experimental/CollectiveX/tests/probe_precision.py
@@ -1043,14 +1043,27 @@ def run_target(target: dict[str, Any], output: Path) -> int:
"scale_out_transport": None, "scope": "scale-out",
"topology_class": "unknown", "transport": "unknown", "world_size": target["ep"],
}
+ os.environ.pop("NCCL_ASYNC_ERROR_HANDLING", None)
+ os.environ.setdefault("TORCH_NCCL_ASYNC_ERROR_HANDLING", "1")
+ os.environ.setdefault("NCCL_DEBUG", "INFO")
+ os.environ.setdefault("NCCL_DEBUG_SUBSYS", "ENV,INIT,NET")
+
+ def stage(name: str) -> None:
+ nonlocal diagnostic_stage
+ diagnostic_stage = name
+ if rank == 0:
+ print(f"[collectivex] precision-probe-stage={name}", file=sys.stderr, flush=True)
+
try:
+ stage("distributed-init")
_init_distributed(torch, dist, target["backend"], device, rank, world_size)
- diagnostic_stage = "runtime-context"
+ stage("runtime-context")
topology, placement, fingerprint = _runtime_context(
torch, dist, target, device, local_rank
)
topology_record = _topology_record(topology, bool(placement["valid"]))
args = _runtime_args(target, topology, fingerprint)
+ stage("backend-construction")
try:
backend = _backend_class(target["backend"])(
args, rank, world_size, local_rank, device
@@ -1059,7 +1072,7 @@ def run_target(target: dict[str, Any], output: Path) -> int:
except Exception:
construction = {"ok": False, "reason": "backend-construction-failed"}
gathered: list[Any] = [None] * world_size
- diagnostic_stage = "construction-consensus"
+ stage("construction-consensus")
dist.all_gather_object(gathered, construction)
if not all(record.get("ok") is True for record in gathered):
manifest = build_manifest(
@@ -1067,6 +1080,7 @@ def run_target(target: dict[str, Any], output: Path) -> int:
reason="backend-construction-failed", runtime_executed=True, evidence=None,
)
else:
+ stage("native-operation")
try:
local = {"ok": True, "evidence": _local_probe(
torch, dist, target, backend, args, rank
@@ -1076,7 +1090,7 @@ def run_target(target: dict[str, Any], output: Path) -> int:
except Exception:
local = {"ok": False, "reason": "native-operation-failed"}
gathered = [None] * world_size
- diagnostic_stage = "operation-consensus"
+ stage("operation-consensus")
dist.all_gather_object(gathered, local)
if not all(record.get("ok") is True for record in gathered):
reasons = {record.get("reason") for record in gathered}
@@ -1086,7 +1100,7 @@ def run_target(target: dict[str, Any], output: Path) -> int:
reason=reason, runtime_executed=True, evidence=None,
)
else:
- diagnostic_stage = "evidence-aggregation"
+ stage("evidence-aggregation")
evidence = _aggregate_local([record["evidence"] for record in gathered])
manifest = build_manifest(
target=target, topology=topology_record, disposition="supported",
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 5b9644ea0f..3bb3ab5710 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -630,6 +630,7 @@ def test_scaleout_network_profile_is_explicit_and_allowlisted(self) -> None:
bash -n <<< "$wrapper"
grep -Fq '. /ix/experimental/CollectiveX/runtime/common.sh' <<< "$wrapper"
grep -Fq 'cx_restore_exact_hca_selector || exit 68' <<< "$wrapper"
+ grep -Fq 'cx_write_runtime_stage execution || exit 68' <<< "$wrapper"
test "$NVSHMEM_HCA_LIST" = mlx5_0:1,mlx5_1:1
test "$NVSHMEM_ENABLE_NIC_PE_MAPPING" = 1
test "$MORI_RDMA_DEVICES:$EP_NIC_NAME" = mlx5_0,mlx5_1:mlx5_0
@@ -2441,6 +2442,7 @@ def test_launchers_use_private_logs_and_allowlisted_failure_stages(self) -> None
self.assertIn("cx_private_log_path shard-summary", distributed)
self.assertIn("cx_fail_stage execution", distributed)
self.assertIn('cx_fail_stage execution "$runtime_log"', distributed)
+ self.assertIn("precision probe timed out rc=%s limit=%ss", distributed)
def test_case_failure_diagnostic_precedes_normal_srun_footer(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
From c27e58394661dc08353f5573d343cf92f109959b Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 22:39:38 +0800
Subject: [PATCH 070/210] fix(collectivex): pin validated B200 pair
---
.github/workflows/collectivex-sweep.yml | 2 +-
experimental/CollectiveX/launchers/launch_single-slurm.sh | 1 +
experimental/CollectiveX/runtime/common.sh | 4 +++-
experimental/CollectiveX/tests/test_deepep_v2_contract.py | 5 +++--
experimental/CollectiveX/tests/test_sampling_contract.py | 4 ++++
5 files changed, 12 insertions(+), 4 deletions(-)
diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml
index 5696998d08..ba7941e2b9 100644
--- a/.github/workflows/collectivex-sweep.yml
+++ b/.github/workflows/collectivex-sweep.yml
@@ -497,7 +497,7 @@ jobs:
if overlay_name == "COLLECTIVEX_B200_CONFIG_CONTENT":
if set(overlay["runners"]) != {"b200-dgxc"}:
raise ValueError("invalid B200 overlay runner")
- private_fields.add("qos")
+ private_fields.update({"nodelist", "qos"})
for runner, fields in overlay["runners"].items():
if runner not in base.get("runners", {}) or not isinstance(fields, dict) or not fields:
raise ValueError("invalid network overlay runner")
diff --git a/experimental/CollectiveX/launchers/launch_single-slurm.sh b/experimental/CollectiveX/launchers/launch_single-slurm.sh
index 21284e5f5b..734c730d38 100644
--- a/experimental/CollectiveX/launchers/launch_single-slurm.sh
+++ b/experimental/CollectiveX/launchers/launch_single-slurm.sh
@@ -104,6 +104,7 @@ allocation=(--partition="$CX_PARTITION" --nodes="$NODES" --gres=gpu:"$GPN" --exc
[ "$NODES" = 1 ] || allocation+=(--ntasks-per-node="$GPN")
[ -z "${CX_ACCOUNT:-}" ] || allocation+=(--account="$CX_ACCOUNT")
[ -z "${CX_QOS:-}" ] || allocation+=(--qos="$CX_QOS")
+[ -z "${CX_NODELIST:-}" ] || allocation+=(--nodelist="$CX_NODELIST")
[ -z "${CX_EXCLUDE_NODES:-}" ] || allocation+=(--exclude="$CX_EXCLUDE_NODES")
cx_salloc_jobid "${allocation[@]}"
[ -n "$JOB_ID" ] || cx_die "could not resolve allocated JOB_ID from salloc"
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index c23a435dab..118aab1191 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -252,7 +252,9 @@ REQUIRED = {
ALLOWED = {
"h100-dgxc": REQUIRED["h100-dgxc"] | {"exclude_nodes", "stage_dir"} | NETWORK_FIELDS,
"h200-dgxc": REQUIRED["h200-dgxc"] | {"account", "exclude_nodes", "stage_dir"} | NETWORK_FIELDS,
- "b200-dgxc": REQUIRED["b200-dgxc"] | {"exclude_nodes", "stage_dir", "qos"} | NETWORK_FIELDS,
+ "b200-dgxc": REQUIRED["b200-dgxc"] | {
+ "exclude_nodes", "nodelist", "stage_dir", "qos",
+ } | NETWORK_FIELDS,
"b300": REQUIRED["b300"] | {"exclude_nodes", "stage_dir"} | NETWORK_FIELDS,
"gb200": REQUIRED["gb200"] | NETWORK_FIELDS,
"gb300": REQUIRED["gb300"] | {"stage_dir"} | NETWORK_FIELDS,
diff --git a/experimental/CollectiveX/tests/test_deepep_v2_contract.py b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
index d34dd21e31..280de94d3d 100644
--- a/experimental/CollectiveX/tests/test_deepep_v2_contract.py
+++ b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
@@ -94,8 +94,8 @@ def operator_config(root: Path) -> dict[str, object]:
"partition": "test", "squash_dir": path, "stage_dir": path, **network,
},
"b200-dgxc": {
- "partition": "test", "account": "test", "qos": "test-qos", "squash_dir": path,
- "stage_dir": path, **network,
+ "partition": "test", "account": "test", "qos": "test-qos",
+ "nodelist": "test-[0-1]", "squash_dir": path, "stage_dir": path, **network,
},
"b300": {
"partition": "test", "account": "test", "squash_dir": path, "stage_dir": path,
@@ -1988,6 +1988,7 @@ def test_operator_config_is_strict_per_runner_json(self) -> None:
"trap 'cx_cleanup_private_logs 0' EXIT; cx_load_operator_config; "
'test "$CX_PARTITION" = test; '
'[ "$CX_RUNNER" != b200-dgxc ] || test "$CX_QOS" = test-qos; '
+ '[ "$CX_RUNNER" != b200-dgxc ] || test "$CX_NODELIST" = "test-[0-1]"; '
'test -z "${COLLECTIVEX_OPERATOR_CONFIG_CONTENT+x}"; '
'test -z "${ENROOT_CACHE_PATH+x}"'
)
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 3bb3ab5710..0b4b0bc145 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -1367,9 +1367,13 @@ def test_launcher_signal_traps_cancel_the_allocation(self) -> None:
self.assertIn("secrets.COLLECTIVEX_B200_CONFIG_V1", workflow)
self.assertIn('set(overlay["runners"]) != {"b200-dgxc"}', workflow)
self.assertIn("set(fields) - private_fields", workflow)
+ self.assertIn('private_fields.update({"nodelist", "qos"})', workflow)
self.assertIn('[ -z "${CX_QOS:-}" ] || allocation+=(--qos="$CX_QOS")', (
ROOT / "launchers" / "launch_single-slurm.sh"
).read_text())
+ self.assertIn('[ -z "${CX_NODELIST:-}" ] || allocation+=(--nodelist="$CX_NODELIST")', (
+ ROOT / "launchers" / "launch_single-slurm.sh"
+ ).read_text())
self.assertIn("COLLECTIVEX_OPERATOR_CONFIG_EPHEMERAL=1", workflow)
self.assertLess(
workflow.index("unset COLLECTIVEX_OPERATOR_CONFIG_REQUIRED"),
From 51fca0268e0809ff711f17ab5b3f5bba48782f4a Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 22:40:33 +0800
Subject: [PATCH 071/210] feat(collectivex): filter exact capability probes
---
.github/workflows/collectivex-sweep.yml | 7 +++++++
experimental/CollectiveX/tests/probe_precision.py | 6 +++++-
.../CollectiveX/tests/test_precision_scheduling.py | 6 ++++++
3 files changed, 18 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml
index ba7941e2b9..1eb7b47a06 100644
--- a/.github/workflows/collectivex-sweep.yml
+++ b/.github/workflows/collectivex-sweep.yml
@@ -28,6 +28,10 @@ on:
description: Restrict to one GHA runner pool (h100-dgxc|h200-dgxc|b300|b200-dgxc|gb200|gb300|mi325x|mi355x); blank = all
type: string
default: ''
+ probe_id:
+ description: Optional exact provisional probe ID; valid only for probe-precision
+ type: string
+ default: ''
min_nodes:
description: Keep only shards with at least this node/tray count (2 keeps every EP16 and GB EP8; blank = all)
type: string
@@ -87,6 +91,7 @@ jobs:
INPUT_OPERATION: ${{ inputs.operation }}
INPUT_SUITES: ${{ inputs.suites }}
INPUT_ONLY_SKU: ${{ inputs.only_sku }}
+ INPUT_PROBE_ID: ${{ inputs.probe_id }}
INPUT_MIN_NODES: ${{ inputs.min_nodes }}
INPUT_MAX_NODES: ${{ inputs.max_nodes }}
INPUT_MAX_CASES: ${{ inputs.max_cases }}
@@ -122,10 +127,12 @@ jobs:
if [ "$INPUT_OPERATION" = probe-precision ]; then
args=(--workflow-plan --backend "$INPUT_BACKEND" --out matrix_full.json)
[ -n "$INPUT_ONLY_SKU" ] && args+=(--only-sku "$INPUT_ONLY_SKU")
+ [ -n "$INPUT_PROBE_ID" ] && args+=(--probe-id "$INPUT_PROBE_ID")
[ -n "$INPUT_MIN_NODES" ] && args+=(--min-nodes "$INPUT_MIN_NODES")
[ -n "$INPUT_MAX_NODES" ] && args+=(--max-nodes "$INPUT_MAX_NODES")
python3 tests/probe_precision.py "${args[@]}"
else
+ [ -z "$INPUT_PROBE_ID" ] || { echo 'probe_id requires probe-precision' >&2; exit 1; }
args=(--suites "$INPUT_SUITES" --max-cases "$INPUT_MAX_CASES")
case "$INPUT_BACKEND" in
all) args+=(--backends all) ;;
diff --git a/experimental/CollectiveX/tests/probe_precision.py b/experimental/CollectiveX/tests/probe_precision.py
index 12e1973b44..5581c20e54 100644
--- a/experimental/CollectiveX/tests/probe_precision.py
+++ b/experimental/CollectiveX/tests/probe_precision.py
@@ -320,7 +320,7 @@ def _workflow_row(target: dict[str, Any]) -> dict[str, Any]:
def workflow_plan(
*, backend: str = "all", only_sku: str = "", min_nodes: int = 0,
- max_nodes: int = 0,
+ max_nodes: int = 0, probe_id: str = "",
) -> dict[str, Any]:
if min_nodes < 0 or max_nodes < 0 or (min_nodes and max_nodes and min_nodes > max_nodes):
raise ValueError("precision probe node filters are invalid")
@@ -328,6 +328,7 @@ def workflow_plan(
target for target in provisional_targets()
if (backend == "all" or target["backend"] == backend)
and (not only_sku or target["sku"] == only_sku)
+ and (not probe_id or _probe_id(target) == probe_id)
and (
not min_nodes
or capability.topology_for(target["sku"], target["ep"])["nodes"] >= min_nodes
@@ -341,6 +342,8 @@ def workflow_plan(
raise ValueError("precision probe backend is not registered")
if only_sku and only_sku not in capability.PLATFORMS:
raise ValueError("precision probe SKU is not registered")
+ if probe_id and not re.fullmatch(r"probe-[0-9a-f]{20}", probe_id):
+ raise ValueError("precision probe ID is invalid")
if not targets:
raise ValueError("precision probe filters select no provisional cells")
return {
@@ -1154,6 +1157,7 @@ def main() -> int:
plan = workflow_plan(
backend=args.backend or "all", only_sku=args.only_sku,
min_nodes=args.min_nodes, max_nodes=args.max_nodes,
+ probe_id=args.probe_id or "",
)
if args.out is None:
print(json.dumps(plan, allow_nan=False, sort_keys=True, separators=(",", ":")))
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
index 2af87722a0..05012bc1b5 100644
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -60,6 +60,12 @@ def test_precision_probe_workflow_plan_binds_exact_controls(self) -> None:
for row in plan["include"]
))
row = plan["include"][0]
+ exact = probe_precision.workflow_plan(probe_id=row["id"])
+ self.assertEqual(exact["include"], [row])
+ with self.assertRaisesRegex(ValueError, "ID is invalid"):
+ probe_precision.workflow_plan(probe_id="probe-invalid")
+ with self.assertRaisesRegex(ValueError, "select no provisional"):
+ probe_precision.workflow_plan(probe_id="probe-00000000000000000000")
control = probe_precision.extract_control(
plan, probe_id=row["id"], sku=row["sku"], backend=row["backend"],
nodes=row["nodes"],
From 9b20b4bb085efa96d627c3a73cbd00be95c7fc15 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 22:55:39 +0800
Subject: [PATCH 072/210] fix(collectivex): restore rank network profile
---
experimental/CollectiveX/runtime/common.sh | 2 +-
experimental/CollectiveX/tests/test_sampling_contract.py | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 118aab1191..ceb6cf949d 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -852,7 +852,7 @@ esac
[ "$SLURM_LOCALID" -lt "$CX_GPUS_PER_NODE" ] || exit 67
if [ "${CX_NODES:-1}" -gt 1 ] && [ "${CX_TRANSPORT:-}" != mnnvl ]; then
. /ix/experimental/CollectiveX/runtime/common.sh || exit 68
- cx_restore_exact_hca_selector || exit 68
+ cx_apply_network_profile "$CX_NODES" "$CX_TRANSPORT" || exit 68
fi
cx_write_runtime_stage execution || exit 68
export RANK="$SLURM_PROCID" WORLD_SIZE="$SLURM_NTASKS"
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 0b4b0bc145..c1a454a73f 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -629,7 +629,7 @@ def test_scaleout_network_profile_is_explicit_and_allowlisted(self) -> None:
wrapper="$(cx_slurm_rank_wrapper)"
bash -n <<< "$wrapper"
grep -Fq '. /ix/experimental/CollectiveX/runtime/common.sh' <<< "$wrapper"
- grep -Fq 'cx_restore_exact_hca_selector || exit 68' <<< "$wrapper"
+ grep -Fq 'cx_apply_network_profile "$CX_NODES" "$CX_TRANSPORT" || exit 68' <<< "$wrapper"
grep -Fq 'cx_write_runtime_stage execution || exit 68' <<< "$wrapper"
test "$NVSHMEM_HCA_LIST" = mlx5_0:1,mlx5_1:1
test "$NVSHMEM_ENABLE_NIC_PE_MAPPING" = 1
From 57e9f3e3b20ab76af612768c2039c072965081ee Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 23:07:59 +0800
Subject: [PATCH 073/210] fix(collectivex): reopen H100 EP16 probes
---
.github/workflows/collectivex-sweep.yml | 9 ++++++++-
experimental/CollectiveX/capability.py | 13 +------------
.../CollectiveX/tests/test_precision_scheduling.py | 14 +++++++-------
.../CollectiveX/tests/test_sampling_contract.py | 6 ++++--
4 files changed, 20 insertions(+), 22 deletions(-)
diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml
index 1eb7b47a06..8609670644 100644
--- a/.github/workflows/collectivex-sweep.yml
+++ b/.github/workflows/collectivex-sweep.yml
@@ -459,6 +459,7 @@ jobs:
env:
COLLECTIVEX_OPERATOR_CONFIG_CONTENT: ${{ secrets.COLLECTIVEX_OPERATOR_CONFIG_V1 }}
COLLECTIVEX_NETWORK_CONFIG_CONTENT: ${{ secrets.COLLECTIVEX_NETWORK_CONFIG_V1 }}
+ COLLECTIVEX_H100_CONFIG_CONTENT: ${{ secrets.COLLECTIVEX_H100_CONFIG_V1 }}
COLLECTIVEX_B200_CONFIG_CONTENT: ${{ secrets.COLLECTIVEX_B200_CONFIG_V1 }}
COLLECTIVEX_OPERATOR_AUDIT_SALT: ${{ secrets.COLLECTIVEX_AUDIT_SALT_V1 }}
COLLECTIVEX_OPERATOR_CONFIG_REQUIRED: '1'
@@ -488,7 +489,8 @@ jobs:
base = load("COLLECTIVEX_OPERATOR_CONFIG_CONTENT")
for overlay_name in (
- "COLLECTIVEX_NETWORK_CONFIG_CONTENT", "COLLECTIVEX_B200_CONFIG_CONTENT",
+ "COLLECTIVEX_NETWORK_CONFIG_CONTENT", "COLLECTIVEX_H100_CONFIG_CONTENT",
+ "COLLECTIVEX_B200_CONFIG_CONTENT",
):
overlay_raw = os.environ.get(overlay_name, "")
if not overlay_raw:
@@ -505,6 +507,10 @@ jobs:
if set(overlay["runners"]) != {"b200-dgxc"}:
raise ValueError("invalid B200 overlay runner")
private_fields.update({"nodelist", "qos"})
+ elif overlay_name == "COLLECTIVEX_H100_CONFIG_CONTENT":
+ if set(overlay["runners"]) != {"h100-dgxc"}:
+ raise ValueError("invalid H100 overlay runner")
+ private_fields.add("exclude_nodes")
for runner, fields in overlay["runners"].items():
if runner not in base.get("runners", {}) or not isinstance(fields, dict) or not fields:
raise ValueError("invalid network overlay runner")
@@ -520,6 +526,7 @@ jobs:
stream.write(payload)
PY
unset COLLECTIVEX_OPERATOR_CONFIG_CONTENT COLLECTIVEX_NETWORK_CONFIG_CONTENT
+ unset COLLECTIVEX_H100_CONFIG_CONTENT
unset COLLECTIVEX_B200_CONFIG_CONTENT
unset COLLECTIVEX_OPERATOR_CONFIG_REQUIRED
export COLLECTIVEX_OPERATOR_CONFIG="$operator_config"
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
index 25ad036a54..1f1f08b15e 100644
--- a/experimental/CollectiveX/capability.py
+++ b/experimental/CollectiveX/capability.py
@@ -302,16 +302,7 @@ def _precision_rule(
),
}
-_H100_EP16_NO_RDMA_BASIS = "current-runner-has-no-active-rdma-device"
-PRECISION_CELL_OVERRIDES = {
- (profile, rule["backend"], "h100-dgxc", 16, rule["mode"]): {
- "basis": _H100_EP16_NO_RDMA_BASIS,
- "disposition": "unsupported",
- }
- for profile, rules in PRECISION_CAPABILITIES.items()
- for rule in rules
- if "h100-dgxc" in rule["skus"] and 16 in rule["ep_degrees"]
-}
+PRECISION_CELL_OVERRIDES: dict[tuple[str, str, str, int, str], dict[str, str]] = {}
_VALIDATED_NATIVE_PROBE_CELLS = (
# run, SKU, EP, backend, mode, profile, disposition, result
@@ -450,8 +441,6 @@ def _resolve_base(
topology = topology_for(sku, ep)
if topology is None or (nodes is not None and nodes != topology["nodes"]):
return False, f"{sku} does not register EP{ep} on {nodes} nodes"
- if sku == "h100-dgxc" and ep == 16:
- return False, _H100_EP16_NO_RDMA_BASIS
if routing not in {"uniform", "zipf"} or (eplb and routing != "zipf"):
return False, "v1 routing is uniform or zipf, with EPLB only on zipf"
if platform["vendor"] not in implementation["vendors"]:
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
index 05012bc1b5..2999538e8e 100644
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -28,7 +28,7 @@ def test_precision_probe_inventory_is_exact_and_non_mutating(self) -> None:
item["precision_profile"],
)
self.assertEqual(targets, sorted(capability.provisional_precision_targets(), key=key))
- self.assertEqual(len(targets), 18)
+ self.assertEqual(len(targets), 27)
self.assertEqual(capability.PRECISION_CAPABILITIES, before)
self.assertEqual(
len({
@@ -228,15 +228,15 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
{"provisional", "supported", "unsupported"},
)
self.assertEqual(
- len(targets) - len(capability.provisional_precision_targets()), 76
+ len(targets) - len(capability.provisional_precision_targets()), 67
)
self.assertEqual(
sum(item["disposition"] == "supported" for item in targets), 51
)
self.assertEqual(
- sum(item["disposition"] == "unsupported" for item in targets), 25
+ sum(item["disposition"] == "unsupported" for item in targets), 16
)
- self.assertEqual(len(capability.provisional_precision_targets()), 18)
+ self.assertEqual(len(capability.provisional_precision_targets()), 27)
keys = {
(
item["precision_profile"],
@@ -261,7 +261,7 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
(("mi355x", "mori", 16, "normal", direct), "not-applicable"),
(("mi325x", "mori", 8, "normal", fnuz_direct), "provisional"),
(("h200-dgxc", "deepep", 8, "low-latency", low_latency), "supported"),
- (("h100-dgxc", "deepep", 16, "low-latency", low_latency), "unsupported"),
+ (("h100-dgxc", "deepep", 16, "low-latency", low_latency), "provisional"),
(("h200-dgxc", "deepep-hybrid", 8, "low-latency", low_latency),
"not-applicable"),
)
@@ -291,8 +291,8 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
"h100-dgxc", "nccl-ep", ep=16, nodes=2,
precision_profile=identity.V1_CONTROL_PRECISION_PROFILE,
)
- self.assertEqual(h100_ep16, "unsupported")
- self.assertEqual(reason, capability._H100_EP16_NO_RDMA_BASIS)
+ self.assertEqual(h100_ep16, "supported")
+ self.assertEqual(reason, "ok")
def test_split_suites_are_provisional_and_do_not_duplicate_bf16(self) -> None:
suites = sweep_matrix._load("suites.yaml")
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index c1a454a73f..2812ef735e 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -208,7 +208,7 @@ def test_matrix_cases_and_shards_are_identity_bound(self) -> None:
sum(len(item["case"]["ladder"].split()) for item in runnable_cases),
sum(len(item["case"]["ladder"].split()) for item in unsupported_cases),
),
- (54, 608, 338, 270, 1600, 868, 732),
+ (58, 608, 364, 244, 1600, 940, 660),
)
expected_topologies = {}
for sku, product in (
@@ -263,7 +263,7 @@ def test_matrix_cases_and_shards_are_identity_bound(self) -> None:
{shard["n"] for shard in matrix["include"]}, {6, 7}
)
self.assertEqual(
- sum(shard["n"] == 7 for shard in matrix["include"]), 14
+ sum(shard["n"] == 7 for shard in matrix["include"]), 16
)
ll_cases = [
item for item in matrix["requested_cases"]
@@ -1367,6 +1367,8 @@ def test_launcher_signal_traps_cancel_the_allocation(self) -> None:
self.assertIn("secrets.COLLECTIVEX_B200_CONFIG_V1", workflow)
self.assertIn('set(overlay["runners"]) != {"b200-dgxc"}', workflow)
self.assertIn("set(fields) - private_fields", workflow)
+ self.assertIn('private_fields.add("exclude_nodes")', workflow)
+ self.assertIn('invalid H100 overlay runner', workflow)
self.assertIn('private_fields.update({"nodelist", "qos"})', workflow)
self.assertIn('[ -z "${CX_QOS:-}" ] || allocation+=(--qos="$CX_QOS")', (
ROOT / "launchers" / "launch_single-slurm.sh"
From b95821490609e54b0562fd9ca4276df9de14abd6 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 23:10:19 +0800
Subject: [PATCH 074/210] fix(collectivex): classify rail preflight failures
---
experimental/CollectiveX/runtime/common.sh | 31 ++++++++++++-------
.../tests/test_sampling_contract.py | 2 ++
2 files changed, 22 insertions(+), 11 deletions(-)
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index ceb6cf949d..ab8eed3337 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -767,39 +767,48 @@ if [ -n "${CX_SOCKET_IFNAME:-}" ]; then
done
fi
check_port() {
- local port_path="$1" state gid link_layer
- [ -d "$port_path" ] || return 1
+ local port_path="$1" ordinal="$2" state gid link_layer
+ [ -d "$port_path" ] \
+ || { printf '[collectivex-private] rdma-port-%s=missing\n' "$ordinal"; return 1; }
read -r state _ < "$port_path/state"
- [ "$state" = 4: ] || return 1
+ [ "$state" = 4: ] \
+ || { printf '[collectivex-private] rdma-port-%s=inactive\n' "$ordinal"; return 1; }
if [ -n "${CX_IB_GID_INDEX:-}" ]; then
- [ -r "$port_path/gids/$CX_IB_GID_INDEX" ] || return 1
+ [ -r "$port_path/gids/$CX_IB_GID_INDEX" ] \
+ || { printf '[collectivex-private] rdma-port-%s=gid-missing\n' "$ordinal"; return 1; }
gid="$(tr -d ':0[:space:]' < "$port_path/gids/$CX_IB_GID_INDEX")"
- [ -n "$gid" ] || return 1
+ [ -n "$gid" ] \
+ || { printf '[collectivex-private] rdma-port-%s=gid-empty\n' "$ordinal"; return 1; }
fi
- [ -r "$port_path/link_layer" ] || return 1
+ [ -r "$port_path/link_layer" ] \
+ || { printf '[collectivex-private] rdma-port-%s=link-layer-missing\n' "$ordinal"; return 1; }
link_layer="$(< "$port_path/link_layer")"
case "$link_layer" in
Ethernet) link_layer=roce ;;
InfiniBand) link_layer=infiniband ;;
- *) return 1 ;;
+ *) printf '[collectivex-private] rdma-port-%s=link-layer-invalid\n' "$ordinal"; return 1 ;;
esac
- [ -z "${profile:-}" ] || [ "$profile" = "$link_layer" ] || return 1
+ [ -z "${profile:-}" ] || [ "$profile" = "$link_layer" ] \
+ || { printf '[collectivex-private] rdma-port-%s=link-layer-mixed\n' "$ordinal"; return 1; }
profile="$link_layer"
}
profile=""
IFS=, read -r -a devices <<< "$CX_RDMA_DEVICES"
+ordinal=0
for selector in "${devices[@]}"; do
+ ordinal=$((ordinal + 1))
device="${selector%%:*}"
configured_port=""
[ "$selector" = "$device" ] || configured_port="${selector#*:}"
ports="/sys/class/infiniband/$device/ports"
- [ -d "$ports" ]
+ [ -d "$ports" ] \
+ || { printf '[collectivex-private] rdma-device-%s=missing\n' "$ordinal"; exit 1; }
if [ -n "$configured_port" ]; then
- check_port "$ports/$configured_port"
+ check_port "$ports/$configured_port" "$ordinal"
else
active=0
for port_path in "$ports"/*; do
- if check_port "$port_path"; then
+ if check_port "$port_path" "$ordinal"; then
active=1
fi
done
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 2812ef735e..db3d9bf0f3 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -2449,6 +2449,8 @@ def test_launchers_use_private_logs_and_allowlisted_failure_stages(self) -> None
self.assertIn("cx_fail_stage execution", distributed)
self.assertIn('cx_fail_stage execution "$runtime_log"', distributed)
self.assertIn("precision probe timed out rc=%s limit=%ss", distributed)
+ self.assertIn("rdma-port-%s=inactive", common)
+ self.assertIn("rdma-device-%s=missing", common)
def test_case_failure_diagnostic_precedes_normal_srun_footer(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
From 9b031c6892202b07ca260d8420fe37288f6f6a97 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 23:19:29 +0800
Subject: [PATCH 075/210] test(collectivex): record H100 EP16 support
---
experimental/CollectiveX/capability.py | 1 +
.../CollectiveX/tests/test_precision_scheduling.py | 8 ++++----
2 files changed, 5 insertions(+), 4 deletions(-)
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
index 1f1f08b15e..06bde8bf92 100644
--- a/experimental/CollectiveX/capability.py
+++ b/experimental/CollectiveX/capability.py
@@ -338,6 +338,7 @@ def _precision_rule(
("28738113606", "h100-dgxc", 8, "uccl", "low-latency", _LL_LOGFMT_PROFILE, "supported", "native-probe-passed"),
("28738113606", "h100-dgxc", 8, "uccl", "low-latency", _LL_FP8_PROFILE, "unsupported", "native-operation-timeout"),
("28738113606", "h100-dgxc", 8, "uccl", "low-latency", _LL_FP8_LOGFMT_PROFILE, "unsupported", "native-operation-timeout"),
+ ("28745208954", "h100-dgxc", 16, "deepep", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed"),
*(("28738445591", "gb300", ep, "deepep", mode, profile, "supported", "native-probe-passed")
for ep in (8, 16)
for profile, mode in ((_NORMAL_E4M3FN_PROFILE, "normal"), (_LL_FP8_PROFILE, "low-latency"),
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
index 2999538e8e..9a224b9bcb 100644
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -28,7 +28,7 @@ def test_precision_probe_inventory_is_exact_and_non_mutating(self) -> None:
item["precision_profile"],
)
self.assertEqual(targets, sorted(capability.provisional_precision_targets(), key=key))
- self.assertEqual(len(targets), 27)
+ self.assertEqual(len(targets), 26)
self.assertEqual(capability.PRECISION_CAPABILITIES, before)
self.assertEqual(
len({
@@ -228,15 +228,15 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
{"provisional", "supported", "unsupported"},
)
self.assertEqual(
- len(targets) - len(capability.provisional_precision_targets()), 67
+ len(targets) - len(capability.provisional_precision_targets()), 68
)
self.assertEqual(
- sum(item["disposition"] == "supported" for item in targets), 51
+ sum(item["disposition"] == "supported" for item in targets), 52
)
self.assertEqual(
sum(item["disposition"] == "unsupported" for item in targets), 16
)
- self.assertEqual(len(capability.provisional_precision_targets()), 27)
+ self.assertEqual(len(capability.provisional_precision_targets()), 26)
keys = {
(
item["precision_profile"],
From ab2e3ff61166f5899b1fe953524ecf6c999c6672 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 23:32:29 +0800
Subject: [PATCH 076/210] test(collectivex): finalize H100 EP16 capabilities
---
experimental/CollectiveX/capability.py | 8 ++++++++
.../CollectiveX/tests/test_precision_scheduling.py | 12 ++++++------
2 files changed, 14 insertions(+), 6 deletions(-)
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
index 06bde8bf92..f1289ca8df 100644
--- a/experimental/CollectiveX/capability.py
+++ b/experimental/CollectiveX/capability.py
@@ -339,6 +339,14 @@ def _precision_rule(
("28738113606", "h100-dgxc", 8, "uccl", "low-latency", _LL_FP8_PROFILE, "unsupported", "native-operation-timeout"),
("28738113606", "h100-dgxc", 8, "uccl", "low-latency", _LL_FP8_LOGFMT_PROFILE, "unsupported", "native-operation-timeout"),
("28745208954", "h100-dgxc", 16, "deepep", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed"),
+ ("28745423523", "h100-dgxc", 16, "deepep", "low-latency", _LL_FP8_PROFILE, "supported", "native-probe-passed"),
+ ("28745423523", "h100-dgxc", 16, "deepep", "low-latency", _LL_LOGFMT_PROFILE, "supported", "native-probe-passed"),
+ ("28745423523", "h100-dgxc", 16, "deepep", "low-latency", _LL_FP8_LOGFMT_PROFILE, "supported", "native-probe-passed"),
+ ("28745423523", "h100-dgxc", 16, "deepep-hybrid", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "native-operation-failed"),
+ ("28745423523", "h100-dgxc", 16, "uccl", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed"),
+ ("28745423523", "h100-dgxc", 16, "uccl", "low-latency", _LL_FP8_PROFILE, "supported", "native-probe-passed"),
+ ("28745423523", "h100-dgxc", 16, "uccl", "low-latency", _LL_LOGFMT_PROFILE, "supported", "native-probe-passed"),
+ ("28745423523", "h100-dgxc", 16, "uccl", "low-latency", _LL_FP8_LOGFMT_PROFILE, "supported", "native-probe-passed"),
*(("28738445591", "gb300", ep, "deepep", mode, profile, "supported", "native-probe-passed")
for ep in (8, 16)
for profile, mode in ((_NORMAL_E4M3FN_PROFILE, "normal"), (_LL_FP8_PROFILE, "low-latency"),
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
index 9a224b9bcb..bc4e44be96 100644
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -28,7 +28,7 @@ def test_precision_probe_inventory_is_exact_and_non_mutating(self) -> None:
item["precision_profile"],
)
self.assertEqual(targets, sorted(capability.provisional_precision_targets(), key=key))
- self.assertEqual(len(targets), 26)
+ self.assertEqual(len(targets), 18)
self.assertEqual(capability.PRECISION_CAPABILITIES, before)
self.assertEqual(
len({
@@ -228,15 +228,15 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
{"provisional", "supported", "unsupported"},
)
self.assertEqual(
- len(targets) - len(capability.provisional_precision_targets()), 68
+ len(targets) - len(capability.provisional_precision_targets()), 76
)
self.assertEqual(
- sum(item["disposition"] == "supported" for item in targets), 52
+ sum(item["disposition"] == "supported" for item in targets), 59
)
self.assertEqual(
- sum(item["disposition"] == "unsupported" for item in targets), 16
+ sum(item["disposition"] == "unsupported" for item in targets), 17
)
- self.assertEqual(len(capability.provisional_precision_targets()), 26)
+ self.assertEqual(len(capability.provisional_precision_targets()), 18)
keys = {
(
item["precision_profile"],
@@ -261,7 +261,7 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
(("mi355x", "mori", 16, "normal", direct), "not-applicable"),
(("mi325x", "mori", 8, "normal", fnuz_direct), "provisional"),
(("h200-dgxc", "deepep", 8, "low-latency", low_latency), "supported"),
- (("h100-dgxc", "deepep", 16, "low-latency", low_latency), "provisional"),
+ (("h100-dgxc", "deepep", 16, "low-latency", low_latency), "supported"),
(("h200-dgxc", "deepep-hybrid", 8, "low-latency", low_latency),
"not-applicable"),
)
From 5e333a0c6a67b2ec3adab916e65744f945de876d Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 23:33:32 +0800
Subject: [PATCH 077/210] test(collectivex): classify B200 EP16 timeout
---
experimental/CollectiveX/capability.py | 1 +
.../CollectiveX/tests/test_precision_scheduling.py | 8 ++++----
2 files changed, 5 insertions(+), 4 deletions(-)
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
index f1289ca8df..f9738bb88d 100644
--- a/experimental/CollectiveX/capability.py
+++ b/experimental/CollectiveX/capability.py
@@ -312,6 +312,7 @@ def _precision_rule(
("28737315879", "b200-dgxc", 8, "deepep", "low-latency", _LL_FP8_LOGFMT_PROFILE, "supported", "native-probe-passed"),
("28737315879", "b200-dgxc", 8, "deepep-hybrid", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "native-operation-failed"),
("28737315879", "b200-dgxc", 8, "deepep-v2", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "backend-construction-failed"),
+ ("28745114766", "b200-dgxc", 16, "deepep", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "native-operation-timeout"),
("28737422303", "h200-dgxc", 8, "deepep", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed"),
("28737422303", "h200-dgxc", 8, "deepep", "low-latency", _LL_FP8_PROFILE, "supported", "native-probe-passed"),
("28737422303", "h200-dgxc", 8, "deepep", "low-latency", _LL_LOGFMT_PROFILE, "supported", "native-probe-passed"),
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
index bc4e44be96..e063c39983 100644
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -28,7 +28,7 @@ def test_precision_probe_inventory_is_exact_and_non_mutating(self) -> None:
item["precision_profile"],
)
self.assertEqual(targets, sorted(capability.provisional_precision_targets(), key=key))
- self.assertEqual(len(targets), 18)
+ self.assertEqual(len(targets), 17)
self.assertEqual(capability.PRECISION_CAPABILITIES, before)
self.assertEqual(
len({
@@ -228,15 +228,15 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
{"provisional", "supported", "unsupported"},
)
self.assertEqual(
- len(targets) - len(capability.provisional_precision_targets()), 76
+ len(targets) - len(capability.provisional_precision_targets()), 77
)
self.assertEqual(
sum(item["disposition"] == "supported" for item in targets), 59
)
self.assertEqual(
- sum(item["disposition"] == "unsupported" for item in targets), 17
+ sum(item["disposition"] == "unsupported" for item in targets), 18
)
- self.assertEqual(len(capability.provisional_precision_targets()), 18)
+ self.assertEqual(len(capability.provisional_precision_targets()), 17)
keys = {
(
item["precision_profile"],
From 8981efbc405214a3353338b0bb44830d78096fc0 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 23:36:27 +0800
Subject: [PATCH 078/210] fix(collectivex): isolate exact probe concurrency
---
.github/workflows/collectivex-sweep.yml | 2 +-
experimental/CollectiveX/tests/test_sampling_contract.py | 2 ++
2 files changed, 3 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml
index 8609670644..c71ced7ec2 100644
--- a/.github/workflows/collectivex-sweep.yml
+++ b/.github/workflows/collectivex-sweep.yml
@@ -67,7 +67,7 @@ on:
type: string
default: ''
concurrency:
- group: cx-${{ inputs.operation }}-${{ github.ref }}-${{ inputs.release_tag }}-${{ inputs.backend }}-${{ inputs.only_sku }}
+ group: cx-${{ inputs.operation }}-${{ github.ref }}-${{ inputs.release_tag }}-${{ inputs.backend }}-${{ inputs.only_sku }}-${{ inputs.probe_id }}
cancel-in-progress: false
jobs:
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index db3d9bf0f3..6598be3304 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -1486,6 +1486,8 @@ def test_v1_publication_requires_explicit_release_markers(self) -> None:
self.assertFalse((workflows / "collectivex-publish.yml").exists())
self.assertIn("options: [sweep, probe-precision, publish-v1, refresh-v1]", sweep)
+ self.assertIn("${{ inputs.only_sku }}-${{ inputs.probe_id }}", sweep)
+ self.assertIn("cancel-in-progress: false", sweep)
self.assertIn("collectivex.precision-probe-plan.v1", (ROOT / "tests" / "probe_precision.py").read_text())
self.assertIn("cxprecision-probes-${{ github.run_id }}-${{ github.run_attempt }}", sweep)
self.assertIn("--validate-bundle", sweep)
From 285ff58a275d2b1f51d0e9777ad75e755eb7cf85 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 23:44:21 +0800
Subject: [PATCH 079/210] fix(collectivex): carry MoRI RDMA QoS
---
.github/workflows/collectivex-sweep.yml | 1 +
experimental/CollectiveX/README.md | 3 ++-
experimental/CollectiveX/runtime/common.sh | 22 ++++++++++++++++++-
experimental/CollectiveX/tests/run_ep.py | 1 +
.../tests/test_sampling_contract.py | 5 ++++-
5 files changed, 29 insertions(+), 3 deletions(-)
diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml
index c71ced7ec2..d3f05b7ffe 100644
--- a/.github/workflows/collectivex-sweep.yml
+++ b/.github/workflows/collectivex-sweep.yml
@@ -498,6 +498,7 @@ jobs:
overlay = load(overlay_name)
private_fields = {
"socket_ifname", "rdma_devices", "ib_gid_index", "rdma_service_level",
+ "rdma_traffic_class",
}
if set(overlay) != {"schema_version", "runners"} or overlay["schema_version"] != 1:
raise ValueError("invalid network overlay envelope")
diff --git a/experimental/CollectiveX/README.md b/experimental/CollectiveX/README.md
index 16b89f495f..fc86761f6c 100644
--- a/experimental/CollectiveX/README.md
+++ b/experimental/CollectiveX/README.md
@@ -141,7 +141,8 @@ temporary copy before allocation. Required JSON fields are:
Every selected non-MNNVL EP16 placement additionally requires `socket_ifname` and `rdma_devices`
for its operator-approved fabric; optional
-`ib_gid_index` and `rdma_service_level` are also allowlisted. CollectiveX does not heuristically
+`ib_gid_index`, `rdma_service_level`, and `rdma_traffic_class` are also allowlisted. Service level
+and traffic class are mapped into MoRI's RDMA/IO QoS environment. CollectiveX does not heuristically
select a management route or HCA. After allocation, every non-MNNVL scale-out node must prove that
all configured interfaces and active HCA ports exist before backend setup. Scale-up and MNNVL jobs
clear these overrides. Scale-out NCCL/RCCL is pinned to `IB` with exact-match HCA selectors so a
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index ab8eed3337..d4cf46219b 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -167,6 +167,7 @@ cx_load_operator_config() {
unset ENROOT_CACHE_PATH
unset CX_EXCLUDE_NODES CX_NODELIST CX_LOCK_DIR CX_MASTER_PORT
unset CX_SOCKET_IFNAME CX_RDMA_DEVICES CX_IB_GID_INDEX CX_RDMA_SERVICE_LEVEL
+ unset CX_RDMA_TRAFFIC_CLASS
unset CX_AUDIT_SALT
unset MASTER_ADDR MASTER_PORT RANK WORLD_SIZE LOCAL_RANK LOCAL_WORLD_SIZE
config_path="${COLLECTIVEX_OPERATOR_CONFIG:-${XDG_CONFIG_HOME:-${HOME}/.config}/inferencex/collectivex.json}"
@@ -235,9 +236,11 @@ FIELDS = {
"rdma_devices": "CX_RDMA_DEVICES",
"ib_gid_index": "CX_IB_GID_INDEX",
"rdma_service_level": "CX_RDMA_SERVICE_LEVEL",
+ "rdma_traffic_class": "CX_RDMA_TRAFFIC_CLASS",
}
NETWORK_FIELDS = {
"socket_ifname", "rdma_devices", "ib_gid_index", "rdma_service_level",
+ "rdma_traffic_class",
}
REQUIRED = {
"h100-dgxc": {"partition", "account", "squash_dir"},
@@ -371,6 +374,9 @@ try:
elif field == "rdma_service_level":
if type(value) is not int or not 0 <= value <= 15:
raise ValueError
+ elif field == "rdma_traffic_class":
+ if type(value) is not int or not 0 <= value <= 255:
+ raise ValueError
elif field.endswith(("_dir", "_path")):
if not valid_path(value):
raise ValueError
@@ -539,7 +545,7 @@ PY
# Explicit Slurm export boundary. Operator config, runner credentials, HOME,
# workspace paths, and unrelated service secrets never enter the container.
cx_container_exports() {
- printf '%s' 'COLLECTIVEX_SOURCE_SHA,COLLECTIVEX_ARTIFACT_NAME,COLLECTIVEX_EXECUTION_ID,COLLECTIVEX_CONTROL_SHA256,COLLECTIVEX_IMAGE,COLLECTIVEX_IMAGE_DIGEST,COLLECTIVEX_IMAGE_DIGEST_VERIFIED,COLLECTIVEX_SQUASH_SHA256,GITHUB_REF_NAME,GITHUB_REF,GITHUB_REPOSITORY,GITHUB_JOB,GITHUB_RUN_ID,GITHUB_RUN_ATTEMPT,GITHUB_SHA,CX_RUNNER,CX_BENCH,CX_NODES,CX_GPUS_PER_NODE,CX_SCALE_UP_DOMAIN,CX_SHARD_FILE,CX_SHARD_SKU,CX_PRECISION_PROBE,CX_NGPUS,CX_TS,CX_TOPO,CX_SCOPE,CX_TRANSPORT,CX_SCALE_UP_TRANSPORT,CX_SCALE_OUT_TRANSPORT,CX_MODE,CX_PHASE,CX_ROUTING,CX_EPLB,CX_CASE_ID,CX_SUITE,CX_WORKLOAD_NAME,CX_REQUIRED_PUBLICATION,CX_PRECISION_PROFILE,CX_QUALIFICATION_INDEX,CX_HIDDEN,CX_TOPK,CX_EXPERTS,CX_TOKENS_LADDER,CX_CANONICAL,CX_ITERS,CX_TRIALS,CX_WARMUP,CX_SAMPLES_PER_POINT,CX_WARMUP_SEMANTICS,CX_SEED,CX_RUN_TIMEOUT,CX_NCCL_HOME,CX_ALLOW_MNNVL,CX_ATTEMPT_ID,CX_RUNTIME_MARKER,CX_MORI_KERNEL_TYPE,CX_WORKLOAD_DIR,CX_BACKEND_CACHE_ROOT,CX_BACKEND_CACHE_SENTINEL_SHA256,CX_BACKEND_SOURCE_ROOT,CX_AUDIT_SALT,CX_SOCKET_IFNAME,CX_RDMA_DEVICES,CX_IB_GID_INDEX,CX_RDMA_SERVICE_LEVEL,CX_RDMA_LINK_LAYER,MASTER_ADDR,MASTER_PORT,RANK,WORLD_SIZE,LOCAL_RANK,LOCAL_WORLD_SIZE,NCCL_NET,NCCL_SOCKET_IFNAME,GLOO_SOCKET_IFNAME,NCCL_IB_HCA,NCCL_IB_GID_INDEX,NCCL_IB_SL,NVSHMEM_DISABLE_IB,NVSHMEM_ENABLE_NIC_PE_MAPPING,NVSHMEM_HCA_LIST,NVSHMEM_IB_GID_INDEX,NVSHMEM_IB_SL,NVSHMEM_IB_ENABLE_IBGDA,NVSHMEM_IBGDA_NIC_HANDLER,EP_NIC_NAME,EP_OVERRIDE_RDMA_SL,UCCL_SOCKET_IFNAME,UCCL_IB_GID_INDEX,UCCL_IB_SL,MORI_RDMA_DEVICES,HYBRID_EP_MULTINODE,USE_NIXL,RDMA_CORE_HOME,DEEPEP_HYBRID_BUILD_MODE,NCCL_CUMEM_ENABLE,NCCL_MNNVL_ENABLE,MC_FORCE_MNNVL,MORI_DISABLE_AUTO_XGMI,MORI_ENABLE_SDMA,MORI_APP_LOG_LEVEL,MORI_SHMEM_LOG_LEVEL,MORI_IO_LOG_LEVEL'
+ printf '%s' 'COLLECTIVEX_SOURCE_SHA,COLLECTIVEX_ARTIFACT_NAME,COLLECTIVEX_EXECUTION_ID,COLLECTIVEX_CONTROL_SHA256,COLLECTIVEX_IMAGE,COLLECTIVEX_IMAGE_DIGEST,COLLECTIVEX_IMAGE_DIGEST_VERIFIED,COLLECTIVEX_SQUASH_SHA256,GITHUB_REF_NAME,GITHUB_REF,GITHUB_REPOSITORY,GITHUB_JOB,GITHUB_RUN_ID,GITHUB_RUN_ATTEMPT,GITHUB_SHA,CX_RUNNER,CX_BENCH,CX_NODES,CX_GPUS_PER_NODE,CX_SCALE_UP_DOMAIN,CX_SHARD_FILE,CX_SHARD_SKU,CX_PRECISION_PROBE,CX_NGPUS,CX_TS,CX_TOPO,CX_SCOPE,CX_TRANSPORT,CX_SCALE_UP_TRANSPORT,CX_SCALE_OUT_TRANSPORT,CX_MODE,CX_PHASE,CX_ROUTING,CX_EPLB,CX_CASE_ID,CX_SUITE,CX_WORKLOAD_NAME,CX_REQUIRED_PUBLICATION,CX_PRECISION_PROFILE,CX_QUALIFICATION_INDEX,CX_HIDDEN,CX_TOPK,CX_EXPERTS,CX_TOKENS_LADDER,CX_CANONICAL,CX_ITERS,CX_TRIALS,CX_WARMUP,CX_SAMPLES_PER_POINT,CX_WARMUP_SEMANTICS,CX_SEED,CX_RUN_TIMEOUT,CX_NCCL_HOME,CX_ALLOW_MNNVL,CX_ATTEMPT_ID,CX_RUNTIME_MARKER,CX_MORI_KERNEL_TYPE,CX_WORKLOAD_DIR,CX_BACKEND_CACHE_ROOT,CX_BACKEND_CACHE_SENTINEL_SHA256,CX_BACKEND_SOURCE_ROOT,CX_AUDIT_SALT,CX_SOCKET_IFNAME,CX_RDMA_DEVICES,CX_IB_GID_INDEX,CX_RDMA_SERVICE_LEVEL,CX_RDMA_TRAFFIC_CLASS,CX_RDMA_LINK_LAYER,MASTER_ADDR,MASTER_PORT,RANK,WORLD_SIZE,LOCAL_RANK,LOCAL_WORLD_SIZE,NCCL_NET,NCCL_SOCKET_IFNAME,GLOO_SOCKET_IFNAME,NCCL_IB_HCA,NCCL_IB_GID_INDEX,NCCL_IB_SL,NVSHMEM_DISABLE_IB,NVSHMEM_ENABLE_NIC_PE_MAPPING,NVSHMEM_HCA_LIST,NVSHMEM_IB_GID_INDEX,NVSHMEM_IB_SL,NVSHMEM_IB_ENABLE_IBGDA,NVSHMEM_IBGDA_NIC_HANDLER,EP_NIC_NAME,EP_OVERRIDE_RDMA_SL,UCCL_SOCKET_IFNAME,UCCL_IB_GID_INDEX,UCCL_IB_SL,MORI_RDMA_DEVICES,MORI_RDMA_TC,MORI_IO_TC,MORI_RDMA_SL,MORI_IO_SL,HYBRID_EP_MULTINODE,USE_NIXL,RDMA_CORE_HOME,DEEPEP_HYBRID_BUILD_MODE,NCCL_CUMEM_ENABLE,NCCL_MNNVL_ENABLE,MC_FORCE_MNNVL,MORI_DISABLE_AUTO_XGMI,MORI_ENABLE_SDMA,MORI_APP_LOG_LEVEL,MORI_SHMEM_LOG_LEVEL,MORI_IO_LOG_LEVEL'
printf '%s' ',MORI_COMMIT'
}
@@ -654,6 +660,7 @@ cx_apply_network_profile() {
unset NVSHMEM_HCA_PE_MAPPING NVSHMEM_REMOTE_TRANSPORT
unset EP_NIC_NAME EP_OVERRIDE_RDMA_SL
unset UCCL_SOCKET_IFNAME UCCL_IB_GID_INDEX UCCL_IB_SL MORI_RDMA_DEVICES
+ unset MORI_RDMA_TC MORI_IO_TC MORI_RDMA_SL MORI_IO_SL
if [ "$nodes" -gt 1 ] && [ "$transport" != mnnvl ]; then
scaleout=1
elif [ "${CX_SHARD_SKU:-}" = b300 ] && [ "$nodes" = 1 ] \
@@ -702,8 +709,15 @@ cx_apply_network_profile() {
if [ "$scaleout" = 1 ]; then
export NCCL_IB_SL="$CX_RDMA_SERVICE_LEVEL" UCCL_IB_SL="$CX_RDMA_SERVICE_LEVEL"
export EP_OVERRIDE_RDMA_SL="$CX_RDMA_SERVICE_LEVEL"
+ export MORI_RDMA_SL="$CX_RDMA_SERVICE_LEVEL" MORI_IO_SL="$CX_RDMA_SERVICE_LEVEL"
fi
fi
+ if [ -n "${CX_RDMA_TRAFFIC_CLASS:-}" ]; then
+ [[ "$CX_RDMA_TRAFFIC_CLASS" =~ ^[0-9]+$ ]] && [ "$CX_RDMA_TRAFFIC_CLASS" -le 255 ] \
+ || cx_die "invalid private RDMA traffic class"
+ [ "$scaleout" = 1 ] \
+ && export MORI_RDMA_TC="$CX_RDMA_TRAFFIC_CLASS" MORI_IO_TC="$CX_RDMA_TRAFFIC_CLASS"
+ fi
export NVSHMEM_IB_ENABLE_IBGDA=1 NVSHMEM_IBGDA_NIC_HANDLER=gpu
if [ -n "${CX_RDMA_LINK_LAYER:-}" ]; then
case "$CX_RDMA_LINK_LAYER" in
@@ -1824,6 +1838,7 @@ cx_lock_canonical_gha_env() {
local trusted_qos=""
local trusted_socket_ifname="" trusted_rdma_devices=""
local trusted_ib_gid_index="" trusted_rdma_service_level=""
+ local trusted_rdma_traffic_class=""
local trusted_audit_salt=""
[ "${COLLECTIVEX_CANONICAL_GHA:-0}" = 1 ] || return 0
[ "${GITHUB_ACTIONS:-}" = true ] \
@@ -1845,6 +1860,7 @@ cx_lock_canonical_gha_env() {
trusted_rdma_devices="${CX_RDMA_DEVICES:-}"
trusted_ib_gid_index="${CX_IB_GID_INDEX:-}"
trusted_rdma_service_level="${CX_RDMA_SERVICE_LEVEL:-}"
+ trusted_rdma_traffic_class="${CX_RDMA_TRAFFIC_CLASS:-}"
trusted_audit_salt="${CX_AUDIT_SALT:-}"
fi
# The legacy B300 operator row contains a root-owned stage path. B300's
@@ -1854,6 +1870,7 @@ cx_lock_canonical_gha_env() {
unset CX_STAGE_PARENT_OWNER_OK
unset MASTER_ADDR MASTER_PORT RANK WORLD_SIZE LOCAL_RANK LOCAL_WORLD_SIZE
unset CX_SOCKET_IFNAME CX_RDMA_DEVICES CX_IB_GID_INDEX CX_RDMA_SERVICE_LEVEL
+ unset CX_RDMA_TRAFFIC_CLASS
unset CX_AUDIT_SALT
unset NCCL_NET NCCL_SOCKET_IFNAME GLOO_SOCKET_IFNAME NCCL_IB_HCA
unset NCCL_IB_GID_INDEX NCCL_IB_SL
@@ -1863,6 +1880,7 @@ cx_lock_canonical_gha_env() {
unset NVSHMEM_HCA_PE_MAPPING NVSHMEM_REMOTE_TRANSPORT
unset EP_NIC_NAME EP_OVERRIDE_RDMA_SL
unset UCCL_SOCKET_IFNAME UCCL_IB_GID_INDEX UCCL_IB_SL MORI_RDMA_DEVICES
+ unset MORI_RDMA_TC MORI_IO_TC MORI_RDMA_SL MORI_IO_SL
unset HYBRID_EP_MULTINODE USE_NIXL RDMA_CORE_HOME DEEPEP_HYBRID_BUILD_MODE
unset MORI_COMMIT MORI_DISABLE_AUTO_XGMI MORI_ENABLE_SDMA
unset MORI_APP_LOG_LEVEL MORI_SHMEM_LOG_LEVEL MORI_IO_LOG_LEVEL
@@ -1972,6 +1990,8 @@ cx_lock_canonical_gha_env() {
|| export CX_IB_GID_INDEX="$trusted_ib_gid_index"
[ -z "$trusted_rdma_service_level" ] \
|| export CX_RDMA_SERVICE_LEVEL="$trusted_rdma_service_level"
+ [ -z "$trusted_rdma_traffic_class" ] \
+ || export CX_RDMA_TRAFFIC_CLASS="$trusted_rdma_traffic_class"
CX_AUDIT_SALT="$trusted_audit_salt"
export CX_STAGE_DIR CX_AUDIT_SALT
[ "${CX_NODES:-}" = "$expected_nodes" ] \
diff --git a/experimental/CollectiveX/tests/run_ep.py b/experimental/CollectiveX/tests/run_ep.py
index 7f3ca79d0b..4fb9de03c4 100644
--- a/experimental/CollectiveX/tests/run_ep.py
+++ b/experimental/CollectiveX/tests/run_ep.py
@@ -30,6 +30,7 @@
"ib_gid_index": "CX_IB_GID_INDEX",
"rdma_devices": "CX_RDMA_DEVICES",
"rdma_service_level": "CX_RDMA_SERVICE_LEVEL",
+ "rdma_traffic_class": "CX_RDMA_TRAFFIC_CLASS",
"socket_ifname": "CX_SOCKET_IFNAME",
}
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 6598be3304..63227432ed 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -517,6 +517,7 @@ def test_canonical_operator_config_requires_a_private_audit_salt(self) -> None:
"h100-dgxc": {
"partition": "test", "account": "test",
"squash_dir": str(root), "stage_dir": str(root),
+ "rdma_traffic_class": 104,
},
},
}
@@ -618,7 +619,7 @@ def test_scaleout_network_profile_is_explicit_and_allowlisted(self) -> None:
test -z "${NCCL_NET+x}${NCCL_IB_HCA+x}${NVSHMEM_DISABLE_IB+x}${NVSHMEM_HCA_LIST+x}"
cx_apply_network_profile 4 mnnvl
test -z "${NCCL_NET+x}${NCCL_IB_HCA+x}${NVSHMEM_HCA_LIST+x}"
- export CX_IB_GID_INDEX=3 CX_RDMA_SERVICE_LEVEL=2
+ export CX_IB_GID_INDEX=3 CX_RDMA_SERVICE_LEVEL=2 CX_RDMA_TRAFFIC_CLASS=104
cx_apply_network_profile 2 nvlink-rdma
test "$NCCL_SOCKET_IFNAME:$GLOO_SOCKET_IFNAME:$UCCL_SOCKET_IFNAME" = ib0:ib0:ib0
test "$NCCL_NET:$NCCL_IB_HCA" = 'IB:=mlx5_0:1,mlx5_1:1'
@@ -634,6 +635,7 @@ def test_scaleout_network_profile_is_explicit_and_allowlisted(self) -> None:
test "$NVSHMEM_HCA_LIST" = mlx5_0:1,mlx5_1:1
test "$NVSHMEM_ENABLE_NIC_PE_MAPPING" = 1
test "$MORI_RDMA_DEVICES:$EP_NIC_NAME" = mlx5_0,mlx5_1:mlx5_0
+ test "$MORI_RDMA_TC:$MORI_IO_TC:$MORI_RDMA_SL:$MORI_IO_SL" = 104:104:2:2
test -z "${NCCL_IB_GID_INDEX+x}${NVSHMEM_IB_GID_INDEX+x}${UCCL_IB_GID_INDEX+x}"
cx_export_gid_index_for_link_layer roce 1
test "$NCCL_IB_GID_INDEX:$NCCL_IB_SL" = 3:2
@@ -1804,6 +1806,7 @@ def test_private_allocation_stratum_is_salted_ordered_and_rank_consistent(self)
"ib_gid_index": "3",
"rdma_devices": "private-hca0:1,private-hca1:1",
"rdma_service_level": "2",
+ "rdma_traffic_class": "104",
"socket_ifname": "private-if0",
}
digest = run_ep._allocation_stratum_sha256(
From b4140893048ca40022362395817aa712a6b80798 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 23:51:57 +0800
Subject: [PATCH 080/210] fix(collectivex): expose bounded probe timeout stage
---
experimental/CollectiveX/runtime/common.sh | 7 +++++-
.../tests/test_sampling_contract.py | 22 +++++++++++++++++++
2 files changed, 28 insertions(+), 1 deletion(-)
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index d4cf46219b..8dd7134d37 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -36,9 +36,11 @@ cx_set_failure_stage() {
}
cx_fail_stage() {
- local stage="$1" log_path="${2:-}" diagnostic="unknown"
+ local stage="$1" log_path="${2:-}" diagnostic="unknown" probe_stage=""
cx_set_failure_stage "$stage"
if [ -n "$log_path" ] && [ -f "$log_path" ]; then
+ probe_stage="$(grep -aoE 'precision-probe-stage=(distributed-init|runtime-context|backend-construction|construction-consensus|native-operation|operation-consensus|evidence-aggregation)' "$log_path" \
+ | tail -n 1 | cut -d= -f2 || true)"
if grep -aEqi 'no space left|disk quota|quota exceeded' "$log_path"; then
diagnostic="storage-capacity"
elif grep -aEqi 'permission denied|operation not permitted|read-only file system|source mount (creation|ownership validation|permission inspection|permission normalization|permission validation) failed' "$log_path"; then
@@ -71,6 +73,9 @@ cx_fail_stage() {
diagnostic="missing-runtime"
elif grep -aEqi 'too many requests|rate.?limit' "$log_path"; then
diagnostic="registry-rate-limit"
+ elif [ -n "$probe_stage" ] \
+ && grep -aEqi 'timed out|operation timeout|wait timeout after|watchdog.*timeout|timeout: sending signal' "$log_path"; then
+ diagnostic="${probe_stage}-timeout"
elif grep -aEqi 'timed out|operation timeout|wait timeout after|watchdog.*timeout|timeout: sending signal|connection reset|could not resolve|TLS|certificate' "$log_path"; then
diagnostic="network-or-timeout"
elif grep -aEqi 'salloc:|srun:.*(unable to create step|step creation|invalid partition|invalid account)|unable to create step|job allocation' "$log_path"; then
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 63227432ed..10c2537a10 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -2494,6 +2494,28 @@ def test_non_timeout_failure_warning_is_classified_as_case_failure(self) -> None
self.assertNotIn("diagnostic=network-or-timeout", result.stderr)
self.assertIn("diagnostic=benchmark-case-failure", result.stderr)
+ def test_precision_probe_timeout_reports_only_the_last_closed_stage(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ log = Path(temporary) / "runtime.log"
+ log.write_text(
+ "[collectivex] precision-probe-stage=distributed-init\n"
+ "[collectivex] precision-probe-stage=backend-construction\n"
+ "[collectivex] precision-probe-stage=native-operation\n"
+ "[collectivex] precision probe timed out rc=124 limit=900s\n"
+ )
+ result = subprocess.run(
+ [
+ "bash", "-c",
+ 'source "$1"; cx_fail_stage execution "$2"',
+ "_", str(ROOT / "runtime" / "common.sh"), str(log),
+ ],
+ text=True,
+ capture_output=True,
+ )
+ self.assertEqual(result.returncode, 1)
+ self.assertIn("diagnostic=native-operation-timeout", result.stderr)
+ self.assertNotIn("distributed-init", result.stderr)
+
def test_private_runtime_failure_signatures_override_case_footer(self) -> None:
signatures = {
"DeepEP V2 no-GIN run is outside one realized LSA domain":
From 7b9094a6ebe0e0b9352b2596ec65e274de751611 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 5 Jul 2026 23:53:37 +0800
Subject: [PATCH 081/210] feat(collectivex): isolate MI355 network overlay
---
.github/workflows/collectivex-sweep.yml | 6 +++++-
experimental/CollectiveX/tests/test_sampling_contract.py | 2 ++
2 files changed, 7 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml
index d3f05b7ffe..134dc629f5 100644
--- a/.github/workflows/collectivex-sweep.yml
+++ b/.github/workflows/collectivex-sweep.yml
@@ -461,6 +461,7 @@ jobs:
COLLECTIVEX_NETWORK_CONFIG_CONTENT: ${{ secrets.COLLECTIVEX_NETWORK_CONFIG_V1 }}
COLLECTIVEX_H100_CONFIG_CONTENT: ${{ secrets.COLLECTIVEX_H100_CONFIG_V1 }}
COLLECTIVEX_B200_CONFIG_CONTENT: ${{ secrets.COLLECTIVEX_B200_CONFIG_V1 }}
+ COLLECTIVEX_MI355_CONFIG_CONTENT: ${{ secrets.COLLECTIVEX_MI355_CONFIG_V1 }}
COLLECTIVEX_OPERATOR_AUDIT_SALT: ${{ secrets.COLLECTIVEX_AUDIT_SALT_V1 }}
COLLECTIVEX_OPERATOR_CONFIG_REQUIRED: '1'
run: |
@@ -490,7 +491,7 @@ jobs:
base = load("COLLECTIVEX_OPERATOR_CONFIG_CONTENT")
for overlay_name in (
"COLLECTIVEX_NETWORK_CONFIG_CONTENT", "COLLECTIVEX_H100_CONFIG_CONTENT",
- "COLLECTIVEX_B200_CONFIG_CONTENT",
+ "COLLECTIVEX_B200_CONFIG_CONTENT", "COLLECTIVEX_MI355_CONFIG_CONTENT",
):
overlay_raw = os.environ.get(overlay_name, "")
if not overlay_raw:
@@ -512,6 +513,9 @@ jobs:
if set(overlay["runners"]) != {"h100-dgxc"}:
raise ValueError("invalid H100 overlay runner")
private_fields.add("exclude_nodes")
+ elif overlay_name == "COLLECTIVEX_MI355_CONFIG_CONTENT":
+ if set(overlay["runners"]) != {"mi355x"}:
+ raise ValueError("invalid MI355 overlay runner")
for runner, fields in overlay["runners"].items():
if runner not in base.get("runners", {}) or not isinstance(fields, dict) or not fields:
raise ValueError("invalid network overlay runner")
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 10c2537a10..adc8ed108c 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -1490,6 +1490,8 @@ def test_v1_publication_requires_explicit_release_markers(self) -> None:
self.assertIn("options: [sweep, probe-precision, publish-v1, refresh-v1]", sweep)
self.assertIn("${{ inputs.only_sku }}-${{ inputs.probe_id }}", sweep)
self.assertIn("cancel-in-progress: false", sweep)
+ self.assertIn("COLLECTIVEX_MI355_CONFIG_V1", sweep)
+ self.assertIn('set(overlay["runners"]) != {"mi355x"}', sweep)
self.assertIn("collectivex.precision-probe-plan.v1", (ROOT / "tests" / "probe_precision.py").read_text())
self.assertIn("cxprecision-probes-${{ github.run_id }}-${{ github.run_attempt }}", sweep)
self.assertIn("--validate-bundle", sweep)
From 887df295256bb2ddfb042771d9f7165290a434c4 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 00:12:09 +0800
Subject: [PATCH 082/210] fix(collectivex): localize probe memory failures
---
experimental/CollectiveX/runtime/common.sh | 3 +++
.../tests/test_sampling_contract.py | 21 +++++++++++++++++++
2 files changed, 24 insertions(+)
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 8dd7134d37..8f0f58ab91 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -57,6 +57,9 @@ cx_fail_stage() {
diagnostic="nccl-device-api"
elif grep -aEqi 'NVCC (PTX )?compilation failed|cuobjdump failed|invalid device (kernel )?image|no kernel image is available' "$log_path"; then
diagnostic="jit-toolchain"
+ elif [ -n "$probe_stage" ] \
+ && grep -aEqi 'cuda out of memory|CUDA_ERROR_OUT_OF_MEMORY|out of memory.*cuda' "$log_path"; then
+ diagnostic="${probe_stage}-accelerator-memory"
elif grep -aEqi 'cuda out of memory|CUDA_ERROR_OUT_OF_MEMORY|out of memory.*cuda' "$log_path"; then
diagnostic="accelerator-memory"
elif grep -aEqi 'does not match its pinned image contract|requires the exact pinned|version mismatch' "$log_path"; then
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index adc8ed108c..fce9ed86ff 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -2518,6 +2518,27 @@ def test_precision_probe_timeout_reports_only_the_last_closed_stage(self) -> Non
self.assertIn("diagnostic=native-operation-timeout", result.stderr)
self.assertNotIn("distributed-init", result.stderr)
+ def test_precision_probe_memory_failure_reports_the_last_closed_stage(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ log = Path(temporary) / "runtime.log"
+ log.write_text(
+ "[collectivex] precision-probe-stage=backend-construction\n"
+ "CUDA out of memory\n"
+ )
+ result = subprocess.run(
+ [
+ "bash", "-c",
+ 'source "$1"; cx_fail_stage execution "$2"',
+ "_", str(ROOT / "runtime" / "common.sh"), str(log),
+ ],
+ text=True,
+ capture_output=True,
+ )
+ self.assertEqual(result.returncode, 1)
+ self.assertIn(
+ "diagnostic=backend-construction-accelerator-memory", result.stderr
+ )
+
def test_private_runtime_failure_signatures_override_case_footer(self) -> None:
signatures = {
"DeepEP V2 no-GIN run is outside one realized LSA domain":
From 917a4044a28fc0b2926f376e36b3ae65241c1e97 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 00:58:38 +0800
Subject: [PATCH 083/210] test(collectivex): classify B200 logfmt timeout
---
experimental/CollectiveX/capability.py | 1 +
.../CollectiveX/tests/test_precision_scheduling.py | 8 ++++----
2 files changed, 5 insertions(+), 4 deletions(-)
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
index f9738bb88d..a895aedcee 100644
--- a/experimental/CollectiveX/capability.py
+++ b/experimental/CollectiveX/capability.py
@@ -313,6 +313,7 @@ def _precision_rule(
("28737315879", "b200-dgxc", 8, "deepep-hybrid", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "native-operation-failed"),
("28737315879", "b200-dgxc", 8, "deepep-v2", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "backend-construction-failed"),
("28745114766", "b200-dgxc", 16, "deepep", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "native-operation-timeout"),
+ ("28746354426", "b200-dgxc", 16, "deepep", "low-latency", _LL_LOGFMT_PROFILE, "unsupported", "native-operation-timeout"),
("28737422303", "h200-dgxc", 8, "deepep", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed"),
("28737422303", "h200-dgxc", 8, "deepep", "low-latency", _LL_FP8_PROFILE, "supported", "native-probe-passed"),
("28737422303", "h200-dgxc", 8, "deepep", "low-latency", _LL_LOGFMT_PROFILE, "supported", "native-probe-passed"),
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
index e063c39983..0174f8ef6f 100644
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -28,7 +28,7 @@ def test_precision_probe_inventory_is_exact_and_non_mutating(self) -> None:
item["precision_profile"],
)
self.assertEqual(targets, sorted(capability.provisional_precision_targets(), key=key))
- self.assertEqual(len(targets), 17)
+ self.assertEqual(len(targets), 16)
self.assertEqual(capability.PRECISION_CAPABILITIES, before)
self.assertEqual(
len({
@@ -228,15 +228,15 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
{"provisional", "supported", "unsupported"},
)
self.assertEqual(
- len(targets) - len(capability.provisional_precision_targets()), 77
+ len(targets) - len(capability.provisional_precision_targets()), 78
)
self.assertEqual(
sum(item["disposition"] == "supported" for item in targets), 59
)
self.assertEqual(
- sum(item["disposition"] == "unsupported" for item in targets), 18
+ sum(item["disposition"] == "unsupported" for item in targets), 19
)
- self.assertEqual(len(capability.provisional_precision_targets()), 17)
+ self.assertEqual(len(capability.provisional_precision_targets()), 16)
keys = {
(
item["precision_profile"],
From 2b958707e28bc3ae04ee99c0a09a39ce9e16de40 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 01:12:39 +0800
Subject: [PATCH 084/210] fix(collectivex): bound unknown probe failures
---
experimental/CollectiveX/runtime/common.sh | 2 ++
.../tests/test_sampling_contract.py | 20 +++++++++++++++++++
2 files changed, 22 insertions(+)
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 8f0f58ab91..8aad78053e 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -149,6 +149,8 @@ cx_fail_stage() {
diagnostic="python-subprocess"
elif grep -aEqi 'Traceback \(most recent call last\)' "$log_path"; then
diagnostic="python-exception"
+ elif [ -n "$probe_stage" ]; then
+ diagnostic="${probe_stage}-failed"
elif grep -aEqi 'SHARD done: [0-9]+/[0-9]+ case\(s\) failed|WARN: .* run failed rc=|completed with invalid semantic evidence' "$log_path"; then
diagnostic="benchmark-case-failure"
elif [ -s "$log_path" ]; then
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index fce9ed86ff..8fd31d9185 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -2539,6 +2539,26 @@ def test_precision_probe_memory_failure_reports_the_last_closed_stage(self) -> N
"diagnostic=backend-construction-accelerator-memory", result.stderr
)
+ def test_precision_probe_unknown_failure_reports_only_the_last_closed_stage(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ log = Path(temporary) / "runtime.log"
+ log.write_text(
+ "[collectivex] precision-probe-stage=native-operation\n"
+ "private backend failure details\n"
+ )
+ result = subprocess.run(
+ [
+ "bash", "-c",
+ 'source "$1"; cx_fail_stage execution "$2"',
+ "_", str(ROOT / "runtime" / "common.sh"), str(log),
+ ],
+ text=True,
+ capture_output=True,
+ )
+ self.assertEqual(result.returncode, 1)
+ self.assertIn("diagnostic=native-operation-failed", result.stderr)
+ self.assertNotIn("private backend", result.stderr)
+
def test_private_runtime_failure_signatures_override_case_footer(self) -> None:
signatures = {
"DeepEP V2 no-GIN run is outside one realized LSA domain":
From cb8137fcaf3b98ba079741720a7c5f89d372c1b2 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 01:19:06 +0800
Subject: [PATCH 085/210] test(collectivex): classify B200 V2 memory limit
---
experimental/CollectiveX/capability.py | 1 +
.../CollectiveX/tests/test_precision_scheduling.py | 8 ++++----
2 files changed, 5 insertions(+), 4 deletions(-)
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
index a895aedcee..f19169b71f 100644
--- a/experimental/CollectiveX/capability.py
+++ b/experimental/CollectiveX/capability.py
@@ -314,6 +314,7 @@ def _precision_rule(
("28737315879", "b200-dgxc", 8, "deepep-v2", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "backend-construction-failed"),
("28745114766", "b200-dgxc", 16, "deepep", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "native-operation-timeout"),
("28746354426", "b200-dgxc", 16, "deepep", "low-latency", _LL_LOGFMT_PROFILE, "unsupported", "native-operation-timeout"),
+ ("28746910633", "b200-dgxc", 16, "deepep-v2", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "construction-consensus-accelerator-memory"),
("28737422303", "h200-dgxc", 8, "deepep", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed"),
("28737422303", "h200-dgxc", 8, "deepep", "low-latency", _LL_FP8_PROFILE, "supported", "native-probe-passed"),
("28737422303", "h200-dgxc", 8, "deepep", "low-latency", _LL_LOGFMT_PROFILE, "supported", "native-probe-passed"),
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
index 0174f8ef6f..133724fcce 100644
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -28,7 +28,7 @@ def test_precision_probe_inventory_is_exact_and_non_mutating(self) -> None:
item["precision_profile"],
)
self.assertEqual(targets, sorted(capability.provisional_precision_targets(), key=key))
- self.assertEqual(len(targets), 16)
+ self.assertEqual(len(targets), 15)
self.assertEqual(capability.PRECISION_CAPABILITIES, before)
self.assertEqual(
len({
@@ -228,15 +228,15 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
{"provisional", "supported", "unsupported"},
)
self.assertEqual(
- len(targets) - len(capability.provisional_precision_targets()), 78
+ len(targets) - len(capability.provisional_precision_targets()), 79
)
self.assertEqual(
sum(item["disposition"] == "supported" for item in targets), 59
)
self.assertEqual(
- sum(item["disposition"] == "unsupported" for item in targets), 19
+ sum(item["disposition"] == "unsupported" for item in targets), 20
)
- self.assertEqual(len(capability.provisional_precision_targets()), 16)
+ self.assertEqual(len(capability.provisional_precision_targets()), 15)
keys = {
(
item["precision_profile"],
From 63d1ae9b9ab5e6de13f7e29664d350a7af0984d8 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 01:33:11 +0800
Subject: [PATCH 086/210] test(collectivex): classify B200 FP8 timeout
---
experimental/CollectiveX/capability.py | 1 +
.../CollectiveX/tests/test_precision_scheduling.py | 8 ++++----
2 files changed, 5 insertions(+), 4 deletions(-)
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
index f19169b71f..3c79d2472d 100644
--- a/experimental/CollectiveX/capability.py
+++ b/experimental/CollectiveX/capability.py
@@ -314,6 +314,7 @@ def _precision_rule(
("28737315879", "b200-dgxc", 8, "deepep-v2", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "backend-construction-failed"),
("28745114766", "b200-dgxc", 16, "deepep", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "native-operation-timeout"),
("28746354426", "b200-dgxc", 16, "deepep", "low-latency", _LL_LOGFMT_PROFILE, "unsupported", "native-operation-timeout"),
+ ("28747290376", "b200-dgxc", 16, "deepep", "low-latency", _LL_FP8_PROFILE, "unsupported", "native-operation-timeout"),
("28746910633", "b200-dgxc", 16, "deepep-v2", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "construction-consensus-accelerator-memory"),
("28737422303", "h200-dgxc", 8, "deepep", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed"),
("28737422303", "h200-dgxc", 8, "deepep", "low-latency", _LL_FP8_PROFILE, "supported", "native-probe-passed"),
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
index 133724fcce..570f5310fa 100644
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -28,7 +28,7 @@ def test_precision_probe_inventory_is_exact_and_non_mutating(self) -> None:
item["precision_profile"],
)
self.assertEqual(targets, sorted(capability.provisional_precision_targets(), key=key))
- self.assertEqual(len(targets), 15)
+ self.assertEqual(len(targets), 14)
self.assertEqual(capability.PRECISION_CAPABILITIES, before)
self.assertEqual(
len({
@@ -228,15 +228,15 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
{"provisional", "supported", "unsupported"},
)
self.assertEqual(
- len(targets) - len(capability.provisional_precision_targets()), 79
+ len(targets) - len(capability.provisional_precision_targets()), 80
)
self.assertEqual(
sum(item["disposition"] == "supported" for item in targets), 59
)
self.assertEqual(
- sum(item["disposition"] == "unsupported" for item in targets), 20
+ sum(item["disposition"] == "unsupported" for item in targets), 21
)
- self.assertEqual(len(capability.provisional_precision_targets()), 15)
+ self.assertEqual(len(capability.provisional_precision_targets()), 14)
keys = {
(
item["precision_profile"],
From a56b5be8788e86368772284b8ba12514e5e1847b Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 01:49:22 +0800
Subject: [PATCH 087/210] test(collectivex): classify B200 fused logfmt timeout
---
experimental/CollectiveX/capability.py | 1 +
.../CollectiveX/tests/test_precision_scheduling.py | 14 ++++++++------
2 files changed, 9 insertions(+), 6 deletions(-)
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
index 3c79d2472d..f77a578c62 100644
--- a/experimental/CollectiveX/capability.py
+++ b/experimental/CollectiveX/capability.py
@@ -315,6 +315,7 @@ def _precision_rule(
("28745114766", "b200-dgxc", 16, "deepep", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "native-operation-timeout"),
("28746354426", "b200-dgxc", 16, "deepep", "low-latency", _LL_LOGFMT_PROFILE, "unsupported", "native-operation-timeout"),
("28747290376", "b200-dgxc", 16, "deepep", "low-latency", _LL_FP8_PROFILE, "unsupported", "native-operation-timeout"),
+ ("28747292565", "b200-dgxc", 16, "deepep", "low-latency", _LL_FP8_LOGFMT_PROFILE, "unsupported", "native-operation-timeout"),
("28746910633", "b200-dgxc", 16, "deepep-v2", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "construction-consensus-accelerator-memory"),
("28737422303", "h200-dgxc", 8, "deepep", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed"),
("28737422303", "h200-dgxc", 8, "deepep", "low-latency", _LL_FP8_PROFILE, "supported", "native-probe-passed"),
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
index 570f5310fa..ae333d95d0 100644
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -28,7 +28,7 @@ def test_precision_probe_inventory_is_exact_and_non_mutating(self) -> None:
item["precision_profile"],
)
self.assertEqual(targets, sorted(capability.provisional_precision_targets(), key=key))
- self.assertEqual(len(targets), 14)
+ self.assertEqual(len(targets), 13)
self.assertEqual(capability.PRECISION_CAPABILITIES, before)
self.assertEqual(
len({
@@ -53,10 +53,12 @@ def test_precision_probe_selects_only_one_exact_provisional_cell(self) -> None:
)
def test_precision_probe_workflow_plan_binds_exact_controls(self) -> None:
- plan = probe_precision.workflow_plan(backend="deepep", only_sku="b200-dgxc")
+ plan = probe_precision.workflow_plan(
+ backend="deepep-hybrid", only_sku="b200-dgxc"
+ )
self.assertTrue(plan["include"])
self.assertTrue(all(
- row["backend"] == "deepep" and row["sku"] == "b200-dgxc"
+ row["backend"] == "deepep-hybrid" and row["sku"] == "b200-dgxc"
for row in plan["include"]
))
row = plan["include"][0]
@@ -228,15 +230,15 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
{"provisional", "supported", "unsupported"},
)
self.assertEqual(
- len(targets) - len(capability.provisional_precision_targets()), 80
+ len(targets) - len(capability.provisional_precision_targets()), 81
)
self.assertEqual(
sum(item["disposition"] == "supported" for item in targets), 59
)
self.assertEqual(
- sum(item["disposition"] == "unsupported" for item in targets), 21
+ sum(item["disposition"] == "unsupported" for item in targets), 22
)
- self.assertEqual(len(capability.provisional_precision_targets()), 14)
+ self.assertEqual(len(capability.provisional_precision_targets()), 13)
keys = {
(
item["precision_profile"],
From e56019299160a08f30c192bc03764c18362cd56f Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 02:03:23 +0800
Subject: [PATCH 088/210] test(collectivex): finalize B200 EP16 capabilities
---
experimental/CollectiveX/capability.py | 1 +
.../CollectiveX/tests/test_precision_scheduling.py | 14 ++++++--------
2 files changed, 7 insertions(+), 8 deletions(-)
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
index f77a578c62..753cbfd8bc 100644
--- a/experimental/CollectiveX/capability.py
+++ b/experimental/CollectiveX/capability.py
@@ -317,6 +317,7 @@ def _precision_rule(
("28747290376", "b200-dgxc", 16, "deepep", "low-latency", _LL_FP8_PROFILE, "unsupported", "native-operation-timeout"),
("28747292565", "b200-dgxc", 16, "deepep", "low-latency", _LL_FP8_LOGFMT_PROFILE, "unsupported", "native-operation-timeout"),
("28746910633", "b200-dgxc", 16, "deepep-v2", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "construction-consensus-accelerator-memory"),
+ ("28748550531", "b200-dgxc", 16, "deepep-hybrid", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "backend-construction-failed"),
("28737422303", "h200-dgxc", 8, "deepep", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed"),
("28737422303", "h200-dgxc", 8, "deepep", "low-latency", _LL_FP8_PROFILE, "supported", "native-probe-passed"),
("28737422303", "h200-dgxc", 8, "deepep", "low-latency", _LL_LOGFMT_PROFILE, "supported", "native-probe-passed"),
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
index ae333d95d0..3513651721 100644
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -28,7 +28,7 @@ def test_precision_probe_inventory_is_exact_and_non_mutating(self) -> None:
item["precision_profile"],
)
self.assertEqual(targets, sorted(capability.provisional_precision_targets(), key=key))
- self.assertEqual(len(targets), 13)
+ self.assertEqual(len(targets), 12)
self.assertEqual(capability.PRECISION_CAPABILITIES, before)
self.assertEqual(
len({
@@ -53,12 +53,10 @@ def test_precision_probe_selects_only_one_exact_provisional_cell(self) -> None:
)
def test_precision_probe_workflow_plan_binds_exact_controls(self) -> None:
- plan = probe_precision.workflow_plan(
- backend="deepep-hybrid", only_sku="b200-dgxc"
- )
+ plan = probe_precision.workflow_plan(backend="deepep", only_sku="b300")
self.assertTrue(plan["include"])
self.assertTrue(all(
- row["backend"] == "deepep-hybrid" and row["sku"] == "b200-dgxc"
+ row["backend"] == "deepep" and row["sku"] == "b300"
for row in plan["include"]
))
row = plan["include"][0]
@@ -230,15 +228,15 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
{"provisional", "supported", "unsupported"},
)
self.assertEqual(
- len(targets) - len(capability.provisional_precision_targets()), 81
+ len(targets) - len(capability.provisional_precision_targets()), 82
)
self.assertEqual(
sum(item["disposition"] == "supported" for item in targets), 59
)
self.assertEqual(
- sum(item["disposition"] == "unsupported" for item in targets), 22
+ sum(item["disposition"] == "unsupported" for item in targets), 23
)
- self.assertEqual(len(capability.provisional_precision_targets()), 13)
+ self.assertEqual(len(capability.provisional_precision_targets()), 12)
keys = {
(
item["precision_profile"],
From aef703089b009d84475e83f01e049da10b746437 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 02:07:05 +0800
Subject: [PATCH 089/210] feat(collectivex): close B300 V1 publication cells
---
experimental/CollectiveX/capability.py | 13 ++++++++
experimental/CollectiveX/configs/suites.yaml | 2 +-
.../tests/test_precision_scheduling.py | 23 ++++++++------
.../tests/test_sampling_contract.py | 31 +++++++++++++------
4 files changed, 49 insertions(+), 20 deletions(-)
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
index 753cbfd8bc..a664ce4636 100644
--- a/experimental/CollectiveX/capability.py
+++ b/experimental/CollectiveX/capability.py
@@ -304,6 +304,19 @@ def _precision_rule(
PRECISION_CELL_OVERRIDES: dict[tuple[str, str, str, int, str], dict[str, str]] = {}
+for _profile, _backend, _mode in (
+ (_NORMAL_E4M3FN_PROFILE, "deepep", "normal"),
+ (_NORMAL_E4M3FN_PROFILE, "deepep-v2", "normal"),
+ (_NORMAL_E4M3FN_PROFILE, "deepep-hybrid", "normal"),
+ (_LL_FP8_PROFILE, "deepep", "low-latency"),
+ (_LL_LOGFMT_PROFILE, "deepep", "low-latency"),
+ (_LL_FP8_LOGFMT_PROFILE, "deepep", "low-latency"),
+):
+ PRECISION_CELL_OVERRIDES[(_profile, _backend, "b300", 16, _mode)] = {
+ "disposition": "unsupported",
+ "basis": "v1-publication-fabric-unavailable",
+ }
+
_VALIDATED_NATIVE_PROBE_CELLS = (
# run, SKU, EP, backend, mode, profile, disposition, result
("28737315879", "b200-dgxc", 8, "deepep", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed"),
diff --git a/experimental/CollectiveX/configs/suites.yaml b/experimental/CollectiveX/configs/suites.yaml
index 2f52147248..27d368e166 100644
--- a/experimental/CollectiveX/configs/suites.yaml
+++ b/experimental/CollectiveX/configs/suites.yaml
@@ -68,5 +68,5 @@ suites:
- d-fp8-e4m3fn-b128-f32-fused.c-bf16
- d-bf16.c-logfmt10-dynamic64
- d-fp8-e4m3fn-b128-f32-fused.c-logfmt10-dynamic64
- provisional: true
+ provisional: false
required_publication: comparable-experimental
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
index 3513651721..d39b4bcfc4 100644
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -28,7 +28,7 @@ def test_precision_probe_inventory_is_exact_and_non_mutating(self) -> None:
item["precision_profile"],
)
self.assertEqual(targets, sorted(capability.provisional_precision_targets(), key=key))
- self.assertEqual(len(targets), 12)
+ self.assertEqual(len(targets), 6)
self.assertEqual(capability.PRECISION_CAPABILITIES, before)
self.assertEqual(
len({
@@ -53,10 +53,10 @@ def test_precision_probe_selects_only_one_exact_provisional_cell(self) -> None:
)
def test_precision_probe_workflow_plan_binds_exact_controls(self) -> None:
- plan = probe_precision.workflow_plan(backend="deepep", only_sku="b300")
+ plan = probe_precision.workflow_plan(backend="mori", only_sku="mi355x")
self.assertTrue(plan["include"])
self.assertTrue(all(
- row["backend"] == "deepep" and row["sku"] == "b300"
+ row["backend"] == "mori" and row["sku"] == "mi355x"
for row in plan["include"]
))
row = plan["include"][0]
@@ -228,15 +228,15 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
{"provisional", "supported", "unsupported"},
)
self.assertEqual(
- len(targets) - len(capability.provisional_precision_targets()), 82
+ len(targets) - len(capability.provisional_precision_targets()), 88
)
self.assertEqual(
sum(item["disposition"] == "supported" for item in targets), 59
)
self.assertEqual(
- sum(item["disposition"] == "unsupported" for item in targets), 23
+ sum(item["disposition"] == "unsupported" for item in targets), 29
)
- self.assertEqual(len(capability.provisional_precision_targets()), 12)
+ self.assertEqual(len(capability.provisional_precision_targets()), 6)
keys = {
(
item["precision_profile"],
@@ -294,7 +294,7 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
self.assertEqual(h100_ep16, "supported")
self.assertEqual(reason, "ok")
- def test_split_suites_are_provisional_and_do_not_duplicate_bf16(self) -> None:
+ def test_split_suites_track_provisional_state_and_do_not_duplicate_bf16(self) -> None:
suites = sweep_matrix._load("suites.yaml")
workloads = sweep_matrix._load("workloads.yaml")
sweep_matrix.validate_config_documents(suites, workloads)
@@ -318,7 +318,7 @@ def test_split_suites_are_provisional_and_do_not_duplicate_bf16(self) -> None:
("low-latency", ["decode"], [128]),
)
self.assertTrue(normal["provisional"])
- self.assertTrue(low_latency["provisional"])
+ self.assertFalse(low_latency["provisional"])
self.assertEqual(
normal["required_publication"], "comparable-experimental"
)
@@ -332,7 +332,12 @@ def test_split_suites_are_provisional_and_do_not_duplicate_bf16(self) -> None:
matrix = sweep_matrix.validate_matrix_document(
sweep_matrix.resolve_matrix(backends="all")
)
- self.assertFalse(any("precision_profile" in item["case"] for item in matrix["requested_cases"]))
+ scheduled_profiles = {
+ item["case"].get("precision_profile")
+ for item in matrix["requested_cases"]
+ if "precision_profile" in item["case"]
+ }
+ self.assertEqual(scheduled_profiles, set(low_latency["precision_profiles"]))
with self.assertRaisesRegex(SystemExit, "provisional precision suites"):
sweep_matrix.resolve_matrix(
suites="ep-precision-normal-v1", backends="all"
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 8fd31d9185..cfdb408792 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -5,6 +5,7 @@
import argparse
import ast
import copy
+from collections import Counter
import hashlib
import io
import json
@@ -208,7 +209,7 @@ def test_matrix_cases_and_shards_are_identity_bound(self) -> None:
sum(len(item["case"]["ladder"].split()) for item in runnable_cases),
sum(len(item["case"]["ladder"].split()) for item in unsupported_cases),
),
- (58, 608, 364, 244, 1600, 940, 660),
+ (58, 656, 404, 252, 1648, 980, 668),
)
expected_topologies = {}
for sku, product in (
@@ -260,23 +261,29 @@ def test_matrix_cases_and_shards_are_identity_bound(self) -> None:
expected_topologies,
)
self.assertEqual(
- {shard["n"] for shard in matrix["include"]}, {6, 7}
- )
- self.assertEqual(
- sum(shard["n"] == 7 for shard in matrix["include"]), 16
+ Counter(shard["n"] for shard in matrix["include"]),
+ Counter({6: 42, 7: 2, 8: 1, 10: 13}),
)
ll_cases = [
item for item in matrix["requested_cases"]
if item["case"]["mode"] == "low-latency"
]
- self.assertEqual(len(ll_cases), 32)
+ self.assertEqual(len(ll_cases), 80)
self.assertTrue(all(
- item["case"]["suite"] == "ep-low-latency-v1"
- and item["case"]["backend"] in {"deepep", "uccl"}
+ item["case"]["backend"] in {"deepep", "uccl"}
and item["case"]["phase"] == "decode"
and item["case"]["routing"] == "uniform"
and not item["case"]["eplb"]
- and item["case"]["ladder"] == "1 2 4 8 16 32 64 128"
+ and (
+ (
+ item["case"]["suite"] == "ep-low-latency-v1"
+ and item["case"]["ladder"] == "1 2 4 8 16 32 64 128"
+ )
+ or (
+ item["case"]["suite"] == "ep-precision-low-latency-v1"
+ and item["case"]["ladder"] == "128"
+ )
+ )
for item in ll_cases
))
for shard in matrix["include"]:
@@ -2193,7 +2200,11 @@ def test_terminal_fail_safe_fills_only_missing_shard_cases(self) -> None:
out_dir.mkdir()
existing = contracts.make_terminal_document(
allocation_factors=allocation, attempt_ordinal=1, case=first,
- case_factors={"case": first, "profile": identity.V1_CASE_PROFILE, "sku": shard["sku"]},
+ case_factors={
+ "case": first,
+ "profile": identity.profile_for_case(first),
+ "sku": shard["sku"],
+ },
control_sha256=hashlib.sha256(control_path.read_bytes()).hexdigest(),
failure_mode="setup", generated_at="2026-07-04T00:00:00Z", git_run=git_run,
reason="launcher-setup-failed", return_code=7, source="runtime-emitter",
From e3302c57aa3e0998b1daca186b7b24764a77c57d Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 02:35:24 +0800
Subject: [PATCH 090/210] fix(collectivex): prioritize probe stage diagnostics
---
experimental/CollectiveX/runtime/common.sh | 3 +++
.../tests/test_sampling_contract.py | 20 +++++++++++++++++++
2 files changed, 23 insertions(+)
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 8aad78053e..42e9e143ab 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -83,6 +83,9 @@ cx_fail_stage() {
diagnostic="network-or-timeout"
elif grep -aEqi 'salloc:|srun:.*(unable to create step|step creation|invalid partition|invalid account)|unable to create step|job allocation' "$log_path"; then
diagnostic="scheduler"
+ elif [ -n "$probe_stage" ] \
+ && grep -aEqi 'Traceback \(most recent call last\)|[A-Za-z]+(Error|Exception):' "$log_path"; then
+ diagnostic="${probe_stage}-failed"
elif grep -aEqi 'ModuleNotFoundError|ImportError:' "$log_path"; then
diagnostic="python-import"
elif grep -aEqi 'AttributeError:|TypeError:.*(unexpected|argument|operand)|has no attribute' "$log_path"; then
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index cfdb408792..96e0c3a24b 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -2570,6 +2570,26 @@ def test_precision_probe_unknown_failure_reports_only_the_last_closed_stage(self
self.assertIn("diagnostic=native-operation-failed", result.stderr)
self.assertNotIn("private backend", result.stderr)
+ def test_precision_probe_traceback_prefers_the_last_closed_stage(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ log = Path(temporary) / "runtime.log"
+ log.write_text(
+ "[collectivex] precision-probe-stage=native-operation\n"
+ "Traceback (most recent call last):\nRuntimeError: private details\n"
+ )
+ result = subprocess.run(
+ [
+ "bash", "-c",
+ 'source "$1"; cx_fail_stage execution "$2"',
+ "_", str(ROOT / "runtime" / "common.sh"), str(log),
+ ],
+ text=True,
+ capture_output=True,
+ )
+ self.assertEqual(result.returncode, 1)
+ self.assertIn("diagnostic=native-operation-failed", result.stderr)
+ self.assertNotIn("python-exception", result.stderr)
+
def test_private_runtime_failure_signatures_override_case_footer(self) -> None:
signatures = {
"DeepEP V2 no-GIN run is outside one realized LSA domain":
From 44b6f867e598b6729ceeafb26d478d97e97cd5bf Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 02:52:14 +0800
Subject: [PATCH 091/210] test(collectivex): classify MI355 EP8 construction
---
experimental/CollectiveX/capability.py | 1 +
.../tests/test_precision_scheduling.py | 16 ++++++++--------
2 files changed, 9 insertions(+), 8 deletions(-)
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
index a664ce4636..b8e66a5e24 100644
--- a/experimental/CollectiveX/capability.py
+++ b/experimental/CollectiveX/capability.py
@@ -386,6 +386,7 @@ def _precision_rule(
("28740154697", "h200-dgxc", 16, "uccl", "low-latency", _LL_FP8_PROFILE, "supported", "native-probe-passed"),
("28740154697", "h200-dgxc", 16, "uccl", "low-latency", _LL_FP8_LOGFMT_PROFILE, "supported", "native-probe-passed"),
("28740154697", "h200-dgxc", 16, "uccl", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed"),
+ ("28750823474", "mi355x", 8, "mori", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "backend-construction-failed"),
("28740154697", "h200-dgxc", 16, "deepep-v2", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "backend-setup-timeout"),
("28740533382", "mi355x", 8, "mori", "normal", _MORI_E4M3FN_BOTH_PROFILE, "unsupported", "backend-construction-failed"),
("28740533382", "mi355x", 8, "mori", "normal", _MORI_E4M3FN_DIRECT_PROFILE, "unsupported", "backend-construction-failed"),
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
index d39b4bcfc4..3bf5a21932 100644
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -28,7 +28,7 @@ def test_precision_probe_inventory_is_exact_and_non_mutating(self) -> None:
item["precision_profile"],
)
self.assertEqual(targets, sorted(capability.provisional_precision_targets(), key=key))
- self.assertEqual(len(targets), 6)
+ self.assertEqual(len(targets), 5)
self.assertEqual(capability.PRECISION_CAPABILITIES, before)
self.assertEqual(
len({
@@ -83,11 +83,11 @@ def test_precision_probe_workflow_plan_binds_exact_controls(self) -> None:
)
with self.assertRaisesRegex(ValueError, "select no provisional"):
probe_precision.workflow_plan(backend="mori", only_sku="b200-dgxc")
- ep8 = probe_precision.workflow_plan(
- backend="mori", only_sku="mi355x", max_nodes=1,
+ ep16 = probe_precision.workflow_plan(
+ backend="mori", only_sku="mi355x", min_nodes=2,
)
- self.assertTrue(ep8["include"])
- self.assertEqual({row["ep"] for row in ep8["include"]}, {8})
+ self.assertTrue(ep16["include"])
+ self.assertEqual({row["ep"] for row in ep16["include"]}, {16})
with self.assertRaisesRegex(ValueError, "node filters"):
probe_precision.workflow_plan(min_nodes=2, max_nodes=1)
@@ -228,15 +228,15 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
{"provisional", "supported", "unsupported"},
)
self.assertEqual(
- len(targets) - len(capability.provisional_precision_targets()), 88
+ len(targets) - len(capability.provisional_precision_targets()), 89
)
self.assertEqual(
sum(item["disposition"] == "supported" for item in targets), 59
)
self.assertEqual(
- sum(item["disposition"] == "unsupported" for item in targets), 29
+ sum(item["disposition"] == "unsupported" for item in targets), 30
)
- self.assertEqual(len(capability.provisional_precision_targets()), 6)
+ self.assertEqual(len(capability.provisional_precision_targets()), 5)
keys = {
(
item["precision_profile"],
From 0e300a820ff1da2d660f7285c73faf923bbf3fca Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 03:07:11 +0800
Subject: [PATCH 092/210] fix(collectivex): exclude non-native B300 EP16
---
experimental/CollectiveX/capability.py | 10 +++++++++
experimental/CollectiveX/docs/methodology.md | 21 ++++++++++---------
.../tests/test_precision_scheduling.py | 16 ++++++++++++++
.../tests/test_sampling_contract.py | 16 ++++++++++++--
4 files changed, 51 insertions(+), 12 deletions(-)
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
index b8e66a5e24..14d9277d8c 100644
--- a/experimental/CollectiveX/capability.py
+++ b/experimental/CollectiveX/capability.py
@@ -160,6 +160,13 @@ def _platform(
}
SWEEP_BACKENDS = tuple(BACKENDS)
+# Publication-quality topology exceptions apply after ordinary backend support
+# checks. They describe the currently usable benchmark fabric, not a library
+# implementation limit, and can be removed when the named topology is repaired.
+TOPOLOGY_CELL_OVERRIDES: dict[tuple[str, int], str] = {
+ ("b300", 16): "v1 publication fabric unavailable for B300 EP16",
+}
+
PRECISION_DISPOSITIONS = {
"supported", "unsupported", "not-applicable", "provisional",
}
@@ -481,6 +488,9 @@ def _resolve_base(
return False, f"{backend} does not support {platform['machine']}"
if sku in implementation.get("excluded_skus", set()):
return False, f"{backend} is unavailable on {sku}"
+ topology_override = TOPOLOGY_CELL_OVERRIDES.get((sku, ep))
+ if topology_override is not None:
+ return False, topology_override
return True, "ok"
diff --git a/experimental/CollectiveX/docs/methodology.md b/experimental/CollectiveX/docs/methodology.md
index 6368c92faf..8dda677466 100644
--- a/experimental/CollectiveX/docs/methodology.md
+++ b/experimental/CollectiveX/docs/methodology.md
@@ -10,9 +10,8 @@ This document describes the implemented BF16 baseline. It is not yet the V1 qual
Before any V1-tagged run, this English document must match the implemented precision, measurement,
publication, and frontend contracts; counts and digests remain unfrozen. Chinese documentation
synchronization is explicitly deferred for the V1 implementation phase.
-H100, H200, GB200, and GB300 precision cells are terminal. B200 and B300 EP8 are terminal, and
-B300 EP16 has terminal native outcomes. Twelve B200/MI325X/MI355X precision cells remain
-provisional.
+H100, H200, B200, B300, GB200, and GB300 precision cells are terminal. Five MI325X/MI355X
+precision cells remain provisional.
## Product Boundary
@@ -39,9 +38,8 @@ normal `layout-and-dispatch-v1` or low-latency `expert-packed-weighted-combine-v
- `ep-precision-normal-v1`: nonbaseline native dispatch/combine profiles at decode T=128 and prefill
T=512; BF16/BF16 endpoint controls are referenced rather than duplicated.
- `ep-precision-low-latency-v1`: nonbaseline native low-latency profiles at decode T=128.
-- BF16 planning baseline: 608 requested cases / 1,600 token points; 338 runnable cases / 868
- points in
- 54 executable workflow shards/allocation cells; 270 unsupported cases / 732 points.
+- Current planning matrix: 656 requested cases / 1,648 token points; 379 runnable cases / 916
+ points in 54 executable workflow shards/allocation cells; 277 unsupported cases / 732 points.
| Systems | EP8 | EP16 |
|---|---|---|
@@ -64,10 +62,13 @@ declared unsupported combination in v1 because NCCL 2.30.4 reports no Device API
support for its EP8 communicator; that pool can return only after all-rank CUDA P2P/LSA support is
restored. This baseline omits `[cl]`, `[rv]`, quantization, alternate activation/routing profiles,
uneven allocation, placement permutations, model envelopes, and scaling.
-H100 EP16 is also planned unsupported on the current runner pool because allocated compute nodes
-expose no active RDMA device. H100 EP8 remains in scope with a private stage beside its configured
-shared container directory; unlike the B300 runner account home, the H100 account home is not
-compute-visible.
+H100 EP16 is supported on the healthy runner subset. The private runner overlay excludes pods that
+do not expose the host RDMA devices in their network namespace, and allocation validation requires
+the complete operator-pinned RoCE profile on every selected node before image import. H100 EP8
+remains in scope with a private stage beside its configured shared container directory; unlike the
+B300 runner account home, the H100 account home is not compute-visible. B300 EP16 is terminal
+unsupported for V1 publication because its currently functional fallback HCAs are not the
+GPU-adjacent product fabric; this is an operational topology decision, not a library limitation.
FlashInfer is excluded from v1 after repeatable intermittent execution failures; those failures are
not converted into planned-unsupported coverage.
MoRI EP8 uses MI325X AsyncLL or MI355X IntraNode in normal mode. EP16 uses pinned InterNodeV1 over
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
index 3bf5a21932..b2ab3898a0 100644
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -294,6 +294,22 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
self.assertEqual(h100_ep16, "supported")
self.assertEqual(reason, "ok")
+ for backend in ("deepep", "deepep-v2", "deepep-hybrid", "nccl-ep"):
+ disposition, reason = capability.resolve_disposition(
+ "b300", backend, ep=16, nodes=2,
+ precision_profile=identity.V1_CONTROL_PRECISION_PROFILE,
+ )
+ self.assertEqual(disposition, "unsupported")
+ self.assertEqual(
+ reason, "v1 publication fabric unavailable for B300 EP16"
+ )
+ b300_ep8, reason = capability.resolve_disposition(
+ "b300", "nccl-ep", ep=8, nodes=1,
+ precision_profile=identity.V1_CONTROL_PRECISION_PROFILE,
+ )
+ self.assertEqual(b300_ep8, "supported")
+ self.assertEqual(reason, "ok")
+
def test_split_suites_track_provisional_state_and_do_not_duplicate_bf16(self) -> None:
suites = sweep_matrix._load("suites.yaml")
workloads = sweep_matrix._load("workloads.yaml")
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 96e0c3a24b..00e821f84f 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -209,8 +209,20 @@ def test_matrix_cases_and_shards_are_identity_bound(self) -> None:
sum(len(item["case"]["ladder"].split()) for item in runnable_cases),
sum(len(item["case"]["ladder"].split()) for item in unsupported_cases),
),
- (58, 656, 404, 252, 1648, 980, 668),
+ (54, 656, 379, 277, 1648, 916, 732),
)
+ b300_ep16 = [
+ item for item in unsupported_cases
+ if item["sku"] == "b300" and item["case"]["ep"] == 16
+ and item["case"]["backend"] in {
+ "deepep", "deepep-v2", "deepep-hybrid", "nccl-ep",
+ }
+ ]
+ self.assertTrue(b300_ep16)
+ self.assertTrue(all(
+ item["detail"] == "v1 publication fabric unavailable for B300 EP16"
+ for item in b300_ep16
+ ))
expected_topologies = {}
for sku, product in (
("h100-dgxc", "h100"), ("h200-dgxc", "h200"),
@@ -262,7 +274,7 @@ def test_matrix_cases_and_shards_are_identity_bound(self) -> None:
)
self.assertEqual(
Counter(shard["n"] for shard in matrix["include"]),
- Counter({6: 42, 7: 2, 8: 1, 10: 13}),
+ Counter({6: 39, 7: 1, 8: 1, 10: 13}),
)
ll_cases = [
item for item in matrix["requested_cases"]
From 6131cabb556d35fcccc1d06a169cfbcc68ebe5e5 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 03:17:50 +0800
Subject: [PATCH 093/210] test(collectivex): close MI355 EP16 precision
---
experimental/CollectiveX/capability.py | 1 +
experimental/CollectiveX/docs/methodology.md | 2 +-
.../CollectiveX/tests/test_precision_scheduling.py | 14 +++++++-------
3 files changed, 9 insertions(+), 8 deletions(-)
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
index 14d9277d8c..b70c5d24b6 100644
--- a/experimental/CollectiveX/capability.py
+++ b/experimental/CollectiveX/capability.py
@@ -394,6 +394,7 @@ def _precision_rule(
("28740154697", "h200-dgxc", 16, "uccl", "low-latency", _LL_FP8_LOGFMT_PROFILE, "supported", "native-probe-passed"),
("28740154697", "h200-dgxc", 16, "uccl", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed"),
("28750823474", "mi355x", 8, "mori", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "backend-construction-failed"),
+ ("28750825814", "mi355x", 16, "mori", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "backend-construction-failed"),
("28740154697", "h200-dgxc", 16, "deepep-v2", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "backend-setup-timeout"),
("28740533382", "mi355x", 8, "mori", "normal", _MORI_E4M3FN_BOTH_PROFILE, "unsupported", "backend-construction-failed"),
("28740533382", "mi355x", 8, "mori", "normal", _MORI_E4M3FN_DIRECT_PROFILE, "unsupported", "backend-construction-failed"),
diff --git a/experimental/CollectiveX/docs/methodology.md b/experimental/CollectiveX/docs/methodology.md
index 8dda677466..46c9e4b21e 100644
--- a/experimental/CollectiveX/docs/methodology.md
+++ b/experimental/CollectiveX/docs/methodology.md
@@ -10,7 +10,7 @@ This document describes the implemented BF16 baseline. It is not yet the V1 qual
Before any V1-tagged run, this English document must match the implemented precision, measurement,
publication, and frontend contracts; counts and digests remain unfrozen. Chinese documentation
synchronization is explicitly deferred for the V1 implementation phase.
-H100, H200, B200, B300, GB200, and GB300 precision cells are terminal. Five MI325X/MI355X
+H100, H200, B200, B300, GB200, GB300, and MI355X precision cells are terminal. Four MI325X
precision cells remain provisional.
## Product Boundary
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
index b2ab3898a0..d0527f9c8f 100644
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -28,7 +28,7 @@ def test_precision_probe_inventory_is_exact_and_non_mutating(self) -> None:
item["precision_profile"],
)
self.assertEqual(targets, sorted(capability.provisional_precision_targets(), key=key))
- self.assertEqual(len(targets), 5)
+ self.assertEqual(len(targets), 4)
self.assertEqual(capability.PRECISION_CAPABILITIES, before)
self.assertEqual(
len({
@@ -53,10 +53,10 @@ def test_precision_probe_selects_only_one_exact_provisional_cell(self) -> None:
)
def test_precision_probe_workflow_plan_binds_exact_controls(self) -> None:
- plan = probe_precision.workflow_plan(backend="mori", only_sku="mi355x")
+ plan = probe_precision.workflow_plan(backend="mori", only_sku="mi325x")
self.assertTrue(plan["include"])
self.assertTrue(all(
- row["backend"] == "mori" and row["sku"] == "mi355x"
+ row["backend"] == "mori" and row["sku"] == "mi325x"
for row in plan["include"]
))
row = plan["include"][0]
@@ -84,7 +84,7 @@ def test_precision_probe_workflow_plan_binds_exact_controls(self) -> None:
with self.assertRaisesRegex(ValueError, "select no provisional"):
probe_precision.workflow_plan(backend="mori", only_sku="b200-dgxc")
ep16 = probe_precision.workflow_plan(
- backend="mori", only_sku="mi355x", min_nodes=2,
+ backend="mori", only_sku="mi325x", min_nodes=2,
)
self.assertTrue(ep16["include"])
self.assertEqual({row["ep"] for row in ep16["include"]}, {16})
@@ -228,15 +228,15 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
{"provisional", "supported", "unsupported"},
)
self.assertEqual(
- len(targets) - len(capability.provisional_precision_targets()), 89
+ len(targets) - len(capability.provisional_precision_targets()), 90
)
self.assertEqual(
sum(item["disposition"] == "supported" for item in targets), 59
)
self.assertEqual(
- sum(item["disposition"] == "unsupported" for item in targets), 30
+ sum(item["disposition"] == "unsupported" for item in targets), 31
)
- self.assertEqual(len(capability.provisional_precision_targets()), 5)
+ self.assertEqual(len(capability.provisional_precision_targets()), 4)
keys = {
(
item["precision_profile"],
From f997ac2071c4ed2932102ec0658477b675f46a0f Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 03:26:14 +0800
Subject: [PATCH 094/210] docs(collectivex): align pre-v1 status
---
experimental/CollectiveX/README.md | 44 ++++++++++++++++--------------
1 file changed, 23 insertions(+), 21 deletions(-)
diff --git a/experimental/CollectiveX/README.md b/experimental/CollectiveX/README.md
index fc86761f6c..01ceee1510 100644
--- a/experimental/CollectiveX/README.md
+++ b/experimental/CollectiveX/README.md
@@ -13,9 +13,8 @@ combine, and paired roundtrip latency across EP libraries and accelerator system
> rankings, recommendations, or regression baselines.
> Development status: the V1 precision, point-level publication, branch-only delivery, and frontend
-> contracts are implemented. H100, H200, GB200, and GB300 are terminal; B200 and B300 EP8 are
-> terminal, and B300 EP16 has terminal native outcomes. Twelve B200/MI325X/MI355X precision cells
-> remain provisional, so V1 counts and digests are not frozen.
+> contracts are implemented. Every NVIDIA and MI355X precision cell is terminal. Four MI325X
+> precision cells remain provisional, so V1 counts and digests are not frozen.
## Implemented Pre-V1 Execution Profile
@@ -41,12 +40,13 @@ logfmt10, and `rtol=0.08, atol=0.04` for native FP8 direct-cast combine. These a
publication thresholds, not estimates of codec error. FP8 direct-cast evidence also counts
saturation on the exact transformed native combine input; any saturated value fails the point.
-The BF16 planning baseline covers H100, H200, B200, B300, GB200, GB300, MI325X, and MI355X. It
-requests
-608 cases / 1,600 token points: 338 runnable cases / 868 points, emitted as 54 executable workflow
-shards/allocation cells, plus 270 explicit unsupported cases / 732 points. `sweep_matrix.py`
-materializes every token ladder and rejects missing, stale, malformed, or altered shard controls.
-Shards are emitted round-robin by SKU so the bounded GHA matrix uses every runner pool early.
+The current unfrozen planning graph covers H100, H200, B200, B300, GB200, GB300, MI325X, and
+MI355X. It requests 656 cases / 1,648 token points: 379 runnable cases / 916 points, emitted as 54
+executable workflow shards/allocation cells, plus 277 explicit unsupported cases / 732 points. The
+final normal-precision cases remain probe-gated and these counts will change when the four MI325X
+cells become terminal. `sweep_matrix.py` materializes every token ladder and rejects missing, stale,
+malformed, or altered shard controls. Shards are emitted round-robin by SKU so the bounded GHA
+matrix uses every runner pool early.
| Systems | EP8 | EP16 |
|---|---|---|
@@ -80,19 +80,21 @@ unless the realized LSA team covers the full EP world. x86 EP16 scale-out cases
hybrid path with GIN, two logical scale-out domains represented by two physical RDMA ranks, and eight
scale-up ranks per domain; GB EP16 remains MNNVL scale-up and therefore uses LSA. The isolated build
records the API, source, loaded libraries, generated JIT source, executable SASS, and raw CUBIN
-diagnostics. The current H100 runner pool is explicitly unsupported for V2 because NCCL 2.30.4
+diagnostics. H100 is explicitly unsupported for V2 because NCCL 2.30.4
reports that its EP8 communicator lacks Device API symmetric-memory support; re-enabling that pool
-requires an all-rank CUDA P2P/LSA-capable runtime. Other NVIDIA SKUs remain unvalidated until their
-GPU outcomes pass the native correctness and publication gates.
-
-H100 EP16 is planned unsupported on the current runner pool because allocated compute nodes expose
-no active RDMA device. EP8 remains in scope and derives its private stage beside the existing shared
-container directory because the runner account home is not mounted on compute nodes.
-
-Axes not implemented in this baseline include cached-layout `[cl]`, runtime-visible `[rv]`, FP8,
-quantized combine,
-extra routing distributions, activation profiles, uneven allocation, placement permutations, model
-envelopes, and scaling studies.
+requires an all-rank CUDA P2P/LSA-capable runtime. Every other NVIDIA precision cell now records an
+exact native supported or unsupported outcome from the pinned runtime.
+
+H100 EP16 is supported on the healthy runner subset. The private overlay excludes pods whose network
+namespaces lack the host RDMA devices, and every allocation must validate the complete pinned RoCE
+profile before image import. EP8 remains in scope and derives its private stage beside the existing
+shared container directory because the runner account home is not mounted on compute nodes. B300
+EP16 is terminal unsupported for V1 publication because its functional fallback HCAs are not the
+GPU-adjacent product fabric; this is an operational topology decision, not a library limitation.
+
+Axes not implemented in this baseline include cached-layout `[cl]`, runtime-visible `[rv]`, precision
+formats outside the exact native allowlist, extra routing distributions, activation profiles, uneven
+allocation, placement permutations, model envelopes, and scaling studies.
## Workflow And Artifacts
From ecc4747a2e7c4477fce343ca8b5a4ef3994334ca Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 09:21:00 +0800
Subject: [PATCH 095/210] feat(collectivex): freeze seven-sku v1 matrix
---
.github/workflows/collectivex-sweep.yml | 8 +-
experimental/CollectiveX/README.md | 25 ++---
experimental/CollectiveX/capability.py | 18 ----
experimental/CollectiveX/configs/suites.yaml | 15 ++-
experimental/CollectiveX/docs/methodology.md | 29 +++---
experimental/CollectiveX/identity.py | 18 ----
experimental/CollectiveX/publisher.py | 6 +-
.../schemas/public-dataset-v1.schema.json | 5 +-
.../schemas/raw-case-v1.schema.json | 5 +-
.../schemas/terminal-outcome-v1.schema.json | 5 +-
experimental/CollectiveX/sweep_matrix.py | 5 +
.../tests/test_precision_scheduling.py | 96 +++++++++++--------
.../CollectiveX/tests/test_publisher.py | 2 +-
.../tests/test_qualification_planning.py | 15 +++
.../tests/test_sampling_contract.py | 10 +-
15 files changed, 130 insertions(+), 132 deletions(-)
diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml
index 134dc629f5..9d94838263 100644
--- a/.github/workflows/collectivex-sweep.yml
+++ b/.github/workflows/collectivex-sweep.yml
@@ -25,7 +25,7 @@ on:
type: string
default: all
only_sku:
- description: Restrict to one GHA runner pool (h100-dgxc|h200-dgxc|b300|b200-dgxc|gb200|gb300|mi325x|mi355x); blank = all
+ description: Restrict to one V1 GHA runner pool (h100-dgxc|h200-dgxc|b300|b200-dgxc|gb200|gb300|mi355x); blank = all
type: string
default: ''
probe_id:
@@ -215,7 +215,7 @@ jobs:
- name: Create V1 release marker
if: ${{ inputs.operation == 'sweep' && inputs.release_tag == 'v1' }}
env:
- EXPECTED_MATRIX_SHA256: f1ca85f9689922b90edd5767b9ff2a902f6b896f32f68b2ca086dde3fd2157d0
+ EXPECTED_MATRIX_SHA256: c6988c5e81239ace699541322a88a37bfd80819d8bc1a2446f928665cd3ebba0
RUN_ID: ${{ github.run_id }}
RUN_ATTEMPT: ${{ github.run_attempt }}
SOURCE_SHA: ${{ github.sha }}
@@ -242,6 +242,10 @@ jobs:
execution_plan_sha256 = sweep_matrix.qualification_execution_plan_sha256(
json.loads(matrix), qualification_index
)
+ if execution_plan_sha256 != sweep_matrix.CANONICAL_V1_EXECUTION_PLAN_SHA256.get(
+ qualification_index
+ ):
+ raise SystemExit("V1 release tag requires the locked execution plan")
marker = {
"execution_plan_sha256": execution_plan_sha256,
"format": "collectivex.release-tag.v1",
diff --git a/experimental/CollectiveX/README.md b/experimental/CollectiveX/README.md
index 01ceee1510..4144539d42 100644
--- a/experimental/CollectiveX/README.md
+++ b/experimental/CollectiveX/README.md
@@ -12,9 +12,9 @@ combine, and paired roundtrip latency across EP libraries and accelerator system
> Publication hold: historical schema 3-5 data is diagnostic. No current dataset is approved for
> rankings, recommendations, or regression baselines.
-> Development status: the V1 precision, point-level publication, branch-only delivery, and frontend
-> contracts are implemented. Every NVIDIA and MI355X precision cell is terminal. Four MI325X
-> precision cells remain provisional, so V1 counts and digests are not frozen.
+> Development status: the seven-SKU V1 matrix, precision, point-level publication, branch-only
+> delivery, and frontend contracts are frozen and ready for qualification. MI325X is deferred to a
+> later version because its intended two-node runner pool is unavailable.
## Implemented Pre-V1 Execution Profile
@@ -40,18 +40,16 @@ logfmt10, and `rtol=0.08, atol=0.04` for native FP8 direct-cast combine. These a
publication thresholds, not estimates of codec error. FP8 direct-cast evidence also counts
saturation on the exact transformed native combine input; any saturated value fails the point.
-The current unfrozen planning graph covers H100, H200, B200, B300, GB200, GB300, MI325X, and
-MI355X. It requests 656 cases / 1,648 token points: 379 runnable cases / 916 points, emitted as 54
-executable workflow shards/allocation cells, plus 277 explicit unsupported cases / 732 points. The
-final normal-precision cases remain probe-gated and these counts will change when the four MI325X
-cells become terminal. `sweep_matrix.py` materializes every token ladder and rejects missing, stale,
-malformed, or altered shard controls. Shards are emitted round-robin by SKU so the bounded GHA
-matrix uses every runner pool early.
+The frozen V1 graph covers H100, H200, B200, B300, GB200, GB300, and MI355X. It requests 664 cases /
+1,532 token points: 393 runnable cases / 898 points, emitted as 50 executable workflow
+shards/allocation cells, plus 271 explicit unsupported cases / 634 points. `sweep_matrix.py`
+materializes every token ladder and rejects missing, stale, malformed, or altered shard controls.
+Shards are emitted round-robin by SKU so the bounded GHA matrix uses every runner pool early.
| Systems | EP8 | EP16 |
|---|---|---|
| H100/H200/B200/B300 | 1x8 NVLink, scale-up | 2x8 NVLink + RDMA, scale-out |
-| MI325X/MI355X | 1x8 XGMI, scale-up | 2x8 XGMI + RDMA, scale-out |
+| MI355X | 1x8 XGMI, scale-up | 2x8 XGMI + RDMA, scale-out |
| GB200/GB300 | 2x4 MNNVL, scale-up | 4x4 MNNVL, scale-up |
Physical host count does not determine scope: both GB topologies stay inside one 72-GPU MNNVL
@@ -64,7 +62,7 @@ scale-up domain.
| DeepEP Hybrid | Pinned `HybridEPBuffer`: x86 EP16 multi-domain RDMA/DOCA; GB EP8/EP16 in one MNNVL communication domain |
| UCCL | Pinned 0.1.1 wheel and wrapper with normal and native low-latency APIs on Hopper; Blackwell is explicitly unsupported |
| NCCL/RCCL A2A | Portable rank-deduplicated payload plus expert/routing-metadata reference |
-| MoRI | EP8 uses MI325X AsyncLL or MI355X IntraNode; EP16 pins InterNodeV1 over 2x8 XGMI + RDMA |
+| MoRI | MI355X EP8 uses IntraNode; EP16 pins InterNodeV1 over 2x8 XGMI + RDMA |
FlashInfer is outside v1 because its exercised EP path failed intermittently at runtime. It is not
misreported as a platform capability limitation and can return after a stable pinned path is proven.
@@ -141,6 +139,9 @@ temporary copy before allocation. Required JSON fields are:
| `gb300` | `partition`, `account`, `squash_dir`, `enroot_cache_path` |
| `mi325x`, `mi355x` | `partition`, `squash_dir`, `stage_dir` |
+The MI325X launcher/configuration path is retained for future versions but is not referenced by any
+V1 suite, capability target, workflow shard, or publication cohort.
+
Every selected non-MNNVL EP16 placement additionally requires `socket_ifname` and `rdma_devices`
for its operator-approved fabric; optional
`ib_gid_index`, `rdma_service_level`, and `rdma_traffic_class` are also allowlisted. Service level
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
index b70c5d24b6..7a77e54c0b 100644
--- a/experimental/CollectiveX/capability.py
+++ b/experimental/CollectiveX/capability.py
@@ -241,12 +241,6 @@ def _precision_rule(
basis="mori-gfx950-normal-prequantized-ocp-e4m3fn-block128-f32-scale",
),
),
- _NORMAL_E4M3FNUZ_PROFILE: (
- _precision_rule(
- backend="mori", skus=("mi325x",), ep_degrees=(8, 16), mode="normal",
- basis="mori-gfx942-normal-prequantized-e4m3fnuz-block128-f32-scale",
- ),
- ),
_LL_FP8_PROFILE: (
_precision_rule(
backend="deepep", skus=_NVIDIA_SKUS, ep_degrees=(8, 16),
@@ -295,18 +289,6 @@ def _precision_rule(
basis="mori-gfx950-ep8-intranode-e4m3fn-dispatch-and-direct-cast-combine",
),
),
- _MORI_E4M3FNUZ_DIRECT_PROFILE: (
- _precision_rule(
- backend="mori", skus=("mi325x",), ep_degrees=(8,), mode="normal",
- basis="mori-gfx942-ep8-asyncll-e4m3fnuz-direct-cast-combine",
- ),
- ),
- _MORI_E4M3FNUZ_BOTH_PROFILE: (
- _precision_rule(
- backend="mori", skus=("mi325x",), ep_degrees=(8,), mode="normal",
- basis="mori-gfx942-ep8-asyncll-e4m3fnuz-dispatch-and-direct-cast-combine",
- ),
- ),
}
PRECISION_CELL_OVERRIDES: dict[tuple[str, str, str, int, str], dict[str, str]] = {}
diff --git a/experimental/CollectiveX/configs/suites.yaml b/experimental/CollectiveX/configs/suites.yaml
index 27d368e166..b6c6d7cb43 100644
--- a/experimental/CollectiveX/configs/suites.yaml
+++ b/experimental/CollectiveX/configs/suites.yaml
@@ -5,7 +5,7 @@ suites:
ep-core-v1:
mode: normal
workloads: [deepseek-v3-v1]
- platforms: [h100-dgxc, h200-dgxc, b300, b200-dgxc, gb300, gb200, mi355x, mi325x]
+ platforms: [h100-dgxc, h200-dgxc, b300, b200-dgxc, gb300, gb200, mi355x]
ep_degrees: [8, 16]
routings: [uniform]
phases: [decode, prefill]
@@ -15,7 +15,7 @@ suites:
ep-routing-v1:
mode: normal
workloads: [deepseek-v3-v1]
- platforms: [h100-dgxc, h200-dgxc, b300, b200-dgxc, gb300, gb200, mi355x, mi325x]
+ platforms: [h100-dgxc, h200-dgxc, b300, b200-dgxc, gb300, gb200, mi355x]
ep_degrees: [8, 16]
routings: [zipf]
eplb: [false, true]
@@ -28,7 +28,7 @@ suites:
mode: low-latency
backends: [deepep, uccl]
workloads: [deepseek-v3-v1]
- platforms: [h100-dgxc, h200-dgxc, b300, b200-dgxc, gb300, gb200, mi355x, mi325x]
+ platforms: [h100-dgxc, h200-dgxc, b300, b200-dgxc, gb300, gb200, mi355x]
ep_degrees: [8, 16]
routings: [uniform]
phases: [decode]
@@ -39,7 +39,7 @@ suites:
mode: normal
backends: [deepep, deepep-v2, uccl, deepep-hybrid, mori]
workloads: [deepseek-v3-v1]
- platforms: [h100-dgxc, h200-dgxc, b300, b200-dgxc, gb300, gb200, mi355x, mi325x]
+ platforms: [h100-dgxc, h200-dgxc, b300, b200-dgxc, gb300, gb200, mi355x]
ep_degrees: [8, 16]
routings: [uniform]
phases: [decode, prefill]
@@ -47,19 +47,16 @@ suites:
token_points_prefill: [512]
precision_profiles:
- d-fp8-e4m3fn-b128-f32-prequantized.c-bf16
- - d-fp8-e4m3fnuz-b128-f32-prequantized.c-bf16
- d-bf16.c-fp8-e4m3fn-direct-cast-noscale
- d-fp8-e4m3fn-b128-f32-prequantized.c-fp8-e4m3fn-direct-cast-noscale
- - d-bf16.c-fp8-e4m3fnuz-direct-cast-noscale
- - d-fp8-e4m3fnuz-b128-f32-prequantized.c-fp8-e4m3fnuz-direct-cast-noscale
- provisional: true
+ provisional: false
required_publication: comparable-experimental
ep-precision-low-latency-v1:
mode: low-latency
backends: [deepep, uccl]
workloads: [deepseek-v3-v1]
- platforms: [h100-dgxc, h200-dgxc, b300, b200-dgxc, gb300, gb200, mi355x, mi325x]
+ platforms: [h100-dgxc, h200-dgxc, b300, b200-dgxc, gb300, gb200, mi355x]
ep_degrees: [8, 16]
routings: [uniform]
phases: [decode]
diff --git a/experimental/CollectiveX/docs/methodology.md b/experimental/CollectiveX/docs/methodology.md
index 46c9e4b21e..7b047a879d 100644
--- a/experimental/CollectiveX/docs/methodology.md
+++ b/experimental/CollectiveX/docs/methodology.md
@@ -1,4 +1,4 @@
-# CollectiveX EP Pre-V1 Baseline
+# CollectiveX EP V1 Methodology
@@ -6,12 +6,10 @@
-This document describes the implemented BF16 baseline. It is not yet the V1 qualification contract.
-Before any V1-tagged run, this English document must match the implemented precision, measurement,
-publication, and frontend contracts; counts and digests remain unfrozen. Chinese documentation
-synchronization is explicitly deferred for the V1 implementation phase.
-H100, H200, B200, B300, GB200, GB300, and MI355X precision cells are terminal. Four MI325X
-precision cells remain provisional.
+This document is the frozen V1 qualification contract. No V1 dataset is approved until three
+complete first-attempt executions pass publication and frontend audit. Chinese documentation
+synchronization is explicitly deferred for the V1 implementation phase. MI325X is deferred to a
+later version because its intended two-node runner pool is unavailable.
## Product Boundary
@@ -38,13 +36,22 @@ normal `layout-and-dispatch-v1` or low-latency `expert-packed-weighted-combine-v
- `ep-precision-normal-v1`: nonbaseline native dispatch/combine profiles at decode T=128 and prefill
T=512; BF16/BF16 endpoint controls are referenced rather than duplicated.
- `ep-precision-low-latency-v1`: nonbaseline native low-latency profiles at decode T=128.
-- Current planning matrix: 656 requested cases / 1,648 token points; 379 runnable cases / 916
- points in 54 executable workflow shards/allocation cells; 277 unsupported cases / 732 points.
+- Frozen V1 matrix: 664 requested cases / 1,532 token points; 393 runnable cases / 898 points in 50
+ executable workflow shards/allocation cells; 271 unsupported cases / 634 points.
+- Canonical matrix SHA-256: `c6988c5e81239ace699541322a88a37bfd80819d8bc1a2446f928665cd3ebba0`.
+- Case/disposition catalog SHA-256: `98b0c5e85c223a2edcbf9ea580a48bea115294c83691650bb25cb85904ed8be4`.
+- Qualification execution-plan SHA-256 values for repeats 1/2/3 are respectively
+ `dd35bee6909a85761e069ebe83444a6f982eb571a4e61ad604a845ea7e4d10dc`,
+ `fbc7262b3399225493f6c025c3bd2ceca52e90762bda1b6f6002683037094caf`, and
+ `5edc837858df2d7498229db30ec0d364f5b6cd7ca703790a8c83aabb09e6aff4`.
+- The backend-produced frontend catalog is 601,970 bytes with SHA-256
+ `86ac7bb3f9f6310385db52f6f77554d705985bd99e387ecb5fedd627c11994d7`, below the 1 MiB fixture
+ budget and the 32 MiB publication limit.
| Systems | EP8 | EP16 |
|---|---|---|
| H100/H200/B200/B300 | 1x8 NVLink, scale-up | 2x8 NVLink + RDMA, scale-out |
-| MI325X/MI355X | 1x8 XGMI, scale-up | 2x8 XGMI + RDMA, scale-out |
+| MI355X | 1x8 XGMI, scale-up | 2x8 XGMI + RDMA, scale-out |
| GB200/GB300 | 2x4 MNNVL, scale-up | 4x4 MNNVL, scale-up |
Physical host count does not define scope. Both GB cells remain inside one 72-GPU MNNVL scale-up
@@ -71,7 +78,7 @@ unsupported for V1 publication because its currently functional fallback HCAs ar
GPU-adjacent product fabric; this is an operational topology decision, not a library limitation.
FlashInfer is excluded from v1 after repeatable intermittent execution failures; those failures are
not converted into planned-unsupported coverage.
-MoRI EP8 uses MI325X AsyncLL or MI355X IntraNode in normal mode. EP16 uses pinned InterNodeV1 over
+MoRI EP8 uses MI355X IntraNode in normal mode. EP16 uses pinned InterNodeV1 over
2x8 XGMI + RDMA with 96 blocks, 64 RDMA blocks, 8 warps, one QP per PE, and external input. MoRI's
AsyncLL transport is not the genuine low-latency suite contract and is never labeled as such.
diff --git a/experimental/CollectiveX/identity.py b/experimental/CollectiveX/identity.py
index 9a0b300dbd..6851dbe3c8 100644
--- a/experimental/CollectiveX/identity.py
+++ b/experimental/CollectiveX/identity.py
@@ -69,11 +69,8 @@
V1_CONTROL_PRECISION_PROFILE = "d-bf16.c-bf16"
V1_NORMAL_PRECISION_PROFILE_IDS = (
"d-fp8-e4m3fn-b128-f32-prequantized.c-bf16",
- "d-fp8-e4m3fnuz-b128-f32-prequantized.c-bf16",
"d-bf16.c-fp8-e4m3fn-direct-cast-noscale",
"d-fp8-e4m3fn-b128-f32-prequantized.c-fp8-e4m3fn-direct-cast-noscale",
- "d-bf16.c-fp8-e4m3fnuz-direct-cast-noscale",
- "d-fp8-e4m3fnuz-b128-f32-prequantized.c-fp8-e4m3fnuz-direct-cast-noscale",
)
V1_LOW_LATENCY_PRECISION_PROFILE_IDS = (
"d-fp8-e4m3fn-b128-f32-fused.c-bf16",
@@ -205,11 +202,6 @@ def _communication_axis(
"dispatch": _FP8_E4M3FN_PREQUANTIZED_DISPATCH,
"combine": _BF16_AXIS,
},
- "d-fp8-e4m3fnuz-b128-f32-prequantized.c-bf16": {
- "modes": ["normal"],
- "dispatch": _FP8_E4M3FNUZ_PREQUANTIZED_DISPATCH,
- "combine": _BF16_AXIS,
- },
"d-fp8-e4m3fn-b128-f32-fused.c-bf16": {
"modes": ["low-latency"],
"dispatch": _FP8_E4M3FN_FUSED_DISPATCH,
@@ -235,16 +227,6 @@ def _communication_axis(
"dispatch": _FP8_E4M3FN_PREQUANTIZED_DISPATCH,
"combine": _FP8_E4M3FN_DIRECT_CAST_COMBINE,
},
- "d-bf16.c-fp8-e4m3fnuz-direct-cast-noscale": {
- "modes": ["normal"],
- "dispatch": _BF16_AXIS,
- "combine": _FP8_E4M3FNUZ_DIRECT_CAST_COMBINE,
- },
- "d-fp8-e4m3fnuz-b128-f32-prequantized.c-fp8-e4m3fnuz-direct-cast-noscale": {
- "modes": ["normal"],
- "dispatch": _FP8_E4M3FNUZ_PREQUANTIZED_DISPATCH,
- "combine": _FP8_E4M3FNUZ_DIRECT_CAST_COMBINE,
- },
}
V1_COMBINE_ORACLE_TOLERANCES = {
diff --git a/experimental/CollectiveX/publisher.py b/experimental/CollectiveX/publisher.py
index 2e57b76be6..fcf462b42d 100644
--- a/experimental/CollectiveX/publisher.py
+++ b/experimental/CollectiveX/publisher.py
@@ -44,12 +44,12 @@
PRECISION_COHORT_KINDS = (
"dispatch-precision", "combine-precision", "precision-pair",
)
-REQUIRED_PROMOTION_COHORT_COUNTS = {"library": 76, "system": 12, "routing": 116}
+REQUIRED_PROMOTION_COHORT_COUNTS = {"library": 70, "system": 12, "routing": 62}
CANONICAL_FULL_V1_MATRIX_SHA256 = (
- "f1ca85f9689922b90edd5767b9ff2a902f6b896f32f68b2ca086dde3fd2157d0"
+ "c6988c5e81239ace699541322a88a37bfd80819d8bc1a2446f928665cd3ebba0"
)
CANONICAL_FULL_V1_CASE_CATALOG_SHA256 = (
- "8e262178f770b0cdde12b7ec71604afd87251fa55685d4594f29717153ad6bbd"
+ "98b0c5e85c223a2edcbf9ea580a48bea115294c83691650bb25cb85904ed8be4"
)
P50_STABILITY_LIMIT = 1.10
P99_STABILITY_LIMIT = 1.25
diff --git a/experimental/CollectiveX/schemas/public-dataset-v1.schema.json b/experimental/CollectiveX/schemas/public-dataset-v1.schema.json
index cf5a5eed42..0ad660a37d 100644
--- a/experimental/CollectiveX/schemas/public-dataset-v1.schema.json
+++ b/experimental/CollectiveX/schemas/public-dataset-v1.schema.json
@@ -104,14 +104,11 @@
"enum": [
"d-bf16.c-bf16",
"d-fp8-e4m3fn-b128-f32-prequantized.c-bf16",
- "d-fp8-e4m3fnuz-b128-f32-prequantized.c-bf16",
"d-fp8-e4m3fn-b128-f32-fused.c-bf16",
"d-bf16.c-logfmt10-dynamic64",
"d-fp8-e4m3fn-b128-f32-fused.c-logfmt10-dynamic64",
"d-bf16.c-fp8-e4m3fn-direct-cast-noscale",
- "d-fp8-e4m3fn-b128-f32-prequantized.c-fp8-e4m3fn-direct-cast-noscale",
- "d-bf16.c-fp8-e4m3fnuz-direct-cast-noscale",
- "d-fp8-e4m3fnuz-b128-f32-prequantized.c-fp8-e4m3fnuz-direct-cast-noscale"
+ "d-fp8-e4m3fn-b128-f32-prequantized.c-fp8-e4m3fn-direct-cast-noscale"
]
},
"byteAccounting": {
diff --git a/experimental/CollectiveX/schemas/raw-case-v1.schema.json b/experimental/CollectiveX/schemas/raw-case-v1.schema.json
index 6b12afc21b..87bf577621 100644
--- a/experimental/CollectiveX/schemas/raw-case-v1.schema.json
+++ b/experimental/CollectiveX/schemas/raw-case-v1.schema.json
@@ -81,14 +81,11 @@
"enum": [
"d-bf16.c-bf16",
"d-fp8-e4m3fn-b128-f32-prequantized.c-bf16",
- "d-fp8-e4m3fnuz-b128-f32-prequantized.c-bf16",
"d-fp8-e4m3fn-b128-f32-fused.c-bf16",
"d-bf16.c-logfmt10-dynamic64",
"d-fp8-e4m3fn-b128-f32-fused.c-logfmt10-dynamic64",
"d-bf16.c-fp8-e4m3fn-direct-cast-noscale",
- "d-fp8-e4m3fn-b128-f32-prequantized.c-fp8-e4m3fn-direct-cast-noscale",
- "d-bf16.c-fp8-e4m3fnuz-direct-cast-noscale",
- "d-fp8-e4m3fnuz-b128-f32-prequantized.c-fp8-e4m3fnuz-direct-cast-noscale"
+ "d-fp8-e4m3fn-b128-f32-prequantized.c-fp8-e4m3fn-direct-cast-noscale"
]
},
"communication_axis": {
diff --git a/experimental/CollectiveX/schemas/terminal-outcome-v1.schema.json b/experimental/CollectiveX/schemas/terminal-outcome-v1.schema.json
index 30d9bcebd1..fb97ed2a94 100644
--- a/experimental/CollectiveX/schemas/terminal-outcome-v1.schema.json
+++ b/experimental/CollectiveX/schemas/terminal-outcome-v1.schema.json
@@ -144,14 +144,11 @@
"enum":[
"d-bf16.c-bf16",
"d-fp8-e4m3fn-b128-f32-prequantized.c-bf16",
- "d-fp8-e4m3fnuz-b128-f32-prequantized.c-bf16",
"d-fp8-e4m3fn-b128-f32-fused.c-bf16",
"d-bf16.c-logfmt10-dynamic64",
"d-fp8-e4m3fn-b128-f32-fused.c-logfmt10-dynamic64",
"d-bf16.c-fp8-e4m3fn-direct-cast-noscale",
- "d-fp8-e4m3fn-b128-f32-prequantized.c-fp8-e4m3fn-direct-cast-noscale",
- "d-bf16.c-fp8-e4m3fnuz-direct-cast-noscale",
- "d-fp8-e4m3fnuz-b128-f32-prequantized.c-fp8-e4m3fnuz-direct-cast-noscale"
+ "d-fp8-e4m3fn-b128-f32-prequantized.c-fp8-e4m3fn-direct-cast-noscale"
]
},
"caseId": {"type": "string","pattern": "^cxcase-v1-[0-9a-f]{64}$"},
diff --git a/experimental/CollectiveX/sweep_matrix.py b/experimental/CollectiveX/sweep_matrix.py
index 2cebf3be6e..91aed7bd4b 100644
--- a/experimental/CollectiveX/sweep_matrix.py
+++ b/experimental/CollectiveX/sweep_matrix.py
@@ -104,6 +104,11 @@
"scale_out_transport", "transport", "topology_class",
)
QUALIFICATION_INDICES = range(1, 4)
+CANONICAL_V1_EXECUTION_PLAN_SHA256 = {
+ 1: "dd35bee6909a85761e069ebe83444a6f982eb571a4e61ad604a845ea7e4d10dc",
+ 2: "fbc7262b3399225493f6c025c3bd2ceca52e90762bda1b6f6002683037094caf",
+ 3: "5edc837858df2d7498229db30ec0d364f5b6cd7ca703790a8c83aabb09e6aff4",
+}
class MatrixError(ValueError):
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
index d0527f9c8f..10fa4b0fb2 100644
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -20,6 +20,19 @@
class PrecisionSchedulingTest(unittest.TestCase):
+ @staticmethod
+ def _synthetic_provisional_targets() -> list[dict[str, object]]:
+ targets = []
+ for ep in (8, 16):
+ target = next(
+ item for item in capability.precision_targets()
+ if item["sku"] == "h100-dgxc" and item["backend"] == "deepep"
+ and item["ep"] == ep and item["mode"] == "normal"
+ ).copy()
+ target.update(disposition="provisional", basis="test-provisional-cell")
+ targets.append(target)
+ return targets
+
def test_precision_probe_inventory_is_exact_and_non_mutating(self) -> None:
before = copy.deepcopy(capability.PRECISION_CAPABILITIES)
targets = probe_precision.provisional_targets()
@@ -28,7 +41,7 @@ def test_precision_probe_inventory_is_exact_and_non_mutating(self) -> None:
item["precision_profile"],
)
self.assertEqual(targets, sorted(capability.provisional_precision_targets(), key=key))
- self.assertEqual(len(targets), 4)
+ self.assertEqual(targets, [])
self.assertEqual(capability.PRECISION_CAPABILITIES, before)
self.assertEqual(
len({
@@ -40,11 +53,12 @@ def test_precision_probe_inventory_is_exact_and_non_mutating(self) -> None:
)
def test_precision_probe_selects_only_one_exact_provisional_cell(self) -> None:
- target = probe_precision.provisional_targets()[0]
- selected = probe_precision.select_target(
- backend=target["backend"], sku=target["sku"], ep=target["ep"],
- mode=target["mode"], precision_profile=target["precision_profile"],
- )
+ target = self._synthetic_provisional_targets()[0]
+ with mock.patch.object(probe_precision, "provisional_targets", return_value=[target]):
+ selected = probe_precision.select_target(
+ backend=target["backend"], sku=target["sku"], ep=target["ep"],
+ mode=target["mode"], precision_profile=target["precision_profile"],
+ )
self.assertEqual(selected, target)
with self.assertRaisesRegex(probe_precision.ProbeError, "target-not-provisional"):
probe_precision.select_target(
@@ -53,46 +67,52 @@ def test_precision_probe_selects_only_one_exact_provisional_cell(self) -> None:
)
def test_precision_probe_workflow_plan_binds_exact_controls(self) -> None:
- plan = probe_precision.workflow_plan(backend="mori", only_sku="mi325x")
+ targets = self._synthetic_provisional_targets()
+ with mock.patch.object(probe_precision, "provisional_targets", return_value=targets):
+ plan = probe_precision.workflow_plan(backend="deepep", only_sku="h100-dgxc")
self.assertTrue(plan["include"])
self.assertTrue(all(
- row["backend"] == "mori" and row["sku"] == "mi325x"
+ row["backend"] == "deepep" and row["sku"] == "h100-dgxc"
for row in plan["include"]
))
row = plan["include"][0]
- exact = probe_precision.workflow_plan(probe_id=row["id"])
+ with mock.patch.object(probe_precision, "provisional_targets", return_value=targets):
+ exact = probe_precision.workflow_plan(probe_id=row["id"])
self.assertEqual(exact["include"], [row])
with self.assertRaisesRegex(ValueError, "ID is invalid"):
probe_precision.workflow_plan(probe_id="probe-invalid")
with self.assertRaisesRegex(ValueError, "select no provisional"):
probe_precision.workflow_plan(probe_id="probe-00000000000000000000")
- control = probe_precision.extract_control(
- plan, probe_id=row["id"], sku=row["sku"], backend=row["backend"],
- nodes=row["nodes"],
- )
- self.assertEqual(
- probe_precision.validate_control(
- control, sku=row["sku"], backend=row["backend"], nodes=row["nodes"],
- ),
- control,
- )
- with self.assertRaisesRegex(ValueError, "workflow matrix"):
+ with mock.patch.object(probe_precision, "provisional_targets", return_value=targets):
+ control = probe_precision.extract_control(
+ plan, probe_id=row["id"], sku=row["sku"], backend=row["backend"],
+ nodes=row["nodes"],
+ )
+ self.assertEqual(
+ probe_precision.validate_control(
+ control, sku=row["sku"], backend=row["backend"], nodes=row["nodes"],
+ ),
+ control,
+ )
+ with mock.patch.object(probe_precision, "provisional_targets", return_value=targets), \
+ self.assertRaisesRegex(ValueError, "workflow matrix"):
probe_precision.extract_control(
plan, probe_id=row["id"], sku=row["sku"], backend=row["backend"],
nodes=row["nodes"] + 1,
)
with self.assertRaisesRegex(ValueError, "select no provisional"):
probe_precision.workflow_plan(backend="mori", only_sku="b200-dgxc")
- ep16 = probe_precision.workflow_plan(
- backend="mori", only_sku="mi325x", min_nodes=2,
- )
+ with mock.patch.object(probe_precision, "provisional_targets", return_value=targets):
+ ep16 = probe_precision.workflow_plan(
+ backend="deepep", only_sku="h100-dgxc", min_nodes=2,
+ )
self.assertTrue(ep16["include"])
self.assertEqual({row["ep"] for row in ep16["include"]}, {16})
with self.assertRaisesRegex(ValueError, "node filters"):
probe_precision.workflow_plan(min_nodes=2, max_nodes=1)
def test_precision_probe_manifest_is_sanitized_and_runtime_evidence_is_required(self) -> None:
- target = probe_precision.provisional_targets()[0]
+ target = self._synthetic_provisional_targets()[0]
topology = capability.topology_for(target["sku"], target["ep"])
self.assertIsNotNone(topology)
topology_record = probe_precision._topology_record(topology, False)
@@ -180,10 +200,6 @@ def test_precision_profiles_bind_exact_formats_and_timing_boundaries(self) -> No
self.assertEqual(fused["conversion_boundary"], "inside-dispatch-timing")
self.assertEqual(prequantized["scale_group_size"], 128)
- mi325 = identity.precision_profile(
- "d-fp8-e4m3fnuz-b128-f32-prequantized.c-bf16"
- )["dispatch"]
- self.assertEqual(mi325["communication_format"], "fp8-e4m3fnuz")
logfmt = identity.precision_profile("d-bf16.c-logfmt10-dynamic64")["combine"]
self.assertEqual(
(logfmt["communication_format"], logfmt["scale_group_size"]),
@@ -225,10 +241,10 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
self.assertTrue(targets)
self.assertEqual(
{item["disposition"] for item in targets},
- {"provisional", "supported", "unsupported"},
+ {"supported", "unsupported"},
)
self.assertEqual(
- len(targets) - len(capability.provisional_precision_targets()), 90
+ len(targets), 90
)
self.assertEqual(
sum(item["disposition"] == "supported" for item in targets), 59
@@ -236,7 +252,7 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
self.assertEqual(
sum(item["disposition"] == "unsupported" for item in targets), 31
)
- self.assertEqual(len(capability.provisional_precision_targets()), 4)
+ self.assertEqual(len(capability.provisional_precision_targets()), 0)
keys = {
(
item["precision_profile"],
@@ -251,7 +267,6 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
normal = "d-fp8-e4m3fn-b128-f32-prequantized.c-bf16"
direct = "d-bf16.c-fp8-e4m3fn-direct-cast-noscale"
- fnuz_direct = "d-bf16.c-fp8-e4m3fnuz-direct-cast-noscale"
low_latency = "d-bf16.c-logfmt10-dynamic64"
cases = (
(("h200-dgxc", "deepep-v2", 8, "normal", normal), "unsupported"),
@@ -259,7 +274,6 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
(("h200-dgxc", "nccl-ep", 8, "normal", normal), "not-applicable"),
(("mi355x", "mori", 8, "normal", direct), "unsupported"),
(("mi355x", "mori", 16, "normal", direct), "not-applicable"),
- (("mi325x", "mori", 8, "normal", fnuz_direct), "provisional"),
(("h200-dgxc", "deepep", 8, "low-latency", low_latency), "supported"),
(("h100-dgxc", "deepep", 16, "low-latency", low_latency), "supported"),
(("h200-dgxc", "deepep-hybrid", 8, "low-latency", low_latency),
@@ -333,7 +347,7 @@ def test_split_suites_track_provisional_state_and_do_not_duplicate_bf16(self) ->
),
("low-latency", ["decode"], [128]),
)
- self.assertTrue(normal["provisional"])
+ self.assertFalse(normal["provisional"])
self.assertFalse(low_latency["provisional"])
self.assertEqual(
normal["required_publication"], "comparable-experimental"
@@ -353,14 +367,13 @@ def test_split_suites_track_provisional_state_and_do_not_duplicate_bf16(self) ->
for item in matrix["requested_cases"]
if "precision_profile" in item["case"]
}
- self.assertEqual(scheduled_profiles, set(low_latency["precision_profiles"]))
- with self.assertRaisesRegex(SystemExit, "provisional precision suites"):
- sweep_matrix.resolve_matrix(
- suites="ep-precision-normal-v1", backends="all"
- )
+ self.assertEqual(
+ scheduled_profiles,
+ set(normal["precision_profiles"]) | set(low_latency["precision_profiles"]),
+ )
stale = copy.deepcopy(suites)
- stale["suites"]["ep-precision-normal-v1"]["provisional"] = False
+ stale["suites"]["ep-precision-normal-v1"]["provisional"] = True
with self.assertRaisesRegex(SystemExit, "must track unresolved"):
sweep_matrix.validate_config_documents(stale, workloads)
@@ -394,8 +407,7 @@ def load_config(name: str) -> dict[str, object]:
for target in capability.precision_targets()
)
with mock.patch.object(capability, "PRECISION_CAPABILITIES", promoted):
- with self.assertRaisesRegex(SystemExit, "must track unresolved"):
- sweep_matrix.validate_config_documents(suites, workloads)
+ sweep_matrix.validate_config_documents(suites, workloads)
with mock.patch.object(sweep_matrix, "_load", side_effect=load_config):
matrix = sweep_matrix.validate_matrix_document(
sweep_matrix.resolve_matrix(suites=suite_names, backends="all")
diff --git a/experimental/CollectiveX/tests/test_publisher.py b/experimental/CollectiveX/tests/test_publisher.py
index b4fd0ffb30..bb08318a9e 100644
--- a/experimental/CollectiveX/tests/test_publisher.py
+++ b/experimental/CollectiveX/tests/test_publisher.py
@@ -2024,7 +2024,7 @@ def test_workflow_matrix_and_catalog_digests_do_not_drift(self) -> None:
if item["disposition"] == "unsupported"
),
),
- (54, 608, 338, 270, 1600, 868, 732),
+ (50, 664, 393, 271, 1532, 898, 634),
)
library: dict[tuple, set[str]] = {}
system: dict[tuple, set[str]] = {}
diff --git a/experimental/CollectiveX/tests/test_qualification_planning.py b/experimental/CollectiveX/tests/test_qualification_planning.py
index 9cb1fd1e87..66c42ad37e 100644
--- a/experimental/CollectiveX/tests/test_qualification_planning.py
+++ b/experimental/CollectiveX/tests/test_qualification_planning.py
@@ -171,6 +171,21 @@ def test_matrix_execution_plan_digest_is_repeat_specific_and_stable(self) -> Non
):
sweep_matrix.qualification_execution_plan_sha256(tampered, 1)
+ def test_full_v1_execution_plans_match_the_frozen_digests(self) -> None:
+ matrix = sweep_matrix.validate_matrix_document(
+ sweep_matrix.resolve_matrix(suites="all", backends="all", max_cases=128)
+ )
+ self.assertEqual(
+ [
+ sweep_matrix.qualification_execution_plan_sha256(matrix, index)
+ for index in (1, 2, 3)
+ ],
+ [
+ sweep_matrix.CANONICAL_V1_EXECUTION_PLAN_SHA256[index]
+ for index in (1, 2, 3)
+ ],
+ )
+
def test_frontend_catalog_covers_every_requested_case_and_point(self) -> None:
catalog = sweep_matrix.frontend_catalog(self.matrix)
self.assertEqual(catalog["format"], "collectivex.frontend-catalog.v1")
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 00e821f84f..59434f94a7 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -209,7 +209,7 @@ def test_matrix_cases_and_shards_are_identity_bound(self) -> None:
sum(len(item["case"]["ladder"].split()) for item in runnable_cases),
sum(len(item["case"]["ladder"].split()) for item in unsupported_cases),
),
- (54, 656, 379, 277, 1648, 916, 732),
+ (50, 664, 393, 271, 1532, 898, 634),
)
b300_ep16 = [
item for item in unsupported_cases
@@ -244,7 +244,7 @@ def test_matrix_cases_and_shards_are_identity_bound(self) -> None:
expected_topologies[sku, 16] = (
4, 4, 72, "scale-up", "mnnvl", None, "mnnvl", topology_class,
)
- for sku in ("mi325x", "mi355x"):
+ for sku in ("mi355x",):
expected_topologies[sku, 8] = (
1, 8, 8, "scale-up", "xgmi", None, "xgmi", f"{sku}-xgmi",
)
@@ -268,19 +268,21 @@ def test_matrix_cases_and_shards_are_identity_bound(self) -> None:
{
(sku, ep): tuple(topology[field] for field in topology_fields)
for sku, platform in capability.PLATFORMS.items()
+ if sku != "mi325x"
for ep, topology in platform["topologies"].items()
},
expected_topologies,
)
+ self.assertIsNotNone(capability.topology_for("mi325x", 8))
self.assertEqual(
Counter(shard["n"] for shard in matrix["include"]),
- Counter({6: 39, 7: 1, 8: 1, 10: 13}),
+ Counter({6: 30, 7: 1, 8: 5, 10: 1, 12: 13}),
)
ll_cases = [
item for item in matrix["requested_cases"]
if item["case"]["mode"] == "low-latency"
]
- self.assertEqual(len(ll_cases), 80)
+ self.assertEqual(len(ll_cases), 76)
self.assertTrue(all(
item["case"]["backend"] in {"deepep", "uccl"}
and item["case"]["phase"] == "decode"
From e09570223777605020a6b09a953b48cdd1516e56 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 09:34:44 +0800
Subject: [PATCH 096/210] fix(collectivex): register precision unsupported
outcomes
---
experimental/CollectiveX/contracts.py | 1 +
.../tests/test_qualification_planning.py | 38 +++++++++++++++++++
2 files changed, 39 insertions(+)
diff --git a/experimental/CollectiveX/contracts.py b/experimental/CollectiveX/contracts.py
index b196a4fcbf..ca357cac18 100644
--- a/experimental/CollectiveX/contracts.py
+++ b/experimental/CollectiveX/contracts.py
@@ -66,6 +66,7 @@
CAPABILITY_FAILURE_REASONS = frozenset({
"backend-platform-unsupported",
"backend-token-capacity",
+ "precision-profile-unsupported",
})
RETURN_CODE_FAILURE_MODES = {
5: "runtime-identity",
diff --git a/experimental/CollectiveX/tests/test_qualification_planning.py b/experimental/CollectiveX/tests/test_qualification_planning.py
index 66c42ad37e..516b10bd15 100644
--- a/experimental/CollectiveX/tests/test_qualification_planning.py
+++ b/experimental/CollectiveX/tests/test_qualification_planning.py
@@ -186,6 +186,44 @@ def test_full_v1_execution_plans_match_the_frozen_digests(self) -> None:
],
)
+ def test_full_v1_matrix_emits_every_unsupported_terminal_outcome(self) -> None:
+ matrix = sweep_matrix.validate_matrix_document(
+ sweep_matrix.resolve_matrix(suites="all", backends="all", max_cases=128)
+ )
+ expected = [
+ item for item in matrix["requested_cases"]
+ if item["disposition"] == "unsupported"
+ ]
+ environment = {
+ "COLLECTIVEX_ARTIFACT_NAME": "cxunsupported-123-1",
+ "COLLECTIVEX_EXECUTION_ID": "123_1_unsupported",
+ "COLLECTIVEX_SOURCE_SHA": "a" * 40,
+ "CX_QUALIFICATION_INDEX": "1",
+ "GITHUB_JOB": "setup",
+ "GITHUB_REF_NAME": "collectivex",
+ "GITHUB_REPOSITORY": "SemiAnalysisAI/InferenceX",
+ "GITHUB_RUN_ATTEMPT": "1",
+ "GITHUB_RUN_ID": "123",
+ }
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary)
+ matrix_path = root / "matrix.json"
+ matrix_path.write_bytes(_canonical(matrix))
+ with mock.patch.dict(os.environ, environment, clear=False):
+ written = sweep_matrix.emit_unsupported(matrix_path, root / "unsupported")
+ documents = [json.loads(path.read_text()) for path in written]
+
+ self.assertEqual(len(documents), len(expected))
+ self.assertEqual(len(documents), 271)
+ self.assertEqual(
+ {document["outcome"]["reason"] for document in documents},
+ {item["reason"] for item in expected},
+ )
+ self.assertEqual(
+ {document["identity"]["case_id"] for document in documents},
+ {item["case"]["case_id"] for item in expected},
+ )
+
def test_frontend_catalog_covers_every_requested_case_and_point(self) -> None:
catalog = sweep_matrix.frontend_catalog(self.matrix)
self.assertEqual(catalog["format"], "collectivex.frontend-catalog.v1")
From 58d94e3a922e876b4f28189872d199e3132023f2 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 09:38:11 +0800
Subject: [PATCH 097/210] fix(collectivex): normalize private numeric selectors
---
experimental/CollectiveX/runtime/common.sh | 20 +++++++++++++------
.../tests/test_sampling_contract.py | 16 +++++++++++++--
2 files changed, 28 insertions(+), 8 deletions(-)
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 42e9e143ab..fb26e32277 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -298,6 +298,17 @@ def valid_path(value):
and posixpath.normpath(value) == value and not IPV4.search(value)
)
+def bounded_integer(value, maximum):
+ if type(value) is int:
+ result = value
+ elif isinstance(value, str) and re.fullmatch(r"0|[1-9][0-9]{0,2}", value):
+ result = int(value)
+ else:
+ raise ValueError
+ if not 0 <= result <= maximum:
+ raise ValueError
+ return result
+
try:
path, runner, audit_required, audit_override = sys.argv[1:]
if runner not in RUNNERS or audit_required not in {"0", "1"}:
@@ -382,14 +393,11 @@ try:
if not isinstance(value, str) or not RDMA_DEVICES.fullmatch(value):
raise ValueError
elif field == "ib_gid_index":
- if type(value) is not int or not 0 <= value <= 255:
- raise ValueError
+ config[field] = bounded_integer(value, 255)
elif field == "rdma_service_level":
- if type(value) is not int or not 0 <= value <= 15:
- raise ValueError
+ config[field] = bounded_integer(value, 15)
elif field == "rdma_traffic_class":
- if type(value) is not int or not 0 <= value <= 255:
- raise ValueError
+ config[field] = bounded_integer(value, 255)
elif field.endswith(("_dir", "_path")):
if not valid_path(value):
raise ValueError
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 59434f94a7..155ce391d4 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -538,14 +538,16 @@ def test_canonical_operator_config_requires_a_private_audit_salt(self) -> None:
"h100-dgxc": {
"partition": "test", "account": "test",
"squash_dir": str(root), "stage_dir": str(root),
- "rdma_traffic_class": 104,
+ "ib_gid_index": "3", "rdma_service_level": "2",
+ "rdma_traffic_class": "104",
},
},
}
command = (
'source "$1"; export COLLECTIVEX_EXECUTION_ID="audit-config-$$"; '
"trap 'cx_cleanup_private_logs 0' EXIT; cx_load_operator_config; "
- 'test "$CX_AUDIT_SALT" = "$EXPECTED_AUDIT_SALT"'
+ 'test "$CX_AUDIT_SALT" = "$EXPECTED_AUDIT_SALT"; '
+ 'test "$CX_IB_GID_INDEX:$CX_RDMA_SERVICE_LEVEL:$CX_RDMA_TRAFFIC_CLASS" = 3:2:104'
)
def invoke(
@@ -611,6 +613,16 @@ def invoke(
self.assertNotEqual(rejected.returncode, 0)
self.assertIn("validation-missing-required-account", rejected.stderr)
+ for field, value in (
+ ("ib_gid_index", "03"),
+ ("rdma_service_level", "16"),
+ ("rdma_traffic_class", "256"),
+ ):
+ invalid_selector = copy.deepcopy(document)
+ invalid_selector["runners"]["h100-dgxc"][field] = value
+ rejected = invoke(invalid_selector, canonical=True)
+ self.assertNotEqual(rejected.returncode, 0)
+
def test_scaleout_network_profile_is_explicit_and_allowlisted(self) -> None:
command = r'''
set -euo pipefail
From 20b51e0121ff0a0c08bc1a5eb16048e3e3975447 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 17:00:50 +0800
Subject: [PATCH 098/210] fix(collectivex): preserve shard runtime identity
---
experimental/CollectiveX/runtime/common.sh | 15 ++++++++++-----
.../CollectiveX/runtime/run_in_container.sh | 1 +
.../CollectiveX/tests/test_sampling_contract.py | 9 +++++++++
3 files changed, 20 insertions(+), 5 deletions(-)
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index fb26e32277..3753dd92f3 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -2697,7 +2697,8 @@ cx_run_distributed_shard() {
build_rc=$?
if [ "$build_rc" = 0 ]; then
srun --jobid="$JOB_ID" --nodes="$NODES" --ntasks-per-node=1 --chdir=/tmp \
- --container-name="$container_name" "${container_args[@]}" \
+ --container-name="$container_name" --container-image="$SQUASH_FILE" \
+ "${container_args[@]}" \
--export="$(cx_container_exports)" bash -c "$BACKEND_PROBE" \
>"$build_log" 2>&1
build_rc=$?
@@ -2724,7 +2725,8 @@ cx_run_distributed_shard() {
set +e
timeout -k 30 "${CX_RUN_TIMEOUT:-900}" srun --jobid="$JOB_ID" --nodes="$NODES" \
--ntasks="$NGPUS" --ntasks-per-node="$GPN" --chdir=/tmp \
- --container-name="$container_name" "${container_args[@]}" \
+ --container-name="$container_name" --container-image="$SQUASH_FILE" \
+ "${container_args[@]}" \
--export="$(cx_container_exports)" \
bash -c "$WRAP" _ --backend "$backend" --sku "$sku" --ep "$ep" \
--mode "$mode" --precision-profile "$profile" --out "$out" \
@@ -2804,7 +2806,8 @@ PY
case_trials="${case_trials:-64}"
case_warmup="${case_warmup:-32}"
ep="${ep:-$NGPUS}"
- export CX_MODE="$mode" CX_CASE_ID="$case_id" CX_SUITE="$suite" CX_WORKLOAD_NAME="$workload"
+ export CX_MODE="$mode" CX_PHASE="$ph" CX_CASE_ID="$case_id" CX_SUITE="$suite"
+ export CX_WORKLOAD_NAME="$workload"
export CX_REQUIRED_PUBLICATION="$required_pub" CX_CANONICAL="$canonical" CX_EP="$ep"
export CX_PRECISION_PROFILE="$precision_profile"
export CX_ROUTING="$routing" CX_EPLB="$eplb" CX_TOKENS_LADDER="$ladder"
@@ -2839,7 +2842,8 @@ PY
workload_log="$(cx_private_log_path "workload-c$(printf '%03d' "$ci")")"
set +e
srun --jobid="$JOB_ID" --nodes=1 --ntasks=1 --chdir=/tmp \
- --container-name="$container_name" "${container_args[@]}" \
+ --container-name="$container_name" --container-image="$SQUASH_FILE" \
+ "${container_args[@]}" \
--export="$(cx_container_exports)" "${workload_args[@]}" \
"$workload_log" 2>&1
stage_rc=$?
@@ -2873,7 +2877,8 @@ PY
set +e
timeout -k 30 "${CX_RUN_TIMEOUT:-900}" srun --jobid="$JOB_ID" --nodes="$NODES" \
--ntasks="$NGPUS" --ntasks-per-node="$GPN" --chdir=/tmp \
- --container-name="$container_name" "${container_args[@]}" \
+ --container-name="$container_name" --container-image="$SQUASH_FILE" \
+ "${container_args[@]}" \
--export="$(cx_container_exports)" \
bash -c "$WRAP" _ "${ep_args[@]}" --out "$out" \
"$runtime_log" 2>&1
diff --git a/experimental/CollectiveX/runtime/run_in_container.sh b/experimental/CollectiveX/runtime/run_in_container.sh
index b48a4d3d53..6c970e10e6 100644
--- a/experimental/CollectiveX/runtime/run_in_container.sh
+++ b/experimental/CollectiveX/runtime/run_in_container.sh
@@ -1064,6 +1064,7 @@ env = {
"CX_EPLB": "1" if c.get("eplb") else "",
"CX_CASE_ID": g("case_id"), "CX_SUITE": g("suite"), "CX_WORKLOAD_NAME": g("workload"),
"CX_REQUIRED_PUBLICATION": g("required_publication"),
+ "CX_PRECISION_PROFILE": g("precision_profile"),
"CX_HIDDEN": g("hidden"), "CX_TOPK": g("topk"), "CX_EXPERTS": g("experts"),
"CX_TOKENS_LADDER": g("ladder"), "CX_CANONICAL": ("1" if c.get("canonical") else ""),
"CX_NODES": g("nodes"), "CX_GPUS_PER_NODE": g("gpus_per_node"),
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 155ce391d4..cbc8be3dc3 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -2493,6 +2493,15 @@ def test_launchers_use_private_logs_and_allowlisted_failure_stages(self) -> None
self.assertIn("cx_fail_stage execution", distributed)
self.assertIn('cx_fail_stage execution "$runtime_log"', distributed)
self.assertIn("precision probe timed out rc=%s limit=%ss", distributed)
+ self.assertIn('export CX_MODE="$mode" CX_PHASE="$ph"', distributed)
+ self.assertEqual(
+ distributed.count(
+ '--container-name="$container_name" --container-image="$SQUASH_FILE"'
+ ),
+ 5,
+ )
+ shard_runtime = runtime[runtime.index('elif [ -n "${CX_SHARD_FILE:-}" ]') :]
+ self.assertIn('"CX_PRECISION_PROFILE": g("precision_profile")', shard_runtime)
self.assertIn("rdma-port-%s=inactive", common)
self.assertIn("rdma-device-%s=missing", common)
From 72008d3f8af45f5ed449e7992c3a5320f6c07df6 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 17:08:02 +0800
Subject: [PATCH 099/210] fix(collectivex): initialize MNNVL rank wrappers
---
experimental/CollectiveX/runtime/common.sh | 2 +-
experimental/CollectiveX/tests/test_sampling_contract.py | 9 +++++++++
2 files changed, 10 insertions(+), 1 deletion(-)
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 3753dd92f3..241c907869 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -894,8 +894,8 @@ case "${SLURM_PROCID:-}:${SLURM_NTASKS:-}:${SLURM_LOCALID:-}:${SLURM_NODEID:-}"
esac
[ "$SLURM_NTASKS" = "$CX_NGPUS" ] || exit 67
[ "$SLURM_LOCALID" -lt "$CX_GPUS_PER_NODE" ] || exit 67
+. /ix/experimental/CollectiveX/runtime/common.sh || exit 68
if [ "${CX_NODES:-1}" -gt 1 ] && [ "${CX_TRANSPORT:-}" != mnnvl ]; then
- . /ix/experimental/CollectiveX/runtime/common.sh || exit 68
cx_apply_network_profile "$CX_NODES" "$CX_TRANSPORT" || exit 68
fi
cx_write_runtime_stage execution || exit 68
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index cbc8be3dc3..6b442cbf38 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -2494,6 +2494,15 @@ def test_launchers_use_private_logs_and_allowlisted_failure_stages(self) -> None
self.assertIn('cx_fail_stage execution "$runtime_log"', distributed)
self.assertIn("precision probe timed out rc=%s limit=%ss", distributed)
self.assertIn('export CX_MODE="$mode" CX_PHASE="$ph"', distributed)
+ rank_wrapper = common[
+ common.index("cx_slurm_rank_wrapper()") : common.index(
+ "cx_validate_shard_control()"
+ )
+ ]
+ self.assertLess(
+ rank_wrapper.index(". /ix/experimental/CollectiveX/runtime/common.sh"),
+ rank_wrapper.index('if [ "${CX_NODES:-1}" -gt 1 ]'),
+ )
self.assertEqual(
distributed.count(
'--container-name="$container_name" --container-image="$SQUASH_FILE"'
From 907bed60fc133cd1347b92b01362704a5004b8c5 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 17:13:33 +0800
Subject: [PATCH 100/210] fix(collectivex): validate precision oracle
tolerances
---
.../CollectiveX/schemas/raw-case-v1.schema.json | 4 ++--
.../CollectiveX/tests/test_schema_v1_contract.py | 10 ++++++++++
2 files changed, 12 insertions(+), 2 deletions(-)
diff --git a/experimental/CollectiveX/schemas/raw-case-v1.schema.json b/experimental/CollectiveX/schemas/raw-case-v1.schema.json
index 87bf577621..fb0d748534 100644
--- a/experimental/CollectiveX/schemas/raw-case-v1.schema.json
+++ b/experimental/CollectiveX/schemas/raw-case-v1.schema.json
@@ -249,7 +249,7 @@
"required": ["combine_values","counts","metadata","multiplicity","payload","source_set","weights"],
"type": "object"
},
- "atol": {"const":0.02},
+ "atol": {"enum":[0.02,0.03,0.04]},
"combine_weight_semantics": {"enum":["unweighted-rank-sum","gate-weighted-sum"]},
"contract": {"enum":["expert-specific-transform-v1","expert-assignment-transform-v1"]},
"dispatch_sha256": {"$ref":"#/$defs/nullable_sha256"},
@@ -261,7 +261,7 @@
"ordering_contract": {"minLength":1,"type":"string"},
"passed": {"type":"boolean"},
"receive_count": {"minimum":0,"type":"integer"},
- "rtol": {"const":0.05}
+ "rtol": {"enum":[0.05,0.06,0.08]}
},
"required": [
"atol",
diff --git a/experimental/CollectiveX/tests/test_schema_v1_contract.py b/experimental/CollectiveX/tests/test_schema_v1_contract.py
index a517cd5d56..ee52af7630 100644
--- a/experimental/CollectiveX/tests/test_schema_v1_contract.py
+++ b/experimental/CollectiveX/tests/test_schema_v1_contract.py
@@ -115,6 +115,16 @@ def test_precision_catalog_and_axes_are_exact_and_strict(self) -> None:
self.assertNotIn("dispatch_dtype", workload["properties"])
self.assertNotIn("combine_dtype", workload["properties"])
+ oracle = self.raw["$defs"]["oracle"]["properties"]
+ tolerances = {
+ tuple(identity.combine_oracle_tolerances(profile).values())
+ for profile in (
+ identity.precision_profile(name) for name in sorted(expected)
+ )
+ }
+ self.assertEqual(set(oracle["atol"]["enum"]), {value[0] for value in tolerances})
+ self.assertEqual(set(oracle["rtol"]["enum"]), {value[1] for value in tolerances})
+
profile = self.raw["$defs"]["case_profile"]
self.assertEqual(
profile["properties"]["activation_generator"]["const"],
From d403bf09b01c195e7267dba963c382c18b600081 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 17:20:59 +0800
Subject: [PATCH 101/210] fix(collectivex): retry unhealthy H100 allocations
---
.../launchers/launch_single-slurm.sh | 28 +++++++--
experimental/CollectiveX/runtime/common.sh | 22 ++++++-
.../tests/test_sampling_contract.py | 60 +++++++++++++++++++
3 files changed, 102 insertions(+), 8 deletions(-)
diff --git a/experimental/CollectiveX/launchers/launch_single-slurm.sh b/experimental/CollectiveX/launchers/launch_single-slurm.sh
index 734c730d38..72fa8beb37 100644
--- a/experimental/CollectiveX/launchers/launch_single-slurm.sh
+++ b/experimental/CollectiveX/launchers/launch_single-slurm.sh
@@ -105,11 +105,29 @@ allocation=(--partition="$CX_PARTITION" --nodes="$NODES" --gres=gpu:"$GPN" --exc
[ -z "${CX_ACCOUNT:-}" ] || allocation+=(--account="$CX_ACCOUNT")
[ -z "${CX_QOS:-}" ] || allocation+=(--qos="$CX_QOS")
[ -z "${CX_NODELIST:-}" ] || allocation+=(--nodelist="$CX_NODELIST")
-[ -z "${CX_EXCLUDE_NODES:-}" ] || allocation+=(--exclude="$CX_EXCLUDE_NODES")
-cx_salloc_jobid "${allocation[@]}"
-[ -n "$JOB_ID" ] || cx_die "could not resolve allocated JOB_ID from salloc"
-cx_set_failure_stage setup
-cx_validate_network_profile_on_job "$JOB_ID" "$NODES" "$CX_TRANSPORT"
+excluded_nodes="${CX_EXCLUDE_NODES:-}"
+for allocation_attempt in 1 2 3; do
+ attempt_allocation=("${allocation[@]}")
+ [ -z "$excluded_nodes" ] || attempt_allocation+=(--exclude="$excluded_nodes")
+ cx_salloc_jobid "${attempt_allocation[@]}"
+ [ -n "$JOB_ID" ] || cx_die "could not resolve allocated JOB_ID from salloc"
+ cx_set_failure_stage setup
+ if cx_validate_network_profile_on_job "$JOB_ID" "$NODES" "$CX_TRANSPORT" 0; then
+ break
+ fi
+ if [ "$RUNNER" != h100-dgxc ] || [ "$allocation_attempt" = 3 ]; then
+ cx_fail_stage setup "$(cx_private_log_path network-profile)" || true
+ cx_die "allocated nodes failed the network profile"
+ fi
+ rejected_nodes="$(cx_allocation_nodes_csv "$JOB_ID")" \
+ || cx_die "cannot identify nodes from a rejected allocation"
+ cx_log "allocated H100 nodes failed network validation; retrying elsewhere"
+ cx_cancel_job "$JOB_ID" || cx_die "cannot release a rejected allocation"
+ cx_clear_allocation_jobid || cx_die "cannot reset rejected allocation state"
+ JOB_ID=""
+ [ -z "$excluded_nodes" ] || excluded_nodes+=,
+ excluded_nodes+="$rejected_nodes"
+done
if [ "$LOCAL_IMPORT" = 1 ]; then
cx_set_failure_stage container-import
SQUASH_FILE="$(CX_ENROOT_LOCAL_IMPORT=1 cx_ensure_squash "$CX_SQUASH_DIR" "$IMAGE")"
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 241c907869..b89dda3150 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -767,7 +767,8 @@ cx_restore_exact_hca_selector() {
# node before image import or backend initialization. Selector values and node
# diagnostics stay in the runner-private log.
cx_validate_network_profile_on_job() {
- local job_id="$1" nodes="$2" transport="$3" log rc=0 scaleout=0 marker_count link_layer
+ local job_id="$1" nodes="$2" transport="$3" report_failure="${4:-1}"
+ local log rc=0 scaleout=0 marker_count link_layer
local single_node_rdma=0
if [ "$nodes" -gt 1 ] && [ "$transport" != mnnvl ]; then
scaleout=1
@@ -854,7 +855,7 @@ done
printf '[collectivex-private] rdma-link-layer=%s\n' "$profile"
BASH
if [ "$rc" != 0 ]; then
- cx_fail_stage setup "$log" || true
+ [ "$report_failure" = 0 ] || cx_fail_stage setup "$log" || true
return "$rc"
fi
link_layer="$(
@@ -864,12 +865,27 @@ BASH
marker_count="$(grep -Ec '^\[collectivex-private\] rdma-link-layer=(roce|infiniband)$' "$log")"
case "$marker_count:$link_layer" in
"$nodes":roce|"$nodes":infiniband) ;;
- *) cx_fail_stage setup "$log" || true; return 1 ;;
+ *) [ "$report_failure" = 0 ] || cx_fail_stage setup "$log" || true; return 1 ;;
esac
export CX_RDMA_LINK_LAYER="$link_layer"
cx_export_gid_index_for_link_layer "$link_layer" "$scaleout"
}
+cx_allocation_nodes_csv() {
+ local job_id="$1" nodelist node output=""
+ [[ "$job_id" =~ ^[1-9][0-9]*$ ]] || return 1
+ nodelist="$(squeue -h -j "$job_id" -o %N 2>/dev/null)" || return 1
+ [[ "$nodelist" =~ ^[][A-Za-z0-9._,-]+$ ]] || return 1
+ while IFS= read -r node; do
+ [ -n "$node" ] || continue
+ [[ "$node" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] || return 1
+ [ -z "$output" ] || output+=,
+ output+="$node"
+ done < <(scontrol show hostnames "$nodelist" 2>/dev/null)
+ [ -n "$output" ] || return 1
+ printf '%s' "$output"
+}
+
cx_resolve_slurm_rendezvous() {
local job_id="$1" master_addr master_port
[[ "$job_id" =~ ^[1-9][0-9]*$ ]] || cx_die "invalid rendezvous allocation"
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 6b442cbf38..3672d894aa 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -795,6 +795,19 @@ def test_network_profile_validation_is_private_and_all_node(self) -> None:
self.assertNotIn("privateif0", failed.stdout + failed.stderr)
self.assertNotIn("privatehca0", failed.stdout + failed.stderr)
+ quiet_failed = subprocess.run(
+ [
+ "bash", "-c",
+ command + " 0",
+ "_", str(ROOT / "runtime" / "common.sh"),
+ ],
+ text=True,
+ capture_output=True,
+ env={**environment, "SRUN_RC": "9"},
+ )
+ self.assertNotEqual(quiet_failed.returncode, 0)
+ self.assertNotIn("failure-class=", quiet_failed.stderr)
+
arguments.unlink()
subprocess.run(
[
@@ -823,6 +836,49 @@ def test_network_profile_validation_is_private_and_all_node(self) -> None:
self.assertIn("--nodes=1", arguments.read_text())
self.assertIn("--ntasks=1", arguments.read_text())
+ def test_rejected_allocation_nodes_are_expanded_safely(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary).resolve()
+ for name, payload in (
+ ("squeue", "#!/bin/sh\nprintf '%s\\n' 'rack-[1,3]'\n"),
+ ("scontrol", "#!/bin/sh\nprintf '%s\\n' rack-1 rack-3\n"),
+ ):
+ binary = root / name
+ binary.write_text(payload)
+ binary.chmod(0o700)
+ result = subprocess.run(
+ [
+ "bash", "-c",
+ 'source "$1"; cx_allocation_nodes_csv 42',
+ "_", str(ROOT / "runtime" / "common.sh"),
+ ],
+ text=True,
+ capture_output=True,
+ env={
+ **os.environ,
+ "PATH": f"{root}:{os.environ['PATH']}",
+ "COLLECTIVEX_OPERATOR_CONFIG": "/dev/null",
+ },
+ )
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertEqual(result.stdout, "rack-1,rack-3")
+
+ (root / "squeue").write_text("#!/bin/sh\nprintf 'rack;unsafe\\n'\n")
+ (root / "squeue").chmod(0o700)
+ rejected = subprocess.run(
+ [
+ "bash", "-c",
+ 'source "$1"; cx_allocation_nodes_csv 42',
+ "_", str(ROOT / "runtime" / "common.sh"),
+ ],
+ env={
+ **os.environ,
+ "PATH": f"{root}:{os.environ['PATH']}",
+ "COLLECTIVEX_OPERATOR_CONFIG": "/dev/null",
+ },
+ )
+ self.assertNotEqual(rejected.returncode, 0)
+
def test_allocation_preflight_proves_shared_write_visibility(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary).resolve()
@@ -2513,6 +2569,10 @@ def test_launchers_use_private_logs_and_allowlisted_failure_stages(self) -> None
self.assertIn('"CX_PRECISION_PROFILE": g("precision_profile")', shard_runtime)
self.assertIn("rdma-port-%s=inactive", common)
self.assertIn("rdma-device-%s=missing", common)
+ single_slurm = (ROOT / "launchers" / "launch_single-slurm.sh").read_text()
+ self.assertIn("for allocation_attempt in 1 2 3", single_slurm)
+ self.assertIn('if [ "$RUNNER" != h100-dgxc ]', single_slurm)
+ self.assertIn('rejected_nodes="$(cx_allocation_nodes_csv "$JOB_ID")"', single_slurm)
def test_case_failure_diagnostic_precedes_normal_srun_footer(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
From 435f883e6bf3b2c17c94b1f4f9a9ca629ec47daa Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 17:22:46 +0800
Subject: [PATCH 102/210] fix(collectivex): accept opaque logfmt scale evidence
---
experimental/CollectiveX/contracts.py | 3 ++-
.../CollectiveX/tests/test_sampling_contract.py | 17 +++++++++++++++++
2 files changed, 19 insertions(+), 1 deletion(-)
diff --git a/experimental/CollectiveX/contracts.py b/experimental/CollectiveX/contracts.py
index ca357cac18..f0f7090e88 100644
--- a/experimental/CollectiveX/contracts.py
+++ b/experimental/CollectiveX/contracts.py
@@ -1539,7 +1539,8 @@ def _validate_precision_evidence(
for field in ("dequantized_semantics", "encoded_payload_valid", "passed"):
if type(axis[field]) is not bool:
raise ContractError(f"{axis_path}.{field} must be boolean")
- expects_scales = communication_precision[direction]["scale_dtype"] is not None
+ scale_dtype = communication_precision[direction]["scale_dtype"]
+ expects_scales = scale_dtype not in (None, "implicit-logfmt10")
for field in ("scales_finite", "scales_positive"):
if expects_scales:
if type(axis[field]) is not bool:
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 3672d894aa..9a6b10e603 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -2234,6 +2234,23 @@ def test_precision_evidence_rejects_direct_cast_saturation(self) -> None:
saturated, profile_id, communication_precision, "precision"
)
+ logfmt_profile_id = "d-bf16.c-logfmt10-dynamic64"
+ logfmt_precision = identity.precision_profile(logfmt_profile_id)
+ logfmt_evidence = copy.deepcopy(evidence)
+ logfmt_evidence["profile_id"] = logfmt_profile_id
+ contracts._validate_precision_evidence(
+ logfmt_evidence, logfmt_profile_id, logfmt_precision, "precision"
+ )
+ invented_scales = copy.deepcopy(logfmt_evidence)
+ invented_scales["combine"].update({
+ "scales_finite": True,
+ "scales_positive": True,
+ })
+ with self.assertRaisesRegex(contracts.ContractError, "must be null"):
+ contracts._validate_precision_evidence(
+ invented_scales, logfmt_profile_id, logfmt_precision, "precision"
+ )
+
def test_oracle_stability_canonicalizes_native_receive_order(self) -> None:
source = (HERE / "ep_harness.py").read_text()
begin = source.index("canonical_order = torch.argsort")
From 45a420019e74ebad56ff9490f4683d05ac1d30de Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 17:24:18 +0800
Subject: [PATCH 103/210] fix(collectivex): preserve allocation retry logs
---
.../CollectiveX/launchers/launch_single-slurm.sh | 2 ++
experimental/CollectiveX/runtime/common.sh | 9 +++++++--
experimental/CollectiveX/tests/test_sampling_contract.py | 2 ++
3 files changed, 11 insertions(+), 2 deletions(-)
diff --git a/experimental/CollectiveX/launchers/launch_single-slurm.sh b/experimental/CollectiveX/launchers/launch_single-slurm.sh
index 72fa8beb37..57a06a6419 100644
--- a/experimental/CollectiveX/launchers/launch_single-slurm.sh
+++ b/experimental/CollectiveX/launchers/launch_single-slurm.sh
@@ -109,6 +109,7 @@ excluded_nodes="${CX_EXCLUDE_NODES:-}"
for allocation_attempt in 1 2 3; do
attempt_allocation=("${allocation[@]}")
[ -z "$excluded_nodes" ] || attempt_allocation+=(--exclude="$excluded_nodes")
+ export CX_SALLOC_ATTEMPT="$allocation_attempt"
cx_salloc_jobid "${attempt_allocation[@]}"
[ -n "$JOB_ID" ] || cx_die "could not resolve allocated JOB_ID from salloc"
cx_set_failure_stage setup
@@ -128,6 +129,7 @@ for allocation_attempt in 1 2 3; do
[ -z "$excluded_nodes" ] || excluded_nodes+=,
excluded_nodes+="$rejected_nodes"
done
+unset CX_SALLOC_ATTEMPT
if [ "$LOCAL_IMPORT" = 1 ]; then
cx_set_failure_stage container-import
SQUASH_FILE="$(CX_ENROOT_LOCAL_IMPORT=1 cx_ensure_squash "$CX_SQUASH_DIR" "$IMAGE")"
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index b89dda3150..f8e671bf36 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -1076,8 +1076,13 @@ cx_reconcile_salloc_jobid() {
# Allocate via salloc's stable grant message and assign JOB_ID in this shell.
# Raw scheduler output remains in the bounded private execution log.
cx_salloc_jobid() {
- local log job_id job_name argument salloc_rc=0
- log="$(cx_private_log_path scheduler-allocation)"
+ local log_label=scheduler-allocation log job_id job_name argument salloc_rc=0
+ case "${CX_SALLOC_ATTEMPT:-1}" in
+ 1) ;;
+ 2|3) log_label+="-a${CX_SALLOC_ATTEMPT}" ;;
+ *) return 1 ;;
+ esac
+ log="$(cx_private_log_path "$log_label")"
for argument in "$@"; do
case "$argument" in
--job-name|--job-name=*|-J|-J*)
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 9a6b10e603..875f7de3db 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -2590,6 +2590,8 @@ def test_launchers_use_private_logs_and_allowlisted_failure_stages(self) -> None
self.assertIn("for allocation_attempt in 1 2 3", single_slurm)
self.assertIn('if [ "$RUNNER" != h100-dgxc ]', single_slurm)
self.assertIn('rejected_nodes="$(cx_allocation_nodes_csv "$JOB_ID")"', single_slurm)
+ self.assertIn('export CX_SALLOC_ATTEMPT="$allocation_attempt"', single_slurm)
+ self.assertIn('log_label+="-a${CX_SALLOC_ATTEMPT}"', common)
def test_case_failure_diagnostic_precedes_normal_srun_footer(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
From c48a29f7f703429d47cf43b85ac3e28c449fef8c Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 17:27:54 +0800
Subject: [PATCH 104/210] fix(collectivex): isolate network retry logs
---
.../CollectiveX/launchers/launch_single-slurm.sh | 5 +++--
experimental/CollectiveX/runtime/common.sh | 10 ++++++++--
.../CollectiveX/tests/test_sampling_contract.py | 2 ++
3 files changed, 13 insertions(+), 4 deletions(-)
diff --git a/experimental/CollectiveX/launchers/launch_single-slurm.sh b/experimental/CollectiveX/launchers/launch_single-slurm.sh
index 57a06a6419..8515482014 100644
--- a/experimental/CollectiveX/launchers/launch_single-slurm.sh
+++ b/experimental/CollectiveX/launchers/launch_single-slurm.sh
@@ -110,6 +110,7 @@ for allocation_attempt in 1 2 3; do
attempt_allocation=("${allocation[@]}")
[ -z "$excluded_nodes" ] || attempt_allocation+=(--exclude="$excluded_nodes")
export CX_SALLOC_ATTEMPT="$allocation_attempt"
+ export CX_NETWORK_VALIDATION_ATTEMPT="$allocation_attempt"
cx_salloc_jobid "${attempt_allocation[@]}"
[ -n "$JOB_ID" ] || cx_die "could not resolve allocated JOB_ID from salloc"
cx_set_failure_stage setup
@@ -117,7 +118,7 @@ for allocation_attempt in 1 2 3; do
break
fi
if [ "$RUNNER" != h100-dgxc ] || [ "$allocation_attempt" = 3 ]; then
- cx_fail_stage setup "$(cx_private_log_path network-profile)" || true
+ cx_fail_stage setup "$CX_NETWORK_PROFILE_LOG" || true
cx_die "allocated nodes failed the network profile"
fi
rejected_nodes="$(cx_allocation_nodes_csv "$JOB_ID")" \
@@ -129,7 +130,7 @@ for allocation_attempt in 1 2 3; do
[ -z "$excluded_nodes" ] || excluded_nodes+=,
excluded_nodes+="$rejected_nodes"
done
-unset CX_SALLOC_ATTEMPT
+unset CX_SALLOC_ATTEMPT CX_NETWORK_VALIDATION_ATTEMPT
if [ "$LOCAL_IMPORT" = 1 ]; then
cx_set_failure_stage container-import
SQUASH_FILE="$(CX_ENROOT_LOCAL_IMPORT=1 cx_ensure_squash "$CX_SQUASH_DIR" "$IMAGE")"
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index f8e671bf36..c9ea577e73 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -768,7 +768,7 @@ cx_restore_exact_hca_selector() {
# diagnostics stay in the runner-private log.
cx_validate_network_profile_on_job() {
local job_id="$1" nodes="$2" transport="$3" report_failure="${4:-1}"
- local log rc=0 scaleout=0 marker_count link_layer
+ local log_label=network-profile log rc=0 scaleout=0 marker_count link_layer
local single_node_rdma=0
if [ "$nodes" -gt 1 ] && [ "$transport" != mnnvl ]; then
scaleout=1
@@ -781,7 +781,13 @@ cx_validate_network_profile_on_job() {
|| return 1
[ -n "${CX_RDMA_DEVICES:-}" ] || return 1
[ "$scaleout" = 0 ] || [ -n "${CX_SOCKET_IFNAME:-}" ] || return 1
- log="$(cx_private_log_path network-profile)" || return 1
+ case "${CX_NETWORK_VALIDATION_ATTEMPT:-1}" in
+ 1) ;;
+ 2|3) log_label+="-a${CX_NETWORK_VALIDATION_ATTEMPT}" ;;
+ *) return 1 ;;
+ esac
+ log="$(cx_private_log_path "$log_label")" || return 1
+ CX_NETWORK_PROFILE_LOG="$log"
srun --jobid="$job_id" --nodes="$nodes" --ntasks="$nodes" --ntasks-per-node=1 \
--chdir=/tmp --input=all \
--export="$(cx_host_exports),CX_SOCKET_IFNAME,CX_RDMA_DEVICES,CX_IB_GID_INDEX" \
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 875f7de3db..346c93187c 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -2591,7 +2591,9 @@ def test_launchers_use_private_logs_and_allowlisted_failure_stages(self) -> None
self.assertIn('if [ "$RUNNER" != h100-dgxc ]', single_slurm)
self.assertIn('rejected_nodes="$(cx_allocation_nodes_csv "$JOB_ID")"', single_slurm)
self.assertIn('export CX_SALLOC_ATTEMPT="$allocation_attempt"', single_slurm)
+ self.assertIn('export CX_NETWORK_VALIDATION_ATTEMPT="$allocation_attempt"', single_slurm)
self.assertIn('log_label+="-a${CX_SALLOC_ATTEMPT}"', common)
+ self.assertIn('log_label+="-a${CX_NETWORK_VALIDATION_ATTEMPT}"', common)
def test_case_failure_diagnostic_precedes_normal_srun_footer(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
From fa671a0649472e62b5d20c6f9ebd76a413a87cee Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 17:39:22 +0800
Subject: [PATCH 105/210] feat(collectivex): add MI300X qualification
---
.github/workflows/collectivex-sweep.yml | 17 ++++++++--
experimental/CollectiveX/capability.py | 24 +++++++++++++
experimental/CollectiveX/configs/suites.yaml | 15 ++++----
experimental/CollectiveX/identity.py | 18 ++++++++++
.../CollectiveX/launchers/launch_mi-amds.sh | 10 +++---
experimental/CollectiveX/runtime/common.sh | 16 +++++----
.../schemas/public-dataset-v1.schema.json | 5 ++-
.../schemas/raw-case-v1.schema.json | 5 ++-
.../schemas/terminal-outcome-v1.schema.json | 5 ++-
experimental/CollectiveX/tests/ep_mori.py | 2 +-
.../CollectiveX/tests/probe_precision.py | 2 +-
.../tests/test_deepep_v2_contract.py | 3 ++
.../tests/test_precision_scheduling.py | 34 ++++++++++++++-----
.../tests/test_qualification_planning.py | 15 +++++++-
.../tests/test_sampling_contract.py | 8 ++---
15 files changed, 139 insertions(+), 40 deletions(-)
diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml
index 9d94838263..ceeb0851a0 100644
--- a/.github/workflows/collectivex-sweep.yml
+++ b/.github/workflows/collectivex-sweep.yml
@@ -25,7 +25,7 @@ on:
type: string
default: all
only_sku:
- description: Restrict to one V1 GHA runner pool (h100-dgxc|h200-dgxc|b300|b200-dgxc|gb200|gb300|mi355x); blank = all
+ description: Restrict to one V1 GHA runner pool (h100-dgxc|h200-dgxc|b300|b200-dgxc|gb200|gb300|mi300x|mi355x); blank = all
type: string
default: ''
probe_id:
@@ -465,6 +465,7 @@ jobs:
COLLECTIVEX_NETWORK_CONFIG_CONTENT: ${{ secrets.COLLECTIVEX_NETWORK_CONFIG_V1 }}
COLLECTIVEX_H100_CONFIG_CONTENT: ${{ secrets.COLLECTIVEX_H100_CONFIG_V1 }}
COLLECTIVEX_B200_CONFIG_CONTENT: ${{ secrets.COLLECTIVEX_B200_CONFIG_V1 }}
+ COLLECTIVEX_MI300_CONFIG_CONTENT: ${{ secrets.COLLECTIVEX_MI300_CONFIG_V1 }}
COLLECTIVEX_MI355_CONFIG_CONTENT: ${{ secrets.COLLECTIVEX_MI355_CONFIG_V1 }}
COLLECTIVEX_OPERATOR_AUDIT_SALT: ${{ secrets.COLLECTIVEX_AUDIT_SALT_V1 }}
COLLECTIVEX_OPERATOR_CONFIG_REQUIRED: '1'
@@ -495,7 +496,8 @@ jobs:
base = load("COLLECTIVEX_OPERATOR_CONFIG_CONTENT")
for overlay_name in (
"COLLECTIVEX_NETWORK_CONFIG_CONTENT", "COLLECTIVEX_H100_CONFIG_CONTENT",
- "COLLECTIVEX_B200_CONFIG_CONTENT", "COLLECTIVEX_MI355_CONFIG_CONTENT",
+ "COLLECTIVEX_B200_CONFIG_CONTENT", "COLLECTIVEX_MI300_CONFIG_CONTENT",
+ "COLLECTIVEX_MI355_CONFIG_CONTENT",
):
overlay_raw = os.environ.get(overlay_name, "")
if not overlay_raw:
@@ -520,9 +522,17 @@ jobs:
elif overlay_name == "COLLECTIVEX_MI355_CONFIG_CONTENT":
if set(overlay["runners"]) != {"mi355x"}:
raise ValueError("invalid MI355 overlay runner")
+ elif overlay_name == "COLLECTIVEX_MI300_CONFIG_CONTENT":
+ if set(overlay["runners"]) != {"mi300x"}:
+ raise ValueError("invalid MI300 overlay runner")
+ private_fields.update({"partition", "squash_dir", "exclude_nodes", "nodelist", "lock_dir"})
for runner, fields in overlay["runners"].items():
- if runner not in base.get("runners", {}) or not isinstance(fields, dict) or not fields:
+ if not isinstance(fields, dict) or not fields:
raise ValueError("invalid network overlay runner")
+ if runner not in base.get("runners", {}):
+ if overlay_name != "COLLECTIVEX_MI300_CONFIG_CONTENT":
+ raise ValueError("invalid network overlay runner")
+ base.setdefault("runners", {})[runner] = {}
if set(fields) - private_fields:
raise ValueError("private overlay contains a disallowed field")
base["runners"][runner].update(fields)
@@ -537,6 +547,7 @@ jobs:
unset COLLECTIVEX_OPERATOR_CONFIG_CONTENT COLLECTIVEX_NETWORK_CONFIG_CONTENT
unset COLLECTIVEX_H100_CONFIG_CONTENT
unset COLLECTIVEX_B200_CONFIG_CONTENT
+ unset COLLECTIVEX_MI300_CONFIG_CONTENT COLLECTIVEX_MI355_CONFIG_CONTENT
unset COLLECTIVEX_OPERATOR_CONFIG_REQUIRED
export COLLECTIVEX_OPERATOR_CONFIG="$operator_config"
export COLLECTIVEX_OPERATOR_CONFIG_EPHEMERAL=1
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
index 7a77e54c0b..b6e3ef3ae0 100644
--- a/experimental/CollectiveX/capability.py
+++ b/experimental/CollectiveX/capability.py
@@ -21,6 +21,7 @@
"b300": {"schedulable": True, "basis": "pinned-pr605-pr630-sm103-maps-sm100f"},
"gb300": {"schedulable": True, "basis": "pinned-pr605-pr630-sm103-maps-sm100f"},
"mi325x": {"schedulable": False, "basis": "nvidia-only"},
+ "mi300x": {"schedulable": False, "basis": "nvidia-only"},
"mi355x": {"schedulable": False, "basis": "nvidia-only"},
}
@@ -130,6 +131,11 @@ def _platform(
gpus_per_node=8, scale_up_domain=8, scale_up_transport="xgmi",
launcher="mi-amds",
),
+ "mi300x": _platform(
+ vendor="amd", arch="gfx942", machine="amd64", product="mi300x",
+ gpus_per_node=8, scale_up_domain=8, scale_up_transport="xgmi",
+ launcher="mi-amds",
+ ),
"mi355x": _platform(
vendor="amd", arch="gfx950", machine="amd64", product="mi355x",
gpus_per_node=8, scale_up_domain=8, scale_up_transport="xgmi",
@@ -289,6 +295,24 @@ def _precision_rule(
basis="mori-gfx950-ep8-intranode-e4m3fn-dispatch-and-direct-cast-combine",
),
),
+ _NORMAL_E4M3FNUZ_PROFILE: (
+ _precision_rule(
+ backend="mori", skus=("mi300x",), ep_degrees=(8, 16), mode="normal",
+ basis="mori-gfx942-normal-prequantized-ocp-e4m3fnuz-block128-f32-scale",
+ ),
+ ),
+ _MORI_E4M3FNUZ_DIRECT_PROFILE: (
+ _precision_rule(
+ backend="mori", skus=("mi300x",), ep_degrees=(8,), mode="normal",
+ basis="mori-gfx942-ep8-asyncll-e4m3fnuz-direct-cast-combine",
+ ),
+ ),
+ _MORI_E4M3FNUZ_BOTH_PROFILE: (
+ _precision_rule(
+ backend="mori", skus=("mi300x",), ep_degrees=(8,), mode="normal",
+ basis="mori-gfx942-ep8-asyncll-e4m3fnuz-dispatch-and-direct-cast-combine",
+ ),
+ ),
}
PRECISION_CELL_OVERRIDES: dict[tuple[str, str, str, int, str], dict[str, str]] = {}
diff --git a/experimental/CollectiveX/configs/suites.yaml b/experimental/CollectiveX/configs/suites.yaml
index b6c6d7cb43..281bbe93c5 100644
--- a/experimental/CollectiveX/configs/suites.yaml
+++ b/experimental/CollectiveX/configs/suites.yaml
@@ -5,7 +5,7 @@ suites:
ep-core-v1:
mode: normal
workloads: [deepseek-v3-v1]
- platforms: [h100-dgxc, h200-dgxc, b300, b200-dgxc, gb300, gb200, mi355x]
+ platforms: [h100-dgxc, h200-dgxc, b300, b200-dgxc, gb300, gb200, mi300x, mi355x]
ep_degrees: [8, 16]
routings: [uniform]
phases: [decode, prefill]
@@ -15,7 +15,7 @@ suites:
ep-routing-v1:
mode: normal
workloads: [deepseek-v3-v1]
- platforms: [h100-dgxc, h200-dgxc, b300, b200-dgxc, gb300, gb200, mi355x]
+ platforms: [h100-dgxc, h200-dgxc, b300, b200-dgxc, gb300, gb200, mi300x, mi355x]
ep_degrees: [8, 16]
routings: [zipf]
eplb: [false, true]
@@ -28,7 +28,7 @@ suites:
mode: low-latency
backends: [deepep, uccl]
workloads: [deepseek-v3-v1]
- platforms: [h100-dgxc, h200-dgxc, b300, b200-dgxc, gb300, gb200, mi355x]
+ platforms: [h100-dgxc, h200-dgxc, b300, b200-dgxc, gb300, gb200, mi300x, mi355x]
ep_degrees: [8, 16]
routings: [uniform]
phases: [decode]
@@ -39,7 +39,7 @@ suites:
mode: normal
backends: [deepep, deepep-v2, uccl, deepep-hybrid, mori]
workloads: [deepseek-v3-v1]
- platforms: [h100-dgxc, h200-dgxc, b300, b200-dgxc, gb300, gb200, mi355x]
+ platforms: [h100-dgxc, h200-dgxc, b300, b200-dgxc, gb300, gb200, mi300x, mi355x]
ep_degrees: [8, 16]
routings: [uniform]
phases: [decode, prefill]
@@ -49,14 +49,17 @@ suites:
- d-fp8-e4m3fn-b128-f32-prequantized.c-bf16
- d-bf16.c-fp8-e4m3fn-direct-cast-noscale
- d-fp8-e4m3fn-b128-f32-prequantized.c-fp8-e4m3fn-direct-cast-noscale
- provisional: false
+ - d-fp8-e4m3fnuz-b128-f32-prequantized.c-bf16
+ - d-bf16.c-fp8-e4m3fnuz-direct-cast-noscale
+ - d-fp8-e4m3fnuz-b128-f32-prequantized.c-fp8-e4m3fnuz-direct-cast-noscale
+ provisional: true
required_publication: comparable-experimental
ep-precision-low-latency-v1:
mode: low-latency
backends: [deepep, uccl]
workloads: [deepseek-v3-v1]
- platforms: [h100-dgxc, h200-dgxc, b300, b200-dgxc, gb300, gb200, mi355x]
+ platforms: [h100-dgxc, h200-dgxc, b300, b200-dgxc, gb300, gb200, mi300x, mi355x]
ep_degrees: [8, 16]
routings: [uniform]
phases: [decode]
diff --git a/experimental/CollectiveX/identity.py b/experimental/CollectiveX/identity.py
index 6851dbe3c8..d860180105 100644
--- a/experimental/CollectiveX/identity.py
+++ b/experimental/CollectiveX/identity.py
@@ -71,6 +71,9 @@
"d-fp8-e4m3fn-b128-f32-prequantized.c-bf16",
"d-bf16.c-fp8-e4m3fn-direct-cast-noscale",
"d-fp8-e4m3fn-b128-f32-prequantized.c-fp8-e4m3fn-direct-cast-noscale",
+ "d-fp8-e4m3fnuz-b128-f32-prequantized.c-bf16",
+ "d-bf16.c-fp8-e4m3fnuz-direct-cast-noscale",
+ "d-fp8-e4m3fnuz-b128-f32-prequantized.c-fp8-e4m3fnuz-direct-cast-noscale",
)
V1_LOW_LATENCY_PRECISION_PROFILE_IDS = (
"d-fp8-e4m3fn-b128-f32-fused.c-bf16",
@@ -227,6 +230,21 @@ def _communication_axis(
"dispatch": _FP8_E4M3FN_PREQUANTIZED_DISPATCH,
"combine": _FP8_E4M3FN_DIRECT_CAST_COMBINE,
},
+ "d-fp8-e4m3fnuz-b128-f32-prequantized.c-bf16": {
+ "modes": ["normal"],
+ "dispatch": _FP8_E4M3FNUZ_PREQUANTIZED_DISPATCH,
+ "combine": _BF16_AXIS,
+ },
+ "d-bf16.c-fp8-e4m3fnuz-direct-cast-noscale": {
+ "modes": ["normal"],
+ "dispatch": _BF16_AXIS,
+ "combine": _FP8_E4M3FNUZ_DIRECT_CAST_COMBINE,
+ },
+ "d-fp8-e4m3fnuz-b128-f32-prequantized.c-fp8-e4m3fnuz-direct-cast-noscale": {
+ "modes": ["normal"],
+ "dispatch": _FP8_E4M3FNUZ_PREQUANTIZED_DISPATCH,
+ "combine": _FP8_E4M3FNUZ_DIRECT_CAST_COMBINE,
+ },
}
V1_COMBINE_ORACLE_TOLERANCES = {
diff --git a/experimental/CollectiveX/launchers/launch_mi-amds.sh b/experimental/CollectiveX/launchers/launch_mi-amds.sh
index 162c22adf5..0ecfed4980 100644
--- a/experimental/CollectiveX/launchers/launch_mi-amds.sh
+++ b/experimental/CollectiveX/launchers/launch_mi-amds.sh
@@ -1,5 +1,5 @@
#!/usr/bin/env bash
-# CollectiveX shared MI325X/MI355X AMD Slurm launcher (one or two nodes).
+# CollectiveX shared AMD Slurm launcher (one or two nodes).
# shellcheck disable=SC2016,SC2034
set -euo pipefail
@@ -11,9 +11,9 @@ source "$HERE/../runtime/common.sh"
RUNNER="${CX_SHARD_SKU:-${CX_PUBLIC_RUNNER:-}}"
case "$RUNNER" in
- mi325x) CPUS_PER_TASK=256; DEVICE_MOUNTS=",/dev/kfd:/dev/kfd,/dev/dri:/dev/dri" ;;
+ mi300x|mi325x) CPUS_PER_TASK=256; DEVICE_MOUNTS=",/dev/kfd:/dev/kfd,/dev/dri:/dev/dri" ;;
mi355x) CPUS_PER_TASK=128; DEVICE_MOUNTS="" ;;
- *) cx_die "set CX_SHARD_SKU or CX_PUBLIC_RUNNER to mi325x or mi355x" ;;
+ *) cx_die "set CX_SHARD_SKU or CX_PUBLIC_RUNNER to a registered AMD SKU" ;;
esac
export CX_RUNNER="$RUNNER" CX_BENCH="${CX_BENCH:-mori}"
export CX_IMAGE_PLATFORM=linux/amd64
@@ -43,7 +43,7 @@ case "$CX_BENCH" in
*) cx_die "unsupported AMD EP backend: $CX_BENCH" ;;
esac
-if [ "$RUNNER" = mi325x ]; then
+if [ "$RUNNER" = mi300x ] || [ "$RUNNER" = mi325x ]; then
export MORI_DISABLE_AUTO_XGMI="${MORI_DISABLE_AUTO_XGMI:-0}"
export MORI_ENABLE_SDMA="${MORI_ENABLE_SDMA:-1}"
export MORI_APP_LOG_LEVEL="${MORI_APP_LOG_LEVEL:-info}"
@@ -55,7 +55,7 @@ fi
if [ "$CX_BENCH" = mori ]; then
if [ "$NODES" -gt 1 ]; then
export CX_MORI_KERNEL_TYPE=internode-v1
- elif [ "$RUNNER" = mi325x ]; then
+ elif [ "$RUNNER" = mi300x ] || [ "$RUNNER" = mi325x ]; then
export CX_MORI_KERNEL_TYPE="${CX_MORI_KERNEL_TYPE:-asyncll}"
else
export CX_MORI_KERNEL_TYPE="${CX_MORI_KERNEL_TYPE:-intranode}"
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index c9ea577e73..01697d5fd8 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -233,7 +233,7 @@ import sys
RUNNERS = {
"h100-dgxc", "h200-dgxc", "b200-dgxc", "b300",
- "gb200", "gb300", "mi325x", "mi355x",
+ "gb200", "gb300", "mi300x", "mi325x", "mi355x",
}
FIELDS = {
"partition": "CX_PARTITION",
@@ -262,6 +262,7 @@ REQUIRED = {
"b300": {"partition", "account", "squash_dir"},
"gb200": {"partition", "account", "storage_roots"},
"gb300": {"partition", "account", "squash_dir", "enroot_cache_path"},
+ "mi300x": {"partition", "squash_dir"},
"mi325x": {"partition", "squash_dir"},
"mi355x": {"partition", "squash_dir"},
}
@@ -274,6 +275,7 @@ ALLOWED = {
"b300": REQUIRED["b300"] | {"exclude_nodes", "stage_dir"} | NETWORK_FIELDS,
"gb200": REQUIRED["gb200"] | NETWORK_FIELDS,
"gb300": REQUIRED["gb300"] | {"stage_dir"} | NETWORK_FIELDS,
+ "mi300x": REQUIRED["mi300x"] | {"exclude_nodes", "nodelist", "stage_dir", "lock_dir"} | NETWORK_FIELDS,
"mi325x": REQUIRED["mi325x"] | {"exclude_nodes", "nodelist", "stage_dir", "lock_dir"} | NETWORK_FIELDS,
"mi355x": REQUIRED["mi355x"] | {"exclude_nodes", "nodelist", "stage_dir", "lock_dir"} | NETWORK_FIELDS,
}
@@ -1245,7 +1247,7 @@ CX_IMAGE_AMD_MORI_MI325_DIGEST="sha256:ea42375343c2ef8f73b3bdb9e1b7b435556e3ca92
CX_MORI_COMMIT_MI325="bf99bdf18fc69887a346913ca01c315c2aa9bd4c" # pragma: allowlist secret
cx_default_image() {
case "$1" in
- mi325x*) echo "$CX_IMAGE_AMD_MORI_MI325" ;;
+ mi300x*|mi325x*) echo "$CX_IMAGE_AMD_MORI_MI325" ;;
mi355x*) echo "$CX_IMAGE_AMD_MORI" ;;
b200*|gb200*|b300*|gb300*|h100*|h200*) echo "$CX_IMAGE_MULTIARCH" ;;
*) cx_die "no default image for runner prefix: $1" ;;
@@ -1958,7 +1960,7 @@ cx_lock_canonical_gha_env() {
trusted_stage_dir="$(cx_prepare_implicit_stage_base)" \
|| cx_die "canonical CollectiveX execution cannot create an isolated stage directory"
;;
- mi325x|mi355x)
+ mi300x|mi325x|mi355x)
# AMD self-hosted runners and compute nodes share the runner filesystem,
# while the image cache may be root-owned. Derive a runner-owned base
# outside _work instead of weakening stage ownership validation.
@@ -1992,7 +1994,7 @@ cx_lock_canonical_gha_env() {
CX_NCCL_HOME=/usr
CX_MASTER_PORT=29551
;;
- mi325x)
+ mi300x|mi325x)
expected_nodes="${CX_NODES:-}"; expected_gpn=8
[ "$expected_nodes" = 1 ] || [ "$expected_nodes" = 2 ] \
|| cx_die "canonical AMD execution requires one or two nodes"
@@ -2026,7 +2028,7 @@ cx_lock_canonical_gha_env() {
*) cx_die "canonical CollectiveX runner is not registered" ;;
esac
case "$runner:$trusted_lock_dir" in
- mi325x:?*|mi355x:?*) export CX_LOCK_DIR="$trusted_lock_dir" ;;
+ mi300x:?*|mi325x:?*|mi355x:?*) export CX_LOCK_DIR="$trusted_lock_dir" ;;
esac
CX_STAGE_DIR="$trusted_stage_dir"
[ -z "$trusted_qos" ] || export CX_QOS="$trusted_qos"
@@ -2048,7 +2050,7 @@ cx_lock_canonical_gha_env() {
expected_world=$((expected_nodes * expected_gpn))
CX_NGPUS="$expected_world"
CX_SEED=67
- case "$runner" in mi325x|mi355x) CX_RUN_TIMEOUT=1800 ;; *) CX_RUN_TIMEOUT=900 ;; esac
+ case "$runner" in mi300x|mi325x|mi355x) CX_RUN_TIMEOUT=1800 ;; *) CX_RUN_TIMEOUT=900 ;; esac
unset CX_PUBLIC_RUNNER CX_GB_PRODUCT CX_DRYRUN CX_TIMING CX_ALLOW_MNNVL
unset CX_ENROOT_LOCAL_IMPORT COLLECTIVEX_IMAGE COLLECTIVEX_IMAGE_DIGEST
unset COLLECTIVEX_IMAGE_DIGEST_VERIFIED COLLECTIVEX_SQUASH_SHA256
@@ -2056,7 +2058,7 @@ cx_lock_canonical_gha_env() {
case "$runner" in
h100-dgxc|h200-dgxc|b200-dgxc|b300) export CX_NCCL_HOME ;;
gb200|gb300) export CX_NCCL_HOME CX_MASTER_PORT ;;
- mi325x)
+ mi300x|mi325x)
export CX_MORI_KERNEL_TYPE MORI_COMMIT MORI_DISABLE_AUTO_XGMI MORI_ENABLE_SDMA
export MORI_APP_LOG_LEVEL MORI_SHMEM_LOG_LEVEL MORI_IO_LOG_LEVEL
;;
diff --git a/experimental/CollectiveX/schemas/public-dataset-v1.schema.json b/experimental/CollectiveX/schemas/public-dataset-v1.schema.json
index 0ad660a37d..b27d3f2dac 100644
--- a/experimental/CollectiveX/schemas/public-dataset-v1.schema.json
+++ b/experimental/CollectiveX/schemas/public-dataset-v1.schema.json
@@ -108,7 +108,10 @@
"d-bf16.c-logfmt10-dynamic64",
"d-fp8-e4m3fn-b128-f32-fused.c-logfmt10-dynamic64",
"d-bf16.c-fp8-e4m3fn-direct-cast-noscale",
- "d-fp8-e4m3fn-b128-f32-prequantized.c-fp8-e4m3fn-direct-cast-noscale"
+ "d-fp8-e4m3fn-b128-f32-prequantized.c-fp8-e4m3fn-direct-cast-noscale",
+ "d-fp8-e4m3fnuz-b128-f32-prequantized.c-bf16",
+ "d-bf16.c-fp8-e4m3fnuz-direct-cast-noscale",
+ "d-fp8-e4m3fnuz-b128-f32-prequantized.c-fp8-e4m3fnuz-direct-cast-noscale"
]
},
"byteAccounting": {
diff --git a/experimental/CollectiveX/schemas/raw-case-v1.schema.json b/experimental/CollectiveX/schemas/raw-case-v1.schema.json
index fb0d748534..f65d71ca72 100644
--- a/experimental/CollectiveX/schemas/raw-case-v1.schema.json
+++ b/experimental/CollectiveX/schemas/raw-case-v1.schema.json
@@ -85,7 +85,10 @@
"d-bf16.c-logfmt10-dynamic64",
"d-fp8-e4m3fn-b128-f32-fused.c-logfmt10-dynamic64",
"d-bf16.c-fp8-e4m3fn-direct-cast-noscale",
- "d-fp8-e4m3fn-b128-f32-prequantized.c-fp8-e4m3fn-direct-cast-noscale"
+ "d-fp8-e4m3fn-b128-f32-prequantized.c-fp8-e4m3fn-direct-cast-noscale",
+ "d-fp8-e4m3fnuz-b128-f32-prequantized.c-bf16",
+ "d-bf16.c-fp8-e4m3fnuz-direct-cast-noscale",
+ "d-fp8-e4m3fnuz-b128-f32-prequantized.c-fp8-e4m3fnuz-direct-cast-noscale"
]
},
"communication_axis": {
diff --git a/experimental/CollectiveX/schemas/terminal-outcome-v1.schema.json b/experimental/CollectiveX/schemas/terminal-outcome-v1.schema.json
index fb97ed2a94..51a190cebb 100644
--- a/experimental/CollectiveX/schemas/terminal-outcome-v1.schema.json
+++ b/experimental/CollectiveX/schemas/terminal-outcome-v1.schema.json
@@ -148,7 +148,10 @@
"d-bf16.c-logfmt10-dynamic64",
"d-fp8-e4m3fn-b128-f32-fused.c-logfmt10-dynamic64",
"d-bf16.c-fp8-e4m3fn-direct-cast-noscale",
- "d-fp8-e4m3fn-b128-f32-prequantized.c-fp8-e4m3fn-direct-cast-noscale"
+ "d-fp8-e4m3fn-b128-f32-prequantized.c-fp8-e4m3fn-direct-cast-noscale",
+ "d-fp8-e4m3fnuz-b128-f32-prequantized.c-bf16",
+ "d-bf16.c-fp8-e4m3fnuz-direct-cast-noscale",
+ "d-fp8-e4m3fnuz-b128-f32-prequantized.c-fp8-e4m3fnuz-direct-cast-noscale"
]
},
"caseId": {"type": "string","pattern": "^cxcase-v1-[0-9a-f]{64}$"},
diff --git a/experimental/CollectiveX/tests/ep_mori.py b/experimental/CollectiveX/tests/ep_mori.py
index 832a69ea28..41940d1771 100644
--- a/experimental/CollectiveX/tests/ep_mori.py
+++ b/experimental/CollectiveX/tests/ep_mori.py
@@ -71,7 +71,7 @@ def __init__(self, args, rank, world_size, local_rank, device):
"d-bf16.c-fp8-e4m3fn-direct-cast-noscale",
"d-fp8-e4m3fn-b128-f32-prequantized.c-fp8-e4m3fn-direct-cast-noscale",
})
- elif runner.startswith("mi325x"):
+ elif runner.startswith(("mi300x", "mi325x")):
fp8_format = "fp8-e4m3fnuz"
supported_profiles = {
"d-bf16.c-bf16",
diff --git a/experimental/CollectiveX/tests/probe_precision.py b/experimental/CollectiveX/tests/probe_precision.py
index 5581c20e54..f69ba158a7 100644
--- a/experimental/CollectiveX/tests/probe_precision.py
+++ b/experimental/CollectiveX/tests/probe_precision.py
@@ -650,7 +650,7 @@ def _transport_evidence(backend_name: str, backend, args) -> dict[str, Any]:
route = str(backend.kernel_generation)
expected_kernel = (
"inter-node-v1" if args.scope == "scale-out"
- else "async-ll" if args.runner == "mi325x" else "intranode"
+ else "async-ll" if args.runner in {"mi300x", "mi325x"} else "intranode"
)
fallback = route != expected_kernel
facts = [
diff --git a/experimental/CollectiveX/tests/test_deepep_v2_contract.py b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
index 280de94d3d..7c70d070e6 100644
--- a/experimental/CollectiveX/tests/test_deepep_v2_contract.py
+++ b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
@@ -109,6 +109,9 @@ def operator_config(root: Path) -> dict[str, object]:
"mi325x": {
"partition": "test", "squash_dir": path, "stage_dir": path, **network,
},
+ "mi300x": {
+ "partition": "test", "squash_dir": path, "stage_dir": path, **network,
+ },
"mi355x": {
"partition": "test", "squash_dir": path, "stage_dir": path, **network,
},
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
index 10fa4b0fb2..65b233e7e7 100644
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -41,7 +41,18 @@ def test_precision_probe_inventory_is_exact_and_non_mutating(self) -> None:
item["precision_profile"],
)
self.assertEqual(targets, sorted(capability.provisional_precision_targets(), key=key))
- self.assertEqual(targets, [])
+ self.assertEqual(
+ [
+ (item["sku"], item["backend"], item["ep"], item["precision_profile"])
+ for item in targets
+ ],
+ [
+ ("mi300x", "mori", 8, "d-bf16.c-fp8-e4m3fnuz-direct-cast-noscale"),
+ ("mi300x", "mori", 8, "d-fp8-e4m3fnuz-b128-f32-prequantized.c-bf16"),
+ ("mi300x", "mori", 8, "d-fp8-e4m3fnuz-b128-f32-prequantized.c-fp8-e4m3fnuz-direct-cast-noscale"),
+ ("mi300x", "mori", 16, "d-fp8-e4m3fnuz-b128-f32-prequantized.c-bf16"),
+ ],
+ )
self.assertEqual(capability.PRECISION_CAPABILITIES, before)
self.assertEqual(
len({
@@ -241,10 +252,10 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
self.assertTrue(targets)
self.assertEqual(
{item["disposition"] for item in targets},
- {"supported", "unsupported"},
+ {"supported", "unsupported", "provisional"},
)
self.assertEqual(
- len(targets), 90
+ len(targets), 94
)
self.assertEqual(
sum(item["disposition"] == "supported" for item in targets), 59
@@ -252,7 +263,7 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
self.assertEqual(
sum(item["disposition"] == "unsupported" for item in targets), 31
)
- self.assertEqual(len(capability.provisional_precision_targets()), 0)
+ self.assertEqual(len(capability.provisional_precision_targets()), 4)
keys = {
(
item["precision_profile"],
@@ -347,7 +358,7 @@ def test_split_suites_track_provisional_state_and_do_not_duplicate_bf16(self) ->
),
("low-latency", ["decode"], [128]),
)
- self.assertFalse(normal["provisional"])
+ self.assertTrue(normal["provisional"])
self.assertFalse(low_latency["provisional"])
self.assertEqual(
normal["required_publication"], "comparable-experimental"
@@ -367,13 +378,18 @@ def test_split_suites_track_provisional_state_and_do_not_duplicate_bf16(self) ->
for item in matrix["requested_cases"]
if "precision_profile" in item["case"]
}
+ self.assertEqual(scheduled_profiles, set(low_latency["precision_profiles"]))
self.assertEqual(
- scheduled_profiles,
- set(normal["precision_profiles"]) | set(low_latency["precision_profiles"]),
+ {item["precision_profile"] for item in capability.provisional_precision_targets()},
+ set(normal["precision_profiles"]) - {
+ "d-fp8-e4m3fn-b128-f32-prequantized.c-bf16",
+ "d-bf16.c-fp8-e4m3fn-direct-cast-noscale",
+ "d-fp8-e4m3fn-b128-f32-prequantized.c-fp8-e4m3fn-direct-cast-noscale",
+ },
)
stale = copy.deepcopy(suites)
- stale["suites"]["ep-precision-normal-v1"]["provisional"] = True
+ stale["suites"]["ep-precision-normal-v1"]["provisional"] = False
with self.assertRaisesRegex(SystemExit, "must track unresolved"):
sweep_matrix.validate_config_documents(stale, workloads)
@@ -407,7 +423,7 @@ def load_config(name: str) -> dict[str, object]:
for target in capability.precision_targets()
)
with mock.patch.object(capability, "PRECISION_CAPABILITIES", promoted):
- sweep_matrix.validate_config_documents(suites, workloads)
+ sweep_matrix.validate_config_documents(resolved_suites, workloads)
with mock.patch.object(sweep_matrix, "_load", side_effect=load_config):
matrix = sweep_matrix.validate_matrix_document(
sweep_matrix.resolve_matrix(suites=suite_names, backends="all")
diff --git a/experimental/CollectiveX/tests/test_qualification_planning.py b/experimental/CollectiveX/tests/test_qualification_planning.py
index 516b10bd15..97b698360a 100644
--- a/experimental/CollectiveX/tests/test_qualification_planning.py
+++ b/experimental/CollectiveX/tests/test_qualification_planning.py
@@ -19,6 +19,7 @@
import identity # noqa: E402
import sweep_matrix # noqa: E402
+import capability # noqa: E402
def _canonical(value: object) -> bytes:
@@ -175,6 +176,18 @@ def test_full_v1_execution_plans_match_the_frozen_digests(self) -> None:
matrix = sweep_matrix.validate_matrix_document(
sweep_matrix.resolve_matrix(suites="all", backends="all", max_cases=128)
)
+ if capability.provisional_precision_targets():
+ self.assertNotEqual(
+ [
+ sweep_matrix.qualification_execution_plan_sha256(matrix, index)
+ for index in (1, 2, 3)
+ ],
+ [
+ sweep_matrix.CANONICAL_V1_EXECUTION_PLAN_SHA256[index]
+ for index in (1, 2, 3)
+ ],
+ )
+ return
self.assertEqual(
[
sweep_matrix.qualification_execution_plan_sha256(matrix, index)
@@ -214,7 +227,7 @@ def test_full_v1_matrix_emits_every_unsupported_terminal_outcome(self) -> None:
documents = [json.loads(path.read_text()) for path in written]
self.assertEqual(len(documents), len(expected))
- self.assertEqual(len(documents), 271)
+ self.assertEqual(len(documents), 277)
self.assertEqual(
{document["outcome"]["reason"] for document in documents},
{item["reason"] for item in expected},
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 346c93187c..59e4ac60cc 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -209,7 +209,7 @@ def test_matrix_cases_and_shards_are_identity_bound(self) -> None:
sum(len(item["case"]["ladder"].split()) for item in runnable_cases),
sum(len(item["case"]["ladder"].split()) for item in unsupported_cases),
),
- (50, 664, 393, 271, 1532, 898, 634),
+ (54, 656, 379, 277, 1648, 916, 732),
)
b300_ep16 = [
item for item in unsupported_cases
@@ -244,7 +244,7 @@ def test_matrix_cases_and_shards_are_identity_bound(self) -> None:
expected_topologies[sku, 16] = (
4, 4, 72, "scale-up", "mnnvl", None, "mnnvl", topology_class,
)
- for sku in ("mi355x",):
+ for sku in ("mi300x", "mi355x"):
expected_topologies[sku, 8] = (
1, 8, 8, "scale-up", "xgmi", None, "xgmi", f"{sku}-xgmi",
)
@@ -276,13 +276,13 @@ def test_matrix_cases_and_shards_are_identity_bound(self) -> None:
self.assertIsNotNone(capability.topology_for("mi325x", 8))
self.assertEqual(
Counter(shard["n"] for shard in matrix["include"]),
- Counter({6: 30, 7: 1, 8: 5, 10: 1, 12: 13}),
+ Counter({6: 39, 7: 1, 8: 1, 10: 13}),
)
ll_cases = [
item for item in matrix["requested_cases"]
if item["case"]["mode"] == "low-latency"
]
- self.assertEqual(len(ll_cases), 76)
+ self.assertEqual(len(ll_cases), 80)
self.assertTrue(all(
item["case"]["backend"] in {"deepep", "uccl"}
and item["case"]["phase"] == "decode"
From 59176763c845e8d5c7cfd8639755b5a0a971e008 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 17:43:30 +0800
Subject: [PATCH 106/210] test(collectivex): cover MI300X operator overlay
---
experimental/CollectiveX/tests/test_sampling_contract.py | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 59e4ac60cc..fecc1e6880 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -1456,7 +1456,10 @@ def test_launcher_signal_traps_cancel_the_allocation(self) -> None:
self.assertIn("steps.gen.outputs.source_backends", workflow)
self.assertIn("secrets.COLLECTIVEX_NETWORK_CONFIG_V1", workflow)
self.assertIn("secrets.COLLECTIVEX_B200_CONFIG_V1", workflow)
+ self.assertIn("secrets.COLLECTIVEX_MI300_CONFIG_V1", workflow)
self.assertIn('set(overlay["runners"]) != {"b200-dgxc"}', workflow)
+ self.assertIn('set(overlay["runners"]) != {"mi300x"}', workflow)
+ self.assertIn('overlay_name != "COLLECTIVEX_MI300_CONFIG_CONTENT"', workflow)
self.assertIn("set(fields) - private_fields", workflow)
self.assertIn('private_fields.add("exclude_nodes")', workflow)
self.assertIn('invalid H100 overlay runner', workflow)
@@ -1580,6 +1583,8 @@ def test_v1_publication_requires_explicit_release_markers(self) -> None:
self.assertIn("${{ inputs.only_sku }}-${{ inputs.probe_id }}", sweep)
self.assertIn("cancel-in-progress: false", sweep)
self.assertIn("COLLECTIVEX_MI355_CONFIG_V1", sweep)
+ self.assertIn("COLLECTIVEX_MI300_CONFIG_V1", sweep)
+ self.assertIn('set(overlay["runners"]) != {"mi300x"}', sweep)
self.assertIn('set(overlay["runners"]) != {"mi355x"}', sweep)
self.assertIn("collectivex.precision-probe-plan.v1", (ROOT / "tests" / "probe_precision.py").read_text())
self.assertIn("cxprecision-probes-${{ github.run_id }}-${{ github.run_attempt }}", sweep)
From fa807ecf542faccbd5c3d8117c0abe7535d7f883 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 17:51:34 +0800
Subject: [PATCH 107/210] fix(collectivex): promote MI300X EP8 precision
---
experimental/CollectiveX/capability.py | 3 +++
.../CollectiveX/tests/test_precision_scheduling.py | 13 +++----------
2 files changed, 6 insertions(+), 10 deletions(-)
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
index b6e3ef3ae0..cba4bcb57b 100644
--- a/experimental/CollectiveX/capability.py
+++ b/experimental/CollectiveX/capability.py
@@ -404,6 +404,9 @@ def _precision_rule(
("28740154697", "h200-dgxc", 16, "deepep-v2", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "backend-setup-timeout"),
("28740533382", "mi355x", 8, "mori", "normal", _MORI_E4M3FN_BOTH_PROFILE, "unsupported", "backend-construction-failed"),
("28740533382", "mi355x", 8, "mori", "normal", _MORI_E4M3FN_DIRECT_PROFILE, "unsupported", "backend-construction-failed"),
+ ("28782309414", "mi300x", 8, "mori", "normal", _NORMAL_E4M3FNUZ_PROFILE, "supported", "native-probe-passed"),
+ ("28782309414", "mi300x", 8, "mori", "normal", _MORI_E4M3FNUZ_DIRECT_PROFILE, "supported", "native-probe-passed"),
+ ("28782309414", "mi300x", 8, "mori", "normal", _MORI_E4M3FNUZ_BOTH_PROFILE, "supported", "native-probe-passed"),
*(("28743235213", "b300", 8, "deepep", "low-latency", profile,
"supported", "native-probe-passed")
for profile in (_LL_FP8_PROFILE, _LL_LOGFMT_PROFILE, _LL_FP8_LOGFMT_PROFILE)),
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
index 65b233e7e7..a2a4184f07 100644
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -47,9 +47,6 @@ def test_precision_probe_inventory_is_exact_and_non_mutating(self) -> None:
for item in targets
],
[
- ("mi300x", "mori", 8, "d-bf16.c-fp8-e4m3fnuz-direct-cast-noscale"),
- ("mi300x", "mori", 8, "d-fp8-e4m3fnuz-b128-f32-prequantized.c-bf16"),
- ("mi300x", "mori", 8, "d-fp8-e4m3fnuz-b128-f32-prequantized.c-fp8-e4m3fnuz-direct-cast-noscale"),
("mi300x", "mori", 16, "d-fp8-e4m3fnuz-b128-f32-prequantized.c-bf16"),
],
)
@@ -258,12 +255,12 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
len(targets), 94
)
self.assertEqual(
- sum(item["disposition"] == "supported" for item in targets), 59
+ sum(item["disposition"] == "supported" for item in targets), 62
)
self.assertEqual(
sum(item["disposition"] == "unsupported" for item in targets), 31
)
- self.assertEqual(len(capability.provisional_precision_targets()), 4)
+ self.assertEqual(len(capability.provisional_precision_targets()), 1)
keys = {
(
item["precision_profile"],
@@ -381,11 +378,7 @@ def test_split_suites_track_provisional_state_and_do_not_duplicate_bf16(self) ->
self.assertEqual(scheduled_profiles, set(low_latency["precision_profiles"]))
self.assertEqual(
{item["precision_profile"] for item in capability.provisional_precision_targets()},
- set(normal["precision_profiles"]) - {
- "d-fp8-e4m3fn-b128-f32-prequantized.c-bf16",
- "d-bf16.c-fp8-e4m3fn-direct-cast-noscale",
- "d-fp8-e4m3fn-b128-f32-prequantized.c-fp8-e4m3fn-direct-cast-noscale",
- },
+ {"d-fp8-e4m3fnuz-b128-f32-prequantized.c-bf16"},
)
stale = copy.deepcopy(suites)
From 8d2501dff3d2d22fc3140ebfb03dac241cdb5437 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 17:57:36 +0800
Subject: [PATCH 108/210] fix(collectivex): accept pinned DeepEP V2 package
metadata
---
experimental/CollectiveX/contracts.py | 10 +++++++++-
.../CollectiveX/tests/test_deepep_v2_contract.py | 13 +++++++++++++
2 files changed, 22 insertions(+), 1 deletion(-)
diff --git a/experimental/CollectiveX/contracts.py b/experimental/CollectiveX/contracts.py
index f0f7090e88..b70e688816 100644
--- a/experimental/CollectiveX/contracts.py
+++ b/experimental/CollectiveX/contracts.py
@@ -98,6 +98,9 @@
"nccl_version": "2.30.4",
"nvshmem_package_version": "3.3.9",
}
+DEEPEP_V2_DISTRIBUTION_VERSIONS = frozenset({
+ "2.0.0+fa8a9b1", "2.0.0+local",
+})
UCCL_DEPENDENCY_VERSIONS = {
"intervaltree": "3.1.0",
"nvidia-cuda-runtime-cu12": "12.9.79",
@@ -575,8 +578,13 @@ def backend_provenance_issues(backend: str, provenance: dict[str, Any]) -> list[
unresolved.append("jit_random_seed")
unresolved.extend(
field for field, expected in DEEPEP_V2_V1_PROVENANCE.items()
- if provenance.get(field) != expected
+ if field != "deepep_distribution_version"
+ and provenance.get(field) != expected
)
+ if provenance.get("deepep_distribution_version") not in (
+ DEEPEP_V2_DISTRIBUTION_VERSIONS
+ ):
+ unresolved.append("deepep_distribution_version")
policy = (
provenance.get("allow_hybrid_mode"),
provenance.get("gin_enabled"),
diff --git a/experimental/CollectiveX/tests/test_deepep_v2_contract.py b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
index 7c70d070e6..cd35d0e024 100644
--- a/experimental/CollectiveX/tests/test_deepep_v2_contract.py
+++ b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
@@ -1188,6 +1188,19 @@ def record(role: str, name: str, digit: str) -> dict[str, str]:
expected = "backend_lineage"
self.assertIn(expected, contracts.backend_provenance_issues(backend, changed))
+ for distribution_version in contracts.DEEPEP_V2_DISTRIBUTION_VERSIONS:
+ changed = copy.deepcopy(v2)
+ changed["deepep_distribution_version"] = distribution_version
+ self.assertEqual(
+ contracts.backend_provenance_issues("deepep-v2", changed), []
+ )
+ changed = copy.deepcopy(v2)
+ changed["deepep_distribution_version"] = "2.0.0+unverified"
+ self.assertIn(
+ "deepep_distribution_version",
+ contracts.backend_provenance_issues("deepep-v2", changed),
+ )
+
changed = copy.deepcopy(uccl)
changed["uccl_dependency_versions"]["intervaltree"] = "3.2.0"
self.assertIn(
From 3a6d258885847c1a22aa13c6bff91e8456e4ce36 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 18:05:33 +0800
Subject: [PATCH 109/210] fix(collectivex): record B200 EP16 backend limits
---
experimental/CollectiveX/capability.py | 17 ++++++++++++++
.../tests/test_precision_scheduling.py | 23 +++++++++++++++++++
.../tests/test_qualification_planning.py | 2 +-
.../tests/test_sampling_contract.py | 4 ++--
4 files changed, 43 insertions(+), 3 deletions(-)
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
index cba4bcb57b..aff64bfd59 100644
--- a/experimental/CollectiveX/capability.py
+++ b/experimental/CollectiveX/capability.py
@@ -173,6 +173,18 @@ def _platform(
("b300", 16): "v1 publication fabric unavailable for B300 EP16",
}
+# Backend-specific topology limits require repeated native execution evidence.
+# Keep these narrower than platform overrides so working reference paths remain
+# measurable on the same fabric.
+BACKEND_TOPOLOGY_CELL_OVERRIDES: dict[tuple[str, str, int], str] = {
+ ("b200-dgxc", "deepep", 16): (
+ "DeepEP V1 EP16 native operations time out on the B200 publication fabric"
+ ),
+ ("b200-dgxc", "deepep-hybrid", 16): (
+ "DeepEP Hybrid EP16 cannot map the DOCA UAR into GPU memory on B200"
+ ),
+}
+
PRECISION_DISPOSITIONS = {
"supported", "unsupported", "not-applicable", "provisional",
}
@@ -498,6 +510,11 @@ def _resolve_base(
return False, f"{backend} does not support {platform['machine']}"
if sku in implementation.get("excluded_skus", set()):
return False, f"{backend} is unavailable on {sku}"
+ backend_topology_override = BACKEND_TOPOLOGY_CELL_OVERRIDES.get(
+ (sku, backend, ep)
+ )
+ if backend_topology_override is not None:
+ return False, backend_topology_override
topology_override = TOPOLOGY_CELL_OVERRIDES.get((sku, ep))
if topology_override is not None:
return False, topology_override
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
index a2a4184f07..7923926545 100644
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -332,6 +332,29 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
self.assertEqual(b300_ep8, "supported")
self.assertEqual(reason, "ok")
+ for backend, expected_reason in (
+ (
+ "deepep",
+ "DeepEP V1 EP16 native operations time out on the B200 publication fabric",
+ ),
+ (
+ "deepep-hybrid",
+ "DeepEP Hybrid EP16 cannot map the DOCA UAR into GPU memory on B200",
+ ),
+ ):
+ disposition, reason = capability.resolve_disposition(
+ "b200-dgxc", backend, ep=16, nodes=2,
+ precision_profile=identity.V1_CONTROL_PRECISION_PROFILE,
+ )
+ self.assertEqual(disposition, "unsupported")
+ self.assertEqual(reason, expected_reason)
+ b200_reference, reason = capability.resolve_disposition(
+ "b200-dgxc", "nccl-ep", ep=16, nodes=2,
+ precision_profile=identity.V1_CONTROL_PRECISION_PROFILE,
+ )
+ self.assertEqual(b200_reference, "supported")
+ self.assertEqual(reason, "ok")
+
def test_split_suites_track_provisional_state_and_do_not_duplicate_bf16(self) -> None:
suites = sweep_matrix._load("suites.yaml")
workloads = sweep_matrix._load("workloads.yaml")
diff --git a/experimental/CollectiveX/tests/test_qualification_planning.py b/experimental/CollectiveX/tests/test_qualification_planning.py
index 97b698360a..61f7943f13 100644
--- a/experimental/CollectiveX/tests/test_qualification_planning.py
+++ b/experimental/CollectiveX/tests/test_qualification_planning.py
@@ -227,7 +227,7 @@ def test_full_v1_matrix_emits_every_unsupported_terminal_outcome(self) -> None:
documents = [json.loads(path.read_text()) for path in written]
self.assertEqual(len(documents), len(expected))
- self.assertEqual(len(documents), 277)
+ self.assertEqual(len(documents), 290)
self.assertEqual(
{document["outcome"]["reason"] for document in documents},
{item["reason"] for item in expected},
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index fecc1e6880..842fddba54 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -209,7 +209,7 @@ def test_matrix_cases_and_shards_are_identity_bound(self) -> None:
sum(len(item["case"]["ladder"].split()) for item in runnable_cases),
sum(len(item["case"]["ladder"].split()) for item in unsupported_cases),
),
- (54, 656, 379, 277, 1648, 916, 732),
+ (52, 656, 366, 290, 1648, 880, 768),
)
b300_ep16 = [
item for item in unsupported_cases
@@ -276,7 +276,7 @@ def test_matrix_cases_and_shards_are_identity_bound(self) -> None:
self.assertIsNotNone(capability.topology_for("mi325x", 8))
self.assertEqual(
Counter(shard["n"] for shard in matrix["include"]),
- Counter({6: 39, 7: 1, 8: 1, 10: 13}),
+ Counter({6: 38, 8: 1, 10: 13}),
)
ll_cases = [
item for item in matrix["requested_cases"]
From 5d6fda1ff829dd60a8ccc9a804741ad169cea816 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 18:17:30 +0800
Subject: [PATCH 110/210] fix(collectivex): allow MI300X shared staging overlay
---
.github/workflows/collectivex-sweep.yml | 5 ++++-
experimental/CollectiveX/tests/test_sampling_contract.py | 1 +
2 files changed, 5 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml
index ceeb0851a0..c8db25ff7d 100644
--- a/.github/workflows/collectivex-sweep.yml
+++ b/.github/workflows/collectivex-sweep.yml
@@ -525,7 +525,10 @@ jobs:
elif overlay_name == "COLLECTIVEX_MI300_CONFIG_CONTENT":
if set(overlay["runners"]) != {"mi300x"}:
raise ValueError("invalid MI300 overlay runner")
- private_fields.update({"partition", "squash_dir", "exclude_nodes", "nodelist", "lock_dir"})
+ private_fields.update({
+ "partition", "squash_dir", "stage_dir", "exclude_nodes",
+ "nodelist", "lock_dir",
+ })
for runner, fields in overlay["runners"].items():
if not isinstance(fields, dict) or not fields:
raise ValueError("invalid network overlay runner")
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 842fddba54..6905195504 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -1460,6 +1460,7 @@ def test_launcher_signal_traps_cancel_the_allocation(self) -> None:
self.assertIn('set(overlay["runners"]) != {"b200-dgxc"}', workflow)
self.assertIn('set(overlay["runners"]) != {"mi300x"}', workflow)
self.assertIn('overlay_name != "COLLECTIVEX_MI300_CONFIG_CONTENT"', workflow)
+ self.assertIn('"partition", "squash_dir", "stage_dir", "exclude_nodes"', workflow)
self.assertIn("set(fields) - private_fields", workflow)
self.assertIn('private_fields.add("exclude_nodes")', workflow)
self.assertIn('invalid H100 overlay runner', workflow)
From 2b980bfebaabe021a0c2650eb21343f6d78ea8d9 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 18:19:11 +0800
Subject: [PATCH 111/210] fix(collectivex): record DeepEP V2 EP16 QP limits
---
experimental/CollectiveX/capability.py | 6 ++++++
.../CollectiveX/tests/test_precision_scheduling.py | 11 +++++++++++
.../CollectiveX/tests/test_qualification_planning.py | 2 +-
.../CollectiveX/tests/test_sampling_contract.py | 4 ++--
4 files changed, 20 insertions(+), 3 deletions(-)
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
index aff64bfd59..46a4d5ac82 100644
--- a/experimental/CollectiveX/capability.py
+++ b/experimental/CollectiveX/capability.py
@@ -183,6 +183,12 @@ def _platform(
("b200-dgxc", "deepep-hybrid", 16): (
"DeepEP Hybrid EP16 cannot map the DOCA UAR into GPU memory on B200"
),
+ ("b200-dgxc", "deepep-v2", 16): (
+ "DeepEP V2 EP16 exhausts pinned NCCL Gin QP resources on the B200 fabric"
+ ),
+ ("h200-dgxc", "deepep-v2", 16): (
+ "DeepEP V2 EP16 exhausts pinned NCCL Gin QP resources on the H200 fabric"
+ ),
}
PRECISION_DISPOSITIONS = {
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
index 7923926545..d16673b60f 100644
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -355,6 +355,17 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
self.assertEqual(b200_reference, "supported")
self.assertEqual(reason, "ok")
+ for sku, product in (("b200-dgxc", "B200"), ("h200-dgxc", "H200")):
+ disposition, reason = capability.resolve_disposition(
+ sku, "deepep-v2", ep=16, nodes=2,
+ precision_profile=identity.V1_CONTROL_PRECISION_PROFILE,
+ )
+ self.assertEqual(disposition, "unsupported")
+ self.assertEqual(
+ reason,
+ f"DeepEP V2 EP16 exhausts pinned NCCL Gin QP resources on the {product} fabric",
+ )
+
def test_split_suites_track_provisional_state_and_do_not_duplicate_bf16(self) -> None:
suites = sweep_matrix._load("suites.yaml")
workloads = sweep_matrix._load("workloads.yaml")
diff --git a/experimental/CollectiveX/tests/test_qualification_planning.py b/experimental/CollectiveX/tests/test_qualification_planning.py
index 61f7943f13..c277971679 100644
--- a/experimental/CollectiveX/tests/test_qualification_planning.py
+++ b/experimental/CollectiveX/tests/test_qualification_planning.py
@@ -227,7 +227,7 @@ def test_full_v1_matrix_emits_every_unsupported_terminal_outcome(self) -> None:
documents = [json.loads(path.read_text()) for path in written]
self.assertEqual(len(documents), len(expected))
- self.assertEqual(len(documents), 290)
+ self.assertEqual(len(documents), 302)
self.assertEqual(
{document["outcome"]["reason"] for document in documents},
{item["reason"] for item in expected},
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 6905195504..b202b73426 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -209,7 +209,7 @@ def test_matrix_cases_and_shards_are_identity_bound(self) -> None:
sum(len(item["case"]["ladder"].split()) for item in runnable_cases),
sum(len(item["case"]["ladder"].split()) for item in unsupported_cases),
),
- (52, 656, 366, 290, 1648, 880, 768),
+ (50, 656, 354, 302, 1648, 852, 796),
)
b300_ep16 = [
item for item in unsupported_cases
@@ -276,7 +276,7 @@ def test_matrix_cases_and_shards_are_identity_bound(self) -> None:
self.assertIsNotNone(capability.topology_for("mi325x", 8))
self.assertEqual(
Counter(shard["n"] for shard in matrix["include"]),
- Counter({6: 38, 8: 1, 10: 13}),
+ Counter({6: 36, 8: 1, 10: 13}),
)
ll_cases = [
item for item in matrix["requested_cases"]
From e36fc43d95555df37cf8b9da2f1019edbafc4833 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 18:25:46 +0800
Subject: [PATCH 112/210] fix(collectivex): resolve MI300X shared stage aliases
---
experimental/CollectiveX/runtime/common.sh | 8 ++++++++
.../CollectiveX/tests/test_sampling_contract.py | 12 ++++++++++++
2 files changed, 20 insertions(+)
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 01697d5fd8..bb18bf3280 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -1969,6 +1969,14 @@ cx_lock_canonical_gha_env() {
;;
*) cx_die "canonical CollectiveX execution requires a configured shared stage directory" ;;
esac
+ elif [ "$runner" = mi300x ]; then
+ # The MI300X runner home is a shared-filesystem symlink. Resolve the
+ # operator-selected base once; cx_stage_path still validates the canonical
+ # directory's ownership, permissions, overlap, and per-run child path.
+ trusted_stage_dir="$(python3 -c \
+ 'import os,sys; p=os.path.realpath(sys.argv[1]); assert os.path.isdir(p); print(p,end="")' \
+ "$trusted_stage_dir")" \
+ || cx_die "canonical MI300X execution cannot resolve the shared stage directory"
fi
[[ "$trusted_audit_salt" =~ ^[0-9a-f]{64}$ ]] \
|| cx_die "canonical CollectiveX execution requires a private audit salt"
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index b202b73426..e10ff4a8db 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -2861,6 +2861,12 @@ def test_canonical_gha_environment_is_locked_but_manual_overrides_survive(self)
test -z "${CX_PREPARED_BACKEND_CACHE+x}${CX_BACKEND_SOURCE_ROOT+x}"
test -z "${CX_DRYRUN+x}"
+ export COLLECTIVEX_OPERATOR_CONFIG_LOADED=$$
+ export CX_SHARD_SKU=mi300x CX_NODES=2 CX_GPUS_PER_NODE=8
+ export CX_STAGE_DIR="$TEST_STAGE_ALIAS" CX_AUDIT_SALT="$(printf 'a%.0s' {1..64})"
+ cx_lock_canonical_gha_env mi300x
+ test "$CX_STAGE_DIR" = "$TEST_STAGE_TARGET"
+
cx_prepare_implicit_stage_base() {
if [ -n "${1:-}" ]; then test "$1" = "$TEST_SHARED_PARENT"; fi
if [ -n "${2:-}" ]; then test "$2" = 123; fi
@@ -2932,9 +2938,13 @@ def test_canonical_gha_environment_is_locked_but_manual_overrides_survive(self)
home = root / "home"
workspace = root / "workspace"
implicit_stage = root / "implicit-stage"
+ stage_target = root / "stage-target"
+ stage_alias = root / "stage-alias"
home.mkdir(mode=0o700)
workspace.mkdir(mode=0o720)
implicit_stage.mkdir(mode=0o700)
+ stage_target.mkdir(mode=0o700)
+ stage_alias.symlink_to(stage_target, target_is_directory=True)
subprocess.run(
["bash", "-c", command, "_", str(common)],
check=True,
@@ -2942,6 +2952,8 @@ def test_canonical_gha_environment_is_locked_but_manual_overrides_survive(self)
**os.environ,
"HOME": str(home),
"TEST_IMPLICIT_STAGE": str(implicit_stage),
+ "TEST_STAGE_ALIAS": str(stage_alias),
+ "TEST_STAGE_TARGET": str(stage_target),
"TEST_SHARED_PARENT": str(root),
"COLLECTIVEX_OPERATOR_CONFIG": "/dev/null",
"GITHUB_WORKSPACE": str(workspace),
From 6df017c763eba5c26436c30a3bd7b0f197b66961 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 18:27:16 +0800
Subject: [PATCH 113/210] fix(collectivex): avoid unsupported stage metadata
copies
---
experimental/CollectiveX/runtime/common.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index bb18bf3280..0c4e4011f3 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -2394,7 +2394,7 @@ cx_stage_repo() {
|| cx_die "cannot create the configured stage directory"
cx_log "staging CollectiveX on compute-visible storage"
log="$(cx_private_log_path repository-stage)"
- if ! rsync -a --delete --delete-excluded \
+ if ! rsync -rlp --delete --delete-excluded \
--exclude='__pycache__/' --exclude='results/' --exclude='.cx_workloads/' \
--exclude='.cx_backend/' --exclude='.cx_sources/' \
--exclude='configs/platforms.yaml' --exclude='private-infra.md' \
From b00b039c7bb23fe118c08d497f1bd9db73f37e05 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 18:28:47 +0800
Subject: [PATCH 114/210] fix(collectivex): avoid stage permission metadata
writes
---
experimental/CollectiveX/runtime/common.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 0c4e4011f3..abccf9ad12 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -2394,7 +2394,7 @@ cx_stage_repo() {
|| cx_die "cannot create the configured stage directory"
cx_log "staging CollectiveX on compute-visible storage"
log="$(cx_private_log_path repository-stage)"
- if ! rsync -rlp --delete --delete-excluded \
+ if ! rsync -rl --delete --delete-excluded \
--exclude='__pycache__/' --exclude='results/' --exclude='.cx_workloads/' \
--exclude='.cx_backend/' --exclude='.cx_sources/' \
--exclude='configs/platforms.yaml' --exclude='private-infra.md' \
From 97a58b585602940c6933d5704ccd6f92735339b0 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 18:30:55 +0800
Subject: [PATCH 115/210] fix(collectivex): stage without an rsync dependency
---
experimental/CollectiveX/runtime/common.sh | 13 +++++++------
.../CollectiveX/tests/test_sampling_contract.py | 8 ++++----
2 files changed, 11 insertions(+), 10 deletions(-)
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index abccf9ad12..369e58f5e1 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -2394,12 +2394,13 @@ cx_stage_repo() {
|| cx_die "cannot create the configured stage directory"
cx_log "staging CollectiveX on compute-visible storage"
log="$(cx_private_log_path repository-stage)"
- if ! rsync -rl --delete --delete-excluded \
- --exclude='__pycache__/' --exclude='results/' --exclude='.cx_workloads/' \
- --exclude='.cx_backend/' --exclude='.cx_sources/' \
- --exclude='configs/platforms.yaml' --exclude='private-infra.md' \
- --exclude='goal.md' --exclude='notes.md' \
- "$repo_root/experimental/CollectiveX" "$stage_dir/experimental/" > "$log" 2>&1; then
+ if ! tar -C "$repo_root/experimental" \
+ --exclude='CollectiveX/__pycache__' --exclude='CollectiveX/results' \
+ --exclude='CollectiveX/.cx_workloads' --exclude='CollectiveX/.cx_backend' \
+ --exclude='CollectiveX/.cx_sources' --exclude='CollectiveX/configs/platforms.yaml' \
+ --exclude='CollectiveX/private-infra.md' --exclude='CollectiveX/goal.md' \
+ --exclude='CollectiveX/notes.md' -cf - CollectiveX 2> "$log" \
+ | tar -C "$stage_dir/experimental" -xf - 2>> "$log"; then
rm -rf -- "$stage_dir" >/dev/null 2>&1 \
|| cx_log "ERROR: cannot remove the incomplete execution stage"
cx_fail_stage repository-stage "$log" || true
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index e10ff4a8db..2034ab78dd 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -3235,7 +3235,7 @@ def test_stage_refuses_to_reuse_an_execution_child(self) -> None:
)
self.assertEqual(sentinel.read_text(), "keep")
- def test_stage_removes_its_execution_child_when_rsync_fails(self) -> None:
+ def test_stage_removes_its_execution_child_when_tar_fails(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary).resolve()
repo = root / "repo"
@@ -3244,10 +3244,10 @@ def test_stage_removes_its_execution_child_when_rsync_fails(self) -> None:
(source / "public.py").write_text("public\n")
base = root / "stage"
base.mkdir(mode=0o700)
- sentinel = root / "rsync-called"
+ sentinel = root / "tar-called"
command = r'''
source "$1"
- rsync() { : > "$RSYNC_CALLED"; return 1; }
+ tar() { : > "$TAR_CALLED"; return 1; }
staged="$(cx_stage_path "$2" "$3")"
! cx_stage_repo "$2" "$staged"
'''
@@ -3263,7 +3263,7 @@ def test_stage_removes_its_execution_child_when_rsync_fails(self) -> None:
"COLLECTIVEX_CANONICAL_GHA": "1",
"COLLECTIVEX_EXECUTION_ID": f"test-{root.name}",
"CX_STAGE_DIR": str(base),
- "RSYNC_CALLED": str(sentinel),
+ "TAR_CALLED": str(sentinel),
},
)
self.assertTrue(sentinel.is_file())
From fccfe9c37b97a00e6d30ed7da0275defcd75ff01 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 18:35:27 +0800
Subject: [PATCH 116/210] fix(collectivex): unblock B200 and MI300X scale-out
---
experimental/CollectiveX/capability.py | 19 ++++++--
experimental/CollectiveX/contracts.py | 7 ++-
experimental/CollectiveX/runtime/common.sh | 46 +++++++++++++++----
experimental/CollectiveX/tests/ep_deepep.py | 3 ++
.../tests/test_deepep_v2_contract.py | 1 +
.../tests/test_precision_scheduling.py | 37 +++++++--------
.../tests/test_qualification_planning.py | 2 +-
.../tests/test_sampling_contract.py | 22 ++++++---
8 files changed, 99 insertions(+), 38 deletions(-)
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
index 46a4d5ac82..0f4f922dc8 100644
--- a/experimental/CollectiveX/capability.py
+++ b/experimental/CollectiveX/capability.py
@@ -25,6 +25,10 @@
"mi355x": {"schedulable": False, "basis": "nvidia-only"},
}
+# B200's publication pods cannot map NIC doorbells into GPU address space.
+# DeepEP V1 scale-out therefore uses NVSHMEM's CPU-proxied IBGDA handler.
+DEEPEP_V1_IBGDA_NIC_HANDLERS = {"b200-dgxc": "cpu"}
+
def _topologies(
product: str, *, gpus_per_node: int, scale_up_domain: int, scale_up_transport: str
@@ -177,11 +181,8 @@ def _platform(
# Keep these narrower than platform overrides so working reference paths remain
# measurable on the same fabric.
BACKEND_TOPOLOGY_CELL_OVERRIDES: dict[tuple[str, str, int], str] = {
- ("b200-dgxc", "deepep", 16): (
- "DeepEP V1 EP16 native operations time out on the B200 publication fabric"
- ),
("b200-dgxc", "deepep-hybrid", 16): (
- "DeepEP Hybrid EP16 cannot map the DOCA UAR into GPU memory on B200"
+ "DeepEP Hybrid EP16 requires unavailable GPU-to-NIC doorbell/UAR mappings on B200"
),
("b200-dgxc", "deepep-v2", 16): (
"DeepEP V2 EP16 exhausts pinned NCCL Gin QP resources on the B200 fabric"
@@ -437,6 +438,16 @@ def _precision_rule(
for run_id, sku, ep, backend, mode, profile, disposition, result
in _VALIDATED_NATIVE_PROBE_CELLS
})
+for _profile, _mode in (
+ (_NORMAL_E4M3FN_PROFILE, "normal"),
+ (_LL_FP8_PROFILE, "low-latency"),
+ (_LL_LOGFMT_PROFILE, "low-latency"),
+ (_LL_FP8_LOGFMT_PROFILE, "low-latency"),
+):
+ PRECISION_CELL_OVERRIDES[(_profile, "deepep", "b200-dgxc", 16, _mode)] = {
+ "basis": "b200-cpu-proxied-ibgda-handler-production-shape-passed",
+ "disposition": "supported",
+ }
def runtime_identity_issues(
diff --git a/experimental/CollectiveX/contracts.py b/experimental/CollectiveX/contracts.py
index b70e688816..23cac74586 100644
--- a/experimental/CollectiveX/contracts.py
+++ b/experimental/CollectiveX/contracts.py
@@ -112,6 +112,7 @@
"deepep": (
"deepep_version", "deepep_commit", "backend_lineage", "allow_mnnvl",
"mnnvl_comm", "mode", "num_nvl_bytes", "num_rdma_bytes",
+ "nvshmem_ibgda_nic_handler",
),
"deepep-v2": (
*DEEPEP_V2_V1_PROVENANCE, "api_signature_sha256", "loaded_libraries",
@@ -149,7 +150,7 @@
"logical_scaleup_ranks", "mapping_variant", "max_num_inp_token_per_rank",
"max_num_tokens", "max_total_recv_tokens", "mnnvl_comm", "mode", "mori_commit",
"nccl_communicator", "nccl_package_version", "nccl_version", "num_experts",
- "nvshmem_package_version",
+ "nvshmem_ibgda_nic_handler", "nvshmem_package_version",
"num_max_tokens_per_rank", "num_nvl_bytes", "num_qps", "num_qps_per_rank",
"num_rdma_bytes", "num_sms", "path",
"physical_nvlink_ranks", "physical_rdma_ranks", "prefer_overlap_with_compute",
@@ -541,6 +542,10 @@ def backend_provenance_issues(backend: str, provenance: dict[str, Any]) -> list[
unresolved.append("mnnvl_comm")
if provenance.get("backend_lineage") != "deepep-v1":
unresolved.append("backend_lineage")
+ if provenance.get("nvshmem_ibgda_nic_handler") not in {
+ "cpu", "gpu", "not-active",
+ }:
+ unresolved.append("nvshmem_ibgda_nic_handler")
if backend in {"deepep", "uccl"}:
mode = provenance.get("mode")
num_nvl_bytes = provenance.get("num_nvl_bytes")
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 369e58f5e1..e6618d2a2a 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -741,7 +741,11 @@ cx_apply_network_profile() {
[ "$scaleout" = 1 ] \
&& export MORI_RDMA_TC="$CX_RDMA_TRAFFIC_CLASS" MORI_IO_TC="$CX_RDMA_TRAFFIC_CLASS"
fi
- export NVSHMEM_IB_ENABLE_IBGDA=1 NVSHMEM_IBGDA_NIC_HANDLER=gpu
+ local nic_handler=gpu
+ if [ "${CX_SHARD_SKU:-}" = b200-dgxc ] && [ "${CX_BENCH:-}" = deepep ]; then
+ nic_handler=cpu
+ fi
+ export NVSHMEM_IB_ENABLE_IBGDA=1 NVSHMEM_IBGDA_NIC_HANDLER="$nic_handler"
if [ -n "${CX_RDMA_LINK_LAYER:-}" ]; then
case "$CX_RDMA_LINK_LAYER" in
roce|infiniband) ;;
@@ -2394,13 +2398,39 @@ cx_stage_repo() {
|| cx_die "cannot create the configured stage directory"
cx_log "staging CollectiveX on compute-visible storage"
log="$(cx_private_log_path repository-stage)"
- if ! tar -C "$repo_root/experimental" \
- --exclude='CollectiveX/__pycache__' --exclude='CollectiveX/results' \
- --exclude='CollectiveX/.cx_workloads' --exclude='CollectiveX/.cx_backend' \
- --exclude='CollectiveX/.cx_sources' --exclude='CollectiveX/configs/platforms.yaml' \
- --exclude='CollectiveX/private-infra.md' --exclude='CollectiveX/goal.md' \
- --exclude='CollectiveX/notes.md' -cf - CollectiveX 2> "$log" \
- | tar -C "$stage_dir/experimental" -xf - 2>> "$log"; then
+ if ! python3 - "$repo_root/experimental/CollectiveX" \
+ "$stage_dir/experimental/CollectiveX" > "$log" 2>&1 <<'PY'
+import os
+from pathlib import Path
+import shutil
+import sys
+
+source, target = map(Path, sys.argv[1:])
+excluded = {
+ Path("__pycache__"), Path("results"), Path(".cx_workloads"),
+ Path(".cx_backend"), Path(".cx_sources"), Path("configs/platforms.yaml"),
+ Path("private-infra.md"), Path("goal.md"), Path("notes.md"),
+}
+for root, directories, files in os.walk(source, followlinks=False):
+ root_path = Path(root)
+ relative_root = root_path.relative_to(source)
+ directories[:] = [
+ name for name in directories
+ if relative_root / name not in excluded and not (root_path / name).is_symlink()
+ ]
+ destination = target / relative_root
+ destination.mkdir(mode=0o700, parents=True, exist_ok=True)
+ for name in files:
+ relative = relative_root / name
+ if relative in excluded:
+ continue
+ source_file = root_path / name
+ if source_file.is_symlink() or not source_file.is_file():
+ raise RuntimeError("unsupported source entry")
+ with source_file.open("rb") as input_file, (destination / name).open("xb") as output_file:
+ shutil.copyfileobj(input_file, output_file)
+PY
+ then
rm -rf -- "$stage_dir" >/dev/null 2>&1 \
|| cx_log "ERROR: cannot remove the incomplete execution stage"
cx_fail_stage repository-stage "$log" || true
diff --git a/experimental/CollectiveX/tests/ep_deepep.py b/experimental/CollectiveX/tests/ep_deepep.py
index 5ca5f47ed9..440d7de606 100644
--- a/experimental/CollectiveX/tests/ep_deepep.py
+++ b/experimental/CollectiveX/tests/ep_deepep.py
@@ -217,6 +217,9 @@ def __init__(self, args, rank, world_size, local_rank, device):
"device_sms": device_sms,
"allow_mnnvl": bool(mnnvl_kwargs),
"mnnvl_comm": mnnvl_comm,
+ "nvshmem_ibgda_nic_handler": os.environ.get(
+ "NVSHMEM_IBGDA_NIC_HANDLER", "not-active"
+ ),
**resource_provenance,
}
diff --git a/experimental/CollectiveX/tests/test_deepep_v2_contract.py b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
index cd35d0e024..bdb33ff143 100644
--- a/experimental/CollectiveX/tests/test_deepep_v2_contract.py
+++ b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
@@ -1144,6 +1144,7 @@ def record(role: str, name: str, digit: str) -> dict[str, str]:
"deepep_version": "1.1.0", "deepep_commit": "a" * 40,
"backend_lineage": "deepep-v1", "allow_mnnvl": False,
"mnnvl_comm": "not-requested", "mode": "normal",
+ "nvshmem_ibgda_nic_handler": "not-active",
"num_nvl_bytes": 1024, "num_rdma_bytes": 0,
}
hybrid = {
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
index d16673b60f..29029df237 100644
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -255,10 +255,10 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
len(targets), 94
)
self.assertEqual(
- sum(item["disposition"] == "supported" for item in targets), 62
+ sum(item["disposition"] == "supported" for item in targets), 66
)
self.assertEqual(
- sum(item["disposition"] == "unsupported" for item in targets), 31
+ sum(item["disposition"] == "unsupported" for item in targets), 27
)
self.assertEqual(len(capability.provisional_precision_targets()), 1)
keys = {
@@ -332,22 +332,23 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
self.assertEqual(b300_ep8, "supported")
self.assertEqual(reason, "ok")
- for backend, expected_reason in (
- (
- "deepep",
- "DeepEP V1 EP16 native operations time out on the B200 publication fabric",
- ),
- (
- "deepep-hybrid",
- "DeepEP Hybrid EP16 cannot map the DOCA UAR into GPU memory on B200",
- ),
- ):
- disposition, reason = capability.resolve_disposition(
- "b200-dgxc", backend, ep=16, nodes=2,
- precision_profile=identity.V1_CONTROL_PRECISION_PROFILE,
- )
- self.assertEqual(disposition, "unsupported")
- self.assertEqual(reason, expected_reason)
+ disposition, reason = capability.resolve_disposition(
+ "b200-dgxc", "deepep", ep=16, nodes=2,
+ precision_profile=identity.V1_CONTROL_PRECISION_PROFILE,
+ )
+ self.assertEqual(disposition, "supported")
+ self.assertEqual(reason, "ok")
+ self.assertEqual(capability.DEEPEP_V1_IBGDA_NIC_HANDLERS["b200-dgxc"], "cpu")
+
+ disposition, reason = capability.resolve_disposition(
+ "b200-dgxc", "deepep-hybrid", ep=16, nodes=2,
+ precision_profile=identity.V1_CONTROL_PRECISION_PROFILE,
+ )
+ self.assertEqual(disposition, "unsupported")
+ self.assertEqual(
+ reason,
+ "DeepEP Hybrid EP16 requires unavailable GPU-to-NIC doorbell/UAR mappings on B200",
+ )
b200_reference, reason = capability.resolve_disposition(
"b200-dgxc", "nccl-ep", ep=16, nodes=2,
precision_profile=identity.V1_CONTROL_PRECISION_PROFILE,
diff --git a/experimental/CollectiveX/tests/test_qualification_planning.py b/experimental/CollectiveX/tests/test_qualification_planning.py
index c277971679..574bde610c 100644
--- a/experimental/CollectiveX/tests/test_qualification_planning.py
+++ b/experimental/CollectiveX/tests/test_qualification_planning.py
@@ -227,7 +227,7 @@ def test_full_v1_matrix_emits_every_unsupported_terminal_outcome(self) -> None:
documents = [json.loads(path.read_text()) for path in written]
self.assertEqual(len(documents), len(expected))
- self.assertEqual(len(documents), 302)
+ self.assertEqual(len(documents), 292)
self.assertEqual(
{document["outcome"]["reason"] for document in documents},
{item["reason"] for item in expected},
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 2034ab78dd..d5e1f50974 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -209,7 +209,7 @@ def test_matrix_cases_and_shards_are_identity_bound(self) -> None:
sum(len(item["case"]["ladder"].split()) for item in runnable_cases),
sum(len(item["case"]["ladder"].split()) for item in unsupported_cases),
),
- (50, 656, 354, 302, 1648, 852, 796),
+ (51, 656, 364, 292, 1648, 877, 771),
)
b300_ep16 = [
item for item in unsupported_cases
@@ -276,7 +276,7 @@ def test_matrix_cases_and_shards_are_identity_bound(self) -> None:
self.assertIsNotNone(capability.topology_for("mi325x", 8))
self.assertEqual(
Counter(shard["n"] for shard in matrix["include"]),
- Counter({6: 36, 8: 1, 10: 13}),
+ Counter({6: 36, 8: 1, 10: 14}),
)
ll_cases = [
item for item in matrix["requested_cases"]
@@ -673,6 +673,10 @@ def test_scaleout_network_profile_is_explicit_and_allowlisted(self) -> None:
cx_export_gid_index_for_link_layer roce 1
test "$NCCL_IB_GID_INDEX:$NCCL_IB_SL" = 3:2
test "$NVSHMEM_IB_ENABLE_IBGDA:$NVSHMEM_IBGDA_NIC_HANDLER" = 1:gpu
+ export CX_SHARD_SKU=b200-dgxc CX_BENCH=deepep
+ cx_apply_network_profile 2 nvlink-rdma
+ test "$NVSHMEM_IB_ENABLE_IBGDA:$NVSHMEM_IBGDA_NIC_HANDLER" = 1:cpu
+ unset CX_SHARD_SKU CX_BENCH
cx_apply_network_profile 2 nvlink-rdma
cx_export_gid_index_for_link_layer infiniband 1
test -z "${NVSHMEM_IB_GID_INDEX+x}${NCCL_IB_GID_INDEX+x}${UCCL_IB_GID_INDEX+x}"
@@ -3235,7 +3239,7 @@ def test_stage_refuses_to_reuse_an_execution_child(self) -> None:
)
self.assertEqual(sentinel.read_text(), "keep")
- def test_stage_removes_its_execution_child_when_tar_fails(self) -> None:
+ def test_stage_removes_its_execution_child_when_copy_fails(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary).resolve()
repo = root / "repo"
@@ -3244,10 +3248,16 @@ def test_stage_removes_its_execution_child_when_tar_fails(self) -> None:
(source / "public.py").write_text("public\n")
base = root / "stage"
base.mkdir(mode=0o700)
- sentinel = root / "tar-called"
+ sentinel = root / "python-called"
command = r'''
source "$1"
- tar() { : > "$TAR_CALLED"; return 1; }
+ python3() {
+ if [[ "${2:-}" == */experimental/CollectiveX ]]; then
+ : > "$PYTHON_CALLED"
+ return 1
+ fi
+ command python3 "$@"
+ }
staged="$(cx_stage_path "$2" "$3")"
! cx_stage_repo "$2" "$staged"
'''
@@ -3263,7 +3273,7 @@ def test_stage_removes_its_execution_child_when_tar_fails(self) -> None:
"COLLECTIVEX_CANONICAL_GHA": "1",
"COLLECTIVEX_EXECUTION_ID": f"test-{root.name}",
"CX_STAGE_DIR": str(base),
- "TAR_CALLED": str(sentinel),
+ "PYTHON_CALLED": str(sentinel),
},
)
self.assertTrue(sentinel.is_file())
From 0f9d6d478993ceaf56b798c2c4e19f1408a4d23b Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 18:37:38 +0800
Subject: [PATCH 117/210] fix(collectivex): classify isolated stage copy errors
---
experimental/CollectiveX/runtime/common.sh | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index e6618d2a2a..7b145258cd 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -2405,6 +2405,11 @@ from pathlib import Path
import shutil
import sys
+def report_error(kind, value, trace):
+ error_number = getattr(value, "errno", 0)
+ print(f"collectivex-stage-copy-error={kind.__name__}:{error_number or 0}", file=sys.stderr)
+
+sys.excepthook = report_error
source, target = map(Path, sys.argv[1:])
excluded = {
Path("__pycache__"), Path("results"), Path(".cx_workloads"),
@@ -2431,6 +2436,9 @@ for root, directories, files in os.walk(source, followlinks=False):
shutil.copyfileobj(input_file, output_file)
PY
then
+ copy_error="$(grep -aoE 'collectivex-stage-copy-error=[A-Za-z]+:[0-9]+' "$log" \
+ | tail -n 1 || true)"
+ [ -z "$copy_error" ] || cx_log "ERROR: repository-stage-$copy_error"
rm -rf -- "$stage_dir" >/dev/null 2>&1 \
|| cx_log "ERROR: cannot remove the incomplete execution stage"
cx_fail_stage repository-stage "$log" || true
From 5561eed5da2e9731d89b4c6644374b6c84e4ba46 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 18:40:57 +0800
Subject: [PATCH 118/210] fix(collectivex): expose sanitized network preflight
failures
---
experimental/CollectiveX/runtime/common.sh | 11 +++++++++--
1 file changed, 9 insertions(+), 2 deletions(-)
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 7b145258cd..4b858237a0 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -808,10 +808,14 @@ if [ -n "${CX_IB_GID_INDEX:-}" ]; then
fi
if [ -n "${CX_SOCKET_IFNAME:-}" ]; then
IFS=, read -r -a interfaces <<< "$CX_SOCKET_IFNAME"
+ socket_ordinal=0
for interface in "${interfaces[@]}"; do
- [ -d "/sys/class/net/$interface" ]
+ socket_ordinal=$((socket_ordinal + 1))
+ [ -d "/sys/class/net/$interface" ] \
+ || { printf '[collectivex-private] socket-interface-%s=missing\n' "$socket_ordinal"; exit 1; }
state="$(cat "/sys/class/net/$interface/operstate")"
- [ "$state" = up ] || [ "$state" = unknown ]
+ [ "$state" = up ] || [ "$state" = unknown ] \
+ || { printf '[collectivex-private] socket-interface-%s=down\n' "$socket_ordinal"; exit 1; }
done
fi
check_port() {
@@ -867,6 +871,9 @@ done
printf '[collectivex-private] rdma-link-layer=%s\n' "$profile"
BASH
if [ "$rc" != 0 ]; then
+ marker="$(grep -aoE '(socket-interface|rdma-(device|port))-[0-9]+=(missing|down|inactive|gid-missing|gid-empty|link-layer-missing|link-layer-invalid|link-layer-mixed)' "$log" \
+ | tail -n 1 || true)"
+ [ -z "$marker" ] || cx_log "ERROR: network-profile-$marker"
[ "$report_failure" = 0 ] || cx_fail_stage setup "$log" || true
return "$rc"
fi
From ef1231f23f4889791d958ccd999ec06c6d0d952f Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 18:43:19 +0800
Subject: [PATCH 119/210] fix(collectivex): resolve scale-out sockets per
allocation
---
experimental/CollectiveX/runtime/common.sh | 32 +++++++++++++++----
.../tests/test_deepep_v2_contract.py | 2 +-
.../tests/test_sampling_contract.py | 1 +
3 files changed, 27 insertions(+), 8 deletions(-)
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 4b858237a0..b873b51668 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -697,9 +697,7 @@ cx_apply_network_profile() {
[ "$scaleout" = 1 ] || [ "$single_node_rdma" = 1 ] || return 0
[ -n "${CX_RDMA_DEVICES:-}" ] \
|| cx_die "RDMA execution requires a private device selector"
- if [ "$scaleout" = 1 ]; then
- [ -n "${CX_SOCKET_IFNAME:-}" ] \
- || cx_die "multi-node execution requires a private socket selector"
+ if [ "$scaleout" = 1 ] && [ -n "${CX_SOCKET_IFNAME:-}" ]; then
[[ "$CX_SOCKET_IFNAME" =~ ^[A-Za-z][A-Za-z0-9_.-]{0,31}(,[A-Za-z][A-Za-z0-9_.-]{0,31})*$ ]] \
|| cx_die "invalid private socket interface selector"
export NCCL_SOCKET_IFNAME="$CX_SOCKET_IFNAME" GLOO_SOCKET_IFNAME="$CX_SOCKET_IFNAME"
@@ -786,7 +784,6 @@ cx_validate_network_profile_on_job() {
[[ "$job_id" =~ ^[1-9][0-9]*$ && "$nodes" =~ ^[1-9][0-9]*$ ]] \
|| return 1
[ -n "${CX_RDMA_DEVICES:-}" ] || return 1
- [ "$scaleout" = 0 ] || [ -n "${CX_SOCKET_IFNAME:-}" ] || return 1
case "${CX_NETWORK_VALIDATION_ATTEMPT:-1}" in
1) ;;
2|3) log_label+="-a${CX_NETWORK_VALIDATION_ATTEMPT}" ;;
@@ -799,9 +796,22 @@ cx_validate_network_profile_on_job() {
--export="$(cx_host_exports),CX_SOCKET_IFNAME,CX_RDMA_DEVICES,CX_IB_GID_INDEX" \
bash -s > "$log" 2>&1 <<'BASH' || rc=$?
set -euo pipefail
-if [ -n "${CX_SOCKET_IFNAME:-}" ]; then
- [[ "$CX_SOCKET_IFNAME" =~ ^[A-Za-z][A-Za-z0-9_.-]{0,31}(,[A-Za-z][A-Za-z0-9_.-]{0,31})*$ ]]
+if [ -z "${CX_SOCKET_IFNAME:-}" ]; then
+ CX_SOCKET_IFNAME="$(python3 - <<'PY'
+from pathlib import Path
+
+for line in Path('/proc/net/route').read_text().splitlines()[1:]:
+ fields = line.split()
+ if len(fields) >= 4 and fields[1] == '00000000' and int(fields[3], 16) & 1:
+ print(fields[0], end='')
+ break
+PY
+)"
+ [ -n "$CX_SOCKET_IFNAME" ] \
+ || { printf '[collectivex-private] socket-interface-1=default-route-missing\n'; exit 1; }
fi
+[[ "$CX_SOCKET_IFNAME" =~ ^[A-Za-z][A-Za-z0-9_.-]{0,31}(,[A-Za-z][A-Za-z0-9_.-]{0,31})*$ ]]
+printf '[collectivex-private] socket-interface-selected=%s\n' "$CX_SOCKET_IFNAME"
[[ "$CX_RDMA_DEVICES" =~ ^[A-Za-z][A-Za-z0-9_.-]{0,31}(:[1-9][0-9]*)?(,[A-Za-z][A-Za-z0-9_.-]{0,31}(:[1-9][0-9]*)?)*$ ]]
if [ -n "${CX_IB_GID_INDEX:-}" ]; then
[[ "$CX_IB_GID_INDEX" =~ ^[0-9]+$ ]] && [ "$CX_IB_GID_INDEX" -le 255 ]
@@ -871,12 +881,20 @@ done
printf '[collectivex-private] rdma-link-layer=%s\n' "$profile"
BASH
if [ "$rc" != 0 ]; then
- marker="$(grep -aoE '(socket-interface|rdma-(device|port))-[0-9]+=(missing|down|inactive|gid-missing|gid-empty|link-layer-missing|link-layer-invalid|link-layer-mixed)' "$log" \
+ marker="$(grep -aoE '(socket-interface|rdma-(device|port))-[0-9]+=(missing|down|inactive|default-route-missing|gid-missing|gid-empty|link-layer-missing|link-layer-invalid|link-layer-mixed)' "$log" \
| tail -n 1 || true)"
[ -z "$marker" ] || cx_log "ERROR: network-profile-$marker"
[ "$report_failure" = 0 ] || cx_fail_stage setup "$log" || true
return "$rc"
fi
+ socket_ifname="$(
+ sed -nE 's/^\[collectivex-private\] socket-interface-selected=([A-Za-z][A-Za-z0-9_.-]{0,31})$/\1/p' "$log" \
+ | sort -u
+ )"
+ marker_count="$(grep -Ec '^\[collectivex-private\] socket-interface-selected=' "$log")"
+ [[ "$socket_ifname" =~ ^[A-Za-z][A-Za-z0-9_.-]{0,31}$ ]] \
+ && [ "$marker_count" = "$nodes" ] || return 1
+ export CX_SOCKET_IFNAME="$socket_ifname"
link_layer="$(
sed -nE 's/^\[collectivex-private\] rdma-link-layer=(roce|infiniband)$/\1/p' "$log" \
| sort -u
diff --git a/experimental/CollectiveX/tests/test_deepep_v2_contract.py b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
index bdb33ff143..f28b50ba25 100644
--- a/experimental/CollectiveX/tests/test_deepep_v2_contract.py
+++ b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
@@ -2162,7 +2162,7 @@ def test_operator_config_is_strict_per_runner_json(self) -> None:
"COLLECTIVEX_OPERATOR_CONFIG": str(config),
},
)
- self.assertNotEqual(scaleout.returncode, 0)
+ self.assertEqual(scaleout.returncode, 0, scaleout.stderr)
config.write_text(json.dumps(missing_amd_stage))
config.chmod(0o600)
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index d5e1f50974..2b755a7766 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -750,6 +750,7 @@ def test_network_profile_validation_is_private_and_all_node(self) -> None:
"done\n"
"if [ \"${SRUN_RC:-0}\" = 0 ]; then\n"
" for ((task=0; task
Date: Mon, 6 Jul 2026 18:45:18 +0800
Subject: [PATCH 120/210] fix(collectivex): classify socket consensus failures
---
experimental/CollectiveX/runtime/common.sh | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index b873b51668..875d506308 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -892,8 +892,12 @@ BASH
| sort -u
)"
marker_count="$(grep -Ec '^\[collectivex-private\] socket-interface-selected=' "$log")"
- [[ "$socket_ifname" =~ ^[A-Za-z][A-Za-z0-9_.-]{0,31}$ ]] \
- && [ "$marker_count" = "$nodes" ] || return 1
+ socket_unique_count="$(printf '%s\n' "$socket_ifname" | sed '/^$/d' | wc -l | tr -d ' ')"
+ if ! [[ "$socket_ifname" =~ ^[A-Za-z][A-Za-z0-9_.-]{0,31}$ ]] \
+ || [ "$marker_count" != "$nodes" ]; then
+ cx_log "ERROR: network-profile-socket-markers=$marker_count/$nodes unique=$socket_unique_count"
+ return 1
+ fi
export CX_SOCKET_IFNAME="$socket_ifname"
link_layer="$(
sed -nE 's/^\[collectivex-private\] rdma-link-layer=(roce|infiniband)$/\1/p' "$log" \
From b9d6828c65b9a554c5ce52bb309c79ef07d39cd2 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 18:47:25 +0800
Subject: [PATCH 121/210] fix(collectivex): support per-node socket interface
names
---
experimental/CollectiveX/runtime/common.sh | 24 +++++++++++++++++++---
1 file changed, 21 insertions(+), 3 deletions(-)
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 875d506308..7d7083b47e 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -767,6 +767,16 @@ cx_restore_exact_hca_selector() {
export NCCL_IB_HCA="=$CX_RDMA_DEVICES"
}
+cx_default_route_interface() {
+ python3 - <<'PY'
+from pathlib import Path
+for line in Path('/proc/net/route').read_text().splitlines()[1:]:
+ fields = line.split()
+ if len(fields) >= 4 and fields[1] == '00000000' and int(fields[3], 16) & 1:
+ print(fields[0], end=''); break
+PY
+}
+
# Prove that the operator-pinned scale-out fabric exists on every allocated
# node before image import or backend initialization. Selector values and node
# diagnostics stay in the runner-private log.
@@ -893,12 +903,15 @@ BASH
)"
marker_count="$(grep -Ec '^\[collectivex-private\] socket-interface-selected=' "$log")"
socket_unique_count="$(printf '%s\n' "$socket_ifname" | sed '/^$/d' | wc -l | tr -d ' ')"
- if ! [[ "$socket_ifname" =~ ^[A-Za-z][A-Za-z0-9_.-]{0,31}$ ]] \
- || [ "$marker_count" != "$nodes" ]; then
+ if [ "$socket_unique_count" -lt 1 ] || [ "$marker_count" != "$nodes" ]; then
cx_log "ERROR: network-profile-socket-markers=$marker_count/$nodes unique=$socket_unique_count"
return 1
fi
- export CX_SOCKET_IFNAME="$socket_ifname"
+ if [ "$socket_unique_count" = 1 ]; then
+ export CX_SOCKET_IFNAME="$socket_ifname"
+ else
+ unset CX_SOCKET_IFNAME
+ fi
link_layer="$(
sed -nE 's/^\[collectivex-private\] rdma-link-layer=(roce|infiniband)$/\1/p' "$log" \
| sort -u
@@ -953,6 +966,11 @@ esac
[ "$SLURM_LOCALID" -lt "$CX_GPUS_PER_NODE" ] || exit 67
. /ix/experimental/CollectiveX/runtime/common.sh || exit 68
if [ "${CX_NODES:-1}" -gt 1 ] && [ "${CX_TRANSPORT:-}" != mnnvl ]; then
+ if [ -z "${CX_SOCKET_IFNAME:-}" ]; then
+ CX_SOCKET_IFNAME="$(cx_default_route_interface)" || exit 68
+ [[ "$CX_SOCKET_IFNAME" =~ ^[A-Za-z][A-Za-z0-9_.-]{0,31}$ ]] || exit 68
+ export CX_SOCKET_IFNAME
+ fi
cx_apply_network_profile "$CX_NODES" "$CX_TRANSPORT" || exit 68
fi
cx_write_runtime_stage execution || exit 68
From b337b1e503ce5f45318ac5b214c5f5699d315888 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 18:53:12 +0800
Subject: [PATCH 122/210] fix(collectivex): record MI355X MoRI construction
limits
---
experimental/CollectiveX/capability.py | 6 ++++++
.../CollectiveX/tests/test_precision_scheduling.py | 11 +++++++++++
.../CollectiveX/tests/test_qualification_planning.py | 2 +-
.../CollectiveX/tests/test_sampling_contract.py | 4 ++--
4 files changed, 20 insertions(+), 3 deletions(-)
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
index 0f4f922dc8..d027726632 100644
--- a/experimental/CollectiveX/capability.py
+++ b/experimental/CollectiveX/capability.py
@@ -190,6 +190,12 @@ def _platform(
("h200-dgxc", "deepep-v2", 16): (
"DeepEP V2 EP16 exhausts pinned NCCL Gin QP resources on the H200 fabric"
),
+ ("mi355x", "mori", 8): (
+ "Pinned MoRI backend construction does not complete on MI355X EP8"
+ ),
+ ("mi355x", "mori", 16): (
+ "Pinned MoRI backend construction does not complete on MI355X EP16"
+ ),
}
PRECISION_DISPOSITIONS = {
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
index 29029df237..102c9d33a0 100644
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -356,6 +356,17 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
self.assertEqual(b200_reference, "supported")
self.assertEqual(reason, "ok")
+ for ep in (8, 16):
+ disposition, reason = capability.resolve_disposition(
+ "mi355x", "mori", ep=ep, nodes=ep // 8,
+ precision_profile=identity.V1_CONTROL_PRECISION_PROFILE,
+ )
+ self.assertEqual(disposition, "unsupported")
+ self.assertEqual(
+ reason,
+ f"Pinned MoRI backend construction does not complete on MI355X EP{ep}",
+ )
+
for sku, product in (("b200-dgxc", "B200"), ("h200-dgxc", "H200")):
disposition, reason = capability.resolve_disposition(
sku, "deepep-v2", ep=16, nodes=2,
diff --git a/experimental/CollectiveX/tests/test_qualification_planning.py b/experimental/CollectiveX/tests/test_qualification_planning.py
index 574bde610c..a6d9229dc7 100644
--- a/experimental/CollectiveX/tests/test_qualification_planning.py
+++ b/experimental/CollectiveX/tests/test_qualification_planning.py
@@ -227,7 +227,7 @@ def test_full_v1_matrix_emits_every_unsupported_terminal_outcome(self) -> None:
documents = [json.loads(path.read_text()) for path in written]
self.assertEqual(len(documents), len(expected))
- self.assertEqual(len(documents), 292)
+ self.assertEqual(len(documents), 304)
self.assertEqual(
{document["outcome"]["reason"] for document in documents},
{item["reason"] for item in expected},
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 2b755a7766..6f7a783178 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -209,7 +209,7 @@ def test_matrix_cases_and_shards_are_identity_bound(self) -> None:
sum(len(item["case"]["ladder"].split()) for item in runnable_cases),
sum(len(item["case"]["ladder"].split()) for item in unsupported_cases),
),
- (51, 656, 364, 292, 1648, 877, 771),
+ (49, 656, 352, 304, 1648, 849, 799),
)
b300_ep16 = [
item for item in unsupported_cases
@@ -276,7 +276,7 @@ def test_matrix_cases_and_shards_are_identity_bound(self) -> None:
self.assertIsNotNone(capability.topology_for("mi325x", 8))
self.assertEqual(
Counter(shard["n"] for shard in matrix["include"]),
- Counter({6: 36, 8: 1, 10: 14}),
+ Counter({6: 34, 8: 1, 10: 14}),
)
ll_cases = [
item for item in matrix["requested_cases"]
From db6df14098426c8555a85abc1d4736449077abcd Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 18:59:32 +0800
Subject: [PATCH 123/210] fix(collectivex): preserve backend import failures
---
experimental/CollectiveX/runtime/common.sh | 4 ++--
.../CollectiveX/runtime/run_in_container.sh | 3 ++-
.../tests/test_sampling_contract.py | 20 +++++++++++++++++++
3 files changed, 24 insertions(+), 3 deletions(-)
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 7d7083b47e..69e4efc479 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -68,10 +68,10 @@ cx_fail_stage() {
diagnostic="backend-dependency"
elif grep -aEqi 'revision fetch failed|submodule fetch failed|package installation failed|staged source is invalid|source (pin resolution|seed validation|seed copy|checkout creation|publication validation|existing source validation) failed' "$log_path"; then
diagnostic="backend-source"
+ elif grep -aEqi 'backend preparation failed|backend import failed|build (failed|is incomplete)|cache (mount identity )?validation failed|import failed' "$log_path"; then
+ diagnostic="backend-build"
elif grep -aEqi 'failed to mount|squashfs|enroot|pyxis|mount.*invalid argument|invalid argument.*mount' "$log_path"; then
diagnostic="container-runtime"
- elif grep -aEqi 'backend preparation failed|build (failed|is incomplete)|cache (mount identity )?validation failed|import failed' "$log_path"; then
- diagnostic="backend-build"
elif grep -aEqi 'command not found|not found on this runner|git lookup failed' "$log_path"; then
diagnostic="missing-runtime"
elif grep -aEqi 'too many requests|rate.?limit' "$log_path"; then
diff --git a/experimental/CollectiveX/runtime/run_in_container.sh b/experimental/CollectiveX/runtime/run_in_container.sh
index 6c970e10e6..e47ed357b1 100644
--- a/experimental/CollectiveX/runtime/run_in_container.sh
+++ b/experimental/CollectiveX/runtime/run_in_container.sh
@@ -957,7 +957,8 @@ cx_prepare_backend() {
cx_build_uccl || return 1
;;
mori)
- python3 -c "import mori" 2>/dev/null || return 1
+ python3 -c "import mori" \
+ || { cx_log "ERROR: MoRI backend import failed"; return 1; }
;;
nccl-ep)
;;
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 6f7a783178..1998ef1281 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -2643,6 +2643,26 @@ def test_non_timeout_failure_warning_is_classified_as_case_failure(self) -> None
self.assertNotIn("diagnostic=network-or-timeout", result.stderr)
self.assertIn("diagnostic=benchmark-case-failure", result.stderr)
+ def test_backend_import_failure_precedes_container_cleanup_noise(self) -> None:
+ with tempfile.TemporaryDirectory() as temporary:
+ log = Path(temporary) / "runtime.log"
+ log.write_text(
+ "ERROR: MoRI backend import failed\n"
+ "pyxis: failed to mount container cleanup path: invalid argument\n"
+ )
+ result = subprocess.run(
+ [
+ "bash", "-c",
+ 'source "$1"; cx_fail_stage backend-setup "$2"',
+ "_", str(ROOT / "runtime" / "common.sh"), str(log),
+ ],
+ text=True,
+ capture_output=True,
+ )
+ self.assertEqual(result.returncode, 1)
+ self.assertIn("diagnostic=backend-build", result.stderr)
+ self.assertNotIn("diagnostic=container-runtime", result.stderr)
+
def test_precision_probe_timeout_reports_only_the_last_closed_stage(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
log = Path(temporary) / "runtime.log"
From eff7965da1b429916dd2c088294fb45bc7a08d97 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 19:31:51 +0800
Subject: [PATCH 124/210] fix(collectivex): align runtime provenance contracts
---
experimental/CollectiveX/capability.py | 2 +-
experimental/CollectiveX/contracts.py | 2 +-
experimental/CollectiveX/runtime/common.sh | 2 +-
.../schemas/raw-case-v1.schema.json | 1 +
experimental/CollectiveX/tests/ep_harness.py | 26 +++++++++----------
.../tests/test_precision_scheduling.py | 5 +++-
.../tests/test_sampling_contract.py | 9 ++++++-
.../tests/test_schema_v1_contract.py | 10 +++++++
8 files changed, 38 insertions(+), 19 deletions(-)
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
index d027726632..f35eb11eae 100644
--- a/experimental/CollectiveX/capability.py
+++ b/experimental/CollectiveX/capability.py
@@ -27,7 +27,7 @@
# B200's publication pods cannot map NIC doorbells into GPU address space.
# DeepEP V1 scale-out therefore uses NVSHMEM's CPU-proxied IBGDA handler.
-DEEPEP_V1_IBGDA_NIC_HANDLERS = {"b200-dgxc": "cpu"}
+DEEPEP_V1_IBGDA_NIC_HANDLERS = {"b200-dgxc": "cpu_host_memory"}
def _topologies(
diff --git a/experimental/CollectiveX/contracts.py b/experimental/CollectiveX/contracts.py
index 23cac74586..5846457ec5 100644
--- a/experimental/CollectiveX/contracts.py
+++ b/experimental/CollectiveX/contracts.py
@@ -543,7 +543,7 @@ def backend_provenance_issues(backend: str, provenance: dict[str, Any]) -> list[
if provenance.get("backend_lineage") != "deepep-v1":
unresolved.append("backend_lineage")
if provenance.get("nvshmem_ibgda_nic_handler") not in {
- "cpu", "gpu", "not-active",
+ "cpu_host_memory", "gpu", "not-active",
}:
unresolved.append("nvshmem_ibgda_nic_handler")
if backend in {"deepep", "uccl"}:
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 69e4efc479..b2c3e5d4f0 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -741,7 +741,7 @@ cx_apply_network_profile() {
fi
local nic_handler=gpu
if [ "${CX_SHARD_SKU:-}" = b200-dgxc ] && [ "${CX_BENCH:-}" = deepep ]; then
- nic_handler=cpu
+ nic_handler=cpu_host_memory
fi
export NVSHMEM_IB_ENABLE_IBGDA=1 NVSHMEM_IBGDA_NIC_HANDLER="$nic_handler"
if [ -n "${CX_RDMA_LINK_LAYER:-}" ]; then
diff --git a/experimental/CollectiveX/schemas/raw-case-v1.schema.json b/experimental/CollectiveX/schemas/raw-case-v1.schema.json
index f65d71ca72..9889d1b752 100644
--- a/experimental/CollectiveX/schemas/raw-case-v1.schema.json
+++ b/experimental/CollectiveX/schemas/raw-case-v1.schema.json
@@ -773,6 +773,7 @@
"num_qps_per_rank",
"num_rdma_bytes",
"num_sms",
+ "nvshmem_ibgda_nic_handler",
"nvshmem_package_version",
"path",
"physical_nvlink_ranks",
diff --git a/experimental/CollectiveX/tests/ep_harness.py b/experimental/CollectiveX/tests/ep_harness.py
index d07f924045..0262a11070 100644
--- a/experimental/CollectiveX/tests/ep_harness.py
+++ b/experimental/CollectiveX/tests/ep_harness.py
@@ -1787,9 +1787,6 @@ def component(percentiles, count, *, derived=False):
if capture_deferred_provenance is not None:
capture_deferred_provenance()
- if rank != 0:
- return 0
-
# status=valid requires correctness AND a proven-identical routing trace across ranks.
all_ok = bool(rows) and all(r["correctness"]["passed"] for r in rows) and routing_consistent
@@ -2162,17 +2159,18 @@ def component(percentiles, count, *, derived=False):
},
}
contracts.validate_raw_document(doc, samples_document)
- _write_bytes_atomic(samples_path, samples_payload)
- _write_json_atomic(args.out, doc)
- dispatch_percentiles = headline["components"]["dispatch"]["percentiles_us"]
- dispatch_p99 = dispatch_percentiles["p99"] if dispatch_percentiles else None
- component_summary = (f"disp_p99={dispatch_p99:.1f}us "
- if dispatch_p99 is not None
- else "components=unavailable ")
- print(f"{backend.name} ep-dispatch-combine [{args.phase}/{mode}/{case_profile['contract']}]: "
- f"status={doc['outcome']['status']} {len(rows)} pts, routing_consistent={routing_consistent}, "
- f"headline T={headline['tokens_per_rank']} {component_summary}"
- f"-> {args.out}")
+ if rank == 0:
+ _write_bytes_atomic(samples_path, samples_payload)
+ _write_json_atomic(args.out, doc)
+ dispatch_percentiles = headline["components"]["dispatch"]["percentiles_us"]
+ dispatch_p99 = dispatch_percentiles["p99"] if dispatch_percentiles else None
+ component_summary = (f"disp_p99={dispatch_p99:.1f}us "
+ if dispatch_p99 is not None
+ else "components=unavailable ")
+ print(f"{backend.name} ep-dispatch-combine [{args.phase}/{mode}/{case_profile['contract']}]: "
+ f"status={doc['outcome']['status']} {len(rows)} pts, routing_consistent={routing_consistent}, "
+ f"headline T={headline['tokens_per_rank']} {component_summary}"
+ f"-> {args.out}")
# A complete invalid document is still a successfully captured terminal outcome. Launchers
# inspect its status to fail the case without conflating it with an execution failure.
return 0
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
index 102c9d33a0..d853801504 100644
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -338,7 +338,10 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
)
self.assertEqual(disposition, "supported")
self.assertEqual(reason, "ok")
- self.assertEqual(capability.DEEPEP_V1_IBGDA_NIC_HANDLERS["b200-dgxc"], "cpu")
+ self.assertEqual(
+ capability.DEEPEP_V1_IBGDA_NIC_HANDLERS["b200-dgxc"],
+ "cpu_host_memory",
+ )
disposition, reason = capability.resolve_disposition(
"b200-dgxc", "deepep-hybrid", ep=16, nodes=2,
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 1998ef1281..a4c4ed08c1 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -675,7 +675,7 @@ def test_scaleout_network_profile_is_explicit_and_allowlisted(self) -> None:
test "$NVSHMEM_IB_ENABLE_IBGDA:$NVSHMEM_IBGDA_NIC_HANDLER" = 1:gpu
export CX_SHARD_SKU=b200-dgxc CX_BENCH=deepep
cx_apply_network_profile 2 nvlink-rdma
- test "$NVSHMEM_IB_ENABLE_IBGDA:$NVSHMEM_IBGDA_NIC_HANDLER" = 1:cpu
+ test "$NVSHMEM_IB_ENABLE_IBGDA:$NVSHMEM_IBGDA_NIC_HANDLER" = 1:cpu_host_memory
unset CX_SHARD_SKU CX_BENCH
cx_apply_network_profile 2 nvlink-rdma
cx_export_gid_index_for_link_layer infiniband 1
@@ -2663,6 +2663,13 @@ def test_backend_import_failure_precedes_container_cleanup_noise(self) -> None:
self.assertIn("diagnostic=backend-build", result.stderr)
self.assertNotIn("diagnostic=container-runtime", result.stderr)
+ def test_all_ranks_validate_raw_evidence_before_rank_zero_writes(self) -> None:
+ harness = (ROOT / "tests" / "ep_harness.py").read_text()
+ validation = harness.index("contracts.validate_raw_document(doc, samples_document)")
+ rank_zero_write = harness.index("if rank == 0:", validation)
+ self.assertLess(validation, rank_zero_write)
+ self.assertNotIn("if rank != 0:", harness[harness.index("def run_sweep("):validation])
+
def test_precision_probe_timeout_reports_only_the_last_closed_stage(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
log = Path(temporary) / "runtime.log"
diff --git a/experimental/CollectiveX/tests/test_schema_v1_contract.py b/experimental/CollectiveX/tests/test_schema_v1_contract.py
index ee52af7630..3073a7e253 100644
--- a/experimental/CollectiveX/tests/test_schema_v1_contract.py
+++ b/experimental/CollectiveX/tests/test_schema_v1_contract.py
@@ -15,6 +15,7 @@
sys.path.insert(0, str(ROOT))
import identity # noqa: E402
+import contracts # noqa: E402
def _load(name: str) -> dict:
@@ -47,6 +48,15 @@ def test_all_checked_in_schemas_are_draft_2020_12_valid(self) -> None:
jsonschema.Draft202012Validator.check_schema(schema)
self.assertFalse(schema["additionalProperties"])
+ def test_raw_backend_provenance_schema_matches_runtime_vocabulary(self) -> None:
+ provenance = self.raw["properties"]["implementation"]["properties"][
+ "provenance"
+ ]
+ self.assertEqual(
+ set(provenance["propertyNames"]["enum"]),
+ contracts.PROVENANCE_KEYS,
+ )
+
def test_precision_catalog_and_axes_are_exact_and_strict(self) -> None:
expected = set(identity.V1_PRECISION_PROFILES)
self.assertEqual(set(self.raw["$defs"]["precision_profile"]["enum"]), expected)
From b12944513187972173db1285fe0aae4b68e198d5 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 19:35:11 +0800
Subject: [PATCH 125/210] fix(collectivex): remove MoRI Gloo scale-out
dependency
---
experimental/CollectiveX/tests/probe_precision.py | 2 +-
experimental/CollectiveX/tests/run_ep.py | 5 +++--
experimental/CollectiveX/tests/test_precision_scheduling.py | 4 ++++
3 files changed, 8 insertions(+), 3 deletions(-)
diff --git a/experimental/CollectiveX/tests/probe_precision.py b/experimental/CollectiveX/tests/probe_precision.py
index f69ba158a7..e1de5f4ff5 100644
--- a/experimental/CollectiveX/tests/probe_precision.py
+++ b/experimental/CollectiveX/tests/probe_precision.py
@@ -821,7 +821,7 @@ def _init_distributed(torch_module, dist, backend_name: str, device, rank: int,
return
if backend_name == "mori":
dist.init_process_group(
- backend="cpu:gloo,cuda:nccl", rank=rank, world_size=world_size, device_id=device
+ backend="nccl", rank=rank, world_size=world_size, device_id=device
)
elif backend_name == "deepep-v2":
dist.init_process_group("nccl", device_id=device)
diff --git a/experimental/CollectiveX/tests/run_ep.py b/experimental/CollectiveX/tests/run_ep.py
index 4fb9de03c4..cc43abffa3 100644
--- a/experimental/CollectiveX/tests/run_ep.py
+++ b/experimental/CollectiveX/tests/run_ep.py
@@ -440,11 +440,12 @@ def main() -> int:
else:
from ep_deepep import DeepEPBackend as Backend
- # MoRI uses the gloo+NCCL group shape from its reference; other adapters use NCCL/RCCL.
+ # MoRI registers the default GPU process group with its SHMEM runtime. Keep that
+ # group device-only so scale-out does not also depend on a host Gloo fabric.
if not dist.is_initialized():
if args.backend == "mori":
dist.init_process_group(
- backend="cpu:gloo,cuda:nccl",
+ backend="nccl",
rank=rank,
world_size=world_size,
device_id=device,
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
index d853801504..5b7c79635f 100644
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -518,6 +518,7 @@ def load_config(name: str) -> dict[str, object]:
def test_probe_runtime_diagnostics_are_closed_stage_codes(self) -> None:
probe = (ROOT / "tests" / "probe_precision.py").read_text()
+ run_ep = (ROOT / "tests" / "run_ep.py").read_text()
for stage in (
"distributed-init", "runtime-context", "construction-consensus",
"operation-consensus", "evidence-aggregation",
@@ -525,6 +526,9 @@ def test_probe_runtime_diagnostics_are_closed_stage_codes(self) -> None:
self.assertIn(f'"{stage}"', probe)
self.assertIn("precision-probe-diagnostic={diagnostic_stage}-exception", probe)
self.assertNotIn("precision-probe-diagnostic={exc}", probe)
+ self.assertNotIn("cpu:gloo,cuda:nccl", probe + run_ep)
+ self.assertIn('backend="nccl", rank=rank, world_size=world_size, device_id=device', probe)
+ self.assertIn('backend="nccl",', run_ep)
if __name__ == "__main__":
From 05727e0896466f00c3ccf26d6cf14e5c4660f99e Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 19:47:55 +0800
Subject: [PATCH 126/210] fix(collectivex): budget DeepEP V2 GIN QP allocation
Upstream ElasticBuffer's hybrid-mode default num_allocated_qps (129 on
CX-7) makes GDAKI create contexts x world_size = 2064 device QPs per
rank, exhausting the per-NIC QP budget at EP16: every rank fails in
ncclDevCommCreate when NCCL's regular receiver QPs no longer fit
(ibv_create_qp ENOMEM), identically on H200 bare-metal and B200 pods.
Measured on the failing H200 pair: 49x16=784 QPs/rank initializes,
65x16=1040 does not.
Pass an explicit allocation from a fixed ~512-QP budget instead:
EP8 resolves to 65 (unchanged from what CX-8 racks already run), EP16
to 33, EP32 to 17. Verified on the failing worker pair: construction
plus a dispatch/combine round trip pass 16/16 at EP16 with 33 QPs.
The explicit value also skips upstream's rank-local ibstat probe, which
could diverge across ranks.
---
.../CollectiveX/tests/ep_deepep_v2.py | 23 +++++++++++++++++++
1 file changed, 23 insertions(+)
diff --git a/experimental/CollectiveX/tests/ep_deepep_v2.py b/experimental/CollectiveX/tests/ep_deepep_v2.py
index e3da3f6090..bfb11c6cde 100644
--- a/experimental/CollectiveX/tests/ep_deepep_v2.py
+++ b/experimental/CollectiveX/tests/ep_deepep_v2.py
@@ -212,6 +212,24 @@ def _require_cross_rank_equal(value, label: str) -> None:
raise RuntimeError(f"DeepEP V2 {label} differs across ranks")
+# GIN/GDAKI allocates num_allocated_qps device QPs per peer rank on the local NIC
+# (contexts x world_size QPs, before NCCL's own connection QPs). Upstream's hybrid
+# default (129, or 65 with fast RDMA atomics) exhausts the per-NIC QP budget at
+# EP16: construction dies in ncclDevCommCreate with ibv_create_qp ENOMEM once
+# NCCL's regular QPs land on top (identical on H200 bare-metal and B200 pods; on
+# CX-7 the budget sits between 784 and 1040 QPs — 49x16 initializes, 65x16 does
+# not). Spending a fixed ~512-QP budget keeps every EP size inside that limit
+# with headroom: EP8 resolves to 65 (the allocation CX-8 racks already run
+# successfully), EP16 to 33 and EP32 to 17 (33 and 49 verified on the failing
+# H200 pair). An explicit value also skips upstream's rank-local ibstat probe,
+# which is not guaranteed to resolve identically across ranks.
+_GIN_QP_BUDGET = 512
+
+
+def _hybrid_num_allocated_qps(world_size: int) -> int:
+ return max(9, 1 + _GIN_QP_BUDGET // world_size)
+
+
def _configure_gin_mode(args, world_size: int) -> bool:
scale_up_domain = int(
getattr(args, "scale_up_domain", None)
@@ -369,6 +387,11 @@ def __init__(self, args, rank, world_size, local_rank, device):
prefer_overlap_with_compute=True,
num_gpu_timeout_secs=100,
explicitly_destroy=True,
+ # 0 is upstream's use-the-default sentinel; only hybrid (GIN) mode
+ # needs the explicit budget-derived allocation.
+ num_allocated_qps=(
+ _hybrid_num_allocated_qps(world_size) if allow_hybrid_mode else 0
+ ),
)
tuning_num_experts = int(getattr(args, "num_logical_experts", args.experts))
self.num_sms = int(
From 689f5bfb775f7ead7e3837da256fe77de7019d1f Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 19:52:26 +0800
Subject: [PATCH 127/210] fix(collectivex): mount GDRCopy for B200 DeepEP
---
experimental/CollectiveX/capability.py | 2 +-
experimental/CollectiveX/contracts.py | 2 +-
.../CollectiveX/launchers/launch_single-slurm.sh | 11 +++++++++++
experimental/CollectiveX/runtime/common.sh | 2 +-
.../CollectiveX/tests/test_precision_scheduling.py | 2 +-
.../CollectiveX/tests/test_sampling_contract.py | 8 +++++++-
6 files changed, 22 insertions(+), 5 deletions(-)
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
index f35eb11eae..d027726632 100644
--- a/experimental/CollectiveX/capability.py
+++ b/experimental/CollectiveX/capability.py
@@ -27,7 +27,7 @@
# B200's publication pods cannot map NIC doorbells into GPU address space.
# DeepEP V1 scale-out therefore uses NVSHMEM's CPU-proxied IBGDA handler.
-DEEPEP_V1_IBGDA_NIC_HANDLERS = {"b200-dgxc": "cpu_host_memory"}
+DEEPEP_V1_IBGDA_NIC_HANDLERS = {"b200-dgxc": "cpu"}
def _topologies(
diff --git a/experimental/CollectiveX/contracts.py b/experimental/CollectiveX/contracts.py
index 5846457ec5..23cac74586 100644
--- a/experimental/CollectiveX/contracts.py
+++ b/experimental/CollectiveX/contracts.py
@@ -543,7 +543,7 @@ def backend_provenance_issues(backend: str, provenance: dict[str, Any]) -> list[
if provenance.get("backend_lineage") != "deepep-v1":
unresolved.append("backend_lineage")
if provenance.get("nvshmem_ibgda_nic_handler") not in {
- "cpu_host_memory", "gpu", "not-active",
+ "cpu", "gpu", "not-active",
}:
unresolved.append("nvshmem_ibgda_nic_handler")
if backend in {"deepep", "uccl"}:
diff --git a/experimental/CollectiveX/launchers/launch_single-slurm.sh b/experimental/CollectiveX/launchers/launch_single-slurm.sh
index 8515482014..17f4e29c4d 100644
--- a/experimental/CollectiveX/launchers/launch_single-slurm.sh
+++ b/experimental/CollectiveX/launchers/launch_single-slurm.sh
@@ -131,6 +131,17 @@ for allocation_attempt in 1 2 3; do
excluded_nodes+="$rejected_nodes"
done
unset CX_SALLOC_ATTEMPT CX_NETWORK_VALIDATION_ATTEMPT
+if [ "$RUNNER" = b200-dgxc ] && [ "$CX_BENCH" = deepep ] && [ "$NODES" -gt 1 ]; then
+ gdrcopy_log="$(cx_private_log_path gdrcopy-preflight)"
+ if ! srun --jobid="$JOB_ID" --nodes="$NODES" --ntasks="$NODES" --ntasks-per-node=1 \
+ --chdir=/tmp --export="$(cx_host_exports)" test -c /dev/gdrdrv \
+ >"$gdrcopy_log" 2>&1; then
+ cx_log "ERROR: allocated B200 nodes do not expose the required GDRCopy device"
+ cx_fail_stage setup "$gdrcopy_log" || true
+ cx_die "B200 DeepEP CPU NIC handling requires GDRCopy"
+ fi
+ CONTAINER_MOUNTS="$CONTAINER_MOUNTS,/dev/gdrdrv:/dev/gdrdrv"
+fi
if [ "$LOCAL_IMPORT" = 1 ]; then
cx_set_failure_stage container-import
SQUASH_FILE="$(CX_ENROOT_LOCAL_IMPORT=1 cx_ensure_squash "$CX_SQUASH_DIR" "$IMAGE")"
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index b2c3e5d4f0..69e4efc479 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -741,7 +741,7 @@ cx_apply_network_profile() {
fi
local nic_handler=gpu
if [ "${CX_SHARD_SKU:-}" = b200-dgxc ] && [ "${CX_BENCH:-}" = deepep ]; then
- nic_handler=cpu_host_memory
+ nic_handler=cpu
fi
export NVSHMEM_IB_ENABLE_IBGDA=1 NVSHMEM_IBGDA_NIC_HANDLER="$nic_handler"
if [ -n "${CX_RDMA_LINK_LAYER:-}" ]; then
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
index 5b7c79635f..1a08d74e67 100644
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -340,7 +340,7 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
self.assertEqual(reason, "ok")
self.assertEqual(
capability.DEEPEP_V1_IBGDA_NIC_HANDLERS["b200-dgxc"],
- "cpu_host_memory",
+ "cpu",
)
disposition, reason = capability.resolve_disposition(
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index a4c4ed08c1..b8422a04cc 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -675,7 +675,7 @@ def test_scaleout_network_profile_is_explicit_and_allowlisted(self) -> None:
test "$NVSHMEM_IB_ENABLE_IBGDA:$NVSHMEM_IBGDA_NIC_HANDLER" = 1:gpu
export CX_SHARD_SKU=b200-dgxc CX_BENCH=deepep
cx_apply_network_profile 2 nvlink-rdma
- test "$NVSHMEM_IB_ENABLE_IBGDA:$NVSHMEM_IBGDA_NIC_HANDLER" = 1:cpu_host_memory
+ test "$NVSHMEM_IB_ENABLE_IBGDA:$NVSHMEM_IBGDA_NIC_HANDLER" = 1:cpu
unset CX_SHARD_SKU CX_BENCH
cx_apply_network_profile 2 nvlink-rdma
cx_export_gid_index_for_link_layer infiniband 1
@@ -2603,6 +2603,12 @@ def test_launchers_use_private_logs_and_allowlisted_failure_stages(self) -> None
self.assertIn('rejected_nodes="$(cx_allocation_nodes_csv "$JOB_ID")"', single_slurm)
self.assertIn('export CX_SALLOC_ATTEMPT="$allocation_attempt"', single_slurm)
self.assertIn('export CX_NETWORK_VALIDATION_ATTEMPT="$allocation_attempt"', single_slurm)
+ self.assertIn('test -c /dev/gdrdrv', single_slurm)
+ self.assertIn('/dev/gdrdrv:/dev/gdrdrv', single_slurm)
+ self.assertIn(
+ '[ "$RUNNER" = b200-dgxc ] && [ "$CX_BENCH" = deepep ] && [ "$NODES" -gt 1 ]',
+ single_slurm,
+ )
self.assertIn('log_label+="-a${CX_SALLOC_ATTEMPT}"', common)
self.assertIn('log_label+="-a${CX_NETWORK_VALIDATION_ATTEMPT}"', common)
From b233ba921000939c5ebf4badacb19eeb10c486c1 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 20:06:21 +0800
Subject: [PATCH 128/210] feat(collectivex): freeze the eight-SKU V1 matrix
---
.github/workflows/collectivex-sweep.yml | 2 +-
experimental/CollectiveX/capability.py | 4 +++
experimental/CollectiveX/configs/suites.yaml | 2 +-
experimental/CollectiveX/publisher.py | 6 ++--
experimental/CollectiveX/sweep_matrix.py | 6 ++--
.../tests/test_precision_scheduling.py | 31 +++++++++++++------
.../CollectiveX/tests/test_publisher.py | 2 +-
.../tests/test_qualification_planning.py | 2 +-
.../tests/test_sampling_contract.py | 4 +--
9 files changed, 37 insertions(+), 22 deletions(-)
diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml
index c8db25ff7d..97c83edfc1 100644
--- a/.github/workflows/collectivex-sweep.yml
+++ b/.github/workflows/collectivex-sweep.yml
@@ -215,7 +215,7 @@ jobs:
- name: Create V1 release marker
if: ${{ inputs.operation == 'sweep' && inputs.release_tag == 'v1' }}
env:
- EXPECTED_MATRIX_SHA256: c6988c5e81239ace699541322a88a37bfd80819d8bc1a2446f928665cd3ebba0
+ EXPECTED_MATRIX_SHA256: 63d566a594f464c8c6cd8f36c37790fe40c7a6bb715037246dd7cbb36b151ed5
RUN_ID: ${{ github.run_id }}
RUN_ATTEMPT: ${{ github.run_attempt }}
SOURCE_SHA: ${{ github.sha }}
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
index d027726632..a699a57b1d 100644
--- a/experimental/CollectiveX/capability.py
+++ b/experimental/CollectiveX/capability.py
@@ -196,6 +196,9 @@ def _platform(
("mi355x", "mori", 16): (
"Pinned MoRI backend construction does not complete on MI355X EP16"
),
+ ("mi300x", "mori", 16): (
+ "Pinned MoRI distributed initialization does not complete on MI300X EP16"
+ ),
}
PRECISION_DISPOSITIONS = {
@@ -432,6 +435,7 @@ def _precision_rule(
("28782309414", "mi300x", 8, "mori", "normal", _NORMAL_E4M3FNUZ_PROFILE, "supported", "native-probe-passed"),
("28782309414", "mi300x", 8, "mori", "normal", _MORI_E4M3FNUZ_DIRECT_PROFILE, "supported", "native-probe-passed"),
("28782309414", "mi300x", 8, "mori", "normal", _MORI_E4M3FNUZ_BOTH_PROFILE, "supported", "native-probe-passed"),
+ ("28788599713", "mi300x", 16, "mori", "normal", _NORMAL_E4M3FNUZ_PROFILE, "unsupported", "distributed-init-timeout"),
*(("28743235213", "b300", 8, "deepep", "low-latency", profile,
"supported", "native-probe-passed")
for profile in (_LL_FP8_PROFILE, _LL_LOGFMT_PROFILE, _LL_FP8_LOGFMT_PROFILE)),
diff --git a/experimental/CollectiveX/configs/suites.yaml b/experimental/CollectiveX/configs/suites.yaml
index 281bbe93c5..d04552672e 100644
--- a/experimental/CollectiveX/configs/suites.yaml
+++ b/experimental/CollectiveX/configs/suites.yaml
@@ -52,7 +52,7 @@ suites:
- d-fp8-e4m3fnuz-b128-f32-prequantized.c-bf16
- d-bf16.c-fp8-e4m3fnuz-direct-cast-noscale
- d-fp8-e4m3fnuz-b128-f32-prequantized.c-fp8-e4m3fnuz-direct-cast-noscale
- provisional: true
+ provisional: false
required_publication: comparable-experimental
ep-precision-low-latency-v1:
diff --git a/experimental/CollectiveX/publisher.py b/experimental/CollectiveX/publisher.py
index fcf462b42d..082ed766e4 100644
--- a/experimental/CollectiveX/publisher.py
+++ b/experimental/CollectiveX/publisher.py
@@ -44,12 +44,12 @@
PRECISION_COHORT_KINDS = (
"dispatch-precision", "combine-precision", "precision-pair",
)
-REQUIRED_PROMOTION_COHORT_COUNTS = {"library": 70, "system": 12, "routing": 62}
+REQUIRED_PROMOTION_COHORT_COUNTS = {"library": 64, "system": 12, "routing": 54}
CANONICAL_FULL_V1_MATRIX_SHA256 = (
- "c6988c5e81239ace699541322a88a37bfd80819d8bc1a2446f928665cd3ebba0"
+ "63d566a594f464c8c6cd8f36c37790fe40c7a6bb715037246dd7cbb36b151ed5"
)
CANONICAL_FULL_V1_CASE_CATALOG_SHA256 = (
- "98b0c5e85c223a2edcbf9ea580a48bea115294c83691650bb25cb85904ed8be4"
+ "c02fa9151ba2802d24e2f51af056f9b3e45e8e8d2706a3641d09e1e93705f695"
)
P50_STABILITY_LIMIT = 1.10
P99_STABILITY_LIMIT = 1.25
diff --git a/experimental/CollectiveX/sweep_matrix.py b/experimental/CollectiveX/sweep_matrix.py
index 91aed7bd4b..a40eb59123 100644
--- a/experimental/CollectiveX/sweep_matrix.py
+++ b/experimental/CollectiveX/sweep_matrix.py
@@ -105,9 +105,9 @@
)
QUALIFICATION_INDICES = range(1, 4)
CANONICAL_V1_EXECUTION_PLAN_SHA256 = {
- 1: "dd35bee6909a85761e069ebe83444a6f982eb571a4e61ad604a845ea7e4d10dc",
- 2: "fbc7262b3399225493f6c025c3bd2ceca52e90762bda1b6f6002683037094caf",
- 3: "5edc837858df2d7498229db30ec0d364f5b6cd7ca703790a8c83aabb09e6aff4",
+ 1: "9b4652ef5828cabc546ca10faa1c5f699de8f5bda8853a69fcc0e44ed9bbea91",
+ 2: "8c268877e0051fb93aea17aa3830f7d9d7feab40e7276d3de7dd6e76f3e05f0d",
+ 3: "1f5293201ff48e4d51e0e0d81135da6fab83a05faac1d44e1efd822f9f654527",
}
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
index 1a08d74e67..68bbbc0e0d 100644
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -46,9 +46,7 @@ def test_precision_probe_inventory_is_exact_and_non_mutating(self) -> None:
(item["sku"], item["backend"], item["ep"], item["precision_profile"])
for item in targets
],
- [
- ("mi300x", "mori", 16, "d-fp8-e4m3fnuz-b128-f32-prequantized.c-bf16"),
- ],
+ [],
)
self.assertEqual(capability.PRECISION_CAPABILITIES, before)
self.assertEqual(
@@ -249,7 +247,7 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
self.assertTrue(targets)
self.assertEqual(
{item["disposition"] for item in targets},
- {"supported", "unsupported", "provisional"},
+ {"supported", "unsupported"},
)
self.assertEqual(
len(targets), 94
@@ -258,9 +256,9 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
sum(item["disposition"] == "supported" for item in targets), 66
)
self.assertEqual(
- sum(item["disposition"] == "unsupported" for item in targets), 27
+ sum(item["disposition"] == "unsupported" for item in targets), 28
)
- self.assertEqual(len(capability.provisional_precision_targets()), 1)
+ self.assertEqual(len(capability.provisional_precision_targets()), 0)
keys = {
(
item["precision_profile"],
@@ -370,6 +368,16 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
f"Pinned MoRI backend construction does not complete on MI355X EP{ep}",
)
+ disposition, reason = capability.resolve_disposition(
+ "mi300x", "mori", ep=16, nodes=2,
+ precision_profile=identity.V1_CONTROL_PRECISION_PROFILE,
+ )
+ self.assertEqual(disposition, "unsupported")
+ self.assertEqual(
+ reason,
+ "Pinned MoRI distributed initialization does not complete on MI300X EP16",
+ )
+
for sku, product in (("b200-dgxc", "B200"), ("h200-dgxc", "H200")):
disposition, reason = capability.resolve_disposition(
sku, "deepep-v2", ep=16, nodes=2,
@@ -404,7 +412,7 @@ def test_split_suites_track_provisional_state_and_do_not_duplicate_bf16(self) ->
),
("low-latency", ["decode"], [128]),
)
- self.assertTrue(normal["provisional"])
+ self.assertFalse(normal["provisional"])
self.assertFalse(low_latency["provisional"])
self.assertEqual(
normal["required_publication"], "comparable-experimental"
@@ -424,14 +432,17 @@ def test_split_suites_track_provisional_state_and_do_not_duplicate_bf16(self) ->
for item in matrix["requested_cases"]
if "precision_profile" in item["case"]
}
- self.assertEqual(scheduled_profiles, set(low_latency["precision_profiles"]))
+ self.assertEqual(
+ scheduled_profiles,
+ set(normal["precision_profiles"]) | set(low_latency["precision_profiles"]),
+ )
self.assertEqual(
{item["precision_profile"] for item in capability.provisional_precision_targets()},
- {"d-fp8-e4m3fnuz-b128-f32-prequantized.c-bf16"},
+ set(),
)
stale = copy.deepcopy(suites)
- stale["suites"]["ep-precision-normal-v1"]["provisional"] = False
+ stale["suites"]["ep-precision-normal-v1"]["provisional"] = True
with self.assertRaisesRegex(SystemExit, "must track unresolved"):
sweep_matrix.validate_config_documents(stale, workloads)
diff --git a/experimental/CollectiveX/tests/test_publisher.py b/experimental/CollectiveX/tests/test_publisher.py
index bb08318a9e..195fce85ee 100644
--- a/experimental/CollectiveX/tests/test_publisher.py
+++ b/experimental/CollectiveX/tests/test_publisher.py
@@ -2024,7 +2024,7 @@ def test_workflow_matrix_and_catalog_digests_do_not_drift(self) -> None:
if item["disposition"] == "unsupported"
),
),
- (50, 664, 393, 271, 1532, 898, 634),
+ (48, 748, 392, 356, 1740, 881, 859),
)
library: dict[tuple, set[str]] = {}
system: dict[tuple, set[str]] = {}
diff --git a/experimental/CollectiveX/tests/test_qualification_planning.py b/experimental/CollectiveX/tests/test_qualification_planning.py
index a6d9229dc7..30c926797b 100644
--- a/experimental/CollectiveX/tests/test_qualification_planning.py
+++ b/experimental/CollectiveX/tests/test_qualification_planning.py
@@ -227,7 +227,7 @@ def test_full_v1_matrix_emits_every_unsupported_terminal_outcome(self) -> None:
documents = [json.loads(path.read_text()) for path in written]
self.assertEqual(len(documents), len(expected))
- self.assertEqual(len(documents), 304)
+ self.assertEqual(len(documents), 356)
self.assertEqual(
{document["outcome"]["reason"] for document in documents},
{item["reason"] for item in expected},
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index b8422a04cc..a17fb3b112 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -209,7 +209,7 @@ def test_matrix_cases_and_shards_are_identity_bound(self) -> None:
sum(len(item["case"]["ladder"].split()) for item in runnable_cases),
sum(len(item["case"]["ladder"].split()) for item in unsupported_cases),
),
- (49, 656, 352, 304, 1648, 849, 799),
+ (48, 748, 392, 356, 1740, 881, 859),
)
b300_ep16 = [
item for item in unsupported_cases
@@ -276,7 +276,7 @@ def test_matrix_cases_and_shards_are_identity_bound(self) -> None:
self.assertIsNotNone(capability.topology_for("mi325x", 8))
self.assertEqual(
Counter(shard["n"] for shard in matrix["include"]),
- Counter({6: 34, 8: 1, 10: 14}),
+ Counter({6: 27, 8: 5, 10: 1, 12: 15}),
)
ll_cases = [
item for item in matrix["requested_cases"]
From 243bc3b9d94d4f27eb062f0345d1fdd1ad68ade2 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 20:08:52 +0800
Subject: [PATCH 129/210] feat(collectivex): finalize V1 capability
dispositions
---
.github/workflows/collectivex-sweep.yml | 2 +-
experimental/CollectiveX/capability.py | 15 +++------------
experimental/CollectiveX/publisher.py | 4 ++--
experimental/CollectiveX/sweep_matrix.py | 6 +++---
.../tests/test_precision_scheduling.py | 11 +++++++----
experimental/CollectiveX/tests/test_publisher.py | 2 +-
.../tests/test_qualification_planning.py | 2 +-
.../CollectiveX/tests/test_sampling_contract.py | 4 ++--
8 files changed, 20 insertions(+), 26 deletions(-)
diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml
index 97c83edfc1..ebcf2a203e 100644
--- a/.github/workflows/collectivex-sweep.yml
+++ b/.github/workflows/collectivex-sweep.yml
@@ -215,7 +215,7 @@ jobs:
- name: Create V1 release marker
if: ${{ inputs.operation == 'sweep' && inputs.release_tag == 'v1' }}
env:
- EXPECTED_MATRIX_SHA256: 63d566a594f464c8c6cd8f36c37790fe40c7a6bb715037246dd7cbb36b151ed5
+ EXPECTED_MATRIX_SHA256: 7891b84bacc71081aa856a7b8252cea530b5b6ccf87894894c54abb047cd608d
RUN_ID: ${{ github.run_id }}
RUN_ATTEMPT: ${{ github.run_attempt }}
SOURCE_SHA: ${{ github.sha }}
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
index a699a57b1d..ebc928600d 100644
--- a/experimental/CollectiveX/capability.py
+++ b/experimental/CollectiveX/capability.py
@@ -181,6 +181,9 @@ def _platform(
# Keep these narrower than platform overrides so working reference paths remain
# measurable on the same fabric.
BACKEND_TOPOLOGY_CELL_OVERRIDES: dict[tuple[str, str, int], str] = {
+ ("b200-dgxc", "deepep", 16): (
+ "DeepEP V1 EP16 requires unavailable GPU doorbell mapping or GDRCopy on B200"
+ ),
("b200-dgxc", "deepep-hybrid", 16): (
"DeepEP Hybrid EP16 requires unavailable GPU-to-NIC doorbell/UAR mappings on B200"
),
@@ -448,18 +451,6 @@ def _precision_rule(
for run_id, sku, ep, backend, mode, profile, disposition, result
in _VALIDATED_NATIVE_PROBE_CELLS
})
-for _profile, _mode in (
- (_NORMAL_E4M3FN_PROFILE, "normal"),
- (_LL_FP8_PROFILE, "low-latency"),
- (_LL_LOGFMT_PROFILE, "low-latency"),
- (_LL_FP8_LOGFMT_PROFILE, "low-latency"),
-):
- PRECISION_CELL_OVERRIDES[(_profile, "deepep", "b200-dgxc", 16, _mode)] = {
- "basis": "b200-cpu-proxied-ibgda-handler-production-shape-passed",
- "disposition": "supported",
- }
-
-
def runtime_identity_issues(
sku: str, *, vendor: str, arch: str, machine: str, device_name: str,
device_count: int, world_size: int,
diff --git a/experimental/CollectiveX/publisher.py b/experimental/CollectiveX/publisher.py
index 082ed766e4..c4e637e7f4 100644
--- a/experimental/CollectiveX/publisher.py
+++ b/experimental/CollectiveX/publisher.py
@@ -46,10 +46,10 @@
)
REQUIRED_PROMOTION_COHORT_COUNTS = {"library": 64, "system": 12, "routing": 54}
CANONICAL_FULL_V1_MATRIX_SHA256 = (
- "63d566a594f464c8c6cd8f36c37790fe40c7a6bb715037246dd7cbb36b151ed5"
+ "7891b84bacc71081aa856a7b8252cea530b5b6ccf87894894c54abb047cd608d"
)
CANONICAL_FULL_V1_CASE_CATALOG_SHA256 = (
- "c02fa9151ba2802d24e2f51af056f9b3e45e8e8d2706a3641d09e1e93705f695"
+ "df7f0162afd942f95c2da0f5f70ec56ab99e1e7917e940b9a15b7dedf49762ab"
)
P50_STABILITY_LIMIT = 1.10
P99_STABILITY_LIMIT = 1.25
diff --git a/experimental/CollectiveX/sweep_matrix.py b/experimental/CollectiveX/sweep_matrix.py
index a40eb59123..4a74128a51 100644
--- a/experimental/CollectiveX/sweep_matrix.py
+++ b/experimental/CollectiveX/sweep_matrix.py
@@ -105,9 +105,9 @@
)
QUALIFICATION_INDICES = range(1, 4)
CANONICAL_V1_EXECUTION_PLAN_SHA256 = {
- 1: "9b4652ef5828cabc546ca10faa1c5f699de8f5bda8853a69fcc0e44ed9bbea91",
- 2: "8c268877e0051fb93aea17aa3830f7d9d7feab40e7276d3de7dd6e76f3e05f0d",
- 3: "1f5293201ff48e4d51e0e0d81135da6fab83a05faac1d44e1efd822f9f654527",
+ 1: "8ef37665f408f999daad1cf46e74860056d4f6304d0c91e727d006d9f1c2a252",
+ 2: "db7b66eb8e14dbe46189394722999bd801fa9285d2d5a2d9edd7373fde6557d7",
+ 3: "50a5ed0e707005a18cec6c6a4cf6691076325593799307991561a0a8d93749ea",
}
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
index 68bbbc0e0d..2e44d37de1 100644
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -253,10 +253,10 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
len(targets), 94
)
self.assertEqual(
- sum(item["disposition"] == "supported" for item in targets), 66
+ sum(item["disposition"] == "supported" for item in targets), 62
)
self.assertEqual(
- sum(item["disposition"] == "unsupported" for item in targets), 28
+ sum(item["disposition"] == "unsupported" for item in targets), 32
)
self.assertEqual(len(capability.provisional_precision_targets()), 0)
keys = {
@@ -334,8 +334,11 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
"b200-dgxc", "deepep", ep=16, nodes=2,
precision_profile=identity.V1_CONTROL_PRECISION_PROFILE,
)
- self.assertEqual(disposition, "supported")
- self.assertEqual(reason, "ok")
+ self.assertEqual(disposition, "unsupported")
+ self.assertEqual(
+ reason,
+ "DeepEP V1 EP16 requires unavailable GPU doorbell mapping or GDRCopy on B200",
+ )
self.assertEqual(
capability.DEEPEP_V1_IBGDA_NIC_HANDLERS["b200-dgxc"],
"cpu",
diff --git a/experimental/CollectiveX/tests/test_publisher.py b/experimental/CollectiveX/tests/test_publisher.py
index 195fce85ee..22ad2d2bf2 100644
--- a/experimental/CollectiveX/tests/test_publisher.py
+++ b/experimental/CollectiveX/tests/test_publisher.py
@@ -2024,7 +2024,7 @@ def test_workflow_matrix_and_catalog_digests_do_not_drift(self) -> None:
if item["disposition"] == "unsupported"
),
),
- (48, 748, 392, 356, 1740, 881, 859),
+ (47, 748, 380, 368, 1740, 854, 886),
)
library: dict[tuple, set[str]] = {}
system: dict[tuple, set[str]] = {}
diff --git a/experimental/CollectiveX/tests/test_qualification_planning.py b/experimental/CollectiveX/tests/test_qualification_planning.py
index 30c926797b..7f87c4e8f4 100644
--- a/experimental/CollectiveX/tests/test_qualification_planning.py
+++ b/experimental/CollectiveX/tests/test_qualification_planning.py
@@ -227,7 +227,7 @@ def test_full_v1_matrix_emits_every_unsupported_terminal_outcome(self) -> None:
documents = [json.loads(path.read_text()) for path in written]
self.assertEqual(len(documents), len(expected))
- self.assertEqual(len(documents), 356)
+ self.assertEqual(len(documents), 368)
self.assertEqual(
{document["outcome"]["reason"] for document in documents},
{item["reason"] for item in expected},
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index a17fb3b112..c7db26d381 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -209,7 +209,7 @@ def test_matrix_cases_and_shards_are_identity_bound(self) -> None:
sum(len(item["case"]["ladder"].split()) for item in runnable_cases),
sum(len(item["case"]["ladder"].split()) for item in unsupported_cases),
),
- (48, 748, 392, 356, 1740, 881, 859),
+ (47, 748, 380, 368, 1740, 854, 886),
)
b300_ep16 = [
item for item in unsupported_cases
@@ -276,7 +276,7 @@ def test_matrix_cases_and_shards_are_identity_bound(self) -> None:
self.assertIsNotNone(capability.topology_for("mi325x", 8))
self.assertEqual(
Counter(shard["n"] for shard in matrix["include"]),
- Counter({6: 27, 8: 5, 10: 1, 12: 15}),
+ Counter({6: 27, 8: 5, 10: 1, 12: 14}),
)
ll_cases = [
item for item in matrix["requested_cases"]
From a68c386b41b47c6b9d0085458760264dbcafe2c7 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 20:27:04 +0800
Subject: [PATCH 130/210] fix(collectivex): classify precision runtime failures
---
experimental/CollectiveX/runtime/common.sh | 18 ++++++++++++++++++
.../tests/test_sampling_contract.py | 18 ++++++++++++++++++
2 files changed, 36 insertions(+)
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 69e4efc479..b0cf8e2efc 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -90,6 +90,24 @@ cx_fail_stage() {
diagnostic="python-import"
elif grep -aEqi 'AttributeError:|TypeError:.*(unexpected|argument|operand)|has no attribute' "$log_path"; then
diagnostic="backend-api"
+ elif grep -aEqi 'FP8 dispatch payload is missing block-128 scales' "$log_path"; then
+ diagnostic="precision-dispatch-scales-missing"
+ elif grep -aEqi 'native FP8 dispatch payload has an invalid dtype or shape' "$log_path"; then
+ diagnostic="precision-dispatch-payload-shape"
+ elif grep -aEqi 'native FP8 dispatch scales have an invalid dtype or shape' "$log_path"; then
+ diagnostic="precision-dispatch-scale-shape"
+ elif grep -aEqi 'expert-packed FP8 receive count exceeds capacity' "$log_path"; then
+ diagnostic="precision-receive-capacity"
+ elif grep -aEqi 'expert-packed (FP8 receive counts|BF16 stage workspace) has an invalid shape' "$log_path"; then
+ diagnostic="precision-receive-shape"
+ elif grep -aEqi 'active torch build does not expose (torch[.])?float8|active torch build does not expose fp8-' "$log_path"; then
+ diagnostic="precision-runtime-dtype"
+ elif grep -aEqi 'omits EpDispatchCombineQuantType[.]Fp8DirectCast' "$log_path"; then
+ diagnostic="precision-combine-api"
+ elif grep -aEqi 'dispatch FP8 format differs from the pinned GPU architecture' "$log_path"; then
+ diagnostic="precision-architecture-format"
+ elif grep -aEqi 'native FP8 dispatch requires hidden divisible by 128' "$log_path"; then
+ diagnostic="precision-hidden-alignment"
elif grep -aEqi 'PrecisionError:|unsupported precision|precision profile.*(invalid|unsupported|differs)' "$log_path"; then
diagnostic="precision-contract"
elif grep -aEqi 'AssertionError:' "$log_path"; then
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index c7db26d381..34ce68aa68 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -2778,6 +2778,24 @@ def test_private_runtime_failure_signatures_override_case_footer(self) -> None:
"torch rendezvous timed out": "network-or-timeout",
"ModuleNotFoundError: missing module": "python-import",
"AttributeError: backend has no attribute 'probe'": "backend-api",
+ "PrecisionError: FP8 dispatch payload is missing block-128 scales":
+ "precision-dispatch-scales-missing",
+ "PrecisionError: native FP8 dispatch payload has an invalid dtype or shape":
+ "precision-dispatch-payload-shape",
+ "PrecisionError: native FP8 dispatch scales have an invalid dtype or shape":
+ "precision-dispatch-scale-shape",
+ "PrecisionError: expert-packed FP8 receive count exceeds capacity":
+ "precision-receive-capacity",
+ "PrecisionError: expert-packed FP8 receive counts has an invalid shape":
+ "precision-receive-shape",
+ "PrecisionError: active torch build does not expose torch.float8_e4m3fnuz":
+ "precision-runtime-dtype",
+ "PrecisionError: pinned MoRI API omits EpDispatchCombineQuantType.Fp8DirectCast":
+ "precision-combine-api",
+ "PrecisionError: MoRI dispatch FP8 format differs from the pinned GPU architecture":
+ "precision-architecture-format",
+ "PrecisionError: MoRI native FP8 dispatch requires hidden divisible by 128":
+ "precision-hidden-alignment",
"PrecisionError: unsupported precision profile": "precision-contract",
"AssertionError: probe invariant": "python-assertion",
"RuntimeError: probe execution failed": "python-runtime",
From 421274f3345ff0deac67e4b0d4a9338e6cdcbecd Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 20:36:59 +0800
Subject: [PATCH 131/210] fix(collectivex): reenable budgeted DeepEP V2 EP16
---
.github/workflows/collectivex-sweep.yml | 2 +-
experimental/CollectiveX/capability.py | 6 ------
experimental/CollectiveX/publisher.py | 6 +++---
experimental/CollectiveX/sweep_matrix.py | 6 +++---
.../CollectiveX/tests/test_deepep_v2_contract.py | 8 +++++++-
.../CollectiveX/tests/test_precision_scheduling.py | 9 +++------
experimental/CollectiveX/tests/test_publisher.py | 2 +-
.../CollectiveX/tests/test_qualification_planning.py | 2 +-
experimental/CollectiveX/tests/test_sampling_contract.py | 4 ++--
9 files changed, 21 insertions(+), 24 deletions(-)
diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml
index ebcf2a203e..a03efba136 100644
--- a/.github/workflows/collectivex-sweep.yml
+++ b/.github/workflows/collectivex-sweep.yml
@@ -215,7 +215,7 @@ jobs:
- name: Create V1 release marker
if: ${{ inputs.operation == 'sweep' && inputs.release_tag == 'v1' }}
env:
- EXPECTED_MATRIX_SHA256: 7891b84bacc71081aa856a7b8252cea530b5b6ccf87894894c54abb047cd608d
+ EXPECTED_MATRIX_SHA256: 90a6a71d19bf5f6cc07fe5091688fa61f11eb08b1460364a42a55fde56d41d10
RUN_ID: ${{ github.run_id }}
RUN_ATTEMPT: ${{ github.run_attempt }}
SOURCE_SHA: ${{ github.sha }}
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
index ebc928600d..b12e787da5 100644
--- a/experimental/CollectiveX/capability.py
+++ b/experimental/CollectiveX/capability.py
@@ -187,12 +187,6 @@ def _platform(
("b200-dgxc", "deepep-hybrid", 16): (
"DeepEP Hybrid EP16 requires unavailable GPU-to-NIC doorbell/UAR mappings on B200"
),
- ("b200-dgxc", "deepep-v2", 16): (
- "DeepEP V2 EP16 exhausts pinned NCCL Gin QP resources on the B200 fabric"
- ),
- ("h200-dgxc", "deepep-v2", 16): (
- "DeepEP V2 EP16 exhausts pinned NCCL Gin QP resources on the H200 fabric"
- ),
("mi355x", "mori", 8): (
"Pinned MoRI backend construction does not complete on MI355X EP8"
),
diff --git a/experimental/CollectiveX/publisher.py b/experimental/CollectiveX/publisher.py
index c4e637e7f4..431763bb99 100644
--- a/experimental/CollectiveX/publisher.py
+++ b/experimental/CollectiveX/publisher.py
@@ -44,12 +44,12 @@
PRECISION_COHORT_KINDS = (
"dispatch-precision", "combine-precision", "precision-pair",
)
-REQUIRED_PROMOTION_COHORT_COUNTS = {"library": 64, "system": 12, "routing": 54}
+REQUIRED_PROMOTION_COHORT_COUNTS = {"library": 64, "system": 12, "routing": 58}
CANONICAL_FULL_V1_MATRIX_SHA256 = (
- "7891b84bacc71081aa856a7b8252cea530b5b6ccf87894894c54abb047cd608d"
+ "90a6a71d19bf5f6cc07fe5091688fa61f11eb08b1460364a42a55fde56d41d10"
)
CANONICAL_FULL_V1_CASE_CATALOG_SHA256 = (
- "df7f0162afd942f95c2da0f5f70ec56ab99e1e7917e940b9a15b7dedf49762ab"
+ "3018b1648b3f45cfadea672fa5b97eb1e81c5f824739f905db5ccd527e9663a5"
)
P50_STABILITY_LIMIT = 1.10
P99_STABILITY_LIMIT = 1.25
diff --git a/experimental/CollectiveX/sweep_matrix.py b/experimental/CollectiveX/sweep_matrix.py
index 4a74128a51..3c0bd05f4f 100644
--- a/experimental/CollectiveX/sweep_matrix.py
+++ b/experimental/CollectiveX/sweep_matrix.py
@@ -105,9 +105,9 @@
)
QUALIFICATION_INDICES = range(1, 4)
CANONICAL_V1_EXECUTION_PLAN_SHA256 = {
- 1: "8ef37665f408f999daad1cf46e74860056d4f6304d0c91e727d006d9f1c2a252",
- 2: "db7b66eb8e14dbe46189394722999bd801fa9285d2d5a2d9edd7373fde6557d7",
- 3: "50a5ed0e707005a18cec6c6a4cf6691076325593799307991561a0a8d93749ea",
+ 1: "93c9e1490104f3219ee748cd3b3c149ef807bac1e7598893607dbdd697012051",
+ 2: "82d316691ae7fa684799db80865279f1bcced9a7bb920cd75a7bf9ccc1c10572",
+ 3: "5a7ebbdb8594e4fb11d85545806d7c35ebd98d52f7c6b9ee8a82f42dcebaa831",
}
diff --git a/experimental/CollectiveX/tests/test_deepep_v2_contract.py b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
index f28b50ba25..6e5df4516c 100644
--- a/experimental/CollectiveX/tests/test_deepep_v2_contract.py
+++ b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
@@ -388,11 +388,12 @@ def test_v2_gin_mode_uses_the_scale_up_domain_and_safe_fallbacks(self) -> None:
functions = {
node.name: node for node in self.tree.body if isinstance(node, ast.FunctionDef)
}
- namespace = {"os": os}
+ namespace = {"os": os, "_GIN_QP_BUDGET": 512}
exec(
compile(
ast.Module(
body=[
+ functions["_hybrid_num_allocated_qps"],
functions["_configure_gin_mode"],
functions["_lsa_topology_is_valid"],
],
@@ -403,8 +404,13 @@ def test_v2_gin_mode_uses_the_scale_up_domain_and_safe_fallbacks(self) -> None:
),
namespace,
)
+ allocate_qps = namespace["_hybrid_num_allocated_qps"]
configure = namespace["_configure_gin_mode"]
topology_is_valid = namespace["_lsa_topology_is_valid"]
+ self.assertEqual(
+ [allocate_qps(world_size) for world_size in (8, 16, 32, 64)],
+ [65, 33, 17, 9],
+ )
original = os.environ.get("EP_DISABLE_GIN")
try:
args = types.SimpleNamespace(scale_up_domain=72, gpus_per_node=4)
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
index 2e44d37de1..db99179729 100644
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -381,16 +381,13 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
"Pinned MoRI distributed initialization does not complete on MI300X EP16",
)
- for sku, product in (("b200-dgxc", "B200"), ("h200-dgxc", "H200")):
+ for sku in ("b200-dgxc", "h200-dgxc"):
disposition, reason = capability.resolve_disposition(
sku, "deepep-v2", ep=16, nodes=2,
precision_profile=identity.V1_CONTROL_PRECISION_PROFILE,
)
- self.assertEqual(disposition, "unsupported")
- self.assertEqual(
- reason,
- f"DeepEP V2 EP16 exhausts pinned NCCL Gin QP resources on the {product} fabric",
- )
+ self.assertEqual(disposition, "supported")
+ self.assertEqual(reason, "ok")
def test_split_suites_track_provisional_state_and_do_not_duplicate_bf16(self) -> None:
suites = sweep_matrix._load("suites.yaml")
diff --git a/experimental/CollectiveX/tests/test_publisher.py b/experimental/CollectiveX/tests/test_publisher.py
index 22ad2d2bf2..8506ab09e9 100644
--- a/experimental/CollectiveX/tests/test_publisher.py
+++ b/experimental/CollectiveX/tests/test_publisher.py
@@ -2024,7 +2024,7 @@ def test_workflow_matrix_and_catalog_digests_do_not_drift(self) -> None:
if item["disposition"] == "unsupported"
),
),
- (47, 748, 380, 368, 1740, 854, 886),
+ (49, 748, 392, 356, 1740, 882, 858),
)
library: dict[tuple, set[str]] = {}
system: dict[tuple, set[str]] = {}
diff --git a/experimental/CollectiveX/tests/test_qualification_planning.py b/experimental/CollectiveX/tests/test_qualification_planning.py
index 7f87c4e8f4..30c926797b 100644
--- a/experimental/CollectiveX/tests/test_qualification_planning.py
+++ b/experimental/CollectiveX/tests/test_qualification_planning.py
@@ -227,7 +227,7 @@ def test_full_v1_matrix_emits_every_unsupported_terminal_outcome(self) -> None:
documents = [json.loads(path.read_text()) for path in written]
self.assertEqual(len(documents), len(expected))
- self.assertEqual(len(documents), 368)
+ self.assertEqual(len(documents), 356)
self.assertEqual(
{document["outcome"]["reason"] for document in documents},
{item["reason"] for item in expected},
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 34ce68aa68..ca939a89dd 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -209,7 +209,7 @@ def test_matrix_cases_and_shards_are_identity_bound(self) -> None:
sum(len(item["case"]["ladder"].split()) for item in runnable_cases),
sum(len(item["case"]["ladder"].split()) for item in unsupported_cases),
),
- (47, 748, 380, 368, 1740, 854, 886),
+ (49, 748, 392, 356, 1740, 882, 858),
)
b300_ep16 = [
item for item in unsupported_cases
@@ -276,7 +276,7 @@ def test_matrix_cases_and_shards_are_identity_bound(self) -> None:
self.assertIsNotNone(capability.topology_for("mi325x", 8))
self.assertEqual(
Counter(shard["n"] for shard in matrix["include"]),
- Counter({6: 27, 8: 5, 10: 1, 12: 14}),
+ Counter({6: 29, 8: 5, 10: 1, 12: 14}),
)
ll_cases = [
item for item in matrix["requested_cases"]
From 1aba54d5bc60beaaee2eae22a75619bfcbbdca36 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 20:39:24 +0800
Subject: [PATCH 132/210] fix(collectivex): honor MoRI precision probe failures
---
.github/workflows/collectivex-sweep.yml | 2 +-
experimental/CollectiveX/capability.py | 4 ++--
experimental/CollectiveX/publisher.py | 4 ++--
experimental/CollectiveX/sweep_matrix.py | 6 +++---
experimental/CollectiveX/tests/test_precision_scheduling.py | 4 ++--
experimental/CollectiveX/tests/test_publisher.py | 2 +-
.../CollectiveX/tests/test_qualification_planning.py | 2 +-
experimental/CollectiveX/tests/test_sampling_contract.py | 4 ++--
8 files changed, 14 insertions(+), 14 deletions(-)
diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml
index a03efba136..1c425d3ff2 100644
--- a/.github/workflows/collectivex-sweep.yml
+++ b/.github/workflows/collectivex-sweep.yml
@@ -215,7 +215,7 @@ jobs:
- name: Create V1 release marker
if: ${{ inputs.operation == 'sweep' && inputs.release_tag == 'v1' }}
env:
- EXPECTED_MATRIX_SHA256: 90a6a71d19bf5f6cc07fe5091688fa61f11eb08b1460364a42a55fde56d41d10
+ EXPECTED_MATRIX_SHA256: 5986c56db0df60f6f54e5001e0f214acdd628b65d257881bcbb226b6695b808f
RUN_ID: ${{ github.run_id }}
RUN_ATTEMPT: ${{ github.run_attempt }}
SOURCE_SHA: ${{ github.sha }}
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
index b12e787da5..40bbf35e2f 100644
--- a/experimental/CollectiveX/capability.py
+++ b/experimental/CollectiveX/capability.py
@@ -430,8 +430,8 @@ def _precision_rule(
("28740533382", "mi355x", 8, "mori", "normal", _MORI_E4M3FN_BOTH_PROFILE, "unsupported", "backend-construction-failed"),
("28740533382", "mi355x", 8, "mori", "normal", _MORI_E4M3FN_DIRECT_PROFILE, "unsupported", "backend-construction-failed"),
("28782309414", "mi300x", 8, "mori", "normal", _NORMAL_E4M3FNUZ_PROFILE, "supported", "native-probe-passed"),
- ("28782309414", "mi300x", 8, "mori", "normal", _MORI_E4M3FNUZ_DIRECT_PROFILE, "supported", "native-probe-passed"),
- ("28782309414", "mi300x", 8, "mori", "normal", _MORI_E4M3FNUZ_BOTH_PROFILE, "supported", "native-probe-passed"),
+ ("28782309414", "mi300x", 8, "mori", "normal", _MORI_E4M3FNUZ_DIRECT_PROFILE, "unsupported", "backend-construction-failed"),
+ ("28782309414", "mi300x", 8, "mori", "normal", _MORI_E4M3FNUZ_BOTH_PROFILE, "unsupported", "backend-construction-failed"),
("28788599713", "mi300x", 16, "mori", "normal", _NORMAL_E4M3FNUZ_PROFILE, "unsupported", "distributed-init-timeout"),
*(("28743235213", "b300", 8, "deepep", "low-latency", profile,
"supported", "native-probe-passed")
diff --git a/experimental/CollectiveX/publisher.py b/experimental/CollectiveX/publisher.py
index 431763bb99..7735673e5d 100644
--- a/experimental/CollectiveX/publisher.py
+++ b/experimental/CollectiveX/publisher.py
@@ -46,10 +46,10 @@
)
REQUIRED_PROMOTION_COHORT_COUNTS = {"library": 64, "system": 12, "routing": 58}
CANONICAL_FULL_V1_MATRIX_SHA256 = (
- "90a6a71d19bf5f6cc07fe5091688fa61f11eb08b1460364a42a55fde56d41d10"
+ "5986c56db0df60f6f54e5001e0f214acdd628b65d257881bcbb226b6695b808f"
)
CANONICAL_FULL_V1_CASE_CATALOG_SHA256 = (
- "3018b1648b3f45cfadea672fa5b97eb1e81c5f824739f905db5ccd527e9663a5"
+ "e6e91d90c679fc6aec0621034707c5558b9612a566c9437eb75ae9661753f39b"
)
P50_STABILITY_LIMIT = 1.10
P99_STABILITY_LIMIT = 1.25
diff --git a/experimental/CollectiveX/sweep_matrix.py b/experimental/CollectiveX/sweep_matrix.py
index 3c0bd05f4f..787e898b7e 100644
--- a/experimental/CollectiveX/sweep_matrix.py
+++ b/experimental/CollectiveX/sweep_matrix.py
@@ -105,9 +105,9 @@
)
QUALIFICATION_INDICES = range(1, 4)
CANONICAL_V1_EXECUTION_PLAN_SHA256 = {
- 1: "93c9e1490104f3219ee748cd3b3c149ef807bac1e7598893607dbdd697012051",
- 2: "82d316691ae7fa684799db80865279f1bcced9a7bb920cd75a7bf9ccc1c10572",
- 3: "5a7ebbdb8594e4fb11d85545806d7c35ebd98d52f7c6b9ee8a82f42dcebaa831",
+ 1: "7733f956836ecf01849f3cd317fa1d074bec6bdb24f5d2fad0bdf02e70246c0b",
+ 2: "ce4a19f8322a3b0b7d908b80de67361c0ac9e7da7e82c2a7f867d1a33dab31c3",
+ 3: "b5f03fb3fff0e304383dfe0261f16ab8e7972a9f23b0831693c9f12d496ec9e8",
}
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
index db99179729..d1dbf9e690 100644
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -253,10 +253,10 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
len(targets), 94
)
self.assertEqual(
- sum(item["disposition"] == "supported" for item in targets), 62
+ sum(item["disposition"] == "supported" for item in targets), 60
)
self.assertEqual(
- sum(item["disposition"] == "unsupported" for item in targets), 32
+ sum(item["disposition"] == "unsupported" for item in targets), 34
)
self.assertEqual(len(capability.provisional_precision_targets()), 0)
keys = {
diff --git a/experimental/CollectiveX/tests/test_publisher.py b/experimental/CollectiveX/tests/test_publisher.py
index 8506ab09e9..ffc42c3a9d 100644
--- a/experimental/CollectiveX/tests/test_publisher.py
+++ b/experimental/CollectiveX/tests/test_publisher.py
@@ -2024,7 +2024,7 @@ def test_workflow_matrix_and_catalog_digests_do_not_drift(self) -> None:
if item["disposition"] == "unsupported"
),
),
- (49, 748, 392, 356, 1740, 882, 858),
+ (49, 748, 388, 360, 1740, 878, 862),
)
library: dict[tuple, set[str]] = {}
system: dict[tuple, set[str]] = {}
diff --git a/experimental/CollectiveX/tests/test_qualification_planning.py b/experimental/CollectiveX/tests/test_qualification_planning.py
index 30c926797b..a9bea34586 100644
--- a/experimental/CollectiveX/tests/test_qualification_planning.py
+++ b/experimental/CollectiveX/tests/test_qualification_planning.py
@@ -227,7 +227,7 @@ def test_full_v1_matrix_emits_every_unsupported_terminal_outcome(self) -> None:
documents = [json.loads(path.read_text()) for path in written]
self.assertEqual(len(documents), len(expected))
- self.assertEqual(len(documents), 356)
+ self.assertEqual(len(documents), 360)
self.assertEqual(
{document["outcome"]["reason"] for document in documents},
{item["reason"] for item in expected},
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index ca939a89dd..77ccf90acd 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -209,7 +209,7 @@ def test_matrix_cases_and_shards_are_identity_bound(self) -> None:
sum(len(item["case"]["ladder"].split()) for item in runnable_cases),
sum(len(item["case"]["ladder"].split()) for item in unsupported_cases),
),
- (49, 748, 392, 356, 1740, 882, 858),
+ (49, 748, 388, 360, 1740, 878, 862),
)
b300_ep16 = [
item for item in unsupported_cases
@@ -276,7 +276,7 @@ def test_matrix_cases_and_shards_are_identity_bound(self) -> None:
self.assertIsNotNone(capability.topology_for("mi325x", 8))
self.assertEqual(
Counter(shard["n"] for shard in matrix["include"]),
- Counter({6: 29, 8: 5, 10: 1, 12: 14}),
+ Counter({6: 29, 8: 6, 10: 1, 12: 13}),
)
ll_cases = [
item for item in matrix["requested_cases"]
From 4ee0c5c690463b6f9062429f41acaa996adaaa49 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 20:48:18 +0800
Subject: [PATCH 133/210] fix(collectivex): isolate DeepEP V2 JIT caches per
node
---
.../CollectiveX/runtime/run_in_container.sh | 24 ++++++++-----------
.../tests/test_deepep_v2_contract.py | 6 +++--
2 files changed, 14 insertions(+), 16 deletions(-)
diff --git a/experimental/CollectiveX/runtime/run_in_container.sh b/experimental/CollectiveX/runtime/run_in_container.sh
index e47ed357b1..fc2bd5cb34 100644
--- a/experimental/CollectiveX/runtime/run_in_container.sh
+++ b/experimental/CollectiveX/runtime/run_in_container.sh
@@ -332,7 +332,7 @@ cx_deepep_v2_root() {
}
cx_activate_deepep_v2() {
- local root venv stage_root
+ local root venv execution_id
root="$(cx_deepep_v2_root)" || return 1
venv="$root/venv"
[ -x "$venv/bin/python" ] \
@@ -345,16 +345,13 @@ cx_activate_deepep_v2() {
|| { cx_log "ERROR: DeepEP V2 NVSHMEM package root is unavailable"; return 1; }
export EP_NCCL_ROOT_DIR EP_NVSHMEM_ROOT_DIR
export LD_LIBRARY_PATH="$EP_NCCL_ROOT_DIR/lib:$EP_NVSHMEM_ROOT_DIR/lib:${LD_LIBRARY_PATH:-}"
- case "${CX_BACKEND_SOURCE_ROOT:-}" in
- /*/.cx_sources) stage_root="${CX_BACKEND_SOURCE_ROOT%/.cx_sources}" ;;
- *) cx_log "ERROR: DeepEP V2 job-local source root is unavailable"; return 1 ;;
- esac
- [ -d "$stage_root" ] && [ ! -L "$stage_root" ] \
- || { cx_log "ERROR: DeepEP V2 job-local stage is invalid"; return 1; }
- # JIT CUBINs are evidence from this shard, not part of the persistent AOT environment.
- # Keeping them on the isolated staged tree prevents a prior driver/topology attempt
- # from seeding a later run; all ranks and cases in this shard still share one cold build.
- export EP_JIT_CACHE_DIR="$stage_root/.cx_backend/deepep-v2-jit"
+ execution_id="${COLLECTIVEX_EXECUTION_ID:-manual}"
+ [[ "$execution_id" =~ ^[A-Za-z0-9._-]+$ ]] \
+ || { cx_log "ERROR: DeepEP V2 execution identity is invalid"; return 1; }
+ # JIT CUBINs are per-execution evidence and must be node-local. A shared NFS cache lets
+ # ranks on different nodes race the same compiler output and can trip compiler.hpp asserts.
+ # The identical absolute path still lets ranks on one node share their cold build.
+ export EP_JIT_CACHE_DIR="/tmp/collectivex-deepep-v2-jit-$execution_id"
export EP_REUSE_NCCL_COMM=1
export DEEPEP_V2_PR=605 DEEPEP_V2_FIX_PR=630 DEEPEP_V2_NCCL_CHECK_FIX_PR=640
DEEPEP_V2_COMMIT="$CX_DEEPEP_V2_COMMIT"
@@ -363,10 +360,9 @@ cx_activate_deepep_v2() {
DEEPEP_V2_NCCL_CHECK_COMMIT="$CX_DEEPEP_V2_NCCL_CHECK_COMMIT"
export DEEPEP_V2_COMMIT DEEPEP_V2_TREE DEEPEP_V2_FMT_COMMIT
export DEEPEP_V2_NCCL_CHECK_COMMIT
- [ ! -L "$stage_root/.cx_backend" ] && [ ! -L "$EP_JIT_CACHE_DIR" ] \
+ [ ! -L "$EP_JIT_CACHE_DIR" ] \
|| { cx_log "ERROR: DeepEP V2 JIT cache path is unsafe"; return 1; }
- if ! mkdir -p "$EP_JIT_CACHE_DIR" \
- || ! chmod 700 "$stage_root/.cx_backend" "$EP_JIT_CACHE_DIR"; then
+ if ! mkdir -p "$EP_JIT_CACHE_DIR" || ! chmod 700 "$EP_JIT_CACHE_DIR"; then
cx_log "ERROR: DeepEP V2 JIT cache is unavailable"
return 1
fi
diff --git a/experimental/CollectiveX/tests/test_deepep_v2_contract.py b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
index 6e5df4516c..c5dd46a33e 100644
--- a/experimental/CollectiveX/tests/test_deepep_v2_contract.py
+++ b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
@@ -1505,9 +1505,11 @@ def test_runtime_pins_uccl_wheel_and_hybrid_source_tree(self) -> None:
self.assertNotIn("import os,nvidia.nccl", runtime)
self.assertNotIn("import os,nvidia.nvshmem", runtime)
self.assertIn(
- 'export EP_JIT_CACHE_DIR="$stage_root/.cx_backend/deepep-v2-jit"', runtime
+ 'export EP_JIT_CACHE_DIR="/tmp/collectivex-deepep-v2-jit-$execution_id"',
+ runtime,
)
- self.assertIn('stage_root="${CX_BACKEND_SOURCE_ROOT%/.cx_sources}"', runtime)
+ self.assertIn('execution_id="${COLLECTIVEX_EXECUTION_ID:-manual}"', runtime)
+ self.assertNotIn('stage_root="${CX_BACKEND_SOURCE_ROOT%/.cx_sources}"', runtime)
self.assertNotIn('export EP_JIT_CACHE_DIR="$root/jit"', runtime)
self.assertIn('EP_NVSHMEM_ROOT_DIR="$NVSHMEM_DIR"', runtime)
reference = (HERE / "ep_nccl.py").read_text()
From a5a57c74c481ab167cd5d303493fb310c4850292 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 20:53:30 +0800
Subject: [PATCH 134/210] fix(collectivex): consume timed MoRI dispatch state
---
experimental/CollectiveX/tests/ep_harness.py | 29 ++++++++++++++-----
experimental/CollectiveX/tests/ep_mori.py | 1 +
.../tests/test_deepep_v2_contract.py | 23 +++++++++++++++
.../tests/test_ep_precision_adapters.py | 2 ++
4 files changed, 47 insertions(+), 8 deletions(-)
diff --git a/experimental/CollectiveX/tests/ep_harness.py b/experimental/CollectiveX/tests/ep_harness.py
index 0262a11070..5e8736bb65 100644
--- a/experimental/CollectiveX/tests/ep_harness.py
+++ b/experimental/CollectiveX/tests/ep_harness.py
@@ -367,14 +367,15 @@ def _write_json_atomic(path: str, value) -> tuple[str, int]:
return _write_bytes_atomic(path, payload)
-def time_us(torch, fn, warmup: int, iters: int, pre=None) -> list[float]:
+def time_us(torch, fn, warmup: int, iters: int, pre=None, post=None) -> list[float]:
"""Per-iteration CUDA-event latencies (µs) for THIS rank.
Without `pre`: times `fn()`. With `pre`: runs `pre()` UNTIMED each iteration (sync
- before the start event so its GPU work can't bleed in), then times `fn(pre_result)`
- — how combine is isolated when it consumes the dispatch state and needs a fresh
- untimed dispatch+stage before every sample. Returns the raw per-iteration series;
- the caller reduces across ranks per iteration before percentiling.
+ before the start event so its GPU work can't bleed in), then times `fn(pre_result)`.
+ `post(result)` runs after the end event and synchronization, so stateful backends can
+ consume/reset a timed operation without charging that cleanup to its latency. Returns
+ the raw per-iteration series; the caller reduces across ranks per iteration before
+ percentiling.
"""
def sample():
arg = pre() if pre is not None else None
@@ -383,10 +384,14 @@ def sample():
s = torch.cuda.Event(enable_timing=True)
e = torch.cuda.Event(enable_timing=True)
s.record()
- fn(arg) if pre is not None else fn()
+ result = fn(arg) if pre is not None else fn()
e.record()
torch.cuda.synchronize()
- return s.elapsed_time(e) * 1000.0 # ms -> us
+ elapsed = s.elapsed_time(e) * 1000.0 # ms -> us
+ if post is not None:
+ post(result)
+ torch.cuda.synchronize()
+ return elapsed
for _ in range(max(0, warmup)):
if pre is not None:
@@ -1458,6 +1463,9 @@ def warm_roundtrips(problem, count):
stage_device_work = getattr(backend, "stage_device_work", False)
if type(stage_device_work) is not bool:
raise ValueError("backend.stage_device_work must be a boolean")
+ dispatch_needs_cleanup = getattr(backend, "dispatch_needs_combine_cleanup", False)
+ if type(dispatch_needs_cleanup) is not bool:
+ raise ValueError("backend.dispatch_needs_combine_cleanup must be a boolean")
order_identity = args.case_id or _sha256_json({
"backend": backend.name,
"ep_size": ep_size,
@@ -1510,6 +1518,10 @@ def prep_combine(p=problem):
backend.stage(p, hh)
return hh
+ def finish_dispatch(hh, p=problem):
+ backend.stage(p, hh)
+ backend.combine(p, hh)
+
for component_name in component_order:
# Every measured component receives the same 32 synchronized full-roundtrip
# warmups immediately before its timed trial.
@@ -1520,7 +1532,8 @@ def prep_combine(p=problem):
)
elif component_name == "dispatch":
measured[component_name] = time_us(
- torch, lambda p=problem: backend.dispatch(p), 0, args.iters
+ torch, lambda p=problem: backend.dispatch(p), 0, args.iters,
+ post=finish_dispatch if dispatch_needs_cleanup else None,
)
elif component_name == "stage":
measured[component_name] = time_us(
diff --git a/experimental/CollectiveX/tests/ep_mori.py b/experimental/CollectiveX/tests/ep_mori.py
index 41940d1771..a2f36c4870 100644
--- a/experimental/CollectiveX/tests/ep_mori.py
+++ b/experimental/CollectiveX/tests/ep_mori.py
@@ -51,6 +51,7 @@ class MoRIBackend:
name = "mori"
stage_device_work = False
combine_needs_redispatch = True
+ dispatch_needs_combine_cleanup = True
combine_weight_semantics = "unweighted-rank-sum"
def __init__(self, args, rank, world_size, local_rank, device):
diff --git a/experimental/CollectiveX/tests/test_deepep_v2_contract.py b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
index c5dd46a33e..8478254c4d 100644
--- a/experimental/CollectiveX/tests/test_deepep_v2_contract.py
+++ b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
@@ -125,6 +125,29 @@ def setUpClass(cls) -> None:
cls.path = HERE / "ep_deepep_v2.py"
cls.tree = ast.parse(cls.path.read_text(), str(cls.path))
+ def test_timing_post_hook_runs_after_each_isolated_sample(self) -> None:
+ class Event:
+ def record(self) -> None:
+ pass
+
+ def elapsed_time(self, _other) -> float:
+ return 0.125
+
+ cuda = types.SimpleNamespace(
+ Event=lambda **_kwargs: Event(),
+ synchronize=lambda: None,
+ )
+ handles: list[str] = []
+ samples = ep_harness.time_us(
+ types.SimpleNamespace(cuda=cuda),
+ lambda: "dispatch-handle",
+ warmup=0,
+ iters=3,
+ post=handles.append,
+ )
+ self.assertEqual(samples, [125.0, 125.0, 125.0])
+ self.assertEqual(handles, ["dispatch-handle"] * 3)
+
def test_capability_is_explicit_for_every_sku(self) -> None:
backend = capability.BACKENDS["deepep-v2"]
self.assertEqual(
diff --git a/experimental/CollectiveX/tests/test_ep_precision_adapters.py b/experimental/CollectiveX/tests/test_ep_precision_adapters.py
index b2feba3ba2..21fde94256 100644
--- a/experimental/CollectiveX/tests/test_ep_precision_adapters.py
+++ b/experimental/CollectiveX/tests/test_ep_precision_adapters.py
@@ -126,6 +126,8 @@ def test_mori_wires_dispatch_scales_and_direct_cast_config(self):
self.assertIn('"fp8_direct_cast" if self._direct_cast_combine', source)
self.assertIn("p.scales,", source)
self.assertIn("dispatch_scales=_scales", source)
+ self.assertIn("dispatch_needs_combine_cleanup = True", source)
+ self.assertIn("post=finish_dispatch if dispatch_needs_cleanup else None", harness)
self.assertEqual(harness.count("view.combine_input = transformed"), 2)
self.assertIn("transformed = view.combine_input.float()", precision)
From a763498431dafad73cabfe6000e0e7f2bb645aea Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 6 Jul 2026 22:22:09 +0800
Subject: [PATCH 135/210] fix(collectivex): disposition H100 combined LL
precision
---
.github/workflows/collectivex-sweep.yml | 2 +-
experimental/CollectiveX/capability.py | 2 +-
experimental/CollectiveX/publisher.py | 4 ++--
experimental/CollectiveX/sweep_matrix.py | 6 +++---
experimental/CollectiveX/tests/test_precision_scheduling.py | 4 ++--
experimental/CollectiveX/tests/test_publisher.py | 2 +-
.../CollectiveX/tests/test_qualification_planning.py | 2 +-
experimental/CollectiveX/tests/test_sampling_contract.py | 4 ++--
8 files changed, 13 insertions(+), 13 deletions(-)
diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml
index 1c425d3ff2..53cfd4b016 100644
--- a/.github/workflows/collectivex-sweep.yml
+++ b/.github/workflows/collectivex-sweep.yml
@@ -215,7 +215,7 @@ jobs:
- name: Create V1 release marker
if: ${{ inputs.operation == 'sweep' && inputs.release_tag == 'v1' }}
env:
- EXPECTED_MATRIX_SHA256: 5986c56db0df60f6f54e5001e0f214acdd628b65d257881bcbb226b6695b808f
+ EXPECTED_MATRIX_SHA256: 5894bab58d3deb2bcee51baa075ca5f5d324b4292ac1cef9f6bc08a07ab1d9a3
RUN_ID: ${{ github.run_id }}
RUN_ATTEMPT: ${{ github.run_attempt }}
SOURCE_SHA: ${{ github.sha }}
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
index 40bbf35e2f..651c77f245 100644
--- a/experimental/CollectiveX/capability.py
+++ b/experimental/CollectiveX/capability.py
@@ -398,7 +398,7 @@ def _precision_rule(
("28745208954", "h100-dgxc", 16, "deepep", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed"),
("28745423523", "h100-dgxc", 16, "deepep", "low-latency", _LL_FP8_PROFILE, "supported", "native-probe-passed"),
("28745423523", "h100-dgxc", 16, "deepep", "low-latency", _LL_LOGFMT_PROFILE, "supported", "native-probe-passed"),
- ("28745423523", "h100-dgxc", 16, "deepep", "low-latency", _LL_FP8_LOGFMT_PROFILE, "supported", "native-probe-passed"),
+ ("28796549762", "h100-dgxc", 16, "deepep", "low-latency", _LL_FP8_LOGFMT_PROFILE, "unsupported", "native-operation-timeout"),
("28745423523", "h100-dgxc", 16, "deepep-hybrid", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "native-operation-failed"),
("28745423523", "h100-dgxc", 16, "uccl", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed"),
("28745423523", "h100-dgxc", 16, "uccl", "low-latency", _LL_FP8_PROFILE, "supported", "native-probe-passed"),
diff --git a/experimental/CollectiveX/publisher.py b/experimental/CollectiveX/publisher.py
index 7735673e5d..170b2b11b3 100644
--- a/experimental/CollectiveX/publisher.py
+++ b/experimental/CollectiveX/publisher.py
@@ -46,10 +46,10 @@
)
REQUIRED_PROMOTION_COHORT_COUNTS = {"library": 64, "system": 12, "routing": 58}
CANONICAL_FULL_V1_MATRIX_SHA256 = (
- "5986c56db0df60f6f54e5001e0f214acdd628b65d257881bcbb226b6695b808f"
+ "5894bab58d3deb2bcee51baa075ca5f5d324b4292ac1cef9f6bc08a07ab1d9a3"
)
CANONICAL_FULL_V1_CASE_CATALOG_SHA256 = (
- "e6e91d90c679fc6aec0621034707c5558b9612a566c9437eb75ae9661753f39b"
+ "664b742da3e5ce9073a87da9191f2a488b6891a1d3a207ada264fcaa9a4b5280"
)
P50_STABILITY_LIMIT = 1.10
P99_STABILITY_LIMIT = 1.25
diff --git a/experimental/CollectiveX/sweep_matrix.py b/experimental/CollectiveX/sweep_matrix.py
index 787e898b7e..162daabb24 100644
--- a/experimental/CollectiveX/sweep_matrix.py
+++ b/experimental/CollectiveX/sweep_matrix.py
@@ -105,9 +105,9 @@
)
QUALIFICATION_INDICES = range(1, 4)
CANONICAL_V1_EXECUTION_PLAN_SHA256 = {
- 1: "7733f956836ecf01849f3cd317fa1d074bec6bdb24f5d2fad0bdf02e70246c0b",
- 2: "ce4a19f8322a3b0b7d908b80de67361c0ac9e7da7e82c2a7f867d1a33dab31c3",
- 3: "b5f03fb3fff0e304383dfe0261f16ab8e7972a9f23b0831693c9f12d496ec9e8",
+ 1: "b54cdf53496868424848860225f1ea84c87ce60e115c83de4ce2f5aae36625dc",
+ 2: "4fbd22df187578c88b1552747955e85fe97d576d31e0b5be5501840ca5dfb3e6",
+ 3: "e03105b970af3d8f5f0788c5c04ed1f53761460bb432109152bdb22b7b99f165",
}
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
index d1dbf9e690..bc123de94c 100644
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -253,10 +253,10 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
len(targets), 94
)
self.assertEqual(
- sum(item["disposition"] == "supported" for item in targets), 60
+ sum(item["disposition"] == "supported" for item in targets), 59
)
self.assertEqual(
- sum(item["disposition"] == "unsupported" for item in targets), 34
+ sum(item["disposition"] == "unsupported" for item in targets), 35
)
self.assertEqual(len(capability.provisional_precision_targets()), 0)
keys = {
diff --git a/experimental/CollectiveX/tests/test_publisher.py b/experimental/CollectiveX/tests/test_publisher.py
index ffc42c3a9d..e088dc2acd 100644
--- a/experimental/CollectiveX/tests/test_publisher.py
+++ b/experimental/CollectiveX/tests/test_publisher.py
@@ -2024,7 +2024,7 @@ def test_workflow_matrix_and_catalog_digests_do_not_drift(self) -> None:
if item["disposition"] == "unsupported"
),
),
- (49, 748, 388, 360, 1740, 878, 862),
+ (49, 748, 387, 361, 1740, 877, 863),
)
library: dict[tuple, set[str]] = {}
system: dict[tuple, set[str]] = {}
diff --git a/experimental/CollectiveX/tests/test_qualification_planning.py b/experimental/CollectiveX/tests/test_qualification_planning.py
index a9bea34586..bb3b745f00 100644
--- a/experimental/CollectiveX/tests/test_qualification_planning.py
+++ b/experimental/CollectiveX/tests/test_qualification_planning.py
@@ -227,7 +227,7 @@ def test_full_v1_matrix_emits_every_unsupported_terminal_outcome(self) -> None:
documents = [json.loads(path.read_text()) for path in written]
self.assertEqual(len(documents), len(expected))
- self.assertEqual(len(documents), 360)
+ self.assertEqual(len(documents), 361)
self.assertEqual(
{document["outcome"]["reason"] for document in documents},
{item["reason"] for item in expected},
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 77ccf90acd..8551337625 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -209,7 +209,7 @@ def test_matrix_cases_and_shards_are_identity_bound(self) -> None:
sum(len(item["case"]["ladder"].split()) for item in runnable_cases),
sum(len(item["case"]["ladder"].split()) for item in unsupported_cases),
),
- (49, 748, 388, 360, 1740, 878, 862),
+ (49, 748, 387, 361, 1740, 877, 863),
)
b300_ep16 = [
item for item in unsupported_cases
@@ -276,7 +276,7 @@ def test_matrix_cases_and_shards_are_identity_bound(self) -> None:
self.assertIsNotNone(capability.topology_for("mi325x", 8))
self.assertEqual(
Counter(shard["n"] for shard in matrix["include"]),
- Counter({6: 29, 8: 6, 10: 1, 12: 13}),
+ Counter({6: 29, 8: 6, 10: 1, 11: 1, 12: 12}),
)
ll_cases = [
item for item in matrix["requested_cases"]
From 10cb0612164631d64e97430aeff896fed7a19c73 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 7 Jul 2026 00:13:55 +0800
Subject: [PATCH 136/210] fix(collectivex): reject unavailable B300 allocations
---
.../launchers/launch_single-slurm.sh | 23 ++++--
experimental/CollectiveX/runtime/common.sh | 71 +++++++++++++++++++
.../tests/test_sampling_contract.py | 8 ++-
3 files changed, 96 insertions(+), 6 deletions(-)
diff --git a/experimental/CollectiveX/launchers/launch_single-slurm.sh b/experimental/CollectiveX/launchers/launch_single-slurm.sh
index 17f4e29c4d..485b647734 100644
--- a/experimental/CollectiveX/launchers/launch_single-slurm.sh
+++ b/experimental/CollectiveX/launchers/launch_single-slurm.sh
@@ -107,6 +107,7 @@ allocation=(--partition="$CX_PARTITION" --nodes="$NODES" --gres=gpu:"$GPN" --exc
[ -z "${CX_NODELIST:-}" ] || allocation+=(--nodelist="$CX_NODELIST")
excluded_nodes="${CX_EXCLUDE_NODES:-}"
for allocation_attempt in 1 2 3; do
+ validation_failure=""
attempt_allocation=("${allocation[@]}")
[ -z "$excluded_nodes" ] || attempt_allocation+=(--exclude="$excluded_nodes")
export CX_SALLOC_ATTEMPT="$allocation_attempt"
@@ -114,16 +115,28 @@ for allocation_attempt in 1 2 3; do
cx_salloc_jobid "${attempt_allocation[@]}"
[ -n "$JOB_ID" ] || cx_die "could not resolve allocated JOB_ID from salloc"
cx_set_failure_stage setup
- if cx_validate_network_profile_on_job "$JOB_ID" "$NODES" "$CX_TRANSPORT" 0; then
+ if ! cx_validate_network_profile_on_job "$JOB_ID" "$NODES" "$CX_TRANSPORT" 0; then
+ validation_failure=network
+ elif [ "$RUNNER" = b300 ] \
+ && ! cx_validate_cuda_context_on_job "$JOB_ID" "$NODES" "$GPN"; then
+ validation_failure=cuda-context
+ else
break
fi
- if [ "$RUNNER" != h100-dgxc ] || [ "$allocation_attempt" = 3 ]; then
- cx_fail_stage setup "$CX_NETWORK_PROFILE_LOG" || true
- cx_die "allocated nodes failed the network profile"
+ retryable=0
+ [ "$RUNNER:$validation_failure" != h100-dgxc:network ] || retryable=1
+ [ "$RUNNER:$validation_failure" != b300:cuda-context ] || retryable=1
+ if [ "$retryable" = 0 ] || [ "$allocation_attempt" = 3 ]; then
+ if [ "$validation_failure" = network ]; then
+ cx_fail_stage setup "$CX_NETWORK_PROFILE_LOG" || true
+ cx_die "allocated nodes failed the network profile"
+ fi
+ cx_fail_stage setup "$CX_CUDA_CONTEXT_LOG" || true
+ cx_die "allocated nodes failed accelerator context validation"
fi
rejected_nodes="$(cx_allocation_nodes_csv "$JOB_ID")" \
|| cx_die "cannot identify nodes from a rejected allocation"
- cx_log "allocated H100 nodes failed network validation; retrying elsewhere"
+ cx_log "allocated nodes failed $validation_failure validation; retrying elsewhere"
cx_cancel_job "$JOB_ID" || cx_die "cannot release a rejected allocation"
cx_clear_allocation_jobid || cx_die "cannot reset rejected allocation state"
JOB_ID=""
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index b0cf8e2efc..56e2248152 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -51,6 +51,8 @@ cx_fail_stage() {
diagnostic="accelerator-topology"
elif grep -aEqi 'cuda driver version is insufficient|call requires newer driver|cudaErrorCallRequiresNewerDriver|CUDA_ERROR_SYSTEM_DRIVER_MISMATCH|unsupported toolchain' "$log_path"; then
diagnostic="accelerator-driver"
+ elif grep -aEqi 'cudaErrorDevicesUnavailable|CUDA_ERROR_DEVICE_UNAVAILABLE|CUDA-capable device\(s\) is/are busy or unavailable|primary context retain failed' "$log_path"; then
+ diagnostic="accelerator-unavailable"
elif grep -aEqi 'ncclDevCommCreate|ncclCommWindowRegister|ncclGetLsa(Device)?Pointer|Communicator does not support symmetric memory|Symmetric memory is not supported' "$log_path" \
|| { [ "${CX_BENCH:-}" = deepep-v2 ] \
&& grep -aEqi 'nccl[.]cu:(106|127|128|129|135)([^0-9]|$)' "$log_path"; }; then
@@ -2360,6 +2362,75 @@ BASH
return 1
}
+# A clean nvidia-smi inventory does not prove that a prior cancelled workload
+# released every CUDA context. Retaining each primary context catches poisoned
+# allocations before a full shard spends time failing every case.
+cx_validate_cuda_context_on_job() {
+ local job_id="$1" nodes="$2" gpus_per_node="$3" log_label=cuda-context log
+ case "${CX_SALLOC_ATTEMPT:-1}" in
+ 1) ;;
+ 2|3) log_label+="-a${CX_SALLOC_ATTEMPT}" ;;
+ *) return 1 ;;
+ esac
+ log="$(cx_private_log_path "$log_label")"
+ CX_CUDA_CONTEXT_LOG="$log"
+ srun --jobid="$job_id" --nodes="$nodes" --ntasks="$nodes" --ntasks-per-node=1 \
+ --gres=gpu:"$gpus_per_node" --chdir=/tmp --input=all \
+ --export="$(cx_host_exports)" python3 - "$gpus_per_node" \
+ >"$log" 2>&1 <<'PY'
+import ctypes
+import socket
+import sys
+
+expected = int(sys.argv[1])
+cuda = ctypes.CDLL("libcuda.so.1")
+cuda.cuInit.argtypes = [ctypes.c_uint]
+cuda.cuInit.restype = ctypes.c_int
+cuda.cuDeviceGetCount.argtypes = [ctypes.POINTER(ctypes.c_int)]
+cuda.cuDeviceGetCount.restype = ctypes.c_int
+cuda.cuDeviceGet.argtypes = [ctypes.POINTER(ctypes.c_int), ctypes.c_int]
+cuda.cuDeviceGet.restype = ctypes.c_int
+cuda.cuDevicePrimaryCtxRetain.argtypes = [
+ ctypes.POINTER(ctypes.c_void_p), ctypes.c_int,
+]
+cuda.cuDevicePrimaryCtxRetain.restype = ctypes.c_int
+cuda.cuDevicePrimaryCtxRelease.argtypes = [ctypes.c_int]
+cuda.cuDevicePrimaryCtxRelease.restype = ctypes.c_int
+
+
+def check(result: int, operation: str) -> None:
+ if result != 0:
+ raise RuntimeError(f"{operation} failed with CUDA result {result}")
+
+
+check(cuda.cuInit(0), "CUDA initialization")
+count = ctypes.c_int()
+check(cuda.cuDeviceGetCount(ctypes.byref(count)), "CUDA device count")
+if count.value != expected:
+ raise RuntimeError(
+ f"CUDA device count mismatch on {socket.gethostname()}: "
+ f"expected {expected}, found {count.value}"
+ )
+
+devices: list[int] = []
+try:
+ for ordinal in range(expected):
+ device = ctypes.c_int()
+ context = ctypes.c_void_p()
+ check(cuda.cuDeviceGet(ctypes.byref(device), ordinal), "CUDA device lookup")
+ result = cuda.cuDevicePrimaryCtxRetain(ctypes.byref(context), device.value)
+ if result != 0:
+ raise RuntimeError(
+ f"CUDA primary context retain failed on {socket.gethostname()} "
+ f"device {ordinal} with CUDA result {result}"
+ )
+ devices.append(device.value)
+finally:
+ for device in reversed(devices):
+ check(cuda.cuDevicePrimaryCtxRelease(device), "CUDA primary context release")
+PY
+}
+
# Resolve the exact per-execution child before any copy starts, so the parent
# EXIT trap can remove an interrupted partial stage. The configured base must
# already exist on compute-visible storage and must not traverse symlinks.
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 8551337625..2521474fe8 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -2599,7 +2599,10 @@ def test_launchers_use_private_logs_and_allowlisted_failure_stages(self) -> None
self.assertIn("rdma-device-%s=missing", common)
single_slurm = (ROOT / "launchers" / "launch_single-slurm.sh").read_text()
self.assertIn("for allocation_attempt in 1 2 3", single_slurm)
- self.assertIn('if [ "$RUNNER" != h100-dgxc ]', single_slurm)
+ self.assertIn('RUNNER:$validation_failure', single_slurm)
+ self.assertIn('h100-dgxc:network', single_slurm)
+ self.assertIn('b300:cuda-context', single_slurm)
+ self.assertIn('cx_validate_cuda_context_on_job "$JOB_ID" "$NODES" "$GPN"', single_slurm)
self.assertIn('rejected_nodes="$(cx_allocation_nodes_csv "$JOB_ID")"', single_slurm)
self.assertIn('export CX_SALLOC_ATTEMPT="$allocation_attempt"', single_slurm)
self.assertIn('export CX_NETWORK_VALIDATION_ATTEMPT="$allocation_attempt"', single_slurm)
@@ -2611,6 +2614,9 @@ def test_launchers_use_private_logs_and_allowlisted_failure_stages(self) -> None
)
self.assertIn('log_label+="-a${CX_SALLOC_ATTEMPT}"', common)
self.assertIn('log_label+="-a${CX_NETWORK_VALIDATION_ATTEMPT}"', common)
+ self.assertIn('cx_validate_cuda_context_on_job()', common)
+ self.assertIn('cuDevicePrimaryCtxRetain', common)
+ self.assertIn('diagnostic="accelerator-unavailable"', common)
def test_case_failure_diagnostic_precedes_normal_srun_footer(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
From 2f8c0cf12b34c8e54d67df71df6c48f492f1e7f5 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 7 Jul 2026 00:37:39 +0800
Subject: [PATCH 137/210] fix(collectivex): bind rendezvous to validated
interface
---
experimental/CollectiveX/runtime/common.sh | 26 ++++++++++++++-----
.../tests/test_sampling_contract.py | 14 ++++++----
2 files changed, 29 insertions(+), 11 deletions(-)
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 56e2248152..ebff200527 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -961,15 +961,29 @@ cx_allocation_nodes_csv() {
}
cx_resolve_slurm_rendezvous() {
- local job_id="$1" master_addr master_port
+ local job_id="$1" master_addr master_port socket_ifname="${CX_SOCKET_IFNAME:-}"
[[ "$job_id" =~ ^[1-9][0-9]*$ ]] || cx_die "invalid rendezvous allocation"
- # Expanded Slurm nodelists may be sorted differently from task placement.
# Query relative node zero directly so MASTER_ADDR always hosts global rank 0.
- master_addr="$(srun --jobid="$job_id" --nodes=1 --ntasks=1 --relative=0 \
- --chdir=/tmp --export="$(cx_host_exports)" hostname -s 2>/dev/null | head -n1)"
+ # Prefer the address on the already validated cross-node socket interface;
+ # a short hostname may resolve onto a management network that ranks cannot use.
+ if [[ "$socket_ifname" =~ ^[A-Za-z][A-Za-z0-9_.-]{0,31}$ ]]; then
+ master_addr="$(srun --jobid="$job_id" --nodes=1 --ntasks=1 --relative=0 \
+ --chdir=/tmp --export="$(cx_host_exports)" bash -s -- "$socket_ifname" \
+ 2>/dev/null <<'BASH' | head -n1
+set -euo pipefail
+ip -o -4 address show dev "$1" scope global \
+ | awk 'NR == 1 {split($4, address, "/"); print address[1]}'
+BASH
+)"
+ [[ "$master_addr" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]] \
+ || cx_die "could not resolve the allocated primary interface"
+ else
+ master_addr="$(srun --jobid="$job_id" --nodes=1 --ntasks=1 --relative=0 \
+ --chdir=/tmp --export="$(cx_host_exports)" hostname -s 2>/dev/null | head -n1)"
+ [[ "$master_addr" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] \
+ || cx_die "could not resolve the allocated primary node"
+ fi
master_port="${CX_MASTER_PORT:-29551}"
- [[ "$master_addr" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] \
- || cx_die "could not resolve the allocated primary node"
[[ "$master_port" =~ ^[1-9][0-9]*$ ]] && [ "$master_port" -le 65535 ] \
|| cx_die "invalid distributed rendezvous port"
export MASTER_ADDR="$master_addr" MASTER_PORT="$master_port"
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 2521474fe8..75de48db89 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -3023,7 +3023,7 @@ def test_canonical_gha_environment_is_locked_but_manual_overrides_survive(self)
)
self.assertEqual(list(workspace.iterdir()), [])
- def test_slurm_rendezvous_uses_relative_node_zero(self) -> None:
+ def test_slurm_rendezvous_uses_relative_node_zero_and_validated_interface(self) -> None:
common = ROOT / "runtime" / "common.sh"
command = r'''
set -euo pipefail
@@ -3032,12 +3032,16 @@ def test_slurm_rendezvous_uses_relative_node_zero(self) -> None:
squeue() { return 91; }
scontrol() { return 92; }
srun() {
- test "$*" = "--jobid=123 --nodes=1 --ntasks=1 --relative=0 --chdir=/tmp --export=HOME,PATH,USER hostname -s" \
- || return 93
- printf '%s\n' rank-zero-node
+ case "$*" in
+ *" bash -s -- eth0") printf '%s\n' 192.0.2.7 ;;
+ *" hostname -s") printf '%s\n' rank-zero-node ;;
+ *) return 93 ;;
+ esac
}
+ CX_SOCKET_IFNAME=eth0
cx_resolve_slurm_rendezvous 123
- test "$MASTER_ADDR:$MASTER_PORT" = rank-zero-node:29551
+ test "$MASTER_ADDR:$MASTER_PORT" = 192.0.2.7:29551
+ unset CX_SOCKET_IFNAME
CX_MASTER_PORT=30444
cx_resolve_slurm_rendezvous 123
test "$MASTER_ADDR:$MASTER_PORT" = rank-zero-node:30444
From f8f14985618fb01bd6244d38d8d5fb417ffd5d0a Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 7 Jul 2026 00:48:46 +0800
Subject: [PATCH 138/210] fix(collectivex): classify collective backend
failures
---
experimental/CollectiveX/runtime/common.sh | 10 ++++++++++
.../CollectiveX/tests/test_sampling_contract.py | 7 +++++++
2 files changed, 17 insertions(+)
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index ebff200527..cb1746f3bc 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -81,6 +81,14 @@ cx_fail_stage() {
elif [ -n "$probe_stage" ] \
&& grep -aEqi 'timed out|operation timeout|wait timeout after|watchdog.*timeout|timeout: sending signal' "$log_path"; then
diagnostic="${probe_stage}-timeout"
+ elif grep -aEqi 'ncclRemoteError|remote process exited|connection closed by peer' "$log_path"; then
+ diagnostic="collective-remote"
+ elif grep -aEqi 'ncclSystemError|unhandled system error' "$log_path"; then
+ diagnostic="collective-system"
+ elif grep -aEqi 'ncclInternalError|internal check failed' "$log_path"; then
+ diagnostic="collective-internal"
+ elif grep -aEqi 'ncclInvalidUsage|invalid usage' "$log_path"; then
+ diagnostic="collective-invalid-usage"
elif grep -aEqi 'timed out|operation timeout|wait timeout after|watchdog.*timeout|timeout: sending signal|connection reset|could not resolve|TLS|certificate' "$log_path"; then
diagnostic="network-or-timeout"
elif grep -aEqi 'salloc:|srun:.*(unable to create step|step creation|invalid partition|invalid account)|unable to create step|job allocation' "$log_path"; then
@@ -168,6 +176,8 @@ cx_fail_stage() {
diagnostic="python-os"
elif grep -aEqi '(NotImplemented|System)Error:' "$log_path"; then
diagnostic="python-system"
+ elif grep -aEqi 'DistBackendError:' "$log_path"; then
+ diagnostic="collective-backend"
elif grep -aEqi 'CalledProcessError:' "$log_path"; then
diagnostic="python-subprocess"
elif grep -aEqi 'Traceback \(most recent call last\)' "$log_path"; then
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 75de48db89..10beff6def 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -2782,6 +2782,13 @@ def test_private_runtime_failure_signatures_override_case_footer(self) -> None:
"NVCC compilation failed": "jit-toolchain",
"CUDA out of memory": "accelerator-memory",
"torch rendezvous timed out": "network-or-timeout",
+ "torch.distributed.DistBackendError: ncclRemoteError": "collective-remote",
+ "torch.distributed.DistBackendError: ncclSystemError": "collective-system",
+ "torch.distributed.DistBackendError: ncclInternalError": "collective-internal",
+ "torch.distributed.DistBackendError: ncclInvalidUsage":
+ "collective-invalid-usage",
+ "torch.distributed.DistBackendError: unknown collective failure":
+ "collective-backend",
"ModuleNotFoundError: missing module": "python-import",
"AttributeError: backend has no attribute 'probe'": "backend-api",
"PrecisionError: FP8 dispatch payload is missing block-128 scales":
From 4d46e12e8bb3135d7ce22c5a14ea7dcd5d5d6106 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 7 Jul 2026 00:57:23 +0800
Subject: [PATCH 139/210] fix(collectivex): locate invalid collective usage
---
experimental/CollectiveX/runtime/common.sh | 10 +++++++++-
.../CollectiveX/tests/test_sampling_contract.py | 6 ++++++
2 files changed, 15 insertions(+), 1 deletion(-)
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index cb1746f3bc..00d7d27777 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -88,7 +88,15 @@ cx_fail_stage() {
elif grep -aEqi 'ncclInternalError|internal check failed' "$log_path"; then
diagnostic="collective-internal"
elif grep -aEqi 'ncclInvalidUsage|invalid usage' "$log_path"; then
- diagnostic="collective-invalid-usage"
+ if grep -aEqi 'dist[.]init_process_group|init_process_group[(]' "$log_path"; then
+ diagnostic="collective-init-invalid-usage"
+ elif grep -aEqi 'dist[.]all_gather_object|all_gather_object[(]' "$log_path"; then
+ diagnostic="collective-consensus-invalid-usage"
+ elif grep -aEqi 'dist[.]all_to_all_single|all_to_all_single[(]' "$log_path"; then
+ diagnostic="collective-alltoall-invalid-usage"
+ else
+ diagnostic="collective-invalid-usage"
+ fi
elif grep -aEqi 'timed out|operation timeout|wait timeout after|watchdog.*timeout|timeout: sending signal|connection reset|could not resolve|TLS|certificate' "$log_path"; then
diagnostic="network-or-timeout"
elif grep -aEqi 'salloc:|srun:.*(unable to create step|step creation|invalid partition|invalid account)|unable to create step|job allocation' "$log_path"; then
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 10beff6def..551bb3bbca 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -2787,6 +2787,12 @@ def test_private_runtime_failure_signatures_override_case_footer(self) -> None:
"torch.distributed.DistBackendError: ncclInternalError": "collective-internal",
"torch.distributed.DistBackendError: ncclInvalidUsage":
"collective-invalid-usage",
+ "dist.init_process_group('nccl')\nDistBackendError: ncclInvalidUsage":
+ "collective-init-invalid-usage",
+ "dist.all_gather_object(records, local)\nDistBackendError: ncclInvalidUsage":
+ "collective-consensus-invalid-usage",
+ "dist.all_to_all_single(output, input)\nDistBackendError: ncclInvalidUsage":
+ "collective-alltoall-invalid-usage",
"torch.distributed.DistBackendError: unknown collective failure":
"collective-backend",
"ModuleNotFoundError: missing module": "python-import",
From 0d415faf54a7d62e5afee218e5ab02d433760d0a Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 7 Jul 2026 01:09:34 +0800
Subject: [PATCH 140/210] fix(collectivex): make failed-job reruns reusable
---
.github/workflows/collectivex-sweep.yml | 14 ++++++++------
1 file changed, 8 insertions(+), 6 deletions(-)
diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml
index 53cfd4b016..9d68e60dd9 100644
--- a/.github/workflows/collectivex-sweep.yml
+++ b/.github/workflows/collectivex-sweep.yml
@@ -202,15 +202,17 @@ jobs:
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
if: ${{ steps.gen.outputs.source_backends != '' }}
with:
- name: cxbackend-sources-${{ github.run_id }}-${{ github.run_attempt }}
+ name: cxbackend-sources-${{ github.run_id }}
path: ${{ runner.temp }}/collectivex-backend-sources.tar
if-no-files-found: error
+ overwrite: true
retention-days: ${{ inputs.release_tag == 'v1' && 90 || 3 }}
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
- name: cxsweep-matrix-${{ github.run_id }}-${{ github.run_attempt }}
+ name: cxsweep-matrix-${{ github.run_id }}
path: experimental/CollectiveX/matrix_full.json
if-no-files-found: error
+ overwrite: true
retention-days: ${{ inputs.release_tag == 'v1' && 90 || 3 }}
- name: Create V1 release marker
if: ${{ inputs.operation == 'sweep' && inputs.release_tag == 'v1' }}
@@ -407,13 +409,13 @@ jobs:
trap - EXIT
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
- name: cxsweep-matrix-${{ github.run_id }}-${{ github.run_attempt }}
+ name: cxsweep-matrix-${{ github.run_id }}
path: ${{ env.CX_JOB_ROOT }}/control
- name: Download pinned backend source archive
if: ${{ matrix.backend == 'deepep-v2' || matrix.backend == 'deepep-hybrid' }}
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
- name: cxbackend-sources-${{ github.run_id }}-${{ github.run_attempt }}
+ name: cxbackend-sources-${{ github.run_id }}
path: ${{ env.CX_JOB_ROOT }}/control
- name: Install pinned backend source seed
if: ${{ matrix.backend == 'deepep-v2' || matrix.backend == 'deepep-hybrid' }}
@@ -655,7 +657,7 @@ jobs:
with: { clean: true, persist-credentials: false }
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
- name: cxsweep-matrix-${{ github.run_id }}-${{ github.run_attempt }}
+ name: cxsweep-matrix-${{ github.run_id }}
path: ${{ runner.temp }}/collectivex-probes/control
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
@@ -771,7 +773,7 @@ jobs:
run_dir="$downloads/$run_id"
mkdir -m 750 "$run_dir"
gh run download "$run_id" --repo "$REPOSITORY" --dir "$run_dir"
- matrix="$run_dir/cxsweep-matrix-$run_id-$attempt/matrix_full.json"
+ matrix="$run_dir/cxsweep-matrix-$run_id/matrix_full.json"
marker="$run_dir/cxrelease-v1-$run_id-$attempt/release.json"
[ -f "$matrix" ] || {
echo "run $run_id is missing its exact matrix artifact" >&2
From d296df4f646fcb2500d501d3ef9548cbb120bdfc Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 7 Jul 2026 01:16:38 +0800
Subject: [PATCH 141/210] fix(collectivex): retry AMD container imports
---
.../CollectiveX/launchers/launch_mi-amds.sh | 47 +++++++++++++------
experimental/CollectiveX/runtime/common.sh | 10 +++-
.../tests/test_sampling_contract.py | 3 ++
3 files changed, 44 insertions(+), 16 deletions(-)
diff --git a/experimental/CollectiveX/launchers/launch_mi-amds.sh b/experimental/CollectiveX/launchers/launch_mi-amds.sh
index 0ecfed4980..01d2e4e010 100644
--- a/experimental/CollectiveX/launchers/launch_mi-amds.sh
+++ b/experimental/CollectiveX/launchers/launch_mi-amds.sh
@@ -97,20 +97,39 @@ if [ "$NODES" = 1 ]; then
else
allocation+=(--ntasks-per-node="$GPN" --cpus-per-task="$((CPUS_PER_TASK / GPN))")
fi
-if [ -n "$NODELIST" ]; then
- cx_log "using configured node pin"
- allocation+=(--nodelist="$NODELIST")
-elif [ -n "$EXCLUDE_NODES" ]; then
- allocation+=(--exclude="$EXCLUDE_NODES")
-fi
-cx_salloc_jobid "${allocation[@]}"
-[ -n "$JOB_ID" ] || cx_die "could not resolve allocated JOB_ID from salloc"
-cx_set_failure_stage setup
-cx_validate_network_profile_on_job "$JOB_ID" "$NODES" "$CX_TRANSPORT"
-
-cx_set_failure_stage container-import
-SQUASH_FILE="$(cx_ensure_squash_on_job \
- "$JOB_ID" "$SQUASH_DIR" "$IMAGE" "${CX_LOCK_DIR:-}")"
+excluded_nodes="$EXCLUDE_NODES"
+for allocation_attempt in 1 2 3; do
+ attempt_allocation=("${allocation[@]}")
+ if [ -n "$NODELIST" ]; then
+ cx_log "using configured node pin"
+ attempt_allocation+=(--nodelist="$NODELIST")
+ elif [ -n "$excluded_nodes" ]; then
+ attempt_allocation+=(--exclude="$excluded_nodes")
+ fi
+ export CX_SALLOC_ATTEMPT="$allocation_attempt"
+ export CX_NETWORK_VALIDATION_ATTEMPT="$allocation_attempt"
+ cx_salloc_jobid "${attempt_allocation[@]}"
+ [ -n "$JOB_ID" ] || cx_die "could not resolve allocated JOB_ID from salloc"
+ cx_set_failure_stage setup
+ cx_validate_network_profile_on_job "$JOB_ID" "$NODES" "$CX_TRANSPORT"
+ cx_set_failure_stage container-import
+ if SQUASH_FILE="$(cx_ensure_squash_on_job \
+ "$JOB_ID" "$SQUASH_DIR" "$IMAGE" "${CX_LOCK_DIR:-}")"; then
+ break
+ fi
+ if [ -n "$NODELIST" ] || [ "$allocation_attempt" = 3 ]; then
+ cx_die "allocated nodes failed container import"
+ fi
+ rejected_nodes="$(cx_allocation_nodes_csv "$JOB_ID")" \
+ || cx_die "cannot identify nodes from a rejected allocation"
+ cx_log "allocated nodes failed container import; retrying elsewhere"
+ cx_cancel_job "$JOB_ID" || cx_die "cannot release a rejected allocation"
+ cx_clear_allocation_jobid || cx_die "cannot reset rejected allocation state"
+ JOB_ID=""
+ [ -z "$excluded_nodes" ] || excluded_nodes+=,
+ excluded_nodes+="$rejected_nodes"
+done
+unset CX_SALLOC_ATTEMPT CX_NETWORK_VALIDATION_ATTEMPT
cx_set_failure_stage container-hash
import_log="$(cx_private_log_path image-hash)"
if ! COLLECTIVEX_SQUASH_SHA256="$(
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 00d7d27777..6ef4ae9ab5 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -2282,14 +2282,20 @@ cx_ensure_squash() {
# Import on an allocated compute node so multiarch tags resolve for the target
# architecture. The squash directory must be shared with the submit host.
cx_ensure_squash_on_job() {
- local job_id="$1" squash_dir="$2" image="$3" lock_dir="${4:-}" sq key lock log
+ local job_id="$1" squash_dir="$2" image="$3" lock_dir="${4:-}" sq key lock
+ local log_label=container-import log
[[ "$job_id" =~ ^[0-9]+$ ]] || return 1
+ case "${CX_SALLOC_ATTEMPT:-1}" in
+ 1) ;;
+ 2|3) log_label+="-a${CX_SALLOC_ATTEMPT}" ;;
+ *) return 1 ;;
+ esac
sq="$(cx_squash_path "$squash_dir" "$image")" || return 1
key="${sq##*/}"
key="${key%.sqsh}"
[ -n "$lock_dir" ] || lock_dir="$squash_dir/.locks"
lock="$lock_dir/${key}.lock"
- log="$(cx_private_log_path container-import)"
+ log="$(cx_private_log_path "$log_label")"
if ! srun --jobid="$job_id" --nodes=1 --ntasks=1 --chdir=/tmp \
--export="$(cx_host_exports)" \
bash -s -- "$sq" "$lock" "$image" "$CX_SQUASH_SOURCE_DATE_EPOCH" \
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 551bb3bbca..0da4e060d8 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -2548,6 +2548,9 @@ def test_launchers_use_private_logs_and_allowlisted_failure_stages(self) -> None
self.assertIn(f"cx_set_failure_stage {stage}", stage_source)
amd = (ROOT / "launchers" / "launch_mi-amds.sh").read_text()
self.assertIn("cx_ensure_squash_on_job", amd)
+ self.assertIn("for allocation_attempt in 1 2 3", amd)
+ self.assertIn("allocated nodes failed container import; retrying elsewhere", amd)
+ self.assertIn('rejected_nodes="$(cx_allocation_nodes_csv "$JOB_ID")"', amd)
self.assertIn("cx_fail_stage container-hash", amd)
self.assertNotIn('cat "$import_log"', amd)
self.assertIn('bash -s -- "$sq" "$lock" "$image"', common)
From d4a6def95621942510141dcebe936d6deede1d88 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 7 Jul 2026 01:28:21 +0800
Subject: [PATCH 142/210] fix(collectivex): gather metadata with device tensors
---
experimental/CollectiveX/tests/run_ep.py | 64 ++++++++++++++++---
.../tests/test_sampling_contract.py | 4 ++
2 files changed, 58 insertions(+), 10 deletions(-)
diff --git a/experimental/CollectiveX/tests/run_ep.py b/experimental/CollectiveX/tests/run_ep.py
index cc43abffa3..57ca820fc4 100644
--- a/experimental/CollectiveX/tests/run_ep.py
+++ b/experimental/CollectiveX/tests/run_ep.py
@@ -167,6 +167,44 @@ def _common_runtime_fingerprint(records: list[dict]) -> dict:
return records[0]
+def _all_gather_json(
+ value,
+ *,
+ torch_module,
+ dist_module,
+ device,
+ world_size: int,
+) -> list:
+ """Gather bounded JSON values using device tensors, not Python object collectives."""
+ payload = json.dumps(
+ value,
+ allow_nan=False,
+ ensure_ascii=False,
+ separators=(",", ":"),
+ sort_keys=True,
+ ).encode("utf-8")
+ if not payload or len(payload) > 1_048_576:
+ raise ValueError("distributed consensus payload size is invalid")
+ size = torch_module.tensor([len(payload)], dtype=torch_module.int64, device=device)
+ sizes = [torch_module.zeros_like(size) for _ in range(world_size)]
+ dist_module.all_gather(sizes, size)
+ lengths = [int(item.item()) for item in sizes]
+ if any(length < 1 or length > 1_048_576 for length in lengths):
+ raise ValueError("distributed consensus size evidence is invalid")
+ width = max(lengths)
+ encoded = torch_module.zeros(width, dtype=torch_module.uint8, device=device)
+ encoded[: len(payload)] = torch_module.tensor(
+ list(payload), dtype=torch_module.uint8, device=device
+ )
+ gathered = [torch_module.empty_like(encoded) for _ in range(world_size)]
+ dist_module.all_gather(gathered, encoded)
+ records = []
+ for tensor, length in zip(gathered, lengths, strict=True):
+ raw = bytes(tensor[:length].cpu().tolist())
+ records.append(json.loads(raw.decode("utf-8")))
+ return records
+
+
def _allocation_stratum_sha256(
physical_hosts: list[str],
*,
@@ -468,24 +506,25 @@ def main() -> int:
)
except ValueError as exc:
raise ValueError("SLURM_NNODES must be a positive integer") from exc
- realized_records: list[tuple[str, int, dict] | None] = [None] * world_size
- dist.all_gather_object(
- realized_records,
- (socket.gethostname(), local_rank, args.runtime_fingerprint),
+ realized_records = _all_gather_json(
+ [socket.gethostname(), local_rank, args.runtime_fingerprint],
+ torch_module=torch,
+ dist_module=dist,
+ device=device,
+ world_size=world_size,
)
- complete_records = [record for record in realized_records if record is not None]
args.realized_placement = _summarize_realized_placement(
- [(record[0], record[1]) for record in complete_records],
+ [(record[0], record[1]) for record in realized_records],
expected_nodes=expected_nodes,
expected_gpus_per_node=gpus_per_node,
expected_world_size=world_size,
)
args.runtime_fingerprint = _common_runtime_fingerprint(
- [record[2] for record in complete_records]
+ [record[2] for record in realized_records]
)
canonical = bool(args.workload_dir)
local_stratum = _allocation_stratum_sha256(
- [record[0] for record in complete_records],
+ [record[0] for record in realized_records],
audit_salt=os.environ.get("CX_AUDIT_SALT"),
fabric_selectors={
field: os.environ.get(environment) or None
@@ -493,8 +532,13 @@ def main() -> int:
},
required=canonical,
)
- stratum_records: list[str | None] = [None] * world_size
- dist.all_gather_object(stratum_records, local_stratum)
+ stratum_records = _all_gather_json(
+ local_stratum,
+ torch_module=torch,
+ dist_module=dist,
+ device=device,
+ world_size=world_size,
+ )
args.allocation_stratum_sha256 = _common_allocation_stratum(
stratum_records, required=canonical
)
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 0da4e060d8..1930cf77d4 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -2556,6 +2556,10 @@ def test_launchers_use_private_logs_and_allowlisted_failure_stages(self) -> None
self.assertIn('bash -s -- "$sq" "$lock" "$image"', common)
self.assertIn("> \"$log\" 2>&1 <<'BASH'", common)
self.assertIn("cx_fail_stage container-import", common)
+ run_ep = (ROOT / "tests" / "run_ep.py").read_text()
+ self.assertIn("def _all_gather_json(", run_ep)
+ self.assertIn("dist_module.all_gather(gathered, encoded)", run_ep)
+ self.assertNotIn("dist.all_gather_object", run_ep)
runtime = (ROOT / "runtime" / "run_in_container.sh").read_text()
export_start = common.index("\ncx_container_exports() {")
exports = common[export_start:common.index("\n}", export_start)]
From 95f3635330edec295fc2c9b9e31c3f9724f7935b Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 7 Jul 2026 01:40:15 +0800
Subject: [PATCH 143/210] fix(collectivex): bind RCCL ranks eagerly
---
experimental/CollectiveX/tests/run_ep.py | 5 +++--
experimental/CollectiveX/tests/test_sampling_contract.py | 2 ++
2 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/experimental/CollectiveX/tests/run_ep.py b/experimental/CollectiveX/tests/run_ep.py
index 57ca820fc4..9edf44717a 100644
--- a/experimental/CollectiveX/tests/run_ep.py
+++ b/experimental/CollectiveX/tests/run_ep.py
@@ -488,9 +488,10 @@ def main() -> int:
world_size=world_size,
device_id=device,
)
- elif args.backend == "deepep-v2":
+ elif args.backend in {"deepep-v2", "nccl-ep"}:
# PR #605 reuses PyTorch's NCCL communicator through ``_comm_ptr``. Supplying
- # device_id eagerly forms it before ElasticBuffer construction.
+ # device_id eagerly forms it before ElasticBuffer construction. The reference
+ # backend uses the same eager binding so RCCL never infers rank devices lazily.
dist.init_process_group("nccl", device_id=device)
else:
dist.init_process_group("nccl")
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 1930cf77d4..c609738584 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -2560,6 +2560,8 @@ def test_launchers_use_private_logs_and_allowlisted_failure_stages(self) -> None
self.assertIn("def _all_gather_json(", run_ep)
self.assertIn("dist_module.all_gather(gathered, encoded)", run_ep)
self.assertNotIn("dist.all_gather_object", run_ep)
+ self.assertIn('args.backend in {"deepep-v2", "nccl-ep"}', run_ep)
+ self.assertIn('dist.init_process_group("nccl", device_id=device)', run_ep)
runtime = (ROOT / "runtime" / "run_in_container.sh").read_text()
export_start = common.index("\ncx_container_exports() {")
exports = common[export_start:common.index("\n}", export_start)]
From 51267c81772b3312ff46742341c2be49758e2630 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 7 Jul 2026 01:48:18 +0800
Subject: [PATCH 144/210] fix(collectivex): size MoRI buffers for skew
---
experimental/CollectiveX/tests/ep_mori.py | 15 +++++++-
.../tests/test_sampling_contract.py | 34 +++++++++++++++++++
2 files changed, 48 insertions(+), 1 deletion(-)
diff --git a/experimental/CollectiveX/tests/ep_mori.py b/experimental/CollectiveX/tests/ep_mori.py
index a2f36c4870..3e17b9d94b 100644
--- a/experimental/CollectiveX/tests/ep_mori.py
+++ b/experimental/CollectiveX/tests/ep_mori.py
@@ -47,6 +47,19 @@ def _mori_source_commit() -> str:
raise RuntimeError("MoRI image source revision is unavailable")
+def _mori_input_capacity(args, world_size: int) -> int:
+ """Reserve enough rows for the worst legal Zipf destination rank."""
+ base = 512
+ if getattr(args, "routing", "uniform") != "zipf":
+ return base
+ spec = str(getattr(args, "tokens_ladder", "")).replace(",", " ").split()
+ if spec:
+ largest = max(int(token) for token in spec)
+ else:
+ largest = 128 if getattr(args, "phase", "decode") == "decode" else 4096
+ return max(base, largest * world_size)
+
+
class MoRIBackend:
name = "mori"
stage_device_work = False
@@ -352,7 +365,7 @@ def __init__(self, args, rank, world_size, local_rank, device):
}
def buffer_cap(self, args):
- return 512
+ return _mori_input_capacity(args, self.world_size)
def make_problem(self, T, idx, weights, x):
encoding = ep_precision.encode_dispatch(
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index c609738584..ffd7e46598 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -1177,6 +1177,40 @@ def test_mori_masks_global_topk_metadata_to_the_local_rank(self) -> None:
self.assertIsNone(profile["comm_units_kind"])
self.assertIsNone(profile["configured_units"])
+ def test_mori_zipf_capacity_covers_worst_rank_skew(self) -> None:
+ path = HERE / "ep_mori.py"
+ tree = ast.parse(path.read_text(), str(path))
+ helper = next(
+ node
+ for node in tree.body
+ if isinstance(node, ast.FunctionDef) and node.name == "_mori_input_capacity"
+ )
+ namespace: dict[str, object] = {}
+ exec(
+ compile(ast.Module(body=[helper], type_ignores=[]), str(path), "exec"),
+ namespace,
+ )
+ capacity = namespace["_mori_input_capacity"]
+ self.assertEqual(capacity(types.SimpleNamespace(routing="uniform"), 8), 512)
+ self.assertEqual(
+ capacity(
+ types.SimpleNamespace(
+ routing="zipf", tokens_ladder="512", phase="prefill"
+ ),
+ 8,
+ ),
+ 4096,
+ )
+ self.assertEqual(
+ capacity(
+ types.SimpleNamespace(
+ routing="zipf", tokens_ladder="128", phase="decode"
+ ),
+ 8,
+ ),
+ 1024,
+ )
+
def test_squash_identity_rehashes_instead_of_trusting_a_sidecar(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
image = Path(temporary) / "image.sqsh"
From b7bc16385eb0fec8b3fa1dd132dfc80ed99f8f00 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 7 Jul 2026 02:12:12 +0800
Subject: [PATCH 145/210] fix(collectivex): let RCCL select pinned RDMA
---
experimental/CollectiveX/runtime/common.sh | 8 +++++++-
experimental/CollectiveX/tests/ep_nccl.py | 10 ++++++++--
.../CollectiveX/tests/test_deepep_v2_contract.py | 3 ++-
.../CollectiveX/tests/test_sampling_contract.py | 4 ++++
4 files changed, 21 insertions(+), 4 deletions(-)
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 6ef4ae9ab5..c2490b4390 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -753,7 +753,13 @@ cx_apply_network_profile() {
export NVSHMEM_HCA_LIST="$CX_RDMA_DEVICES"
export NVSHMEM_ENABLE_NIC_PE_MAPPING=1
if [ "$scaleout" = 1 ]; then
- export NCCL_NET=IB NCCL_IB_HCA="=$CX_RDMA_DEVICES"
+ if [ "${CX_SHARD_SKU:-}" = mi300x ] || [ "${CX_SHARD_SKU:-}" = mi325x ] \
+ || [ "${CX_SHARD_SKU:-}" = mi355x ]; then
+ unset NCCL_NET
+ else
+ export NCCL_NET=IB
+ fi
+ export NCCL_IB_HCA="=$CX_RDMA_DEVICES"
export MORI_RDMA_DEVICES="$rdma_names" EP_NIC_NAME="$ep_nic"
fi
fi
diff --git a/experimental/CollectiveX/tests/ep_nccl.py b/experimental/CollectiveX/tests/ep_nccl.py
index efd4d89b7a..f97442605c 100644
--- a/experimental/CollectiveX/tests/ep_nccl.py
+++ b/experimental/CollectiveX/tests/ep_nccl.py
@@ -67,8 +67,11 @@ def __init__(self, args, rank, world_size, local_rank, device):
_library, _version = _runtime_collective(args, torch)
if args.scale_out_transport:
hcas = os.environ.get("NCCL_IB_HCA", "")
- if os.environ.get("NCCL_NET") != "IB":
- raise RuntimeError("scale-out collective network mode is not IB")
+ network_selection = os.environ.get("NCCL_NET")
+ if _library == "nccl" and network_selection != "IB":
+ raise RuntimeError("scale-out NCCL network mode is not IB")
+ if _library == "rccl" and network_selection:
+ raise RuntimeError("scale-out RCCL must auto-select its HCA-pinned network")
if not re.fullmatch(
r"=[A-Za-z][A-Za-z0-9_.-]{0,31}(?::[1-9][0-9]*)?"
r"(?:,[A-Za-z][A-Za-z0-9_.-]{0,31}(?::[1-9][0-9]*)?)*",
@@ -93,6 +96,9 @@ def __init__(self, args, rank, world_size, local_rank, device):
"collective_library": _library,
"nccl_version": _version,
"transport": f"{_library}-all_to_all_single",
+ "network_selection": (
+ "forced-ib" if _library == "nccl" else "auto-hca-pinned"
+ ),
"resource_mode": "fixed-profile",
"num_sms": None,
"device_sms": torch.cuda.get_device_properties(device).multi_processor_count,
diff --git a/experimental/CollectiveX/tests/test_deepep_v2_contract.py b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
index 8478254c4d..bb1bb4a4a8 100644
--- a/experimental/CollectiveX/tests/test_deepep_v2_contract.py
+++ b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
@@ -1537,7 +1537,8 @@ def test_runtime_pins_uccl_wheel_and_hybrid_source_tree(self) -> None:
self.assertIn('EP_NVSHMEM_ROOT_DIR="$NVSHMEM_DIR"', runtime)
reference = (HERE / "ep_nccl.py").read_text()
self.assertIn("self.kernel_generation = contracts.collective_kernel_generation", reference)
- self.assertIn("scale-out collective network mode is not IB", reference)
+ self.assertIn("scale-out NCCL network mode is not IB", reference)
+ self.assertIn("scale-out RCCL must auto-select its HCA-pinned network", reference)
self.assertIn("scale-out collective HCA allowlist is invalid", reference)
self.assertIn('entries={len(entries)}, shaped={shaped}', reference)
self.assertNotIn("scale-out collective transport is not pinned to RDMA", reference)
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index ffd7e46598..7235183226 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -2596,6 +2596,10 @@ def test_launchers_use_private_logs_and_allowlisted_failure_stages(self) -> None
self.assertNotIn("dist.all_gather_object", run_ep)
self.assertIn('args.backend in {"deepep-v2", "nccl-ep"}', run_ep)
self.assertIn('dist.init_process_group("nccl", device_id=device)', run_ep)
+ nccl = (ROOT / "tests" / "ep_nccl.py").read_text()
+ self.assertIn('if _library == "nccl" and network_selection != "IB"', nccl)
+ self.assertIn('if _library == "rccl" and network_selection:', nccl)
+ self.assertIn('"auto-hca-pinned"', nccl)
runtime = (ROOT / "runtime" / "run_in_container.sh").read_text()
export_start = common.index("\ncx_container_exports() {")
exports = common[export_start:common.index("\n}", export_start)]
From 5f6135d6dcdc272b6077cea0d5d9b824b0edba7f Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 7 Jul 2026 02:27:53 +0800
Subject: [PATCH 146/210] fix(collectivex): retain healthy AMD allocations
---
experimental/CollectiveX/launchers/launch_mi-amds.sh | 5 ++++-
experimental/CollectiveX/tests/test_sampling_contract.py | 3 ++-
2 files changed, 6 insertions(+), 2 deletions(-)
diff --git a/experimental/CollectiveX/launchers/launch_mi-amds.sh b/experimental/CollectiveX/launchers/launch_mi-amds.sh
index 01d2e4e010..2cb60997d7 100644
--- a/experimental/CollectiveX/launchers/launch_mi-amds.sh
+++ b/experimental/CollectiveX/launchers/launch_mi-amds.sh
@@ -120,8 +120,11 @@ for allocation_attempt in 1 2 3; do
if [ -n "$NODELIST" ] || [ "$allocation_attempt" = 3 ]; then
cx_die "allocated nodes failed container import"
fi
- rejected_nodes="$(cx_allocation_nodes_csv "$JOB_ID")" \
+ allocation_nodes="$(cx_allocation_nodes_csv "$JOB_ID")" \
|| cx_die "cannot identify nodes from a rejected allocation"
+ rejected_nodes="${allocation_nodes%%,*}"
+ [ -n "$rejected_nodes" ] \
+ || cx_die "cannot identify the failed container-import node"
cx_log "allocated nodes failed container import; retrying elsewhere"
cx_cancel_job "$JOB_ID" || cx_die "cannot release a rejected allocation"
cx_clear_allocation_jobid || cx_die "cannot reset rejected allocation state"
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 7235183226..d58c7dcdf7 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -2584,7 +2584,8 @@ def test_launchers_use_private_logs_and_allowlisted_failure_stages(self) -> None
self.assertIn("cx_ensure_squash_on_job", amd)
self.assertIn("for allocation_attempt in 1 2 3", amd)
self.assertIn("allocated nodes failed container import; retrying elsewhere", amd)
- self.assertIn('rejected_nodes="$(cx_allocation_nodes_csv "$JOB_ID")"', amd)
+ self.assertIn('allocation_nodes="$(cx_allocation_nodes_csv "$JOB_ID")"', amd)
+ self.assertIn('rejected_nodes="${allocation_nodes%%,*}"', amd)
self.assertIn("cx_fail_stage container-hash", amd)
self.assertNotIn('cat "$import_log"', amd)
self.assertIn('bash -s -- "$sq" "$lock" "$image"', common)
From af7254a7ccc140a17e7344862b91a941c3b95f3a Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 7 Jul 2026 02:31:27 +0800
Subject: [PATCH 147/210] fix(collectivex): retry AMD imports per node
---
.../CollectiveX/launchers/launch_mi-amds.sh | 16 ++++++++++------
experimental/CollectiveX/runtime/common.sh | 7 +++++--
.../CollectiveX/tests/test_sampling_contract.py | 6 ++++--
3 files changed, 19 insertions(+), 10 deletions(-)
diff --git a/experimental/CollectiveX/launchers/launch_mi-amds.sh b/experimental/CollectiveX/launchers/launch_mi-amds.sh
index 2cb60997d7..3d692439c4 100644
--- a/experimental/CollectiveX/launchers/launch_mi-amds.sh
+++ b/experimental/CollectiveX/launchers/launch_mi-amds.sh
@@ -113,18 +113,22 @@ for allocation_attempt in 1 2 3; do
cx_set_failure_stage setup
cx_validate_network_profile_on_job "$JOB_ID" "$NODES" "$CX_TRANSPORT"
cx_set_failure_stage container-import
- if SQUASH_FILE="$(cx_ensure_squash_on_job \
- "$JOB_ID" "$SQUASH_DIR" "$IMAGE" "${CX_LOCK_DIR:-}")"; then
+ import_succeeded=0
+ for ((import_node = 0; import_node < NODES; import_node++)); do
+ if SQUASH_FILE="$(cx_ensure_squash_on_job \
+ "$JOB_ID" "$SQUASH_DIR" "$IMAGE" "${CX_LOCK_DIR:-}" "$import_node")"; then
+ import_succeeded=1
+ break
+ fi
+ done
+ if [ "$import_succeeded" = 1 ]; then
break
fi
if [ -n "$NODELIST" ] || [ "$allocation_attempt" = 3 ]; then
cx_die "allocated nodes failed container import"
fi
- allocation_nodes="$(cx_allocation_nodes_csv "$JOB_ID")" \
+ rejected_nodes="$(cx_allocation_nodes_csv "$JOB_ID")" \
|| cx_die "cannot identify nodes from a rejected allocation"
- rejected_nodes="${allocation_nodes%%,*}"
- [ -n "$rejected_nodes" ] \
- || cx_die "cannot identify the failed container-import node"
cx_log "allocated nodes failed container import; retrying elsewhere"
cx_cancel_job "$JOB_ID" || cx_die "cannot release a rejected allocation"
cx_clear_allocation_jobid || cx_die "cannot reset rejected allocation state"
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index c2490b4390..880e628aa4 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -2288,21 +2288,24 @@ cx_ensure_squash() {
# Import on an allocated compute node so multiarch tags resolve for the target
# architecture. The squash directory must be shared with the submit host.
cx_ensure_squash_on_job() {
- local job_id="$1" squash_dir="$2" image="$3" lock_dir="${4:-}" sq key lock
+ local job_id="$1" squash_dir="$2" image="$3" lock_dir="${4:-}" relative="${5:-0}"
+ local sq key lock
local log_label=container-import log
[[ "$job_id" =~ ^[0-9]+$ ]] || return 1
+ [[ "$relative" =~ ^[0-9]+$ ]] || return 1
case "${CX_SALLOC_ATTEMPT:-1}" in
1) ;;
2|3) log_label+="-a${CX_SALLOC_ATTEMPT}" ;;
*) return 1 ;;
esac
+ [ "$relative" = 0 ] || log_label+="-r$relative"
sq="$(cx_squash_path "$squash_dir" "$image")" || return 1
key="${sq##*/}"
key="${key%.sqsh}"
[ -n "$lock_dir" ] || lock_dir="$squash_dir/.locks"
lock="$lock_dir/${key}.lock"
log="$(cx_private_log_path "$log_label")"
- if ! srun --jobid="$job_id" --nodes=1 --ntasks=1 --chdir=/tmp \
+ if ! srun --jobid="$job_id" --nodes=1 --ntasks=1 --relative="$relative" --chdir=/tmp \
--export="$(cx_host_exports)" \
bash -s -- "$sq" "$lock" "$image" "$CX_SQUASH_SOURCE_DATE_EPOCH" \
"$CX_IMAGE_PLATFORM" \
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index d58c7dcdf7..b9aeafca17 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -2584,8 +2584,9 @@ def test_launchers_use_private_logs_and_allowlisted_failure_stages(self) -> None
self.assertIn("cx_ensure_squash_on_job", amd)
self.assertIn("for allocation_attempt in 1 2 3", amd)
self.assertIn("allocated nodes failed container import; retrying elsewhere", amd)
- self.assertIn('allocation_nodes="$(cx_allocation_nodes_csv "$JOB_ID")"', amd)
- self.assertIn('rejected_nodes="${allocation_nodes%%,*}"', amd)
+ self.assertIn("import_node < NODES", amd)
+ self.assertIn('"${CX_LOCK_DIR:-}" "$import_node"', amd)
+ self.assertIn('rejected_nodes="$(cx_allocation_nodes_csv "$JOB_ID")"', amd)
self.assertIn("cx_fail_stage container-hash", amd)
self.assertNotIn('cat "$import_log"', amd)
self.assertIn('bash -s -- "$sq" "$lock" "$image"', common)
@@ -2661,6 +2662,7 @@ def test_launchers_use_private_logs_and_allowlisted_failure_stages(self) -> None
single_slurm,
)
self.assertIn('log_label+="-a${CX_SALLOC_ATTEMPT}"', common)
+ self.assertIn('--relative="$relative"', common)
self.assertIn('log_label+="-a${CX_NETWORK_VALIDATION_ATTEMPT}"', common)
self.assertIn('cx_validate_cuda_context_on_job()', common)
self.assertIn('cuDevicePrimaryCtxRetain', common)
From c4edfd12192898d998fc07a0ddf7418a49a59cbf Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 7 Jul 2026 02:39:43 +0800
Subject: [PATCH 148/210] fix(collectivex): run RCCL without writable overlay
---
experimental/CollectiveX/launchers/launch_mi-amds.sh | 6 +++++-
experimental/CollectiveX/tests/test_sampling_contract.py | 5 +++--
2 files changed, 8 insertions(+), 3 deletions(-)
diff --git a/experimental/CollectiveX/launchers/launch_mi-amds.sh b/experimental/CollectiveX/launchers/launch_mi-amds.sh
index 3d692439c4..96f0c4b65e 100644
--- a/experimental/CollectiveX/launchers/launch_mi-amds.sh
+++ b/experimental/CollectiveX/launchers/launch_mi-amds.sh
@@ -167,7 +167,11 @@ else
SOURCE_BACKEND_ENV='case "${SLURM_NODEID:-}" in ""|*[!0-9]*) exit 66;; esac; env_file="/ix/experimental/CollectiveX/.cx_backend/env/node-${SLURM_NODEID}.sh"; env_root="${env_file%/*}"; [ -d "$env_root" ] && [ ! -L "$env_root" ] || exit 66; case "$(stat -c "%a" "$env_root")" in 700|[1-7]700) ;; *) exit 66;; esac; [ -f "$env_file" ] && [ -r "$env_file" ] && [ ! -L "$env_file" ] && [ "$(stat -c "%u:%a" "$env_file")" = "$(stat -c "%u" "$env_root"):600" ] || exit 66; . "$env_file" || exit 66'
BACKEND_PROBE="$SOURCE_BACKEND_ENV"'; case "$CX_BENCH" in mori) python3 -c "import mori";; nccl-ep) python3 -c "import torch";; esac'
WRAP="${SOURCE_BACKEND_ENV}"$'\n'"$(cx_slurm_rank_wrapper)"
- CX_DISTRIBUTED_CONTAINER_ARGS=(--container-writable --container-remap-root)
+ if [ "$CX_BENCH" = nccl-ep ]; then
+ CX_DISTRIBUTED_CONTAINER_ARGS=()
+ else
+ CX_DISTRIBUTED_CONTAINER_ARGS=(--container-writable --container-remap-root)
+ fi
run_rc=0
cx_set_failure_stage container-launch
cx_run_distributed_shard || run_rc=$?
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index b9aeafca17..2760ec4d60 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -1044,9 +1044,10 @@ def test_launchers_preserve_platform_specific_runtime_requirements(self) -> None
self.assertIn("mi325x) CPUS_PER_TASK=256", amd)
self.assertIn("/dev/kfd:/dev/kfd,/dev/dri:/dev/dri", amd)
self.assertIn("--container-writable --container-remap-root", amd)
+ self.assertIn('if [ "$CX_BENCH" = nccl-ep ]', amd)
+ self.assertIn("CX_DISTRIBUTED_CONTAINER_ARGS=()", amd)
self.assertIn(
- "CX_DISTRIBUTED_CONTAINER_ARGS=(--container-writable --container-remap-root)",
- amd,
+ "CX_DISTRIBUTED_CONTAINER_ARGS=(--container-writable --container-remap-root)", amd
)
collect = common[common.index("cx_collect_results()"):
common.index("cx_cleanup_stage()")]
From ec4ce1b657c4ddf5ea7aa00452becd8ec91e380c Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 7 Jul 2026 02:46:27 +0800
Subject: [PATCH 149/210] fix(collectivex): make RCCL containers ephemeral
---
.../CollectiveX/launchers/launch_mi-amds.sh | 6 ++-
experimental/CollectiveX/runtime/common.sh | 40 +++++++++++++------
.../tests/test_sampling_contract.py | 10 ++---
3 files changed, 36 insertions(+), 20 deletions(-)
diff --git a/experimental/CollectiveX/launchers/launch_mi-amds.sh b/experimental/CollectiveX/launchers/launch_mi-amds.sh
index 96f0c4b65e..d47ed9dcd1 100644
--- a/experimental/CollectiveX/launchers/launch_mi-amds.sh
+++ b/experimental/CollectiveX/launchers/launch_mi-amds.sh
@@ -164,7 +164,11 @@ if [ "$NODES" = 1 ]; then
bash "$MOUNT_DIR/experimental/CollectiveX/runtime/run_in_container.sh" \
>"$runtime_log" 2>&1 || run_rc=$?
else
- SOURCE_BACKEND_ENV='case "${SLURM_NODEID:-}" in ""|*[!0-9]*) exit 66;; esac; env_file="/ix/experimental/CollectiveX/.cx_backend/env/node-${SLURM_NODEID}.sh"; env_root="${env_file%/*}"; [ -d "$env_root" ] && [ ! -L "$env_root" ] || exit 66; case "$(stat -c "%a" "$env_root")" in 700|[1-7]700) ;; *) exit 66;; esac; [ -f "$env_file" ] && [ -r "$env_file" ] && [ ! -L "$env_file" ] && [ "$(stat -c "%u:%a" "$env_file")" = "$(stat -c "%u" "$env_root"):600" ] || exit 66; . "$env_file" || exit 66'
+ if [ "$CX_BENCH" = nccl-ep ]; then
+ SOURCE_BACKEND_ENV=:
+ else
+ SOURCE_BACKEND_ENV='case "${SLURM_NODEID:-}" in ""|*[!0-9]*) exit 66;; esac; env_file="/ix/experimental/CollectiveX/.cx_backend/env/node-${SLURM_NODEID}.sh"; env_root="${env_file%/*}"; [ -d "$env_root" ] && [ ! -L "$env_root" ] || exit 66; case "$(stat -c "%a" "$env_root")" in 700|[1-7]700) ;; *) exit 66;; esac; [ -f "$env_file" ] && [ -r "$env_file" ] && [ ! -L "$env_file" ] && [ "$(stat -c "%u:%a" "$env_file")" = "$(stat -c "%u" "$env_root"):600" ] || exit 66; . "$env_file" || exit 66'
+ fi
BACKEND_PROBE="$SOURCE_BACKEND_ENV"'; case "$CX_BENCH" in mori) python3 -c "import mori";; nccl-ep) python3 -c "import torch";; esac'
WRAP="${SOURCE_BACKEND_ENV}"$'\n'"$(cx_slurm_rank_wrapper)"
if [ "$CX_BENCH" = nccl-ep ]; then
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 880e628aa4..6e04b2193c 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -2927,7 +2927,7 @@ cx_run_distributed_shard() {
local precision_profile
local workload_dir workload_ladder workload_log stage_rc attempt_tag out failure_out
local runtime_log run_rc expected_out case_ok summary_log
- local -a container_args workload_args ep_args
+ local -a container_args container_identity_args workload_args ep_args
[ "${NODES:-0}" -gt 1 ] && [ "${NGPUS:-0}" = "$((NODES * GPN))" ] \
|| cx_die "invalid distributed launcher placement"
[ -n "${JOB_ID:-}" ] && [ -n "${SQUASH_FILE:-}" ] \
@@ -2943,24 +2943,38 @@ cx_run_distributed_shard() {
container_args+=("${CX_DISTRIBUTED_CONTAINER_ARGS[@]}")
fi
local container_name="cxep_${JOB_ID}"
+ if [ "$CX_BENCH" = nccl-ep ]; then
+ container_identity_args=()
+ else
+ container_identity_args=(--container-name="$container_name")
+ fi
cx_log "distributed backend preparation: bench=$CX_BENCH nodes=$NODES"
cx_set_failure_stage backend-setup
build_log="$(cx_private_log_path backend-prepare)"
set +e
- srun --jobid="$JOB_ID" --nodes="$NODES" --ntasks-per-node=1 --chdir=/tmp \
- --container-name="$container_name" --container-image="$SQUASH_FILE" \
- "${container_args[@]}" --export="$(cx_container_exports),CX_BUILD_ONLY=1" \
- bash /ix/experimental/CollectiveX/runtime/run_in_container.sh \
- "$build_log" 2>&1
- build_rc=$?
- if [ "$build_rc" = 0 ]; then
+ if [ "$CX_BENCH" = nccl-ep ]; then
srun --jobid="$JOB_ID" --nodes="$NODES" --ntasks-per-node=1 --chdir=/tmp \
- --container-name="$container_name" --container-image="$SQUASH_FILE" \
+ --container-image="$SQUASH_FILE" \
"${container_args[@]}" \
--export="$(cx_container_exports)" bash -c "$BACKEND_PROBE" \
- >"$build_log" 2>&1
+ "$build_log" 2>&1
build_rc=$?
+ else
+ srun --jobid="$JOB_ID" --nodes="$NODES" --ntasks-per-node=1 --chdir=/tmp \
+ "${container_identity_args[@]}" --container-image="$SQUASH_FILE" \
+ "${container_args[@]}" --export="$(cx_container_exports),CX_BUILD_ONLY=1" \
+ bash /ix/experimental/CollectiveX/runtime/run_in_container.sh \
+ "$build_log" 2>&1
+ build_rc=$?
+ if [ "$build_rc" = 0 ]; then
+ srun --jobid="$JOB_ID" --nodes="$NODES" --ntasks-per-node=1 --chdir=/tmp \
+ "${container_identity_args[@]}" --container-image="$SQUASH_FILE" \
+ "${container_args[@]}" \
+ --export="$(cx_container_exports)" bash -c "$BACKEND_PROBE" \
+ >"$build_log" 2>&1
+ build_rc=$?
+ fi
fi
set -e
if [ "$build_rc" != 0 ]; then
@@ -2984,7 +2998,7 @@ cx_run_distributed_shard() {
set +e
timeout -k 30 "${CX_RUN_TIMEOUT:-900}" srun --jobid="$JOB_ID" --nodes="$NODES" \
--ntasks="$NGPUS" --ntasks-per-node="$GPN" --chdir=/tmp \
- --container-name="$container_name" --container-image="$SQUASH_FILE" \
+ "${container_identity_args[@]}" --container-image="$SQUASH_FILE" \
"${container_args[@]}" \
--export="$(cx_container_exports)" \
bash -c "$WRAP" _ --backend "$backend" --sku "$sku" --ep "$ep" \
@@ -3101,7 +3115,7 @@ PY
workload_log="$(cx_private_log_path "workload-c$(printf '%03d' "$ci")")"
set +e
srun --jobid="$JOB_ID" --nodes=1 --ntasks=1 --chdir=/tmp \
- --container-name="$container_name" --container-image="$SQUASH_FILE" \
+ "${container_identity_args[@]}" --container-image="$SQUASH_FILE" \
"${container_args[@]}" \
--export="$(cx_container_exports)" "${workload_args[@]}" \
"$workload_log" 2>&1
@@ -3136,7 +3150,7 @@ PY
set +e
timeout -k 30 "${CX_RUN_TIMEOUT:-900}" srun --jobid="$JOB_ID" --nodes="$NODES" \
--ntasks="$NGPUS" --ntasks-per-node="$GPN" --chdir=/tmp \
- --container-name="$container_name" --container-image="$SQUASH_FILE" \
+ "${container_identity_args[@]}" --container-image="$SQUASH_FILE" \
"${container_args[@]}" \
--export="$(cx_container_exports)" \
bash -c "$WRAP" _ "${ep_args[@]}" --out "$out" \
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 2760ec4d60..ecd43aca7e 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -2637,12 +2637,10 @@ def test_launchers_use_private_logs_and_allowlisted_failure_stages(self) -> None
rank_wrapper.index(". /ix/experimental/CollectiveX/runtime/common.sh"),
rank_wrapper.index('if [ "${CX_NODES:-1}" -gt 1 ]'),
)
- self.assertEqual(
- distributed.count(
- '--container-name="$container_name" --container-image="$SQUASH_FILE"'
- ),
- 5,
- )
+ self.assertIn('container_identity_args=(--container-name="$container_name")', distributed)
+ self.assertEqual(distributed.count('"${container_identity_args[@]}"'), 5)
+ self.assertIn('if [ "$CX_BENCH" = nccl-ep ]', distributed)
+ self.assertIn("SOURCE_BACKEND_ENV=:", amd)
shard_runtime = runtime[runtime.index('elif [ -n "${CX_SHARD_FILE:-}" ]') :]
self.assertIn('"CX_PRECISION_PROFILE": g("precision_profile")', shard_runtime)
self.assertIn("rdma-port-%s=inactive", common)
From 0956012c292c818c3b292045e477d7812c95cd86 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 7 Jul 2026 02:50:53 +0800
Subject: [PATCH 150/210] fix(collectivex): isolate RCCL runtime scratch
---
.../CollectiveX/launchers/launch_mi-amds.sh | 11 ++++
experimental/CollectiveX/runtime/common.sh | 58 ++++++++++++++++++-
.../tests/test_sampling_contract.py | 5 ++
3 files changed, 72 insertions(+), 2 deletions(-)
diff --git a/experimental/CollectiveX/launchers/launch_mi-amds.sh b/experimental/CollectiveX/launchers/launch_mi-amds.sh
index d47ed9dcd1..6dc5d1fa31 100644
--- a/experimental/CollectiveX/launchers/launch_mi-amds.sh
+++ b/experimental/CollectiveX/launchers/launch_mi-amds.sh
@@ -151,6 +151,10 @@ fi
export COLLECTIVEX_SQUASH_SHA256
cx_preflight_allocation "$JOB_ID" "$NODES" "$MOUNT_SRC" "$SQUASH_FILE" \
"${CX_SHARD_FILE:-}"
+if [ "$CX_BENCH" = nccl-ep ] && [ "$NODES" -gt 1 ]; then
+ cx_prepare_enroot_scratch_on_job "$JOB_ID" "$NODES" \
+ || cx_die "cannot prepare isolated Enroot runtime scratch"
+fi
CONTAINER_MOUNTS="$MOUNT_SRC:$MOUNT_DIR$DEVICE_MOUNTS"
if [ "$NODES" = 1 ]; then
@@ -181,6 +185,13 @@ else
cx_run_distributed_shard || run_rc=$?
fi
+if [ -n "${CX_ENROOT_SCRATCH_ROOT:-}" ]; then
+ cx_cleanup_enroot_scratch_on_job "$JOB_ID" "$NODES" \
+ || { cx_log "ERROR: cannot remove Enroot runtime scratch"; [ "$run_rc" != 0 ] || run_rc=1; }
+ unset CX_ENROOT_SCRATCH_ROOT ENROOT_TEMP_PATH ENROOT_CACHE_PATH
+ unset ENROOT_DATA_PATH ENROOT_RUNTIME_PATH
+fi
+
cx_adopt_runtime_stage "$MOUNT_SRC"
if [ "$NODES" = 1 ] && [ "$run_rc" != 0 ]; then
cx_fail_stage "$CX_FAILSAFE_MODE" "$runtime_log" || true
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 6e04b2193c..ab94a89b4d 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -215,7 +215,7 @@ cx_load_operator_config() {
audit_salt_override="${COLLECTIVEX_OPERATOR_AUDIT_SALT:-}"
unset COLLECTIVEX_OPERATOR_AUDIT_SALT
unset CX_PARTITION CX_ACCOUNT CX_QOS CX_SQUASH_DIR CX_STAGE_DIR CX_ENROOT_CACHE_PATH
- unset ENROOT_CACHE_PATH
+ unset ENROOT_TEMP_PATH ENROOT_CACHE_PATH ENROOT_DATA_PATH ENROOT_RUNTIME_PATH
unset CX_EXCLUDE_NODES CX_NODELIST CX_LOCK_DIR CX_MASTER_PORT
unset CX_SOCKET_IFNAME CX_RDMA_DEVICES CX_IB_GID_INDEX CX_RDMA_SERVICE_LEVEL
unset CX_RDMA_TRAFFIC_CLASS
@@ -608,12 +608,13 @@ PY
cx_container_exports() {
printf '%s' 'COLLECTIVEX_SOURCE_SHA,COLLECTIVEX_ARTIFACT_NAME,COLLECTIVEX_EXECUTION_ID,COLLECTIVEX_CONTROL_SHA256,COLLECTIVEX_IMAGE,COLLECTIVEX_IMAGE_DIGEST,COLLECTIVEX_IMAGE_DIGEST_VERIFIED,COLLECTIVEX_SQUASH_SHA256,GITHUB_REF_NAME,GITHUB_REF,GITHUB_REPOSITORY,GITHUB_JOB,GITHUB_RUN_ID,GITHUB_RUN_ATTEMPT,GITHUB_SHA,CX_RUNNER,CX_BENCH,CX_NODES,CX_GPUS_PER_NODE,CX_SCALE_UP_DOMAIN,CX_SHARD_FILE,CX_SHARD_SKU,CX_PRECISION_PROBE,CX_NGPUS,CX_TS,CX_TOPO,CX_SCOPE,CX_TRANSPORT,CX_SCALE_UP_TRANSPORT,CX_SCALE_OUT_TRANSPORT,CX_MODE,CX_PHASE,CX_ROUTING,CX_EPLB,CX_CASE_ID,CX_SUITE,CX_WORKLOAD_NAME,CX_REQUIRED_PUBLICATION,CX_PRECISION_PROFILE,CX_QUALIFICATION_INDEX,CX_HIDDEN,CX_TOPK,CX_EXPERTS,CX_TOKENS_LADDER,CX_CANONICAL,CX_ITERS,CX_TRIALS,CX_WARMUP,CX_SAMPLES_PER_POINT,CX_WARMUP_SEMANTICS,CX_SEED,CX_RUN_TIMEOUT,CX_NCCL_HOME,CX_ALLOW_MNNVL,CX_ATTEMPT_ID,CX_RUNTIME_MARKER,CX_MORI_KERNEL_TYPE,CX_WORKLOAD_DIR,CX_BACKEND_CACHE_ROOT,CX_BACKEND_CACHE_SENTINEL_SHA256,CX_BACKEND_SOURCE_ROOT,CX_AUDIT_SALT,CX_SOCKET_IFNAME,CX_RDMA_DEVICES,CX_IB_GID_INDEX,CX_RDMA_SERVICE_LEVEL,CX_RDMA_TRAFFIC_CLASS,CX_RDMA_LINK_LAYER,MASTER_ADDR,MASTER_PORT,RANK,WORLD_SIZE,LOCAL_RANK,LOCAL_WORLD_SIZE,NCCL_NET,NCCL_SOCKET_IFNAME,GLOO_SOCKET_IFNAME,NCCL_IB_HCA,NCCL_IB_GID_INDEX,NCCL_IB_SL,NVSHMEM_DISABLE_IB,NVSHMEM_ENABLE_NIC_PE_MAPPING,NVSHMEM_HCA_LIST,NVSHMEM_IB_GID_INDEX,NVSHMEM_IB_SL,NVSHMEM_IB_ENABLE_IBGDA,NVSHMEM_IBGDA_NIC_HANDLER,EP_NIC_NAME,EP_OVERRIDE_RDMA_SL,UCCL_SOCKET_IFNAME,UCCL_IB_GID_INDEX,UCCL_IB_SL,MORI_RDMA_DEVICES,MORI_RDMA_TC,MORI_IO_TC,MORI_RDMA_SL,MORI_IO_SL,HYBRID_EP_MULTINODE,USE_NIXL,RDMA_CORE_HOME,DEEPEP_HYBRID_BUILD_MODE,NCCL_CUMEM_ENABLE,NCCL_MNNVL_ENABLE,MC_FORCE_MNNVL,MORI_DISABLE_AUTO_XGMI,MORI_ENABLE_SDMA,MORI_APP_LOG_LEVEL,MORI_SHMEM_LOG_LEVEL,MORI_IO_LOG_LEVEL'
printf '%s' ',MORI_COMMIT'
+ printf '%s' ',ENROOT_TEMP_PATH,ENROOT_CACHE_PATH,ENROOT_DATA_PATH,ENROOT_RUNTIME_PATH'
}
# Host-side utility steps need only the basic login paths. They never receive
# the complete Actions or runner environment.
cx_host_exports() {
- printf '%s' 'HOME,PATH,USER,XDG_CACHE_HOME,ENROOT_CACHE_PATH'
+ printf '%s' 'HOME,PATH,USER,XDG_CACHE_HOME,ENROOT_TEMP_PATH,ENROOT_CACHE_PATH,ENROOT_DATA_PATH,ENROOT_RUNTIME_PATH'
}
cx_prepare_runtime_marker() {
@@ -2409,6 +2410,52 @@ BASH
return 1
}
+cx_prepare_enroot_scratch_on_job() {
+ local job_id="$1" nodes="$2" tag root log
+ [[ "$job_id" =~ ^[0-9]+$ ]] && [[ "$nodes" =~ ^[1-9][0-9]*$ ]] || return 1
+ tag="${COLLECTIVEX_EXECUTION_ID:-${GITHUB_RUN_ID:-manual-$$}}"
+ [[ "$tag" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] || return 1
+ root="/dev/shm/inferencex-collectivex-enroot-$(id -u)-$tag"
+ log="$(cx_private_log_path enroot-scratch)"
+ if ! srun --jobid="$job_id" --nodes="$nodes" --ntasks="$nodes" --ntasks-per-node=1 \
+ --chdir=/tmp --input=all --export="$(cx_host_exports)" \
+ bash -s -- "$root" >"$log" 2>&1 <<'BASH'
+set -euo pipefail
+root="$1"
+[ ! -e "$root" ] && [ ! -L "$root" ]
+umask 077
+mkdir -m 700 "$root"
+for name in temp cache data runtime; do
+ mkdir -m 700 "$root/$name"
+done
+[ "$(stat -c '%u:%a' "$root")" = "$(id -u):700" ]
+BASH
+ then
+ cx_fail_stage container-launch "$log"
+ return 1
+ fi
+ export CX_ENROOT_SCRATCH_ROOT="$root"
+ export ENROOT_TEMP_PATH="$root/temp" ENROOT_CACHE_PATH="$root/cache"
+ export ENROOT_DATA_PATH="$root/data" ENROOT_RUNTIME_PATH="$root/runtime"
+}
+
+cx_cleanup_enroot_scratch_on_job() {
+ local job_id="${1:-}" nodes="${2:-}" root="${CX_ENROOT_SCRATCH_ROOT:-}"
+ [ -n "$root" ] || return 0
+ [[ "$job_id" =~ ^[0-9]+$ ]] && [[ "$nodes" =~ ^[1-9][0-9]*$ ]] || return 1
+ [[ "$root" =~ ^/dev/shm/inferencex-collectivex-enroot-[0-9]+-[A-Za-z0-9][A-Za-z0-9._-]*$ ]] \
+ || return 1
+ srun --overlap --jobid="$job_id" --nodes="$nodes" --ntasks="$nodes" --ntasks-per-node=1 \
+ --chdir=/tmp --input=all --export="HOME,PATH,USER" \
+ bash -s -- "$root" >/dev/null 2>&1 <<'BASH'
+set -euo pipefail
+root="$1"
+[ -d "$root" ] && [ ! -L "$root" ]
+[ "$(stat -c '%u:%a' "$root")" = "$(id -u):700" ]
+rm -rf -- "$root"
+BASH
+}
+
# A clean nvidia-smi inventory does not prove that a prior cancelled workload
# released every CUDA context. Retaining each primary context catches poisoned
# allocations before a full shard spends time failing every case.
@@ -3199,6 +3246,13 @@ cx_launcher_cleanup() {
rm -f -- "$COLLECTIVEX_EPHEMERAL_CONFIG_PATH" >/dev/null 2>&1 || true
unset COLLECTIVEX_EPHEMERAL_CONFIG_PATH
fi
+ if [ -n "${JOB_ID:-}" ] && [ -n "${CX_ENROOT_SCRATCH_ROOT:-}" ]; then
+ if ! cx_cleanup_enroot_scratch_on_job "$JOB_ID" "${NODES:-${CX_NODES:-}}"; then
+ [ "$rc" != 0 ] || rc=1
+ fi
+ unset CX_ENROOT_SCRATCH_ROOT ENROOT_TEMP_PATH ENROOT_CACHE_PATH
+ unset ENROOT_DATA_PATH ENROOT_RUNTIME_PATH
+ fi
if [ -n "${JOB_ID:-}" ]; then
if ! cx_cancel_job "$JOB_ID"; then
allocation_stopped=0
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index ecd43aca7e..254d9e95bd 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -2588,6 +2588,8 @@ def test_launchers_use_private_logs_and_allowlisted_failure_stages(self) -> None
self.assertIn("import_node < NODES", amd)
self.assertIn('"${CX_LOCK_DIR:-}" "$import_node"', amd)
self.assertIn('rejected_nodes="$(cx_allocation_nodes_csv "$JOB_ID")"', amd)
+ self.assertIn('cx_prepare_enroot_scratch_on_job "$JOB_ID" "$NODES"', amd)
+ self.assertIn('cx_cleanup_enroot_scratch_on_job "$JOB_ID" "$NODES"', amd)
self.assertIn("cx_fail_stage container-hash", amd)
self.assertNotIn('cat "$import_log"', amd)
self.assertIn('bash -s -- "$sq" "$lock" "$image"', common)
@@ -2664,6 +2666,9 @@ def test_launchers_use_private_logs_and_allowlisted_failure_stages(self) -> None
self.assertIn('--relative="$relative"', common)
self.assertIn('log_label+="-a${CX_NETWORK_VALIDATION_ATTEMPT}"', common)
self.assertIn('cx_validate_cuda_context_on_job()', common)
+ self.assertIn('cx_prepare_enroot_scratch_on_job()', common)
+ self.assertIn('/dev/shm/inferencex-collectivex-enroot-', common)
+ self.assertIn('cx_cleanup_enroot_scratch_on_job()', common)
self.assertIn('cuDevicePrimaryCtxRetain', common)
self.assertIn('diagnostic="accelerator-unavailable"', common)
From 7e880d8f96c15efbad68eb2fd0754c15e3846802 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 7 Jul 2026 02:54:12 +0800
Subject: [PATCH 151/210] chore(collectivex): remove ineffective Enroot
fallbacks
---
.../CollectiveX/launchers/launch_mi-amds.sh | 34 +-----
experimental/CollectiveX/runtime/common.sh | 105 +++---------------
.../tests/test_sampling_contract.py | 23 ++--
3 files changed, 29 insertions(+), 133 deletions(-)
diff --git a/experimental/CollectiveX/launchers/launch_mi-amds.sh b/experimental/CollectiveX/launchers/launch_mi-amds.sh
index 6dc5d1fa31..01d2e4e010 100644
--- a/experimental/CollectiveX/launchers/launch_mi-amds.sh
+++ b/experimental/CollectiveX/launchers/launch_mi-amds.sh
@@ -113,15 +113,8 @@ for allocation_attempt in 1 2 3; do
cx_set_failure_stage setup
cx_validate_network_profile_on_job "$JOB_ID" "$NODES" "$CX_TRANSPORT"
cx_set_failure_stage container-import
- import_succeeded=0
- for ((import_node = 0; import_node < NODES; import_node++)); do
- if SQUASH_FILE="$(cx_ensure_squash_on_job \
- "$JOB_ID" "$SQUASH_DIR" "$IMAGE" "${CX_LOCK_DIR:-}" "$import_node")"; then
- import_succeeded=1
- break
- fi
- done
- if [ "$import_succeeded" = 1 ]; then
+ if SQUASH_FILE="$(cx_ensure_squash_on_job \
+ "$JOB_ID" "$SQUASH_DIR" "$IMAGE" "${CX_LOCK_DIR:-}")"; then
break
fi
if [ -n "$NODELIST" ] || [ "$allocation_attempt" = 3 ]; then
@@ -151,10 +144,6 @@ fi
export COLLECTIVEX_SQUASH_SHA256
cx_preflight_allocation "$JOB_ID" "$NODES" "$MOUNT_SRC" "$SQUASH_FILE" \
"${CX_SHARD_FILE:-}"
-if [ "$CX_BENCH" = nccl-ep ] && [ "$NODES" -gt 1 ]; then
- cx_prepare_enroot_scratch_on_job "$JOB_ID" "$NODES" \
- || cx_die "cannot prepare isolated Enroot runtime scratch"
-fi
CONTAINER_MOUNTS="$MOUNT_SRC:$MOUNT_DIR$DEVICE_MOUNTS"
if [ "$NODES" = 1 ]; then
@@ -168,30 +157,15 @@ if [ "$NODES" = 1 ]; then
bash "$MOUNT_DIR/experimental/CollectiveX/runtime/run_in_container.sh" \
>"$runtime_log" 2>&1 || run_rc=$?
else
- if [ "$CX_BENCH" = nccl-ep ]; then
- SOURCE_BACKEND_ENV=:
- else
- SOURCE_BACKEND_ENV='case "${SLURM_NODEID:-}" in ""|*[!0-9]*) exit 66;; esac; env_file="/ix/experimental/CollectiveX/.cx_backend/env/node-${SLURM_NODEID}.sh"; env_root="${env_file%/*}"; [ -d "$env_root" ] && [ ! -L "$env_root" ] || exit 66; case "$(stat -c "%a" "$env_root")" in 700|[1-7]700) ;; *) exit 66;; esac; [ -f "$env_file" ] && [ -r "$env_file" ] && [ ! -L "$env_file" ] && [ "$(stat -c "%u:%a" "$env_file")" = "$(stat -c "%u" "$env_root"):600" ] || exit 66; . "$env_file" || exit 66'
- fi
+ SOURCE_BACKEND_ENV='case "${SLURM_NODEID:-}" in ""|*[!0-9]*) exit 66;; esac; env_file="/ix/experimental/CollectiveX/.cx_backend/env/node-${SLURM_NODEID}.sh"; env_root="${env_file%/*}"; [ -d "$env_root" ] && [ ! -L "$env_root" ] || exit 66; case "$(stat -c "%a" "$env_root")" in 700|[1-7]700) ;; *) exit 66;; esac; [ -f "$env_file" ] && [ -r "$env_file" ] && [ ! -L "$env_file" ] && [ "$(stat -c "%u:%a" "$env_file")" = "$(stat -c "%u" "$env_root"):600" ] || exit 66; . "$env_file" || exit 66'
BACKEND_PROBE="$SOURCE_BACKEND_ENV"'; case "$CX_BENCH" in mori) python3 -c "import mori";; nccl-ep) python3 -c "import torch";; esac'
WRAP="${SOURCE_BACKEND_ENV}"$'\n'"$(cx_slurm_rank_wrapper)"
- if [ "$CX_BENCH" = nccl-ep ]; then
- CX_DISTRIBUTED_CONTAINER_ARGS=()
- else
- CX_DISTRIBUTED_CONTAINER_ARGS=(--container-writable --container-remap-root)
- fi
+ CX_DISTRIBUTED_CONTAINER_ARGS=(--container-writable --container-remap-root)
run_rc=0
cx_set_failure_stage container-launch
cx_run_distributed_shard || run_rc=$?
fi
-if [ -n "${CX_ENROOT_SCRATCH_ROOT:-}" ]; then
- cx_cleanup_enroot_scratch_on_job "$JOB_ID" "$NODES" \
- || { cx_log "ERROR: cannot remove Enroot runtime scratch"; [ "$run_rc" != 0 ] || run_rc=1; }
- unset CX_ENROOT_SCRATCH_ROOT ENROOT_TEMP_PATH ENROOT_CACHE_PATH
- unset ENROOT_DATA_PATH ENROOT_RUNTIME_PATH
-fi
-
cx_adopt_runtime_stage "$MOUNT_SRC"
if [ "$NODES" = 1 ] && [ "$run_rc" != 0 ]; then
cx_fail_stage "$CX_FAILSAFE_MODE" "$runtime_log" || true
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index ab94a89b4d..c2490b4390 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -215,7 +215,7 @@ cx_load_operator_config() {
audit_salt_override="${COLLECTIVEX_OPERATOR_AUDIT_SALT:-}"
unset COLLECTIVEX_OPERATOR_AUDIT_SALT
unset CX_PARTITION CX_ACCOUNT CX_QOS CX_SQUASH_DIR CX_STAGE_DIR CX_ENROOT_CACHE_PATH
- unset ENROOT_TEMP_PATH ENROOT_CACHE_PATH ENROOT_DATA_PATH ENROOT_RUNTIME_PATH
+ unset ENROOT_CACHE_PATH
unset CX_EXCLUDE_NODES CX_NODELIST CX_LOCK_DIR CX_MASTER_PORT
unset CX_SOCKET_IFNAME CX_RDMA_DEVICES CX_IB_GID_INDEX CX_RDMA_SERVICE_LEVEL
unset CX_RDMA_TRAFFIC_CLASS
@@ -608,13 +608,12 @@ PY
cx_container_exports() {
printf '%s' 'COLLECTIVEX_SOURCE_SHA,COLLECTIVEX_ARTIFACT_NAME,COLLECTIVEX_EXECUTION_ID,COLLECTIVEX_CONTROL_SHA256,COLLECTIVEX_IMAGE,COLLECTIVEX_IMAGE_DIGEST,COLLECTIVEX_IMAGE_DIGEST_VERIFIED,COLLECTIVEX_SQUASH_SHA256,GITHUB_REF_NAME,GITHUB_REF,GITHUB_REPOSITORY,GITHUB_JOB,GITHUB_RUN_ID,GITHUB_RUN_ATTEMPT,GITHUB_SHA,CX_RUNNER,CX_BENCH,CX_NODES,CX_GPUS_PER_NODE,CX_SCALE_UP_DOMAIN,CX_SHARD_FILE,CX_SHARD_SKU,CX_PRECISION_PROBE,CX_NGPUS,CX_TS,CX_TOPO,CX_SCOPE,CX_TRANSPORT,CX_SCALE_UP_TRANSPORT,CX_SCALE_OUT_TRANSPORT,CX_MODE,CX_PHASE,CX_ROUTING,CX_EPLB,CX_CASE_ID,CX_SUITE,CX_WORKLOAD_NAME,CX_REQUIRED_PUBLICATION,CX_PRECISION_PROFILE,CX_QUALIFICATION_INDEX,CX_HIDDEN,CX_TOPK,CX_EXPERTS,CX_TOKENS_LADDER,CX_CANONICAL,CX_ITERS,CX_TRIALS,CX_WARMUP,CX_SAMPLES_PER_POINT,CX_WARMUP_SEMANTICS,CX_SEED,CX_RUN_TIMEOUT,CX_NCCL_HOME,CX_ALLOW_MNNVL,CX_ATTEMPT_ID,CX_RUNTIME_MARKER,CX_MORI_KERNEL_TYPE,CX_WORKLOAD_DIR,CX_BACKEND_CACHE_ROOT,CX_BACKEND_CACHE_SENTINEL_SHA256,CX_BACKEND_SOURCE_ROOT,CX_AUDIT_SALT,CX_SOCKET_IFNAME,CX_RDMA_DEVICES,CX_IB_GID_INDEX,CX_RDMA_SERVICE_LEVEL,CX_RDMA_TRAFFIC_CLASS,CX_RDMA_LINK_LAYER,MASTER_ADDR,MASTER_PORT,RANK,WORLD_SIZE,LOCAL_RANK,LOCAL_WORLD_SIZE,NCCL_NET,NCCL_SOCKET_IFNAME,GLOO_SOCKET_IFNAME,NCCL_IB_HCA,NCCL_IB_GID_INDEX,NCCL_IB_SL,NVSHMEM_DISABLE_IB,NVSHMEM_ENABLE_NIC_PE_MAPPING,NVSHMEM_HCA_LIST,NVSHMEM_IB_GID_INDEX,NVSHMEM_IB_SL,NVSHMEM_IB_ENABLE_IBGDA,NVSHMEM_IBGDA_NIC_HANDLER,EP_NIC_NAME,EP_OVERRIDE_RDMA_SL,UCCL_SOCKET_IFNAME,UCCL_IB_GID_INDEX,UCCL_IB_SL,MORI_RDMA_DEVICES,MORI_RDMA_TC,MORI_IO_TC,MORI_RDMA_SL,MORI_IO_SL,HYBRID_EP_MULTINODE,USE_NIXL,RDMA_CORE_HOME,DEEPEP_HYBRID_BUILD_MODE,NCCL_CUMEM_ENABLE,NCCL_MNNVL_ENABLE,MC_FORCE_MNNVL,MORI_DISABLE_AUTO_XGMI,MORI_ENABLE_SDMA,MORI_APP_LOG_LEVEL,MORI_SHMEM_LOG_LEVEL,MORI_IO_LOG_LEVEL'
printf '%s' ',MORI_COMMIT'
- printf '%s' ',ENROOT_TEMP_PATH,ENROOT_CACHE_PATH,ENROOT_DATA_PATH,ENROOT_RUNTIME_PATH'
}
# Host-side utility steps need only the basic login paths. They never receive
# the complete Actions or runner environment.
cx_host_exports() {
- printf '%s' 'HOME,PATH,USER,XDG_CACHE_HOME,ENROOT_TEMP_PATH,ENROOT_CACHE_PATH,ENROOT_DATA_PATH,ENROOT_RUNTIME_PATH'
+ printf '%s' 'HOME,PATH,USER,XDG_CACHE_HOME,ENROOT_CACHE_PATH'
}
cx_prepare_runtime_marker() {
@@ -2289,24 +2288,21 @@ cx_ensure_squash() {
# Import on an allocated compute node so multiarch tags resolve for the target
# architecture. The squash directory must be shared with the submit host.
cx_ensure_squash_on_job() {
- local job_id="$1" squash_dir="$2" image="$3" lock_dir="${4:-}" relative="${5:-0}"
- local sq key lock
+ local job_id="$1" squash_dir="$2" image="$3" lock_dir="${4:-}" sq key lock
local log_label=container-import log
[[ "$job_id" =~ ^[0-9]+$ ]] || return 1
- [[ "$relative" =~ ^[0-9]+$ ]] || return 1
case "${CX_SALLOC_ATTEMPT:-1}" in
1) ;;
2|3) log_label+="-a${CX_SALLOC_ATTEMPT}" ;;
*) return 1 ;;
esac
- [ "$relative" = 0 ] || log_label+="-r$relative"
sq="$(cx_squash_path "$squash_dir" "$image")" || return 1
key="${sq##*/}"
key="${key%.sqsh}"
[ -n "$lock_dir" ] || lock_dir="$squash_dir/.locks"
lock="$lock_dir/${key}.lock"
log="$(cx_private_log_path "$log_label")"
- if ! srun --jobid="$job_id" --nodes=1 --ntasks=1 --relative="$relative" --chdir=/tmp \
+ if ! srun --jobid="$job_id" --nodes=1 --ntasks=1 --chdir=/tmp \
--export="$(cx_host_exports)" \
bash -s -- "$sq" "$lock" "$image" "$CX_SQUASH_SOURCE_DATE_EPOCH" \
"$CX_IMAGE_PLATFORM" \
@@ -2410,52 +2406,6 @@ BASH
return 1
}
-cx_prepare_enroot_scratch_on_job() {
- local job_id="$1" nodes="$2" tag root log
- [[ "$job_id" =~ ^[0-9]+$ ]] && [[ "$nodes" =~ ^[1-9][0-9]*$ ]] || return 1
- tag="${COLLECTIVEX_EXECUTION_ID:-${GITHUB_RUN_ID:-manual-$$}}"
- [[ "$tag" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] || return 1
- root="/dev/shm/inferencex-collectivex-enroot-$(id -u)-$tag"
- log="$(cx_private_log_path enroot-scratch)"
- if ! srun --jobid="$job_id" --nodes="$nodes" --ntasks="$nodes" --ntasks-per-node=1 \
- --chdir=/tmp --input=all --export="$(cx_host_exports)" \
- bash -s -- "$root" >"$log" 2>&1 <<'BASH'
-set -euo pipefail
-root="$1"
-[ ! -e "$root" ] && [ ! -L "$root" ]
-umask 077
-mkdir -m 700 "$root"
-for name in temp cache data runtime; do
- mkdir -m 700 "$root/$name"
-done
-[ "$(stat -c '%u:%a' "$root")" = "$(id -u):700" ]
-BASH
- then
- cx_fail_stage container-launch "$log"
- return 1
- fi
- export CX_ENROOT_SCRATCH_ROOT="$root"
- export ENROOT_TEMP_PATH="$root/temp" ENROOT_CACHE_PATH="$root/cache"
- export ENROOT_DATA_PATH="$root/data" ENROOT_RUNTIME_PATH="$root/runtime"
-}
-
-cx_cleanup_enroot_scratch_on_job() {
- local job_id="${1:-}" nodes="${2:-}" root="${CX_ENROOT_SCRATCH_ROOT:-}"
- [ -n "$root" ] || return 0
- [[ "$job_id" =~ ^[0-9]+$ ]] && [[ "$nodes" =~ ^[1-9][0-9]*$ ]] || return 1
- [[ "$root" =~ ^/dev/shm/inferencex-collectivex-enroot-[0-9]+-[A-Za-z0-9][A-Za-z0-9._-]*$ ]] \
- || return 1
- srun --overlap --jobid="$job_id" --nodes="$nodes" --ntasks="$nodes" --ntasks-per-node=1 \
- --chdir=/tmp --input=all --export="HOME,PATH,USER" \
- bash -s -- "$root" >/dev/null 2>&1 <<'BASH'
-set -euo pipefail
-root="$1"
-[ -d "$root" ] && [ ! -L "$root" ]
-[ "$(stat -c '%u:%a' "$root")" = "$(id -u):700" ]
-rm -rf -- "$root"
-BASH
-}
-
# A clean nvidia-smi inventory does not prove that a prior cancelled workload
# released every CUDA context. Retaining each primary context catches poisoned
# allocations before a full shard spends time failing every case.
@@ -2974,7 +2924,7 @@ cx_run_distributed_shard() {
local precision_profile
local workload_dir workload_ladder workload_log stage_rc attempt_tag out failure_out
local runtime_log run_rc expected_out case_ok summary_log
- local -a container_args container_identity_args workload_args ep_args
+ local -a container_args workload_args ep_args
[ "${NODES:-0}" -gt 1 ] && [ "${NGPUS:-0}" = "$((NODES * GPN))" ] \
|| cx_die "invalid distributed launcher placement"
[ -n "${JOB_ID:-}" ] && [ -n "${SQUASH_FILE:-}" ] \
@@ -2990,38 +2940,24 @@ cx_run_distributed_shard() {
container_args+=("${CX_DISTRIBUTED_CONTAINER_ARGS[@]}")
fi
local container_name="cxep_${JOB_ID}"
- if [ "$CX_BENCH" = nccl-ep ]; then
- container_identity_args=()
- else
- container_identity_args=(--container-name="$container_name")
- fi
cx_log "distributed backend preparation: bench=$CX_BENCH nodes=$NODES"
cx_set_failure_stage backend-setup
build_log="$(cx_private_log_path backend-prepare)"
set +e
- if [ "$CX_BENCH" = nccl-ep ]; then
+ srun --jobid="$JOB_ID" --nodes="$NODES" --ntasks-per-node=1 --chdir=/tmp \
+ --container-name="$container_name" --container-image="$SQUASH_FILE" \
+ "${container_args[@]}" --export="$(cx_container_exports),CX_BUILD_ONLY=1" \
+ bash /ix/experimental/CollectiveX/runtime/run_in_container.sh \
+ "$build_log" 2>&1
+ build_rc=$?
+ if [ "$build_rc" = 0 ]; then
srun --jobid="$JOB_ID" --nodes="$NODES" --ntasks-per-node=1 --chdir=/tmp \
- --container-image="$SQUASH_FILE" \
+ --container-name="$container_name" --container-image="$SQUASH_FILE" \
"${container_args[@]}" \
--export="$(cx_container_exports)" bash -c "$BACKEND_PROBE" \
- "$build_log" 2>&1
- build_rc=$?
- else
- srun --jobid="$JOB_ID" --nodes="$NODES" --ntasks-per-node=1 --chdir=/tmp \
- "${container_identity_args[@]}" --container-image="$SQUASH_FILE" \
- "${container_args[@]}" --export="$(cx_container_exports),CX_BUILD_ONLY=1" \
- bash /ix/experimental/CollectiveX/runtime/run_in_container.sh \
- "$build_log" 2>&1
+ >"$build_log" 2>&1
build_rc=$?
- if [ "$build_rc" = 0 ]; then
- srun --jobid="$JOB_ID" --nodes="$NODES" --ntasks-per-node=1 --chdir=/tmp \
- "${container_identity_args[@]}" --container-image="$SQUASH_FILE" \
- "${container_args[@]}" \
- --export="$(cx_container_exports)" bash -c "$BACKEND_PROBE" \
- >"$build_log" 2>&1
- build_rc=$?
- fi
fi
set -e
if [ "$build_rc" != 0 ]; then
@@ -3045,7 +2981,7 @@ cx_run_distributed_shard() {
set +e
timeout -k 30 "${CX_RUN_TIMEOUT:-900}" srun --jobid="$JOB_ID" --nodes="$NODES" \
--ntasks="$NGPUS" --ntasks-per-node="$GPN" --chdir=/tmp \
- "${container_identity_args[@]}" --container-image="$SQUASH_FILE" \
+ --container-name="$container_name" --container-image="$SQUASH_FILE" \
"${container_args[@]}" \
--export="$(cx_container_exports)" \
bash -c "$WRAP" _ --backend "$backend" --sku "$sku" --ep "$ep" \
@@ -3162,7 +3098,7 @@ PY
workload_log="$(cx_private_log_path "workload-c$(printf '%03d' "$ci")")"
set +e
srun --jobid="$JOB_ID" --nodes=1 --ntasks=1 --chdir=/tmp \
- "${container_identity_args[@]}" --container-image="$SQUASH_FILE" \
+ --container-name="$container_name" --container-image="$SQUASH_FILE" \
"${container_args[@]}" \
--export="$(cx_container_exports)" "${workload_args[@]}" \
"$workload_log" 2>&1
@@ -3197,7 +3133,7 @@ PY
set +e
timeout -k 30 "${CX_RUN_TIMEOUT:-900}" srun --jobid="$JOB_ID" --nodes="$NODES" \
--ntasks="$NGPUS" --ntasks-per-node="$GPN" --chdir=/tmp \
- "${container_identity_args[@]}" --container-image="$SQUASH_FILE" \
+ --container-name="$container_name" --container-image="$SQUASH_FILE" \
"${container_args[@]}" \
--export="$(cx_container_exports)" \
bash -c "$WRAP" _ "${ep_args[@]}" --out "$out" \
@@ -3246,13 +3182,6 @@ cx_launcher_cleanup() {
rm -f -- "$COLLECTIVEX_EPHEMERAL_CONFIG_PATH" >/dev/null 2>&1 || true
unset COLLECTIVEX_EPHEMERAL_CONFIG_PATH
fi
- if [ -n "${JOB_ID:-}" ] && [ -n "${CX_ENROOT_SCRATCH_ROOT:-}" ]; then
- if ! cx_cleanup_enroot_scratch_on_job "$JOB_ID" "${NODES:-${CX_NODES:-}}"; then
- [ "$rc" != 0 ] || rc=1
- fi
- unset CX_ENROOT_SCRATCH_ROOT ENROOT_TEMP_PATH ENROOT_CACHE_PATH
- unset ENROOT_DATA_PATH ENROOT_RUNTIME_PATH
- fi
if [ -n "${JOB_ID:-}" ]; then
if ! cx_cancel_job "$JOB_ID"; then
allocation_stopped=0
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 254d9e95bd..7235183226 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -1044,10 +1044,9 @@ def test_launchers_preserve_platform_specific_runtime_requirements(self) -> None
self.assertIn("mi325x) CPUS_PER_TASK=256", amd)
self.assertIn("/dev/kfd:/dev/kfd,/dev/dri:/dev/dri", amd)
self.assertIn("--container-writable --container-remap-root", amd)
- self.assertIn('if [ "$CX_BENCH" = nccl-ep ]', amd)
- self.assertIn("CX_DISTRIBUTED_CONTAINER_ARGS=()", amd)
self.assertIn(
- "CX_DISTRIBUTED_CONTAINER_ARGS=(--container-writable --container-remap-root)", amd
+ "CX_DISTRIBUTED_CONTAINER_ARGS=(--container-writable --container-remap-root)",
+ amd,
)
collect = common[common.index("cx_collect_results()"):
common.index("cx_cleanup_stage()")]
@@ -2585,11 +2584,7 @@ def test_launchers_use_private_logs_and_allowlisted_failure_stages(self) -> None
self.assertIn("cx_ensure_squash_on_job", amd)
self.assertIn("for allocation_attempt in 1 2 3", amd)
self.assertIn("allocated nodes failed container import; retrying elsewhere", amd)
- self.assertIn("import_node < NODES", amd)
- self.assertIn('"${CX_LOCK_DIR:-}" "$import_node"', amd)
self.assertIn('rejected_nodes="$(cx_allocation_nodes_csv "$JOB_ID")"', amd)
- self.assertIn('cx_prepare_enroot_scratch_on_job "$JOB_ID" "$NODES"', amd)
- self.assertIn('cx_cleanup_enroot_scratch_on_job "$JOB_ID" "$NODES"', amd)
self.assertIn("cx_fail_stage container-hash", amd)
self.assertNotIn('cat "$import_log"', amd)
self.assertIn('bash -s -- "$sq" "$lock" "$image"', common)
@@ -2639,10 +2634,12 @@ def test_launchers_use_private_logs_and_allowlisted_failure_stages(self) -> None
rank_wrapper.index(". /ix/experimental/CollectiveX/runtime/common.sh"),
rank_wrapper.index('if [ "${CX_NODES:-1}" -gt 1 ]'),
)
- self.assertIn('container_identity_args=(--container-name="$container_name")', distributed)
- self.assertEqual(distributed.count('"${container_identity_args[@]}"'), 5)
- self.assertIn('if [ "$CX_BENCH" = nccl-ep ]', distributed)
- self.assertIn("SOURCE_BACKEND_ENV=:", amd)
+ self.assertEqual(
+ distributed.count(
+ '--container-name="$container_name" --container-image="$SQUASH_FILE"'
+ ),
+ 5,
+ )
shard_runtime = runtime[runtime.index('elif [ -n "${CX_SHARD_FILE:-}" ]') :]
self.assertIn('"CX_PRECISION_PROFILE": g("precision_profile")', shard_runtime)
self.assertIn("rdma-port-%s=inactive", common)
@@ -2663,12 +2660,8 @@ def test_launchers_use_private_logs_and_allowlisted_failure_stages(self) -> None
single_slurm,
)
self.assertIn('log_label+="-a${CX_SALLOC_ATTEMPT}"', common)
- self.assertIn('--relative="$relative"', common)
self.assertIn('log_label+="-a${CX_NETWORK_VALIDATION_ATTEMPT}"', common)
self.assertIn('cx_validate_cuda_context_on_job()', common)
- self.assertIn('cx_prepare_enroot_scratch_on_job()', common)
- self.assertIn('/dev/shm/inferencex-collectivex-enroot-', common)
- self.assertIn('cx_cleanup_enroot_scratch_on_job()', common)
self.assertIn('cuDevicePrimaryCtxRetain', common)
self.assertIn('diagnostic="accelerator-unavailable"', common)
From 8034b523e2e5f3efc698e407f2fcb1f1cf0544ca Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 7 Jul 2026 03:10:32 +0800
Subject: [PATCH 152/210] fix(collectivex): keep RCCL initialization compatible
---
experimental/CollectiveX/tests/run_ep.py | 5 ++---
experimental/CollectiveX/tests/test_sampling_contract.py | 3 ++-
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/experimental/CollectiveX/tests/run_ep.py b/experimental/CollectiveX/tests/run_ep.py
index 9edf44717a..57ca820fc4 100644
--- a/experimental/CollectiveX/tests/run_ep.py
+++ b/experimental/CollectiveX/tests/run_ep.py
@@ -488,10 +488,9 @@ def main() -> int:
world_size=world_size,
device_id=device,
)
- elif args.backend in {"deepep-v2", "nccl-ep"}:
+ elif args.backend == "deepep-v2":
# PR #605 reuses PyTorch's NCCL communicator through ``_comm_ptr``. Supplying
- # device_id eagerly forms it before ElasticBuffer construction. The reference
- # backend uses the same eager binding so RCCL never infers rank devices lazily.
+ # device_id eagerly forms it before ElasticBuffer construction.
dist.init_process_group("nccl", device_id=device)
else:
dist.init_process_group("nccl")
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 7235183226..8fe3854adc 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -2594,7 +2594,8 @@ def test_launchers_use_private_logs_and_allowlisted_failure_stages(self) -> None
self.assertIn("def _all_gather_json(", run_ep)
self.assertIn("dist_module.all_gather(gathered, encoded)", run_ep)
self.assertNotIn("dist.all_gather_object", run_ep)
- self.assertIn('args.backend in {"deepep-v2", "nccl-ep"}', run_ep)
+ self.assertIn('args.backend == "deepep-v2"', run_ep)
+ self.assertNotIn('args.backend in {"deepep-v2", "nccl-ep"}', run_ep)
self.assertIn('dist.init_process_group("nccl", device_id=device)', run_ep)
nccl = (ROOT / "tests" / "ep_nccl.py").read_text()
self.assertIn('if _library == "nccl" and network_selection != "IB"', nccl)
From 1f84b95d3e6984a14170c60036323689b94f03d2 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 7 Jul 2026 03:23:07 +0800
Subject: [PATCH 153/210] fix(collectivex): preserve frozen RCCL provenance
---
experimental/CollectiveX/tests/ep_nccl.py | 3 ---
experimental/CollectiveX/tests/test_sampling_contract.py | 1 -
2 files changed, 4 deletions(-)
diff --git a/experimental/CollectiveX/tests/ep_nccl.py b/experimental/CollectiveX/tests/ep_nccl.py
index f97442605c..ee8143aeee 100644
--- a/experimental/CollectiveX/tests/ep_nccl.py
+++ b/experimental/CollectiveX/tests/ep_nccl.py
@@ -96,9 +96,6 @@ def __init__(self, args, rank, world_size, local_rank, device):
"collective_library": _library,
"nccl_version": _version,
"transport": f"{_library}-all_to_all_single",
- "network_selection": (
- "forced-ib" if _library == "nccl" else "auto-hca-pinned"
- ),
"resource_mode": "fixed-profile",
"num_sms": None,
"device_sms": torch.cuda.get_device_properties(device).multi_processor_count,
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 8fe3854adc..d73b894e49 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -2600,7 +2600,6 @@ def test_launchers_use_private_logs_and_allowlisted_failure_stages(self) -> None
nccl = (ROOT / "tests" / "ep_nccl.py").read_text()
self.assertIn('if _library == "nccl" and network_selection != "IB"', nccl)
self.assertIn('if _library == "rccl" and network_selection:', nccl)
- self.assertIn('"auto-hca-pinned"', nccl)
runtime = (ROOT / "runtime" / "run_in_container.sh").read_text()
export_start = common.index("\ncx_container_exports() {")
exports = common[export_start:common.index("\n}", export_start)]
From 9d5444c5f0158b2e6635f87d89e1af7d4460fa08 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 7 Jul 2026 03:55:12 +0800
Subject: [PATCH 154/210] fix(collectivex): move AMD job state off tmp
---
.github/workflows/collectivex-sweep.yml | 31 +++++++++++++++----
.../tests/test_sampling_contract.py | 13 +++++++-
2 files changed, 37 insertions(+), 7 deletions(-)
diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml
index 9d68e60dd9..26dbf093b0 100644
--- a/.github/workflows/collectivex-sweep.yml
+++ b/.github/workflows/collectivex-sweep.yml
@@ -323,9 +323,10 @@ jobs:
CX_TIME: ${{ inputs.operation == 'probe-precision' && '90' || '300' }}
COLLECTIVEX_EXECUTION_ID: ${{ github.run_id }}_${{ github.run_attempt }}_${{ matrix.id }}
CX_QUALIFICATION_INDEX: ${{ inputs.qualification_index }}
- CX_JOB_ROOT: /tmp/inferencex-collectivex-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.id }}
- CX_SOURCE_ROOT: /tmp/inferencex-collectivex-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.id }}/source
- HOME: /tmp/inferencex-collectivex-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.id }}/home
+ CX_JOB_PARENT: ${{ matrix.launcher == 'mi-amds' && format('{0}/.collectivex-jobs', github.workspace) || '/tmp' }}
+ CX_JOB_ROOT: ${{ matrix.launcher == 'mi-amds' && format('{0}/.collectivex-jobs/inferencex-collectivex-{1}-{2}-{3}', github.workspace, github.run_id, github.run_attempt, matrix.id) || format('/tmp/inferencex-collectivex-{0}-{1}-{2}', github.run_id, github.run_attempt, matrix.id) }}
+ CX_SOURCE_ROOT: ${{ matrix.launcher == 'mi-amds' && format('{0}/.collectivex-jobs/inferencex-collectivex-{1}-{2}-{3}/source', github.workspace, github.run_id, github.run_attempt, matrix.id) || format('/tmp/inferencex-collectivex-{0}-{1}-{2}/source', github.run_id, github.run_attempt, matrix.id) }}
+ HOME: ${{ matrix.launcher == 'mi-amds' && format('{0}/.collectivex-jobs/inferencex-collectivex-{1}-{2}-{3}/home', github.workspace, github.run_id, github.run_attempt, matrix.id) || format('/tmp/inferencex-collectivex-{0}-{1}-{2}/home', github.run_id, github.run_attempt, matrix.id) }}
steps:
- name: Prepare isolated source
id: source
@@ -333,6 +334,19 @@ jobs:
COLLECTIVEX_REPOSITORY: ${{ github.repository }}
run: |
set -euo pipefail
+ [ "$CX_JOB_PARENT" = /tmp ] || {
+ [ "$CX_JOB_PARENT" = '${{ github.workspace }}/.collectivex-jobs' ] \
+ || { echo "CollectiveX isolated parent is invalid" >&2; exit 1; }
+ if [ ! -e "$CX_JOB_PARENT" ] && [ ! -L "$CX_JOB_PARENT" ]; then
+ umask 077
+ mkdir -m 700 -- "$CX_JOB_PARENT"
+ fi
+ [ ! -L "$CX_JOB_PARENT" ] \
+ && [ -d "$CX_JOB_PARENT" ] \
+ && [ -O "$CX_JOB_PARENT" ] \
+ && [ "$(stat -c '%a' "$CX_JOB_PARENT")" = 700 ] \
+ || { echo "CollectiveX isolated parent is unsafe" >&2; exit 1; }
+ }
python3 - <<'PY'
import os
import re
@@ -342,7 +356,7 @@ jobs:
pattern = re.compile(r"inferencex-collectivex-[0-9]+-[0-9]+-[A-Za-z0-9._-]+")
cutoff = time.time() - 86400
- for entry in os.scandir("/tmp"):
+ for entry in os.scandir(os.environ["CX_JOB_PARENT"]):
if not pattern.fullmatch(entry.name):
continue
try:
@@ -374,7 +388,8 @@ jobs:
if marked:
shutil.rmtree(entry.path)
PY
- [[ "$CX_JOB_ROOT" =~ ^/tmp/inferencex-collectivex-[0-9]+-[0-9]+-[A-Za-z0-9._-]+$ ]] \
+ [ "${CX_JOB_ROOT%/*}" = "$CX_JOB_PARENT" ] \
+ && [[ "${CX_JOB_ROOT##*/}" =~ ^inferencex-collectivex-[0-9]+-[0-9]+-[A-Za-z0-9._-]+$ ]] \
|| { echo "CollectiveX isolated root is invalid" >&2; exit 1; }
[ "$CX_SOURCE_ROOT" = "$CX_JOB_ROOT/source" ] \
|| { echo "CollectiveX source root is invalid" >&2; exit 1; }
@@ -626,7 +641,11 @@ jobs:
if: ${{ always() && steps.source.outputs.prepared == 'true' }}
run: |
set -euo pipefail
- [[ "$CX_JOB_ROOT" =~ ^/tmp/inferencex-collectivex-[0-9]+-[0-9]+-[A-Za-z0-9._-]+$ ]] \
+ [ "$CX_JOB_PARENT" = /tmp ] \
+ || [ "$CX_JOB_PARENT" = '${{ github.workspace }}/.collectivex-jobs' ] \
+ || { echo "CollectiveX cleanup parent is invalid" >&2; exit 1; }
+ [ "${CX_JOB_ROOT%/*}" = "$CX_JOB_PARENT" ] \
+ && [[ "${CX_JOB_ROOT##*/}" =~ ^inferencex-collectivex-[0-9]+-[0-9]+-[A-Za-z0-9._-]+$ ]] \
|| { echo "CollectiveX cleanup root is invalid" >&2; exit 1; }
[ "$CX_SOURCE_ROOT" = "$CX_JOB_ROOT/source" ] \
|| { echo "CollectiveX cleanup source is invalid" >&2; exit 1; }
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index d73b894e49..b0c4ccda7f 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -1587,9 +1587,17 @@ def test_workflow_reconciles_a_recorded_allocation_after_launcher_loss(self) ->
sweep_workflow = workflow[workflow.index(" sweep:"):]
self.assertNotIn("GITHUB_WORKSPACE", sweep_workflow)
self.assertNotIn("RUNNER_WORKSPACE", sweep_workflow)
- self.assertIn('CX_SOURCE_ROOT: /tmp/inferencex-collectivex-', sweep_workflow)
+ self.assertIn("matrix.launcher == 'mi-amds'", sweep_workflow)
+ self.assertIn("github.workspace, github.run_id", sweep_workflow)
+ self.assertIn("CX_JOB_PARENT:", sweep_workflow)
+ self.assertIn("/.collectivex-jobs", sweep_workflow)
+ self.assertIn("|| '/tmp'", sweep_workflow)
source_step = sweep_workflow[:sweep_workflow.index("- uses: actions/download-artifact")]
self.assertNotIn("unsafe_guards=", source_step)
+ self.assertIn('os.scandir(os.environ["CX_JOB_PARENT"])', source_step)
+ self.assertIn('[ ! -L "$CX_JOB_PARENT" ]', source_step)
+ self.assertIn('[ -O "$CX_JOB_PARENT" ]', source_step)
+ self.assertIn('stat -c \'%a\' "$CX_JOB_PARENT"', source_step)
self.assertIn("cutoff = time.time() - 86400", source_step)
self.assertIn("stat.S_IMODE(metadata.st_mode) != 0o700", source_step)
self.assertIn('for marker_name in ("cleanup-safe", "cleanup-unsafe")', source_step)
@@ -1604,6 +1612,9 @@ def test_workflow_reconciles_a_recorded_allocation_after_launcher_loss(self) ->
self.assertIn("id: upload_artifact", upload)
self.assertIn("steps.stage_artifact.outcome == 'success'", upload)
cleanup = workflow[workflow.index("- name: Cleanup isolated workspace"):]
+ self.assertIn("CollectiveX cleanup parent is invalid", cleanup)
+ self.assertIn('[ "${CX_JOB_ROOT%/*}" = "$CX_JOB_PARENT" ]', cleanup)
+ self.assertIn('^inferencex-collectivex-', cleanup)
for step in (
"sweep_shard", "allocation_cleanup", "artifact_safety",
"delivery_contracts", "stage_artifact", "upload_artifact",
From 6954acfc1226099f17e1eacb58126452b6ba05bc Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 7 Jul 2026 04:00:26 +0800
Subject: [PATCH 155/210] fix(collectivex): conceal AMD shared workspace paths
---
.github/workflows/collectivex-sweep.yml | 52 ++++++++++++-------
.../tests/test_sampling_contract.py | 13 ++---
2 files changed, 39 insertions(+), 26 deletions(-)
diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml
index 26dbf093b0..1b0e278ae5 100644
--- a/.github/workflows/collectivex-sweep.yml
+++ b/.github/workflows/collectivex-sweep.yml
@@ -323,10 +323,10 @@ jobs:
CX_TIME: ${{ inputs.operation == 'probe-precision' && '90' || '300' }}
COLLECTIVEX_EXECUTION_ID: ${{ github.run_id }}_${{ github.run_attempt }}_${{ matrix.id }}
CX_QUALIFICATION_INDEX: ${{ inputs.qualification_index }}
- CX_JOB_PARENT: ${{ matrix.launcher == 'mi-amds' && format('{0}/.collectivex-jobs', github.workspace) || '/tmp' }}
- CX_JOB_ROOT: ${{ matrix.launcher == 'mi-amds' && format('{0}/.collectivex-jobs/inferencex-collectivex-{1}-{2}-{3}', github.workspace, github.run_id, github.run_attempt, matrix.id) || format('/tmp/inferencex-collectivex-{0}-{1}-{2}', github.run_id, github.run_attempt, matrix.id) }}
- CX_SOURCE_ROOT: ${{ matrix.launcher == 'mi-amds' && format('{0}/.collectivex-jobs/inferencex-collectivex-{1}-{2}-{3}/source', github.workspace, github.run_id, github.run_attempt, matrix.id) || format('/tmp/inferencex-collectivex-{0}-{1}-{2}/source', github.run_id, github.run_attempt, matrix.id) }}
- HOME: ${{ matrix.launcher == 'mi-amds' && format('{0}/.collectivex-jobs/inferencex-collectivex-{1}-{2}-{3}/home', github.workspace, github.run_id, github.run_attempt, matrix.id) || format('/tmp/inferencex-collectivex-{0}-{1}-{2}/home', github.run_id, github.run_attempt, matrix.id) }}
+ CX_JOB_PARENT: ${{ matrix.launcher == 'mi-amds' && format('/tmp/inferencex-collectivex-parent-{0}-{1}-{2}', github.run_id, github.run_attempt, matrix.id) || '/tmp' }}
+ CX_JOB_ROOT: ${{ matrix.launcher == 'mi-amds' && format('/tmp/inferencex-collectivex-parent-{0}-{1}-{2}/inferencex-collectivex-{0}-{1}-{2}', github.run_id, github.run_attempt, matrix.id) || format('/tmp/inferencex-collectivex-{0}-{1}-{2}', github.run_id, github.run_attempt, matrix.id) }}
+ CX_SOURCE_ROOT: ${{ matrix.launcher == 'mi-amds' && format('/tmp/inferencex-collectivex-parent-{0}-{1}-{2}/inferencex-collectivex-{0}-{1}-{2}/source', github.run_id, github.run_attempt, matrix.id) || format('/tmp/inferencex-collectivex-{0}-{1}-{2}/source', github.run_id, github.run_attempt, matrix.id) }}
+ HOME: ${{ matrix.launcher == 'mi-amds' && format('/tmp/inferencex-collectivex-parent-{0}-{1}-{2}/inferencex-collectivex-{0}-{1}-{2}/home', github.run_id, github.run_attempt, matrix.id) || format('/tmp/inferencex-collectivex-{0}-{1}-{2}/home', github.run_id, github.run_attempt, matrix.id) }}
steps:
- name: Prepare isolated source
id: source
@@ -334,19 +334,6 @@ jobs:
COLLECTIVEX_REPOSITORY: ${{ github.repository }}
run: |
set -euo pipefail
- [ "$CX_JOB_PARENT" = /tmp ] || {
- [ "$CX_JOB_PARENT" = '${{ github.workspace }}/.collectivex-jobs' ] \
- || { echo "CollectiveX isolated parent is invalid" >&2; exit 1; }
- if [ ! -e "$CX_JOB_PARENT" ] && [ ! -L "$CX_JOB_PARENT" ]; then
- umask 077
- mkdir -m 700 -- "$CX_JOB_PARENT"
- fi
- [ ! -L "$CX_JOB_PARENT" ] \
- && [ -d "$CX_JOB_PARENT" ] \
- && [ -O "$CX_JOB_PARENT" ] \
- && [ "$(stat -c '%a' "$CX_JOB_PARENT")" = 700 ] \
- || { echo "CollectiveX isolated parent is unsafe" >&2; exit 1; }
- }
python3 - <<'PY'
import os
import re
@@ -355,8 +342,32 @@ jobs:
import time
pattern = re.compile(r"inferencex-collectivex-[0-9]+-[0-9]+-[A-Za-z0-9._-]+")
+ link_pattern = re.compile(
+ r"/tmp/inferencex-collectivex-parent-[0-9]+-[0-9]+-[A-Za-z0-9._-]+"
+ )
+ parent = os.environ["CX_JOB_PARENT"]
+ scan_parent = parent
+ if parent != "/tmp":
+ if not link_pattern.fullmatch(parent):
+ raise SystemExit("CollectiveX isolated parent is invalid")
+ shared_parent = os.path.join(os.getcwd(), ".collectivex-jobs")
+ try:
+ shared_metadata = os.lstat(shared_parent)
+ except FileNotFoundError:
+ os.mkdir(shared_parent, mode=0o700)
+ shared_metadata = os.lstat(shared_parent)
+ if (
+ not stat.S_ISDIR(shared_metadata.st_mode)
+ or shared_metadata.st_uid != os.getuid()
+ or stat.S_IMODE(shared_metadata.st_mode) != 0o700
+ ):
+ raise SystemExit("CollectiveX shared parent is unsafe")
+ if os.path.lexists(parent):
+ raise SystemExit("CollectiveX isolated parent already exists")
+ os.symlink(shared_parent, parent)
+ scan_parent = shared_parent
cutoff = time.time() - 86400
- for entry in os.scandir(os.environ["CX_JOB_PARENT"]):
+ for entry in os.scandir(scan_parent):
if not pattern.fullmatch(entry.name):
continue
try:
@@ -399,7 +410,7 @@ jobs:
fi
umask 077
mkdir -m 700 -- "$CX_JOB_ROOT"
- trap 'rc=$?; [ "$rc" = 0 ] || rm -rf -- "$CX_JOB_ROOT"; exit "$rc"' EXIT
+ trap 'rc=$?; if [ "$rc" != 0 ]; then rm -rf -- "$CX_JOB_ROOT"; [ "$CX_JOB_PARENT" = /tmp ] || rm -f -- "$CX_JOB_PARENT"; fi; exit "$rc"' EXIT
mkdir -m 700 -- "$HOME" "$CX_JOB_ROOT/control" "$CX_JOB_ROOT/artifact" "$CX_SOURCE_ROOT"
: > "$CX_JOB_ROOT/cleanup-safe"
if ! {
@@ -642,7 +653,7 @@ jobs:
run: |
set -euo pipefail
[ "$CX_JOB_PARENT" = /tmp ] \
- || [ "$CX_JOB_PARENT" = '${{ github.workspace }}/.collectivex-jobs' ] \
+ || [[ "$CX_JOB_PARENT" =~ ^/tmp/inferencex-collectivex-parent-[0-9]+-[0-9]+-[A-Za-z0-9._-]+$ ]] \
|| { echo "CollectiveX cleanup parent is invalid" >&2; exit 1; }
[ "${CX_JOB_ROOT%/*}" = "$CX_JOB_PARENT" ] \
&& [[ "${CX_JOB_ROOT##*/}" =~ ^inferencex-collectivex-[0-9]+-[0-9]+-[A-Za-z0-9._-]+$ ]] \
@@ -665,6 +676,7 @@ jobs:
fi
fi
rm -rf -- "$CX_JOB_ROOT"
+ [ "$CX_JOB_PARENT" = /tmp ] || rm -f -- "$CX_JOB_PARENT"
probe-summary:
needs: [setup, sweep]
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index b0c4ccda7f..0a47dea823 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -1587,17 +1587,17 @@ def test_workflow_reconciles_a_recorded_allocation_after_launcher_loss(self) ->
sweep_workflow = workflow[workflow.index(" sweep:"):]
self.assertNotIn("GITHUB_WORKSPACE", sweep_workflow)
self.assertNotIn("RUNNER_WORKSPACE", sweep_workflow)
+ self.assertNotIn("github.workspace", sweep_workflow)
self.assertIn("matrix.launcher == 'mi-amds'", sweep_workflow)
- self.assertIn("github.workspace, github.run_id", sweep_workflow)
self.assertIn("CX_JOB_PARENT:", sweep_workflow)
- self.assertIn("/.collectivex-jobs", sweep_workflow)
+ self.assertIn("/tmp/inferencex-collectivex-parent-", sweep_workflow)
self.assertIn("|| '/tmp'", sweep_workflow)
source_step = sweep_workflow[:sweep_workflow.index("- uses: actions/download-artifact")]
self.assertNotIn("unsafe_guards=", source_step)
- self.assertIn('os.scandir(os.environ["CX_JOB_PARENT"])', source_step)
- self.assertIn('[ ! -L "$CX_JOB_PARENT" ]', source_step)
- self.assertIn('[ -O "$CX_JOB_PARENT" ]', source_step)
- self.assertIn('stat -c \'%a\' "$CX_JOB_PARENT"', source_step)
+ self.assertIn('shared_parent = os.path.join(os.getcwd(), ".collectivex-jobs")', source_step)
+ self.assertIn("os.lstat(shared_parent)", source_step)
+ self.assertIn("os.symlink(shared_parent, parent)", source_step)
+ self.assertIn("for entry in os.scandir(scan_parent)", source_step)
self.assertIn("cutoff = time.time() - 86400", source_step)
self.assertIn("stat.S_IMODE(metadata.st_mode) != 0o700", source_step)
self.assertIn('for marker_name in ("cleanup-safe", "cleanup-unsafe")', source_step)
@@ -1615,6 +1615,7 @@ def test_workflow_reconciles_a_recorded_allocation_after_launcher_loss(self) ->
self.assertIn("CollectiveX cleanup parent is invalid", cleanup)
self.assertIn('[ "${CX_JOB_ROOT%/*}" = "$CX_JOB_PARENT" ]', cleanup)
self.assertIn('^inferencex-collectivex-', cleanup)
+ self.assertIn('rm -f -- "$CX_JOB_PARENT"', cleanup)
for step in (
"sweep_shard", "allocation_cleanup", "artifact_safety",
"delivery_contracts", "stage_artifact", "upload_artifact",
From b662466e58ab7d272f5470b92499b8d193d239b7 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 7 Jul 2026 04:02:36 +0800
Subject: [PATCH 156/210] fix(collectivex): canonicalize AMD source roots
---
experimental/CollectiveX/launchers/launch_mi-amds.sh | 2 +-
experimental/CollectiveX/tests/test_sampling_contract.py | 1 +
2 files changed, 2 insertions(+), 1 deletion(-)
diff --git a/experimental/CollectiveX/launchers/launch_mi-amds.sh b/experimental/CollectiveX/launchers/launch_mi-amds.sh
index 01d2e4e010..47b64c41fe 100644
--- a/experimental/CollectiveX/launchers/launch_mi-amds.sh
+++ b/experimental/CollectiveX/launchers/launch_mi-amds.sh
@@ -3,7 +3,7 @@
# shellcheck disable=SC2016,SC2034
set -euo pipefail
-HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+HERE="$(cd -P -- "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
CX_DIR="$(cd "$HERE/.." && pwd)"
REPO_ROOT="$(cd "$CX_DIR/../.." && pwd)"
# shellcheck source=../runtime/common.sh
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 0a47dea823..b69b0f4983 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -1041,6 +1041,7 @@ def test_launchers_preserve_platform_specific_runtime_requirements(self) -> None
common.index("workload_log=", common.index("workload_args=("))
]
self.assertNotIn("--workload", workload_stage)
+ self.assertIn('cd -P -- "$(dirname "${BASH_SOURCE[0]}")"', amd)
self.assertIn("mi325x) CPUS_PER_TASK=256", amd)
self.assertIn("/dev/kfd:/dev/kfd,/dev/dri:/dev/dri", amd)
self.assertIn("--container-writable --container-remap-root", amd)
From 4e800d1bcdb37c7fe0bdc169c98b4af50ba4cda6 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 7 Jul 2026 04:06:42 +0800
Subject: [PATCH 157/210] fix(collectivex): store AMD diagnostics off tmp
---
.github/workflows/collectivex-sweep.yml | 5 ++
experimental/CollectiveX/runtime/common.sh | 50 +++++++++++++++++++
.../tests/test_deepep_v2_contract.py | 30 +++++++++++
.../tests/test_sampling_contract.py | 2 +
4 files changed, 87 insertions(+)
diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml
index 1b0e278ae5..67de1e9b14 100644
--- a/.github/workflows/collectivex-sweep.yml
+++ b/.github/workflows/collectivex-sweep.yml
@@ -675,6 +675,11 @@ jobs:
cx_cleanup_private_logs 0
fi
fi
+ if [ '${{ steps.sweep_shard.outcome }}' != success ] \
+ && [ "$CX_JOB_PARENT" != /tmp ]; then
+ rm -f -- "$CX_JOB_PARENT"
+ exit 0
+ fi
rm -rf -- "$CX_JOB_ROOT"
[ "$CX_JOB_PARENT" = /tmp ] || rm -f -- "$CX_JOB_PARENT"
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index c2490b4390..bc51b366a3 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -526,6 +526,31 @@ tag, label = sys.argv[1:]
if not all(re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]*", value) for value in (tag, label)):
raise SystemExit(1)
root = f"/tmp/inferencex-collectivex-{os.getuid()}"
+job_root = os.environ.get("CX_JOB_ROOT", "")
+job_parent = os.environ.get("CX_JOB_PARENT", "")
+if (
+ os.environ.get("COLLECTIVEX_CANONICAL_GHA") == "1"
+ and job_parent
+ and job_parent != "/tmp"
+):
+ if (
+ not os.path.isabs(job_root)
+ or os.path.dirname(job_root) != job_parent
+ or not re.fullmatch(
+ r"inferencex-collectivex-[0-9]+-[0-9]+-[A-Za-z0-9._-]+",
+ os.path.basename(job_root),
+ )
+ ):
+ raise SystemExit(1)
+ control = os.path.join(job_root, "control")
+ control_metadata = os.stat(control, follow_symlinks=False)
+ if (
+ not stat.S_ISDIR(control_metadata.st_mode)
+ or control_metadata.st_uid != os.getuid()
+ or stat.S_IMODE(control_metadata.st_mode) != 0o700
+ ):
+ raise SystemExit(1)
+ root = os.path.join(control, "private-logs")
old_umask = os.umask(0o077)
flags = os.O_RDONLY | os.O_DIRECTORY | getattr(os, "O_NOFOLLOW", 0)
try:
@@ -589,6 +614,31 @@ tag = sys.argv[1]
if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]*", tag):
raise SystemExit(1)
root = f"/tmp/inferencex-collectivex-{os.getuid()}"
+job_root = os.environ.get("CX_JOB_ROOT", "")
+job_parent = os.environ.get("CX_JOB_PARENT", "")
+if (
+ os.environ.get("COLLECTIVEX_CANONICAL_GHA") == "1"
+ and job_parent
+ and job_parent != "/tmp"
+):
+ if (
+ not os.path.isabs(job_root)
+ or os.path.dirname(job_root) != job_parent
+ or not re.fullmatch(
+ r"inferencex-collectivex-[0-9]+-[0-9]+-[A-Za-z0-9._-]+",
+ os.path.basename(job_root),
+ )
+ ):
+ raise SystemExit(1)
+ control = os.path.join(job_root, "control")
+ control_metadata = os.stat(control, follow_symlinks=False)
+ if (
+ not stat.S_ISDIR(control_metadata.st_mode)
+ or control_metadata.st_uid != os.getuid()
+ or stat.S_IMODE(control_metadata.st_mode) != 0o700
+ ):
+ raise SystemExit(1)
+ root = os.path.join(control, "private-logs")
flags = os.O_RDONLY | os.O_DIRECTORY | getattr(os, "O_NOFOLLOW", 0)
root_fd = os.open(root, flags)
try:
diff --git a/experimental/CollectiveX/tests/test_deepep_v2_contract.py b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
index bb1bb4a4a8..ee999b7433 100644
--- a/experimental/CollectiveX/tests/test_deepep_v2_contract.py
+++ b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
@@ -1915,6 +1915,36 @@ def test_private_runtime_logs_are_not_public_artifacts(self) -> None:
finally:
shutil.rmtree(Path(path).parent, ignore_errors=True)
+ def test_canonical_amd_runtime_logs_use_the_isolated_control_tree(self) -> None:
+ common = str(ROOT / "runtime" / "common.sh")
+ with tempfile.TemporaryDirectory() as temporary:
+ parent = Path(temporary) / "parent"
+ parent.mkdir(mode=0o700)
+ job = parent / f"inferencex-collectivex-123-1-test-{os.getpid()}"
+ control = job / "control"
+ control.mkdir(parents=True, mode=0o700)
+ environment = {
+ **os.environ,
+ "COLLECTIVEX_CANONICAL_GHA": "1",
+ "COLLECTIVEX_EXECUTION_ID": f"canonical-amd-{os.getpid()}",
+ "CX_JOB_PARENT": str(parent),
+ "CX_JOB_ROOT": str(job),
+ }
+ path = subprocess.check_output(
+ ["bash", "-c", 'source "$1"; cx_private_log_path test', "_", common],
+ text=True,
+ env=environment,
+ ).strip()
+ log = Path(path)
+ self.assertTrue(log.is_relative_to(control / "private-logs"))
+ self.assertEqual(stat.S_IMODE(log.stat().st_mode), 0o600)
+ subprocess.run(
+ ["bash", "-c", 'source "$1"; cx_cleanup_private_logs 0', "_", common],
+ check=True,
+ env=environment,
+ )
+ self.assertFalse(log.parent.exists())
+
def test_private_runtime_logs_reject_traversal_and_symlinks(self) -> None:
common = str(ROOT / "runtime" / "common.sh")
for variable, value in (
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index b69b0f4983..d6b4a2eb21 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -1617,6 +1617,8 @@ def test_workflow_reconciles_a_recorded_allocation_after_launcher_loss(self) ->
self.assertIn('[ "${CX_JOB_ROOT%/*}" = "$CX_JOB_PARENT" ]', cleanup)
self.assertIn('^inferencex-collectivex-', cleanup)
self.assertIn('rm -f -- "$CX_JOB_PARENT"', cleanup)
+ self.assertIn("steps.sweep_shard.outcome }}' != success", cleanup)
+ self.assertIn('[ "$CX_JOB_PARENT" != /tmp ]', cleanup)
for step in (
"sweep_shard", "allocation_cleanup", "artifact_safety",
"delivery_contracts", "stage_artifact", "upload_artifact",
From 48986ca3505d14242dfc792f6ac75eb51c7da90b Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 7 Jul 2026 04:08:50 +0800
Subject: [PATCH 158/210] fix(collectivex): classify scheduler request failures
---
experimental/CollectiveX/runtime/common.sh | 7 ++++++-
experimental/CollectiveX/tests/test_sampling_contract.py | 5 +++++
2 files changed, 11 insertions(+), 1 deletion(-)
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index bc51b366a3..bc04e2e943 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -1249,7 +1249,10 @@ cx_salloc_jobid() {
2|3) log_label+="-a${CX_SALLOC_ATTEMPT}" ;;
*) return 1 ;;
esac
- log="$(cx_private_log_path "$log_label")"
+ if ! log="$(cx_private_log_path "$log_label")"; then
+ cx_log "ERROR: failure-class=scheduler-allocation diagnostic=private-log"
+ return 1
+ fi
for argument in "$@"; do
case "$argument" in
--job-name|--job-name=*|-J|-J*)
@@ -1274,6 +1277,7 @@ cx_salloc_jobid() {
CX_ALLOCATION_UNCERTAIN=0
fi
if [ "$salloc_rc" != 0 ]; then
+ cx_log "ERROR: scheduler-request=rejected"
if [ "$salloc_rc" -ge 128 ] && [ -z "$JOB_ID" ]; then
cx_fail_stage scheduler-allocation "$log"
return 1
@@ -1284,6 +1288,7 @@ cx_salloc_jobid() {
return 1
fi
if [ -z "$JOB_ID" ]; then
+ cx_log "ERROR: scheduler-request=missing-grant"
cx_reconcile_salloc_jobid "$job_name" || true
cx_fail_stage scheduler-allocation "$log"
return 1
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index d6b4a2eb21..0f7154c076 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -1356,6 +1356,11 @@ def test_salloc_verified_rejection_is_cleanup_safe(self) -> None:
result = scenario["result"]
self.assertEqual(result.returncode, 1)
self.assertEqual(result.stdout, "1::0\n")
+ self.assertIn("scheduler-request=rejected", result.stderr)
+ self.assertIn(
+ "failure-class=scheduler-allocation diagnostic=empty-log",
+ result.stderr,
+ )
self.assertEqual(len(scenario["squeue_calls"]), 3)
scheduler_user = subprocess.check_output(["id", "-un"], text=True).strip()
self.assertTrue(all(
From b7962ada09495fbb1aee7bdde77a7cb43a5f3957 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 7 Jul 2026 04:10:40 +0800
Subject: [PATCH 159/210] fix(collectivex): expose scheduler submit boundary
---
experimental/CollectiveX/runtime/common.sh | 6 +++++-
experimental/CollectiveX/tests/test_sampling_contract.py | 1 +
2 files changed, 6 insertions(+), 1 deletion(-)
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index bc04e2e943..e887444297 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -1261,11 +1261,15 @@ cx_salloc_jobid() {
;;
esac
done
- job_name="$(cx_scheduler_job_name)" || return 1
+ if ! job_name="$(cx_scheduler_job_name)"; then
+ cx_log "ERROR: failure-class=scheduler-allocation diagnostic=job-name"
+ return 1
+ fi
CX_ALLOCATION_UNCERTAIN=1
# salloc has no portable --parsable option. Parse the stable grant message
# used by the production launchers, while also accepting a bare ID from
# site wrappers.
+ cx_log "scheduler-request=submit"
salloc "$@" --job-name="$job_name" --no-shell > "$log" 2>&1 || salloc_rc=$?
job_id="$(sed -nE \
-e 's/^([0-9]+)(;[^[:space:]]+)?$/\1/p' \
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 0f7154c076..2588bb5028 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -1349,6 +1349,7 @@ def test_salloc_job_id_parser_uses_the_portable_grant_message(self) -> None:
"--no-shell",
],
)
+ self.assertIn("scheduler-request=submit", result.stderr)
self.assertEqual(scenario["squeue_calls"], [])
def test_salloc_verified_rejection_is_cleanup_safe(self) -> None:
From fe8cc52e0791e89e6dcd62a776c98977b0f8fc19 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 7 Jul 2026 04:12:43 +0800
Subject: [PATCH 160/210] fix(collectivex): parse scheduler grants without
pipefail
---
experimental/CollectiveX/runtime/common.sh | 13 +++++++++----
.../CollectiveX/tests/test_sampling_contract.py | 12 ++++++++++++
2 files changed, 21 insertions(+), 4 deletions(-)
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index e887444297..805680cfec 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -1271,10 +1271,15 @@ cx_salloc_jobid() {
# site wrappers.
cx_log "scheduler-request=submit"
salloc "$@" --job-name="$job_name" --no-shell > "$log" 2>&1 || salloc_rc=$?
- job_id="$(sed -nE \
- -e 's/^([0-9]+)(;[^[:space:]]+)?$/\1/p' \
- -e 's/.*Granted job allocation ([0-9]+).*/\1/p' \
- "$log" | head -n1)"
+ if ! job_id="$(sed -nE \
+ -e 's/^([0-9]+)(;[^[:space:]]+)?$/\1/p; t found' \
+ -e 's/.*Granted job allocation ([0-9]+).*/\1/p; t found' \
+ -e 'b' -e ':found' -e 'q' "$log")"; then
+ cx_log "ERROR: failure-class=scheduler-allocation diagnostic=grant-parse"
+ cx_reconcile_salloc_jobid "$job_name" || true
+ [ -z "$JOB_ID" ] || cx_record_allocation_jobid "$JOB_ID" || true
+ return 1
+ fi
if [ -n "$job_id" ]; then
[[ "$job_id" =~ ^[0-9]+$ ]] || return 1
JOB_ID="$job_id"
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 2588bb5028..72354a92b9 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -1352,6 +1352,18 @@ def test_salloc_job_id_parser_uses_the_portable_grant_message(self) -> None:
self.assertIn("scheduler-request=submit", result.stderr)
self.assertEqual(scenario["squeue_calls"], [])
+ def test_salloc_job_id_parser_ignores_duplicate_wrapper_output(self) -> None:
+ scenario = self._run_salloc_scenario(
+ "for _ in {1..10000}; do "
+ "printf 'salloc: Granted job allocation 4242\\n' >&2; done",
+ "exit 2",
+ cleanup=False,
+ )
+ result = scenario["result"]
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertEqual(result.stdout, "0:4242:0\n")
+ self.assertNotIn("grant-parse", result.stderr)
+
def test_salloc_verified_rejection_is_cleanup_safe(self) -> None:
scenario = self._run_salloc_scenario("exit 1", "exit 0", cleanup=True)
result = scenario["result"]
From dd82818b9581695d75d2a4d36e2dd6258982654f Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 7 Jul 2026 04:14:39 +0800
Subject: [PATCH 161/210] fix(collectivex): contain scheduler shell wrappers
---
experimental/CollectiveX/runtime/common.sh | 5 +++--
.../tests/test_sampling_contract.py | 22 +++++++++++++++++++
2 files changed, 25 insertions(+), 2 deletions(-)
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 805680cfec..feb7ca749a 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -1268,9 +1268,10 @@ cx_salloc_jobid() {
CX_ALLOCATION_UNCERTAIN=1
# salloc has no portable --parsable option. Parse the stable grant message
# used by the production launchers, while also accepting a bare ID from
- # site wrappers.
+ # site wrappers. Contain shell-function wrappers that call exit so the
+ # launcher can still reconcile and cancel an allocation.
cx_log "scheduler-request=submit"
- salloc "$@" --job-name="$job_name" --no-shell > "$log" 2>&1 || salloc_rc=$?
+ (salloc "$@" --job-name="$job_name" --no-shell) > "$log" 2>&1 || salloc_rc=$?
if ! job_id="$(sed -nE \
-e 's/^([0-9]+)(;[^[:space:]]+)?$/\1/p; t found' \
-e 's/.*Granted job allocation ([0-9]+).*/\1/p; t found' \
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 72354a92b9..968abf2071 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -1364,6 +1364,28 @@ def test_salloc_job_id_parser_ignores_duplicate_wrapper_output(self) -> None:
self.assertEqual(result.stdout, "0:4242:0\n")
self.assertNotIn("grant-parse", result.stderr)
+ def test_salloc_contains_a_shell_wrapper_that_calls_exit(self) -> None:
+ common = ROOT / "runtime" / "common.sh"
+ with tempfile.TemporaryDirectory() as temporary:
+ log = Path(temporary) / "scheduler.log"
+ command = r'''
+ set -euo pipefail
+ source "$1"
+ cx_private_log_path() { : > "$CX_TEST_LOG"; printf '%s' "$CX_TEST_LOG"; }
+ salloc() { printf 'salloc: Granted job allocation 4242\n' >&2; exit 0; }
+ export COLLECTIVEX_EXECUTION_ID=wrapper-exit
+ JOB_ID=""
+ cx_salloc_jobid --partition=compute
+ test "$JOB_ID" = 4242
+ '''
+ result = subprocess.run(
+ ["bash", "-c", command, "_", str(common), str(log)],
+ text=True,
+ capture_output=True,
+ env={**os.environ, "CX_TEST_LOG": str(log)},
+ )
+ self.assertEqual(result.returncode, 0, result.stderr)
+
def test_salloc_verified_rejection_is_cleanup_safe(self) -> None:
scenario = self._run_salloc_scenario("exit 1", "exit 0", cleanup=True)
result = scenario["result"]
From 234505b605498a1dc4b18dc6c766badfbc01411e Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 7 Jul 2026 04:53:43 +0800
Subject: [PATCH 162/210] fix(collectivex): classify retained scheduler logs
---
.github/workflows/collectivex-sweep.yml | 8 +++
experimental/CollectiveX/runtime/common.sh | 69 +++++++++++++++++++
.../tests/test_deepep_v2_contract.py | 26 +++++++
.../tests/test_sampling_contract.py | 2 +
4 files changed, 105 insertions(+)
diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml
index 67de1e9b14..f3cabca0ee 100644
--- a/.github/workflows/collectivex-sweep.yml
+++ b/.github/workflows/collectivex-sweep.yml
@@ -595,6 +595,14 @@ jobs:
|| { echo "CollectiveX source is unavailable" >&2; exit 1; }
source experimental/CollectiveX/runtime/common.sh
cx_reconcile_recorded_allocation "$CX_JOB_ROOT"
+ - name: Classify private scheduler failure
+ if: ${{ always() && steps.source.outputs.prepared == 'true' && steps.sweep_shard.outcome == 'failure' }}
+ run: |
+ set -euo pipefail
+ cd "$CX_SOURCE_ROOT" 2>/dev/null \
+ || { echo "CollectiveX source is unavailable" >&2; exit 1; }
+ source experimental/CollectiveX/runtime/common.sh
+ cx_report_private_scheduler_failure
- name: Confirm allocation cleanup
id: allocation_cleanup
if: ${{ always() && steps.source.outputs.prepared == 'true' }}
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index feb7ca749a..5150436b65 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -653,6 +653,75 @@ if os.path.isdir(path) and not os.path.islink(path):
PY
}
+cx_report_private_scheduler_failure() {
+ local root="${CX_JOB_ROOT:-}" tag="${COLLECTIVEX_EXECUTION_ID:-}" diagnostic
+ [ "${COLLECTIVEX_CANONICAL_GHA:-0}" = 1 ] \
+ && [ -n "$root" ] && [ -n "$tag" ] \
+ && [[ "$tag" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] \
+ || return 0
+ diagnostic="$(python3 - "$root" "$tag" <<'PY' 2>/dev/null || true
+import os
+import re
+import stat
+import sys
+
+root, tag = sys.argv[1:]
+directory = os.path.join(root, "control", "private-logs", tag)
+try:
+ metadata = os.stat(directory, follow_symlinks=False)
+except OSError:
+ print("missing")
+ raise SystemExit
+if (
+ not stat.S_ISDIR(metadata.st_mode)
+ or metadata.st_uid != os.getuid()
+ or stat.S_IMODE(metadata.st_mode) != 0o700
+):
+ print("unsafe")
+ raise SystemExit
+payload = b""
+for name in sorted(os.listdir(directory)):
+ if not re.fullmatch(r"scheduler-allocation(?:-a[23])?[.]log", name):
+ continue
+ path = os.path.join(directory, name)
+ item = os.stat(path, follow_symlinks=False)
+ if (
+ not stat.S_ISREG(item.st_mode)
+ or item.st_uid != os.getuid()
+ or stat.S_IMODE(item.st_mode) != 0o600
+ or item.st_size > 65536
+ ):
+ print("unsafe")
+ raise SystemExit
+ with open(path, "rb") as stream:
+ payload += stream.read(65537)
+text = payload.decode("utf-8", "replace")
+if not text:
+ result = "empty"
+elif re.search(r"invalid (?:account|partition|qos|reservation)|association.*not permitted|access denied", text, re.I):
+ result = "policy"
+elif re.search(r"pending job allocation|job .* pending|waiting for resource", text, re.I):
+ result = "pending"
+elif re.search(r"requested node configuration is not available|nodes required.*not available|resources? unavailable", text, re.I):
+ result = "capacity"
+elif re.search(r"allocation (?:revoked|cancelled)|job .* cancelled", text, re.I):
+ result = "revoked"
+elif re.search(r"timed out|timeout", text, re.I):
+ result = "timeout"
+elif re.search(r"no space left|disk quota|quota exceeded", text, re.I):
+ result = "storage-capacity"
+else:
+ result = "unclassified"
+print(result)
+PY
+)"
+ case "$diagnostic" in
+ missing|unsafe|empty|policy|pending|capacity|revoked|timeout|storage-capacity|unclassified) ;;
+ *) diagnostic=unclassified ;;
+ esac
+ cx_log "ERROR: scheduler-diagnostic=$diagnostic"
+}
+
# Explicit Slurm export boundary. Operator config, runner credentials, HOME,
# workspace paths, and unrelated service secrets never enter the container.
cx_container_exports() {
diff --git a/experimental/CollectiveX/tests/test_deepep_v2_contract.py b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
index ee999b7433..978646c39a 100644
--- a/experimental/CollectiveX/tests/test_deepep_v2_contract.py
+++ b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
@@ -1945,6 +1945,32 @@ def test_canonical_amd_runtime_logs_use_the_isolated_control_tree(self) -> None:
)
self.assertFalse(log.parent.exists())
+ def test_private_scheduler_diagnostic_is_value_free(self) -> None:
+ common = str(ROOT / "runtime" / "common.sh")
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary) / "inferencex-collectivex-123-1-test"
+ tag = "scheduler-diagnostic"
+ directory = root / "control" / "private-logs" / tag
+ directory.mkdir(parents=True, mode=0o700)
+ log = directory / "scheduler-allocation.log"
+ private_value = "private-node-123"
+ log.write_text(f"salloc: Pending job allocation on {private_value}\n")
+ log.chmod(0o600)
+ result = subprocess.run(
+ ["bash", "-c", 'source "$1"; cx_report_private_scheduler_failure', "_", common],
+ text=True,
+ capture_output=True,
+ env={
+ **os.environ,
+ "COLLECTIVEX_CANONICAL_GHA": "1",
+ "COLLECTIVEX_EXECUTION_ID": tag,
+ "CX_JOB_ROOT": str(root),
+ },
+ )
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertIn("scheduler-diagnostic=pending", result.stderr)
+ self.assertNotIn(private_value, result.stderr)
+
def test_private_runtime_logs_reject_traversal_and_symlinks(self) -> None:
common = str(ROOT / "runtime" / "common.sh")
for variable, value in (
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 968abf2071..a5b2b31d48 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -1659,6 +1659,8 @@ def test_workflow_reconciles_a_recorded_allocation_after_launcher_loss(self) ->
self.assertIn('rm -f -- "$CX_JOB_PARENT"', cleanup)
self.assertIn("steps.sweep_shard.outcome }}' != success", cleanup)
self.assertIn('[ "$CX_JOB_PARENT" != /tmp ]', cleanup)
+ self.assertIn("- name: Classify private scheduler failure", workflow)
+ self.assertIn("cx_report_private_scheduler_failure", workflow)
for step in (
"sweep_shard", "allocation_cleanup", "artifact_safety",
"delivery_contracts", "stage_artifact", "upload_artifact",
From 3507d1b970155edd9043abf32813c1fee6a0d468 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 7 Jul 2026 05:32:34 +0800
Subject: [PATCH 163/210] fix(collectivex): classify Slurm submit failures
---
experimental/CollectiveX/runtime/common.sh | 15 ++++++++++++++-
.../CollectiveX/tests/test_deepep_v2_contract.py | 15 +++++++++++++++
2 files changed, 29 insertions(+), 1 deletion(-)
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 5150436b65..81343501d5 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -704,6 +704,18 @@ elif re.search(r"pending job allocation|job .* pending|waiting for resource", te
result = "pending"
elif re.search(r"requested node configuration is not available|nodes required.*not available|resources? unavailable", text, re.I):
result = "capacity"
+elif re.search(r"unable to contact slurm controller|communication connection failure|socket timed out|slurmctld.*(?:down|unreachable)", text, re.I):
+ result = "controller"
+elif re.search(r"invalid generic resource|invalid gres|invalid node count|invalid cpu count|memory specification can not be satisfied|requested.*configuration.*invalid", text, re.I):
+ result = "resource-request"
+elif re.search(r"job violates accounting[/ ]qos policy|maximum.*jobs|association.*limit|qos.*limit", text, re.I):
+ result = "account-limit"
+elif re.search(r"invalid credential|authentication failure|authentication error", text, re.I):
+ result = "authentication"
+elif re.search(r"job submit plugin|job_submit", text, re.I):
+ result = "submit-plugin"
+elif re.search(r"unrecognized option|unknown option|invalid option", text, re.I):
+ result = "option"
elif re.search(r"allocation (?:revoked|cancelled)|job .* cancelled", text, re.I):
result = "revoked"
elif re.search(r"timed out|timeout", text, re.I):
@@ -716,7 +728,8 @@ print(result)
PY
)"
case "$diagnostic" in
- missing|unsafe|empty|policy|pending|capacity|revoked|timeout|storage-capacity|unclassified) ;;
+ missing|unsafe|empty|policy|pending|capacity|controller|resource-request|account-limit) ;;
+ authentication|submit-plugin|option|revoked|timeout|storage-capacity|unclassified) ;;
*) diagnostic=unclassified ;;
esac
cx_log "ERROR: scheduler-diagnostic=$diagnostic"
diff --git a/experimental/CollectiveX/tests/test_deepep_v2_contract.py b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
index 978646c39a..eed62bf4ae 100644
--- a/experimental/CollectiveX/tests/test_deepep_v2_contract.py
+++ b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
@@ -1971,6 +1971,21 @@ def test_private_scheduler_diagnostic_is_value_free(self) -> None:
self.assertIn("scheduler-diagnostic=pending", result.stderr)
self.assertNotIn(private_value, result.stderr)
+ log.write_text("salloc: error: Unable to contact Slurm controller\n")
+ result = subprocess.run(
+ ["bash", "-c", 'source "$1"; cx_report_private_scheduler_failure', "_", common],
+ text=True,
+ capture_output=True,
+ env={
+ **os.environ,
+ "COLLECTIVEX_CANONICAL_GHA": "1",
+ "COLLECTIVEX_EXECUTION_ID": tag,
+ "CX_JOB_ROOT": str(root),
+ },
+ )
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertIn("scheduler-diagnostic=controller", result.stderr)
+
def test_private_runtime_logs_reject_traversal_and_symlinks(self) -> None:
common = str(ROOT / "runtime" / "common.sh")
for variable, value in (
From d2d869bd33f9b96fbf2c128cca263f7eef610ce7 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 7 Jul 2026 05:49:07 +0800
Subject: [PATCH 164/210] fix(collectivex): respect MI300X allocation limit
---
.github/workflows/collectivex-sweep.yml | 9 ++++-----
experimental/CollectiveX/runtime/common.sh | 2 +-
.../tests/test_deepep_v2_contract.py | 17 +++++++++++++++++
.../CollectiveX/tests/test_sampling_contract.py | 1 +
4 files changed, 23 insertions(+), 6 deletions(-)
diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml
index f3cabca0ee..450950c71e 100644
--- a/.github/workflows/collectivex-sweep.yml
+++ b/.github/workflows/collectivex-sweep.yml
@@ -316,11 +316,10 @@ jobs:
COLLECTIVEX_CANONICAL_GHA: '1'
COLLECTIVEX_SOURCE_SHA: ${{ github.sha }}
COLLECTIVEX_ARTIFACT_NAME: ${{ inputs.operation == 'probe-precision' && format('cxprobe-{0}-{1}-{2}', matrix.id, github.run_id, github.run_attempt) || format('cxshard-{0}-{1}-{2}', matrix.id, github.run_id, github.run_attempt) }}
- # Consolidated shards run one bounded build-group in one Slurm allocation, so
- # the launcher's default 45-min --time is too short. 300 min covers a cold
- # compute-node image import plus the shard. The allocation releases early
- # when the shard finishes, so short shards don't waste it.
- CX_TIME: ${{ inputs.operation == 'probe-precision' && '90' || '300' }}
+ # Consolidated shards run one bounded build-group in one Slurm allocation.
+ # MI300X's compute partition has a 180-minute ceiling; the other production
+ # pools accept 300 minutes. Allocations release as soon as the shard finishes.
+ CX_TIME: ${{ inputs.operation == 'probe-precision' && '90' || (matrix.sku == 'mi300x' && '180' || '300') }}
COLLECTIVEX_EXECUTION_ID: ${{ github.run_id }}_${{ github.run_attempt }}_${{ matrix.id }}
CX_QUALIFICATION_INDEX: ${{ inputs.qualification_index }}
CX_JOB_PARENT: ${{ matrix.launcher == 'mi-amds' && format('/tmp/inferencex-collectivex-parent-{0}-{1}-{2}', github.run_id, github.run_attempt, matrix.id) || '/tmp' }}
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 81343501d5..a66ee616f4 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -706,7 +706,7 @@ elif re.search(r"requested node configuration is not available|nodes required.*n
result = "capacity"
elif re.search(r"unable to contact slurm controller|communication connection failure|socket timed out|slurmctld.*(?:down|unreachable)", text, re.I):
result = "controller"
-elif re.search(r"invalid generic resource|invalid gres|invalid node count|invalid cpu count|memory specification can not be satisfied|requested.*configuration.*invalid", text, re.I):
+elif re.search(r"invalid generic resource|invalid gres|invalid node count|invalid cpu count|memory specification can not be satisfied|requested.*configuration.*invalid|requested time limit is invalid|time limit.*(?:invalid|exceed)", text, re.I):
result = "resource-request"
elif re.search(r"job violates accounting[/ ]qos policy|maximum.*jobs|association.*limit|qos.*limit", text, re.I):
result = "account-limit"
diff --git a/experimental/CollectiveX/tests/test_deepep_v2_contract.py b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
index eed62bf4ae..2a082a6aea 100644
--- a/experimental/CollectiveX/tests/test_deepep_v2_contract.py
+++ b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
@@ -1986,6 +1986,23 @@ def test_private_scheduler_diagnostic_is_value_free(self) -> None:
self.assertEqual(result.returncode, 0, result.stderr)
self.assertIn("scheduler-diagnostic=controller", result.stderr)
+ log.write_text(
+ "salloc: error: Job submit/allocate failed: Requested time limit is invalid\n"
+ )
+ result = subprocess.run(
+ ["bash", "-c", 'source "$1"; cx_report_private_scheduler_failure', "_", common],
+ text=True,
+ capture_output=True,
+ env={
+ **os.environ,
+ "COLLECTIVEX_CANONICAL_GHA": "1",
+ "COLLECTIVEX_EXECUTION_ID": tag,
+ "CX_JOB_ROOT": str(root),
+ },
+ )
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertIn("scheduler-diagnostic=resource-request", result.stderr)
+
def test_private_runtime_logs_reject_traversal_and_symlinks(self) -> None:
common = str(ROOT / "runtime" / "common.sh")
for variable, value in (
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index a5b2b31d48..d42a45cd75 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -1539,6 +1539,7 @@ def test_launcher_signal_traps_cancel_the_allocation(self) -> None:
self.assertIn("secrets.COLLECTIVEX_MI300_CONFIG_V1", workflow)
self.assertIn('set(overlay["runners"]) != {"b200-dgxc"}', workflow)
self.assertIn('set(overlay["runners"]) != {"mi300x"}', workflow)
+ self.assertIn("matrix.sku == 'mi300x' && '180' || '300'", workflow)
self.assertIn('overlay_name != "COLLECTIVEX_MI300_CONFIG_CONTENT"', workflow)
self.assertIn('"partition", "squash_dir", "stage_dir", "exclude_nodes"', workflow)
self.assertIn("set(fields) - private_fields", workflow)
From 1d49de877db394808887c9848a9481267df8006d Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 7 Jul 2026 05:51:30 +0800
Subject: [PATCH 165/210] fix(collectivex): match MI300X allocation policy
---
experimental/CollectiveX/launchers/launch_mi-amds.sh | 5 ++++-
experimental/CollectiveX/tests/test_sampling_contract.py | 3 +++
2 files changed, 7 insertions(+), 1 deletion(-)
diff --git a/experimental/CollectiveX/launchers/launch_mi-amds.sh b/experimental/CollectiveX/launchers/launch_mi-amds.sh
index 47b64c41fe..ff3552aa34 100644
--- a/experimental/CollectiveX/launchers/launch_mi-amds.sh
+++ b/experimental/CollectiveX/launchers/launch_mi-amds.sh
@@ -90,8 +90,11 @@ cx_verify_registry_image "$IMAGE"
cx_set_failure_stage scheduler-allocation
command -v salloc >/dev/null || cx_die "salloc not found on this runner"
-allocation=(--partition="$PARTITION" --nodes="$NODES" --gres=gpu:"$GPN" --exclusive
+allocation=(--partition="$PARTITION" --nodes="$NODES" --gres=gpu:"$GPN"
--time="$TIME_MIN")
+if [ "$RUNNER" = mi355x ]; then
+ allocation+=(--exclusive)
+fi
if [ "$NODES" = 1 ]; then
allocation+=(--cpus-per-task="$CPUS_PER_TASK")
else
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index d42a45cd75..13c2ebb39a 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -521,6 +521,9 @@ def test_only_three_shared_launchers_are_registered(self) -> None:
common = (ROOT / "runtime" / "common.sh").read_text()
workflow = (ROOT.parent.parent / ".github" / "workflows" / "collectivex-sweep.yml").read_text()
+ amd_launcher = (ROOT / "launchers" / "launch_mi-amds.sh").read_text()
+ self.assertIn('if [ "$RUNNER" = mi355x ]; then\n allocation+=(--exclusive)', amd_launcher)
+ self.assertNotIn('--gres=gpu:"$GPN" --exclusive', amd_launcher)
self.assertNotIn("RUNNER_NAME", common)
self.assertNotIn("RUNNER_NAME:", workflow)
self.assertNotIn("flashinfer", capability.BACKENDS)
From 7574613ded558d3ad488e09f331f26fe5d3e3a27 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 7 Jul 2026 09:36:29 +0800
Subject: [PATCH 166/210] chore(collectivex): encrypt scheduler diagnostics
---
.github/workflows/collectivex-sweep.yml | 42 +++++++++++++++++++
.../tests/test_sampling_contract.py | 4 ++
2 files changed, 46 insertions(+)
diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml
index 450950c71e..a9cd48038d 100644
--- a/.github/workflows/collectivex-sweep.yml
+++ b/.github/workflows/collectivex-sweep.yml
@@ -602,6 +602,48 @@ jobs:
|| { echo "CollectiveX source is unavailable" >&2; exit 1; }
source experimental/CollectiveX/runtime/common.sh
cx_report_private_scheduler_failure
+ - name: Encrypt MI300X scheduler diagnostic
+ id: encrypt_scheduler_diagnostic
+ if: ${{ always() && matrix.sku == 'mi300x' && steps.source.outputs.prepared == 'true' && steps.sweep_shard.outcome == 'failure' }}
+ run: |
+ set -euo pipefail
+ umask 077
+ diagnostic_root="$CX_JOB_ROOT/encrypted-scheduler-diagnostic"
+ log="$CX_JOB_ROOT/control/private-logs/$COLLECTIVEX_EXECUTION_ID/scheduler-allocation.log"
+ [ -f "$log" ] && [ ! -L "$log" ] && [ "$(stat -c '%a' "$log")" = 600 ] \
+ || { echo 'MI300X private scheduler diagnostic is unavailable' >&2; exit 1; }
+ mkdir -m 700 "$diagnostic_root"
+ cat > "$diagnostic_root/public.pem" <<'EOF'
+ -----BEGIN PUBLIC KEY-----
+ MIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEAiT4793KA53HtshUZGjQ3
+ EA1hbL5UvAxPr0imI+VrJEJghMzHoH8KODAnsK0WY6d4aZSLuF6IYCVsPNfUcocE
+ zi/kPoo5tXD20vRZME53uqMknfkBysvyWvskecSh7dmOC7pbi5gFwiAXPGNnGjG8
+ P3N8WxjB9pLeTxKxSQLLL6xOWWWynbMYEB5DHs9ZSY0+LACa9d4XsC69jOTEWnOe
+ icngxehSXZl9ttfgKyEs9GmyylcYI/Z7bV5KeDP6cL2lO17sU2D3h52foZ7vYh8U
+ Z4+JEXdGAWLh9vahdBkSlSYX8Jc1KzekZLoIFAe7YtL56UHjMFpVERzWjuQys11w
+ WEbHBwrrzKyb3Y9O3zieLuo0FhzlfIorYuJFxpBvGXtXEQNr4yuYEDqbcR6Whf2y
+ 7raz4HstwwZMax9P53uJ8r9cg1o6GmySNFB4y/onsYcb8GaTAT3L+/iYEKnGSv6g
+ bsEJBUlnMOn60wcqMNaqNqiVNBbB65o62ZOz76p3V8MLAgMBAAE=
+ -----END PUBLIC KEY-----
+ EOF
+ openssl rand -hex 32 > "$diagnostic_root/passphrase"
+ openssl enc -aes-256-cbc -pbkdf2 -iter 200000 -salt \
+ -in "$log" -out "$diagnostic_root/scheduler.log.enc" \
+ -pass file:"$diagnostic_root/passphrase" >/dev/null 2>&1
+ openssl pkeyutl -encrypt -pubin -inkey "$diagnostic_root/public.pem" \
+ -pkeyopt rsa_padding_mode:oaep -pkeyopt rsa_oaep_md:sha256 \
+ -in "$diagnostic_root/passphrase" \
+ -out "$diagnostic_root/passphrase.enc" >/dev/null 2>&1
+ rm -f -- "$diagnostic_root/passphrase" "$diagnostic_root/public.pem"
+ echo 'prepared=true' >> "$GITHUB_OUTPUT"
+ - name: Upload encrypted MI300X scheduler diagnostic
+ if: ${{ always() && steps.encrypt_scheduler_diagnostic.outputs.prepared == 'true' }}
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: cxdiag-${{ matrix.id }}-${{ github.run_id }}-${{ github.run_attempt }}
+ path: ${{ env.CX_JOB_ROOT }}/encrypted-scheduler-diagnostic/*.enc
+ if-no-files-found: error
+ retention-days: 1
- name: Confirm allocation cleanup
id: allocation_cleanup
if: ${{ always() && steps.source.outputs.prepared == 'true' }}
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 13c2ebb39a..aff25aa8ea 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -1665,6 +1665,10 @@ def test_workflow_reconciles_a_recorded_allocation_after_launcher_loss(self) ->
self.assertIn('[ "$CX_JOB_PARENT" != /tmp ]', cleanup)
self.assertIn("- name: Classify private scheduler failure", workflow)
self.assertIn("cx_report_private_scheduler_failure", workflow)
+ self.assertIn("Encrypt MI300X scheduler diagnostic", workflow)
+ self.assertIn("rsa_padding_mode:oaep", workflow)
+ self.assertIn("retention-days: 1", workflow)
+ self.assertNotIn("BEGIN PRIVATE KEY", workflow)
for step in (
"sweep_shard", "allocation_cleanup", "artifact_safety",
"delivery_contracts", "stage_artifact", "upload_artifact",
From 891bdc2ef44a98ea4bcdf9d63aa61ca5a103579e Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 7 Jul 2026 09:39:19 +0800
Subject: [PATCH 167/210] fix(collectivex): preserve verified Slurm grants
---
.github/workflows/collectivex-sweep.yml | 42 -------------------
experimental/CollectiveX/runtime/common.sh | 17 ++++++++
.../tests/test_sampling_contract.py | 32 ++++++++++++--
3 files changed, 45 insertions(+), 46 deletions(-)
diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml
index a9cd48038d..450950c71e 100644
--- a/.github/workflows/collectivex-sweep.yml
+++ b/.github/workflows/collectivex-sweep.yml
@@ -602,48 +602,6 @@ jobs:
|| { echo "CollectiveX source is unavailable" >&2; exit 1; }
source experimental/CollectiveX/runtime/common.sh
cx_report_private_scheduler_failure
- - name: Encrypt MI300X scheduler diagnostic
- id: encrypt_scheduler_diagnostic
- if: ${{ always() && matrix.sku == 'mi300x' && steps.source.outputs.prepared == 'true' && steps.sweep_shard.outcome == 'failure' }}
- run: |
- set -euo pipefail
- umask 077
- diagnostic_root="$CX_JOB_ROOT/encrypted-scheduler-diagnostic"
- log="$CX_JOB_ROOT/control/private-logs/$COLLECTIVEX_EXECUTION_ID/scheduler-allocation.log"
- [ -f "$log" ] && [ ! -L "$log" ] && [ "$(stat -c '%a' "$log")" = 600 ] \
- || { echo 'MI300X private scheduler diagnostic is unavailable' >&2; exit 1; }
- mkdir -m 700 "$diagnostic_root"
- cat > "$diagnostic_root/public.pem" <<'EOF'
- -----BEGIN PUBLIC KEY-----
- MIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEAiT4793KA53HtshUZGjQ3
- EA1hbL5UvAxPr0imI+VrJEJghMzHoH8KODAnsK0WY6d4aZSLuF6IYCVsPNfUcocE
- zi/kPoo5tXD20vRZME53uqMknfkBysvyWvskecSh7dmOC7pbi5gFwiAXPGNnGjG8
- P3N8WxjB9pLeTxKxSQLLL6xOWWWynbMYEB5DHs9ZSY0+LACa9d4XsC69jOTEWnOe
- icngxehSXZl9ttfgKyEs9GmyylcYI/Z7bV5KeDP6cL2lO17sU2D3h52foZ7vYh8U
- Z4+JEXdGAWLh9vahdBkSlSYX8Jc1KzekZLoIFAe7YtL56UHjMFpVERzWjuQys11w
- WEbHBwrrzKyb3Y9O3zieLuo0FhzlfIorYuJFxpBvGXtXEQNr4yuYEDqbcR6Whf2y
- 7raz4HstwwZMax9P53uJ8r9cg1o6GmySNFB4y/onsYcb8GaTAT3L+/iYEKnGSv6g
- bsEJBUlnMOn60wcqMNaqNqiVNBbB65o62ZOz76p3V8MLAgMBAAE=
- -----END PUBLIC KEY-----
- EOF
- openssl rand -hex 32 > "$diagnostic_root/passphrase"
- openssl enc -aes-256-cbc -pbkdf2 -iter 200000 -salt \
- -in "$log" -out "$diagnostic_root/scheduler.log.enc" \
- -pass file:"$diagnostic_root/passphrase" >/dev/null 2>&1
- openssl pkeyutl -encrypt -pubin -inkey "$diagnostic_root/public.pem" \
- -pkeyopt rsa_padding_mode:oaep -pkeyopt rsa_oaep_md:sha256 \
- -in "$diagnostic_root/passphrase" \
- -out "$diagnostic_root/passphrase.enc" >/dev/null 2>&1
- rm -f -- "$diagnostic_root/passphrase" "$diagnostic_root/public.pem"
- echo 'prepared=true' >> "$GITHUB_OUTPUT"
- - name: Upload encrypted MI300X scheduler diagnostic
- if: ${{ always() && steps.encrypt_scheduler_diagnostic.outputs.prepared == 'true' }}
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
- with:
- name: cxdiag-${{ matrix.id }}-${{ github.run_id }}-${{ github.run_attempt }}
- path: ${{ env.CX_JOB_ROOT }}/encrypted-scheduler-diagnostic/*.enc
- if-no-files-found: error
- retention-days: 1
- name: Confirm allocation cleanup
id: allocation_cleanup
if: ${{ always() && steps.source.outputs.prepared == 'true' }}
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index a66ee616f4..4854e8e4cb 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -1322,6 +1322,18 @@ cx_reconcile_salloc_jobid() {
return 1
}
+cx_verify_salloc_jobid() {
+ local job_id="$1" queue_output line count=0
+ [[ "$job_id" =~ ^[1-9][0-9]*$ ]] || return 1
+ queue_output="$(squeue -h -j "$job_id" -o %A 2>/dev/null)" || return 1
+ while IFS= read -r line; do
+ [[ "$line" =~ ^[[:space:]]*$ ]] && continue
+ [[ "$line" =~ ^[[:space:]]*${job_id}[[:space:]]*$ ]] || return 1
+ count=$((count + 1))
+ done <<< "$queue_output"
+ [ "$count" -eq 1 ]
+}
+
# Allocate via salloc's stable grant message and assign JOB_ID in this shell.
# Raw scheduler output remains in the bounded private execution log.
cx_salloc_jobid() {
@@ -1369,6 +1381,11 @@ cx_salloc_jobid() {
CX_ALLOCATION_UNCERTAIN=0
fi
if [ "$salloc_rc" != 0 ]; then
+ if [ -n "$JOB_ID" ] && cx_verify_salloc_jobid "$JOB_ID"; then
+ cx_log "scheduler-request=verified-grant"
+ cx_record_allocation_jobid "$JOB_ID" || return 1
+ return 0
+ fi
cx_log "ERROR: scheduler-request=rejected"
if [ "$salloc_rc" -ge 128 ] && [ -z "$JOB_ID" ]; then
cx_fail_stage scheduler-allocation "$log"
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index aff25aa8ea..fe3954b670 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -1389,6 +1389,34 @@ def test_salloc_contains_a_shell_wrapper_that_calls_exit(self) -> None:
)
self.assertEqual(result.returncode, 0, result.stderr)
+ def test_salloc_accepts_a_live_grant_despite_wrapper_failure(self) -> None:
+ scenario = self._run_salloc_scenario(
+ "printf 'salloc: Granted job allocation 4242\\n' >&2; exit 1",
+ r'''
+ case " $* " in
+ *" -j 4242 "*) printf '4242\n' ;;
+ *) exit 2 ;;
+ esac
+ ''',
+ cleanup=False,
+ )
+ result = scenario["result"]
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertEqual(result.stdout, "0:4242:0\n")
+ self.assertIn("scheduler-request=verified-grant", result.stderr)
+ self.assertEqual(scenario["squeue_calls"], ["-h -j 4242 -o %A"])
+
+ def test_salloc_rejects_a_vanished_grant_after_wrapper_failure(self) -> None:
+ scenario = self._run_salloc_scenario(
+ "printf 'salloc: Granted job allocation 4242\\n' >&2; exit 1",
+ "exit 0",
+ cleanup=False,
+ )
+ result = scenario["result"]
+ self.assertEqual(result.returncode, 1)
+ self.assertEqual(result.stdout, "1:4242:0\n")
+ self.assertIn("scheduler-request=rejected", result.stderr)
+
def test_salloc_verified_rejection_is_cleanup_safe(self) -> None:
scenario = self._run_salloc_scenario("exit 1", "exit 0", cleanup=True)
result = scenario["result"]
@@ -1665,10 +1693,6 @@ def test_workflow_reconciles_a_recorded_allocation_after_launcher_loss(self) ->
self.assertIn('[ "$CX_JOB_PARENT" != /tmp ]', cleanup)
self.assertIn("- name: Classify private scheduler failure", workflow)
self.assertIn("cx_report_private_scheduler_failure", workflow)
- self.assertIn("Encrypt MI300X scheduler diagnostic", workflow)
- self.assertIn("rsa_padding_mode:oaep", workflow)
- self.assertIn("retention-days: 1", workflow)
- self.assertNotIn("BEGIN PRIVATE KEY", workflow)
for step in (
"sweep_shard", "allocation_cleanup", "artifact_safety",
"delivery_contracts", "stage_artifact", "upload_artifact",
From 210401d797b83793b6d0b892c017e6fe895c3682 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 7 Jul 2026 09:43:19 +0800
Subject: [PATCH 168/210] fix(collectivex): record AMD Slurm allocations
---
experimental/CollectiveX/runtime/common.sh | 35 +++++++++-------
.../tests/test_sampling_contract.py | 42 +++++++++++++++++++
2 files changed, 62 insertions(+), 15 deletions(-)
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 4854e8e4cb..e7a0a8446d 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -204,6 +204,21 @@ cx_fail_stage() {
return 1
}
+cx_job_root_is_safe() {
+ local root="$1"
+ if [[ "$root" =~ ^/tmp/inferencex-collectivex-[0-9]+-[0-9]+-[A-Za-z0-9._-]+$ ]]; then
+ :
+ elif [[ "$root" =~ ^/tmp/inferencex-collectivex-parent-([0-9]+)-([0-9]+)-([A-Za-z0-9._-]+)/inferencex-collectivex-([0-9]+)-([0-9]+)-([A-Za-z0-9._-]+)$ ]]; then
+ [ "${BASH_REMATCH[1]}" = "${BASH_REMATCH[4]}" ] \
+ && [ "${BASH_REMATCH[2]}" = "${BASH_REMATCH[5]}" ] \
+ && [ "${BASH_REMATCH[3]}" = "${BASH_REMATCH[6]}" ] || return 1
+ else
+ return 1
+ fi
+ [ -d "$root" ] && [ ! -L "$root" ] \
+ && [ "$(stat -c '%u:%a' "$root" 2>/dev/null)" = "$(id -u):700" ]
+}
+
# Runner-local deployment settings are strict JSON kept outside the checkout.
# Only the selected runner's allowlisted values are exported; the document is
# never sourced or evaluated as shell.
@@ -224,9 +239,7 @@ cx_load_operator_config() {
config_path="${COLLECTIVEX_OPERATOR_CONFIG:-${XDG_CONFIG_HOME:-${HOME}/.config}/inferencex/collectivex.json}"
if [ -n "${COLLECTIVEX_OPERATOR_CONFIG_CONTENT:-}" ]; then
umask 077
- if [[ "${CX_JOB_ROOT:-}" =~ ^/tmp/inferencex-collectivex-[0-9]+-[0-9]+-[A-Za-z0-9._-]+$ ]] \
- && [ -d "$CX_JOB_ROOT" ] && [ ! -L "$CX_JOB_ROOT" ] \
- && [ "$(stat -c '%u:%a' "$CX_JOB_ROOT" 2>/dev/null)" = "$(id -u):700" ]; then
+ if cx_job_root_is_safe "${CX_JOB_ROOT:-}"; then
config_path="$CX_JOB_ROOT/operator-config.json"
(set -C; : > "$config_path") 2>/dev/null \
|| cx_die "cannot create ephemeral runner configuration"
@@ -1409,9 +1422,7 @@ cx_record_allocation_jobid() {
local job_id="$1" root="${CX_JOB_ROOT:-}" path temporary
[[ "$job_id" =~ ^[1-9][0-9]*$ ]] || return 1
[ -n "$root" ] || return 0
- [[ "$root" =~ ^/tmp/inferencex-collectivex-[0-9]+-[0-9]+-[A-Za-z0-9._-]+$ ]] \
- && [ -d "$root" ] && [ ! -L "$root" ] \
- && [ "$(stat -c '%u:%a' "$root" 2>/dev/null)" = "$(id -u):700" ] || return 1
+ cx_job_root_is_safe "$root" || return 1
path="$root/allocation-job-id"
temporary="$(mktemp "$root/.allocation-job-id.XXXXXX")" || return 1
chmod 600 "$temporary" || { rm -f -- "$temporary"; return 1; }
@@ -1423,9 +1434,7 @@ cx_record_allocation_jobid() {
cx_clear_allocation_jobid() {
local root="${CX_JOB_ROOT:-}" path
[ -n "$root" ] || return 0
- [[ "$root" =~ ^/tmp/inferencex-collectivex-[0-9]+-[0-9]+-[A-Za-z0-9._-]+$ ]] \
- && [ -d "$root" ] && [ ! -L "$root" ] \
- && [ "$(stat -c '%u:%a' "$root" 2>/dev/null)" = "$(id -u):700" ] || return 1
+ cx_job_root_is_safe "$root" || return 1
path="$root/allocation-job-id"
[ ! -e "$path" ] || {
[ -f "$path" ] && [ ! -L "$path" ] \
@@ -1452,9 +1461,7 @@ cx_cancel_job() {
cx_write_cleanup_guard() {
local state="$1" root="${CX_JOB_ROOT:-}" safe unsafe
- [[ "$root" =~ ^/tmp/inferencex-collectivex-[0-9]+-[0-9]+-[A-Za-z0-9._-]+$ ]] \
- && [ -d "$root" ] && [ ! -L "$root" ] \
- && [ "$(stat -c '%u:%a' "$root" 2>/dev/null)" = "$(id -u):700" ] || return 0
+ cx_job_root_is_safe "$root" || return 0
safe="$root/cleanup-safe"
unsafe="$root/cleanup-unsafe"
umask 077
@@ -1470,9 +1477,7 @@ cx_write_cleanup_guard() {
# workflow step before isolated source cleanup is allowed.
cx_reconcile_recorded_allocation() {
local root="$1" path job_id
- [[ "$root" =~ ^/tmp/inferencex-collectivex-[0-9]+-[0-9]+-[A-Za-z0-9._-]+$ ]] \
- && [ -d "$root" ] && [ ! -L "$root" ] \
- && [ "$(stat -c '%u:%a' "$root" 2>/dev/null)" = "$(id -u):700" ] || return 1
+ cx_job_root_is_safe "$root" || return 1
export CX_JOB_ROOT="$root"
path="$root/allocation-job-id"
if [ ! -e "$path" ]; then
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index fe3954b670..a0f59407b6 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -1634,6 +1634,7 @@ def test_workflow_reconciles_a_recorded_allocation_after_launcher_loss(self) ->
self.assertTrue((root / "cleanup-safe").is_file())
self.assertFalse((root / "cleanup-unsafe").exists())
self.assertFalse((root / "allocation-job-id").exists())
+
prepare_start = workflow.index("- name: Prepare pinned backend source archive")
source_archive_step = workflow[
prepare_start:workflow.index("- uses: actions/upload-artifact", prepare_start)
@@ -1703,6 +1704,47 @@ def test_workflow_reconciles_a_recorded_allocation_after_launcher_loss(self) ->
cleanup.index('rm -rf -- "$CX_JOB_ROOT"'),
)
+ def test_allocation_record_accepts_only_matching_amd_nested_roots(self) -> None:
+ common = str(ROOT / "runtime" / "common.sh")
+ tag = f"{os.getpid()}-1-mi300x-test"
+ parent = Path("/tmp") / f"inferencex-collectivex-parent-{tag}"
+ shared = Path(tempfile.mkdtemp(prefix="collectivex-amd-shared-"))
+ root = shared / f"inferencex-collectivex-{tag}"
+ root.mkdir(mode=0o700)
+ try:
+ parent.symlink_to(shared, target_is_directory=True)
+ nested_root = parent / root.name
+ result = subprocess.run(
+ [
+ "bash", "-c",
+ 'source "$1"; '
+ 'stat() { case "$3" in */allocation-job-id) mode=600 ;; *) mode=700 ;; esac; '
+ 'printf "%s:%s\\n" "$(id -u)" "$mode"; }; '
+ 'export CX_JOB_ROOT="$2"; '
+ 'cx_record_allocation_jobid 6262; cat "$2/allocation-job-id"; '
+ 'cx_clear_allocation_jobid',
+ "_", common, str(nested_root),
+ ],
+ text=True,
+ capture_output=True,
+ )
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertEqual(result.stdout, "6262\n")
+ self.assertFalse((root / "allocation-job-id").exists())
+
+ mismatched = parent / f"inferencex-collectivex-{os.getpid()}-2-mi300x-test"
+ result = subprocess.run(
+ [
+ "bash", "-c", 'source "$1"; cx_job_root_is_safe "$2"',
+ "_", common, str(mismatched),
+ ],
+ )
+ self.assertNotEqual(result.returncode, 0)
+ finally:
+ parent.unlink(missing_ok=True)
+ root.rmdir()
+ shared.rmdir()
+
def test_v1_publication_requires_explicit_release_markers(self) -> None:
workflows = ROOT.parent.parent / ".github" / "workflows"
sweep = (workflows / "collectivex-sweep.yml").read_text()
From a39ac6af068af61d21ec46c15ba05352623f9291 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 7 Jul 2026 12:26:46 +0800
Subject: [PATCH 169/210] =?UTF-8?q?refactor(collectivex):=20drop=20Zipf=20?=
=?UTF-8?q?routing=20=E2=86=92=20uniform-only=20(R1)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Remove the Zipf routing distribution and the "routing" cohort kind from the
entire product surface, leaving uniform as the sole v1 routing. This is the
first of five folded simplifications ahead of the EPBackend ABC restructure.
- workload.py: delete the Zipf branch in canonical_routing_rows, tighten the
manifest allowlist and self-test to uniform.
- routing.py: docstring + self-test to a single distribution.
- schemas: routing enums → ["uniform"]; public-dataset cohort kind drops
"routing".
- sweep_matrix.py: remove the ep-routing-v1 suite contract; regenerate the
frozen execution-plan digests; both EPLB gates now read "eplb must be false"
(the Zipf clause was dead once routings is uniform-only). EPLB stays a
suite/case field for R2.
- suites.yaml: drop ep-routing-v1.
- publisher.py: remove the routing cohort kind end to end (control/eligibility/
sensitivity/promotion-anchor paths and helpers). Per-workload routing DATA
facts and routing_label are retained. Promotion counts corrected to the real
uniform-only matrix: library 24, system 4.
- ep_harness.py / make_workloads.py: argparse choices → ["uniform"].
- tests updated in lockstep; suite green at 221.
移除 Zipf 路由分布及“routing”对比群体（cohort kind），使 uniform 成为 v1 唯一路由。
这是 EPBackend 抽象基类重构前折叠的五项精简中的第一项。保留每个 workload 的路由
数据事实与 routing_label；晋升计数按真实 uniform-only 矩阵修正为 library 24、system 4。
EPLB 字段暂留待 R2 处理。测试全绿（221）。
---
experimental/CollectiveX/capability.py | 4 +-
experimental/CollectiveX/configs/suites.yaml | 12 -
experimental/CollectiveX/publisher.py | 221 +++---------------
.../schemas/public-dataset-v1.schema.json | 6 +-
.../schemas/raw-case-v1.schema.json | 4 +-
.../schemas/terminal-outcome-v1.schema.json | 2 +-
experimental/CollectiveX/sweep_matrix.py | 27 +--
experimental/CollectiveX/tests/ep_harness.py | 2 +-
.../CollectiveX/tests/make_workloads.py | 2 +-
experimental/CollectiveX/tests/routing.py | 8 +-
.../CollectiveX/tests/test_publisher.py | 201 +---------------
.../tests/test_qualification_planning.py | 2 +-
.../tests/test_sampling_contract.py | 73 +-----
experimental/CollectiveX/tests/workload.py | 33 +--
14 files changed, 75 insertions(+), 522 deletions(-)
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
index 651c77f245..1220d800e5 100644
--- a/experimental/CollectiveX/capability.py
+++ b/experimental/CollectiveX/capability.py
@@ -511,8 +511,8 @@ def _resolve_base(
topology = topology_for(sku, ep)
if topology is None or (nodes is not None and nodes != topology["nodes"]):
return False, f"{sku} does not register EP{ep} on {nodes} nodes"
- if routing not in {"uniform", "zipf"} or (eplb and routing != "zipf"):
- return False, "v1 routing is uniform or zipf, with EPLB only on zipf"
+ if routing != "uniform" or eplb:
+ return False, "v1 routing is uniform; EPLB is unavailable"
if platform["vendor"] not in implementation["vendors"]:
return False, f"{backend} does not support {platform['vendor']}"
sku_capability = implementation.get("sku_capabilities", {}).get(sku)
diff --git a/experimental/CollectiveX/configs/suites.yaml b/experimental/CollectiveX/configs/suites.yaml
index d04552672e..68a6a1502c 100644
--- a/experimental/CollectiveX/configs/suites.yaml
+++ b/experimental/CollectiveX/configs/suites.yaml
@@ -12,18 +12,6 @@ suites:
token_points_prefill: [256, 512]
required_publication: official
- ep-routing-v1:
- mode: normal
- workloads: [deepseek-v3-v1]
- platforms: [h100-dgxc, h200-dgxc, b300, b200-dgxc, gb300, gb200, mi300x, mi355x]
- ep_degrees: [8, 16]
- routings: [zipf]
- eplb: [false, true]
- phases: [decode, prefill]
- token_points_decode: [128]
- token_points_prefill: [512]
- required_publication: comparable-experimental
-
ep-low-latency-v1:
mode: low-latency
backends: [deepep, uccl]
diff --git a/experimental/CollectiveX/publisher.py b/experimental/CollectiveX/publisher.py
index 170b2b11b3..cda268d66e 100644
--- a/experimental/CollectiveX/publisher.py
+++ b/experimental/CollectiveX/publisher.py
@@ -40,16 +40,16 @@
PUBLISHER_POLICY = "collectivex-publisher-v1"
OUTCOMES = ("success", "unsupported", "failed", "invalid", "diagnostic")
REQUIRED_ALLOCATIONS = 3
-REQUIRED_COHORT_KINDS = ("library", "chip", "system", "routing")
+REQUIRED_COHORT_KINDS = ("library", "chip", "system")
PRECISION_COHORT_KINDS = (
"dispatch-precision", "combine-precision", "precision-pair",
)
-REQUIRED_PROMOTION_COHORT_COUNTS = {"library": 64, "system": 12, "routing": 58}
+REQUIRED_PROMOTION_COHORT_COUNTS = {"library": 24, "system": 4}
CANONICAL_FULL_V1_MATRIX_SHA256 = (
- "5894bab58d3deb2bcee51baa075ca5f5d324b4292ac1cef9f6bc08a07ab1d9a3"
+ "2108f920c4315a9f3292d57d81ac8873e0831b4ed97bc2f4604170d971c95396"
)
CANONICAL_FULL_V1_CASE_CATALOG_SHA256 = (
- "664b742da3e5ce9073a87da9191f2a488b6891a1d3a207ada264fcaa9a4b5280"
+ "78bc6969ee48a1c66bc732ef46d0fc367c5e4316758a0f9f482fe4935c9eef27"
)
P50_STABILITY_LIMIT = 1.10
P99_STABILITY_LIMIT = 1.25
@@ -226,23 +226,6 @@ def _metric_label(measure: str, statistic: str) -> str:
return f"{label} at {statistic} latency"
-def _routing_build_control(build: dict[str, Any]) -> dict[str, Any]:
- return {
- key: build[key]
- for key in (
- "routing_control_sha256", "image_digest", "source_sha", "squash_sha256",
- )
- }
-
-
-def _routing_implementation_mismatch(members: Sequence[dict[str, Any]]) -> bool:
- off_eplb_hashes = {
- member["build"]["implementation_contract_sha256"]
- for member in members if not member["workload"]["eplb"]
- }
- return len(off_eplb_hashes) > 1
-
-
def _public_case_factors(series: dict[str, Any]) -> dict[str, Any]:
workload = series["workload"]
system = series["system"]
@@ -359,13 +342,6 @@ def _public_cohort_factors(kind: str, item: dict[str, Any]) -> tuple[Any, Any]:
return {**common, "workload": workload, "source": build["source_sha"]}, [
item["system"]["sku"], item["backend"]["id"], item["resource"]["profile"]
]
- if kind == "routing":
- return (
- {**common, "backend": item["backend"], "system": item["system"],
- "resource": item["resource"], "build": _routing_build_control(build)},
- [workload["routing"], workload["eplb"],
- build["implementation_contract_sha256"]],
- )
if kind in PRECISION_COHORT_KINDS:
static_shape = {
key: workload[key]
@@ -989,7 +965,7 @@ def validate_public_dataset(doc: Any) -> dict[str, Any]:
raise PublisherError("library cohort contains non-library evidence")
if item["kind"] == "system" and roles != {"reference"}:
raise PublisherError("system cohort is not a portable reference comparison")
- if item["kind"] in {"chip", "routing", *PRECISION_COHORT_KINDS} and len(
+ if item["kind"] in {"chip", *PRECISION_COHORT_KINDS} and len(
{_canonical(member["backend"]) for member in members}
) != 1:
raise PublisherError(f"{item['kind']} cohort mixes backend implementations")
@@ -998,21 +974,6 @@ def validate_public_dataset(doc: Any) -> dict[str, Any]:
raise PublisherError(f"{item['kind']} cohort does not control its public factors")
if len({_canonical(value[1]) for value in public_factors}) < 2:
raise PublisherError(f"{item['kind']} cohort does not vary its declared contrast")
- if item["kind"] == "routing":
- if item["publication_tier"] != "comparable-experimental":
- raise PublisherError("routing cohort must be experimental")
- has_baseline = sum(
- member["workload"]["routing"] == "uniform"
- and not member["workload"]["eplb"]
- for member in members
- ) == 1
- missing_reason = "missing-uniform-baseline" in item["eligibility"]["reasons"]
- if has_baseline == missing_reason:
- raise PublisherError("routing baseline and eligibility reason disagree")
- mismatch = _routing_implementation_mismatch(members)
- mismatch_reason = "implementation-config-mismatch" in item["eligibility"]["reasons"]
- if mismatch != mismatch_reason:
- raise PublisherError("routing implementation control and eligibility disagree")
if item["kind"] in PRECISION_COHORT_KINDS:
if item["publication_tier"] != "comparable-experimental":
raise PublisherError("precision cohorts must be experimental")
@@ -1058,10 +1019,6 @@ def validate_public_dataset(doc: Any) -> dict[str, Any]:
["workload", "mode", "phase", "measurement", "source"],
["system", "backend", "resource"],
),
- "routing": (
- ["backend", "implementation-static-build", "system", "model-shape", "mode", "phase", "measurement", "resource"],
- ["workload.routing", "workload.eplb", "implementation-config"],
- ),
"dispatch-precision": (
[
"backend", "implementation-static-build", "system", "model-shape",
@@ -1226,7 +1183,7 @@ def validate_public_dataset(doc: Any) -> dict[str, Any]:
if (
cohort is None
or cohort["kind"] not in {
- "routing", "dispatch-precision", "combine-precision",
+ "dispatch-precision", "combine-precision",
}
or not cohort["eligibility"]["decision_grade"]
or item["publication_tier"] != cohort["publication_tier"]
@@ -1240,24 +1197,17 @@ def validate_public_dataset(doc: Any) -> dict[str, Any]:
raise PublisherError("sensitivity series differ from its cohort")
_validate_metric(item["metric"])
baseline_series = series[item["baseline_series_id"]]
- if cohort["kind"] == "routing":
- if (
- baseline_series["workload"]["routing"] != "uniform"
- or baseline_series["workload"]["eplb"]
- ):
- raise PublisherError("sensitivity baseline is not uniform without EPLB")
- else:
- axis = (
- "dispatch"
- if cohort["kind"] == "dispatch-precision"
- else "combine"
- )
- field = f"{axis}_precision"
- bf16 = identity.precision_profile(
- identity.V1_CONTROL_PRECISION_PROFILE
- )[axis]
- if _canonical(baseline_series["workload"][field]) != _canonical(bf16):
- raise PublisherError("precision sensitivity baseline is not BF16")
+ axis = (
+ "dispatch"
+ if cohort["kind"] == "dispatch-precision"
+ else "combine"
+ )
+ field = f"{axis}_precision"
+ bf16 = identity.precision_profile(
+ identity.V1_CONTROL_PRECISION_PROFILE
+ )[axis]
+ if _canonical(baseline_series["workload"][field]) != _canonical(bf16):
+ raise PublisherError("precision sensitivity baseline is not BF16")
_, baseline, _ = _metric_value(series[item["baseline_series_id"]], item["metric"])
_, candidate, _ = _metric_value(series[item["candidate_series_id"]], item["metric"])
if not math.isclose(item["signed_change_ratio"], (candidate - baseline) / baseline, rel_tol=1e-12):
@@ -1279,32 +1229,25 @@ def validate_public_dataset(doc: Any) -> dict[str, Any]:
for cohort in doc["cohorts"]:
if (
cohort["kind"] not in {
- "routing", "dispatch-precision", "combine-precision",
+ "dispatch-precision", "combine-precision",
}
or not cohort["eligibility"]["decision_grade"]
):
continue
members = [series[series_id] for series_id in cohort["series_ids"]]
- if cohort["kind"] == "routing":
- baseline = next((
- member for member in members
- if member["workload"]["routing"] == "uniform"
- and not member["workload"]["eplb"]
- ), None)
- else:
- axis = (
- "dispatch"
- if cohort["kind"] == "dispatch-precision"
- else "combine"
- )
- field = f"{axis}_precision"
- bf16 = identity.precision_profile(
- identity.V1_CONTROL_PRECISION_PROFILE
- )[axis]
- baseline = next((
- member for member in members
- if _canonical(member["workload"][field]) == _canonical(bf16)
- ), None)
+ axis = (
+ "dispatch"
+ if cohort["kind"] == "dispatch-precision"
+ else "combine"
+ )
+ field = f"{axis}_precision"
+ bf16 = identity.precision_profile(
+ identity.V1_CONTROL_PRECISION_PROFILE
+ )[axis]
+ baseline = next((
+ member for member in members
+ if _canonical(member["workload"][field]) == _canonical(bf16)
+ ), None)
if baseline is None:
continue
tokens = set.intersection(*(
@@ -3246,24 +3189,6 @@ def _cohort_control(
control = {**common, "workload": workload, "source": source}
varying = [series["system"]["sku"], series["backend"]["id"], series["resource"]["profile"]]
return control, ["workload", "mode", "phase", "measurement", "source"], ["system", "backend", "resource"], varying
- if kind == "routing":
- control = {
- **common,
- "backend": series["backend"],
- "system": series["system"],
- "resource": series["resource"],
- "build": _routing_build_control(binary_build),
- }
- varying = [
- workload["routing"], workload["eplb"],
- binary_build["implementation_contract_sha256"],
- ]
- return (
- control,
- ["backend", "implementation-static-build", "system", "model-shape", "mode", "phase", "measurement", "resource"],
- ["workload.routing", "workload.eplb", "implementation-config"],
- varying,
- )
if kind in PRECISION_COHORT_KINDS:
control, variant = _public_cohort_factors(kind, series)
if kind == "dispatch-precision":
@@ -3424,20 +3349,7 @@ def build_decisions(
extra.add("incomplete-aligned-repeats")
if tokens and not _bootstrap_inputs_ready(members, internals, tokens):
extra.add("missing-trial-blocks")
- if kind == "routing" and sum(
- member["workload"]["routing"] == "uniform"
- and not member["workload"]["eplb"]
- for member in members
- ) != 1:
- extra.add("missing-uniform-baseline")
- if kind == "routing" and {
- (member["workload"]["routing"], member["workload"]["eplb"])
- for member in members
- } != {("uniform", False), ("zipf", False), ("zipf", True)}:
- extra.add("incomplete-routing-anchors")
- if kind == "routing" and _routing_implementation_mismatch(members):
- extra.add("implementation-config-mismatch")
- endpoint_contrast = kind == "routing" or kind in PRECISION_COHORT_KINDS
+ endpoint_contrast = kind in PRECISION_COHORT_KINDS
if not tokens or (not endpoint_contrast and not same_points):
extra.add("unmatched-token-coverage")
if kind in {"dispatch-precision", "combine-precision"}:
@@ -3496,10 +3408,6 @@ def build_decisions(
f"Reference EP{first['system']['ep_size']} / {first['mode']} / "
f"{first['phase']} / {routing_label}"
),
- "routing": (
- f"{first['system']['sku'].upper()} / {first['backend']['label']} / "
- f"EP{first['system']['ep_size']} / {first['mode']} / {first['phase']}"
- ),
"dispatch-precision": (
f"{first['system']['sku'].upper()} / {first['backend']['label']} / "
f"EP{first['system']['ep_size']} / {first['mode']} / {first['phase']}"
@@ -3616,48 +3524,6 @@ def build_decisions(
"publication_tier": cohort["publication_tier"],
"eligibility": cohort["eligibility"],
})
- if cohort["kind"] == "routing":
- baseline = next(
- (member for member in members
- if member["workload"]["routing"] == "uniform" and not member["workload"]["eplb"]),
- None,
- )
- if baseline:
- for candidate in members:
- if candidate is baseline:
- continue
- for token in tokens:
- for measure, objective in (
- ("latency_us", "min"),
- ("activation_data_rate_gbps_at_latency_percentile", "max"),
- ("total_logical_data_rate_gbps_at_latency_percentile", "max"),
- ):
- for statistic in ("p50", "p99"):
- metric = {
- "operation": "roundtrip", "statistic": statistic,
- "measure": measure, "objective": objective,
- "tokens_per_rank": token, "phase": baseline["phase"],
- }
- _, base_value, _ = _metric_value(baseline, metric)
- _, candidate_value, _ = _metric_value(candidate, metric)
- sensitivity_id = _derived_id("cxsensitivity-v1-", {
- "baseline": baseline["series_id"], "candidate": candidate["series_id"],
- "cohort": cohort["cohort_id"], "metric": metric,
- })
- sensitivities.append({
- "sensitivity_id": sensitivity_id,
- "cohort_id": cohort["cohort_id"],
- "label": (
- f"Routing sensitivity: "
- f"{_metric_label(measure, statistic)} T={token}"
- ),
- "baseline_series_id": baseline["series_id"],
- "candidate_series_id": candidate["series_id"],
- "metric": metric,
- "signed_change_ratio": (candidate_value - base_value) / base_value,
- "publication_tier": cohort["publication_tier"],
- "eligibility": cohort["eligibility"],
- })
if cohort["kind"] in {"dispatch-precision", "combine-precision"}:
axis = (
"dispatch"
@@ -3792,29 +3658,6 @@ def _require_promotion_cohorts(
f"promotion requires all {expected_chips} derived chip cohorts to be decision-grade"
)
- by_id = {item["series_id"]: item for item in series}
- anchors = {("uniform", False), ("zipf", False), ("zipf", True)}
- for cohort in (
- item for item in cohorts
- if item["kind"] == "routing" and item["eligibility"]["decision_grade"]
- ):
- observed = {
- (by_id[series_id]["workload"]["routing"], by_id[series_id]["workload"]["eplb"]):
- by_id[series_id]
- for series_id in cohort["series_ids"]
- }
- if len(cohort["series_ids"]) != len(anchors) or set(observed) != anchors:
- raise PublisherError(
- "promotion routing cohorts require exact uniform, zipf, and zipf+EPLB anchors"
- )
- if (
- observed[("uniform", False)]["build"]["implementation_contract_sha256"]
- != observed[("zipf", False)]["build"]["implementation_contract_sha256"]
- ):
- raise PublisherError(
- "promotion routing cohorts require identical off-EPLB generated implementation"
- )
-
def _require_promotion_series(series: Sequence[dict[str, Any]]) -> None:
if not series or any(item["status"] != "decision-grade" for item in series):
diff --git a/experimental/CollectiveX/schemas/public-dataset-v1.schema.json b/experimental/CollectiveX/schemas/public-dataset-v1.schema.json
index b27d3f2dac..cab8c5a80e 100644
--- a/experimental/CollectiveX/schemas/public-dataset-v1.schema.json
+++ b/experimental/CollectiveX/schemas/public-dataset-v1.schema.json
@@ -308,7 +308,7 @@
"backend_generation": {"$ref":"#/$defs/nullableLabel"},
"mode": {"enum": ["normal","low-latency"]},
"phase": {"enum": ["decode","prefill"]},
- "routing": {"enum":["uniform","zipf"]},
+ "routing": {"enum":["uniform"]},
"eplb": {"type":"boolean"},
"precision_profile": {"$ref":"#/$defs/precisionProfile"},
"dispatch_precision": {"$ref":"#/$defs/communicationAxis"},
@@ -642,7 +642,7 @@
"hidden": {"type": "integer","minimum": 1},
"top_k": {"type": "integer","minimum": 1},
"experts": {"type": "integer","minimum": 1},
- "routing": {"enum": ["uniform","zipf"]},
+ "routing": {"enum": ["uniform"]},
"eplb": {"type": "boolean"},
"precision_profile": {"$ref":"#/$defs/precisionProfile"},
"dispatch_precision": {"$ref":"#/$defs/communicationAxis"},
@@ -768,7 +768,7 @@
],
"properties": {
"cohort_id": {"type": "string","pattern": "^cxcohort-v1-[0-9a-f]{64}$"},
- "kind": {"enum": ["library","chip","system","routing","dispatch-precision","combine-precision","precision-pair"]},
+ "kind": {"enum": ["library","chip","system","dispatch-precision","combine-precision","precision-pair"]},
"label": {"$ref": "#/$defs/label"},
"description": {"$ref": "#/$defs/label"},
"series_ids": {"type": "array","minItems": 2,"uniqueItems": true,"items": {"$ref": "#/$defs/seriesId"}},
diff --git a/experimental/CollectiveX/schemas/raw-case-v1.schema.json b/experimental/CollectiveX/schemas/raw-case-v1.schema.json
index 9889d1b752..349e2fe997 100644
--- a/experimental/CollectiveX/schemas/raw-case-v1.schema.json
+++ b/experimental/CollectiveX/schemas/raw-case-v1.schema.json
@@ -355,7 +355,7 @@
"phase": {"enum":["decode","prefill"]},
"precision_profile": {"$ref":"#/$defs/precision_profile"},
"required_publication": {"enum":["official","comparable-experimental"]},
- "routing": {"enum":["uniform","zipf"]},
+ "routing": {"enum":["uniform"]},
"samples_per_point": {"const":512},
"scale_out_transport": {"oneOf":[{"type":"null"},{"maxLength":128,"pattern":"^[a-z0-9][a-z0-9_.-]*$","type":"string"}]},
"scale_up_domain": {"minimum":1,"type":"integer"},
@@ -525,7 +525,7 @@
"kernel_gen": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]},
"num_logical_experts": {"minimum":1,"type":"integer"},
"precision_profile": {"$ref":"#/$defs/precision_profile"},
- "routing": {"enum":["uniform","zipf"]},
+ "routing": {"enum":["uniform"]},
"topk": {"minimum":1,"type":"integer"}
},
"required": [
diff --git a/experimental/CollectiveX/schemas/terminal-outcome-v1.schema.json b/experimental/CollectiveX/schemas/terminal-outcome-v1.schema.json
index 51a190cebb..c70d6b82a8 100644
--- a/experimental/CollectiveX/schemas/terminal-outcome-v1.schema.json
+++ b/experimental/CollectiveX/schemas/terminal-outcome-v1.schema.json
@@ -318,7 +318,7 @@
"phase": {"enum": ["decode","prefill"]},
"precision_profile": {"$ref":"#/$defs/precisionProfile"},
"required_publication": {"enum": ["official","comparable-experimental","diagnostic"]},
- "routing": {"enum": ["uniform","zipf"]},
+ "routing": {"enum": ["uniform"]},
"samples_per_point": {"const": 512},
"scale_out_transport": {"oneOf": [{"type": "null"},{"$ref": "#/$defs/safeId"}]},
"scale_up_domain": {"type": "integer","minimum": 1},
diff --git a/experimental/CollectiveX/sweep_matrix.py b/experimental/CollectiveX/sweep_matrix.py
index 162daabb24..11dff01989 100644
--- a/experimental/CollectiveX/sweep_matrix.py
+++ b/experimental/CollectiveX/sweep_matrix.py
@@ -51,17 +51,6 @@
"prefill": (256, 512),
},
},
- "ep-routing-v1": {
- "mode": "normal",
- "publication": "comparable-experimental",
- "coordinates": {
- ("normal", "decode", "zipf", False),
- ("normal", "decode", "zipf", True),
- ("normal", "prefill", "zipf", False),
- ("normal", "prefill", "zipf", True),
- },
- "ladders": {"decode": (128,), "prefill": (512,)},
- },
"ep-low-latency-v1": {
"mode": "low-latency",
"publication": "official",
@@ -105,9 +94,9 @@
)
QUALIFICATION_INDICES = range(1, 4)
CANONICAL_V1_EXECUTION_PLAN_SHA256 = {
- 1: "b54cdf53496868424848860225f1ea84c87ce60e115c83de4ce2f5aae36625dc",
- 2: "4fbd22df187578c88b1552747955e85fe97d576d31e0b5be5501840ca5dfb3e6",
- 3: "e03105b970af3d8f5f0788c5c04ed1f53761460bb432109152bdb22b7b99f165",
+ 1: "cc6d65b4424630a80ca76fc792ca49142820315601d05979756a03f5e9d920f4",
+ 2: "ce6856db2953a019414629f3130b58361feeae7f7e425f311dbf65b1aea7c1b0",
+ 3: "c5826efac5d91bb5b333bbb721f534ed1cdf7a6c80185fa8c77b2d2c0fa2cf74",
}
@@ -295,10 +284,10 @@ def validate_config_documents(
suite["platforms"], f"suite {name}.platforms", str, set(cap.PLATFORMS)
)
phases = _list(suite["phases"], f"suite {name}.phases", str, {"decode", "prefill"})
- routings = _list(suite["routings"], f"suite {name}.routings", str, {"uniform", "zipf"})
+ routings = _list(suite["routings"], f"suite {name}.routings", str, {"uniform"})
eplb = _list(suite.get("eplb", [False]), f"suite {name}.eplb", bool)
- if True in eplb and routings != ["zipf"]:
- raise SystemExit(f"suite {name}: EPLB is only valid for Zipf routing")
+ if True in eplb:
+ raise SystemExit(f"suite {name}: EPLB is unavailable for v1 uniform routing")
if suite["required_publication"] not in {"official", "comparable-experimental"}:
raise SystemExit(f"suite {name}.required_publication is invalid")
if suite["required_publication"] != contract["publication"]:
@@ -974,9 +963,9 @@ def validate_shard_control(
raise MatrixError(f"case {index} has invalid warmup semantics")
if case["phase"] not in {"decode", "prefill"}:
raise MatrixError(f"case {index} has invalid phase")
- if case["routing"] not in {"uniform", "zipf"}:
+ if case["routing"] != "uniform":
raise MatrixError(f"case {index} has invalid routing")
- if not isinstance(case["eplb"], bool) or (case["eplb"] and case["routing"] != "zipf"):
+ if not isinstance(case["eplb"], bool) or case["eplb"]:
raise MatrixError(f"case {index} has invalid EPLB setting")
if not isinstance(case["canonical"], bool) or not case["canonical"]:
raise MatrixError(f"case {index} must use a canonical workload")
diff --git a/experimental/CollectiveX/tests/ep_harness.py b/experimental/CollectiveX/tests/ep_harness.py
index 5e8736bb65..18018ed4e8 100644
--- a/experimental/CollectiveX/tests/ep_harness.py
+++ b/experimental/CollectiveX/tests/ep_harness.py
@@ -176,7 +176,7 @@ def add_common_args(ap: argparse.ArgumentParser) -> None:
ap.add_argument("--hidden", type=int, default=7168)
ap.add_argument("--topk", type=int, default=8)
ap.add_argument("--experts", type=int, default=256, help="TOTAL experts (fixed across EP degrees)")
- ap.add_argument("--routing", default="uniform", choices=["uniform", "zipf"])
+ ap.add_argument("--routing", default="uniform", choices=["uniform"])
# EPLB (Expert-Parallel Load Balancer): replicate hot experts onto redundant physical
# slots + balanced-place so per-rank load equalizes. A pure routing-trace transform
# (tests/eplb.py); experts becomes num_logical+redundant. The remedy for `zipf` skew.
diff --git a/experimental/CollectiveX/tests/make_workloads.py b/experimental/CollectiveX/tests/make_workloads.py
index 862c3d0375..5ceaba70f0 100644
--- a/experimental/CollectiveX/tests/make_workloads.py
+++ b/experimental/CollectiveX/tests/make_workloads.py
@@ -58,7 +58,7 @@ def main() -> int:
ap = argparse.ArgumentParser(description="Generate canonical CollectiveX workloads")
ap.add_argument("--out-dir", help="required unless --id-only")
ap.add_argument("--workload", help="named manifest in configs/workloads.yaml (sets hidden/topk/experts)")
- ap.add_argument("--routing", default="uniform", choices=["uniform", "zipf"])
+ ap.add_argument("--routing", default="uniform", choices=["uniform"])
ap.add_argument("--ep", type=int, required=True, help="ep_size (global_tokens = T * ep)")
ap.add_argument("--hidden", type=int, help="override (default 7168, or the --workload's hidden)")
ap.add_argument("--topk", type=int, help="override (default 8, or the --workload's topk)")
diff --git a/experimental/CollectiveX/tests/routing.py b/experimental/CollectiveX/tests/routing.py
index ee4eb12a8c..08d2cddae2 100644
--- a/experimental/CollectiveX/tests/routing.py
+++ b/experimental/CollectiveX/tests/routing.py
@@ -8,12 +8,11 @@
are per-rank (same rank ⇒ same x on any platform), so a given global token id has
identical activation everywhere without materializing a global activation tensor.
-The v1 suite keeps two routing distributions:
+The v1 suite uses a single routing distribution:
* uniform — top-k distinct experts drawn uniformly per token. The DEFAULT.
Expected fan-out for top-k=8, 256 experts, EP8 (32 experts/rank) ≈
8·(1 − C(224,8)/C(256,8)) ≈ 5.3 ranks/token. Load ~ Poisson.
- * zipf — expert popularity proportional to 1/rank, producing expert/rank load skew.
Always publish the realized fan-out so the workload is never misread again
(`routing_stats`).
@@ -208,10 +207,9 @@ def _cv(t):
import sys
E, TOPK, EPR, GT = 256, 8, 32, 4096
ui, _ = build_global_routing(GT, E, TOPK, "uniform", 67)
- zi, _ = build_global_routing(GT, E, TOPK, "zipf", 67)
assert all(len(set(row.tolist())) == TOPK for row in ui[:16])
- uniform, zipf = routing_stats(ui, E, EPR), routing_stats(zi, E, EPR)
- assert uniform["hotspot_ratio"] < zipf["hotspot_ratio"]
+ uniform = routing_stats(ui, E, EPR)
+ assert uniform["hotspot_ratio"] >= 1.0
dev = torch.device("cpu")
first = rank_activations(8, 256, 67, 0, dev, dtype=torch.float32)
second = rank_activations(8, 256, 67, 0, dev, dtype=torch.float32)
diff --git a/experimental/CollectiveX/tests/test_publisher.py b/experimental/CollectiveX/tests/test_publisher.py
index e088dc2acd..029db83c8c 100644
--- a/experimental/CollectiveX/tests/test_publisher.py
+++ b/experimental/CollectiveX/tests/test_publisher.py
@@ -822,8 +822,6 @@ def _promoted_dataset(*, precision_profiles: tuple[str, ...] = ()) -> dict:
("chip-peer", "deepep", "h200-dgxc", False, None),
("system-one", "nccl-ep", None, True, None),
("system-two", "nccl-ep", "h200-dgxc", True, None),
- ("routing-zipf", "deepep", None, False, None),
- ("routing-zipf-eplb", "deepep", None, False, None),
]
specifications.extend(
(f"precision-{index}", "deepep", None, False, precision_profile)
@@ -845,26 +843,6 @@ def _promoted_dataset(*, precision_profiles: tuple[str, ...] = ()) -> dict:
})
if reference:
item["backend"]["role"] = "reference"
- if seed.startswith("routing-zipf"):
- item["suite"] = "ep-routing-v1"
- item["publication_tier"] = "comparable-experimental"
- item["workload"]["routing"] = "zipf"
- if seed == "routing-zipf-eplb":
- item["workload"]["eplb"] = True
- plan, calibration = contracts._expected_eplb_calibration(
- "zipf", 7168, 8, 256, 288, item["system"]["ep_size"], 67, 2048
- )
- item["eplb"] = {
- "enabled": True, **calibration, "planner": "greedy-rank-major-v1",
- "mapping_sha256": contracts.eplb_contract.mapping_hash(plan),
- "logical_experts": 256, "physical_experts": 288,
- "redundant_experts": 32, "reference_tokens_per_rank": 2048,
- "replicated_experts": plan["replicated_experts"],
- "max_replicas": plan["max_replicas"],
- "imbalance_before": plan["imbalance_before"],
- "imbalance_after": plan["imbalance_after"],
- }
- item["build"]["implementation_contract_sha256"] = "8" * 64
if precision_profile is not None:
precision = identity.precision_profile(precision_profile)
item["suite"] = "ep-precision-normal-v1"
@@ -1039,7 +1017,7 @@ def _promoted_dataset(*, precision_profiles: tuple[str, ...] = ()) -> dict:
def _cohort_counts(dataset: dict) -> dict[str, int]:
return {
kind: sum(item["kind"] == kind for item in dataset["cohorts"])
- for kind in ("library", "system", "routing")
+ for kind in ("library", "system")
}
@@ -1772,22 +1750,15 @@ def test_promotion_requires_an_eligible_cohort_for_every_comparison_kind(self) -
self.assertTrue(ineligible)
anchor_series = [
{
- "series_id": name,
- "workload": {"routing": routing, "eplb": eplb},
+ "series_id": "uniform",
+ "workload": {"routing": "uniform", "eplb": False},
"build": {"implementation_contract_sha256": "1" * 64},
}
- for name, routing, eplb in (
- ("uniform", "uniform", False),
- ("zipf", "zipf", False),
- ("zipf-eplb", "zipf", True),
- )
]
required = eligible + [
{
"kind": kind,
"eligibility": {"decision_grade": True},
- **({"series_ids": [item["series_id"] for item in anchor_series]}
- if kind == "routing" else {}),
}
for kind in publisher.REQUIRED_COHORT_KINDS
if kind != "library"
@@ -1809,7 +1780,7 @@ def test_promotion_requires_an_eligible_cohort_for_every_comparison_kind(self) -
if item["kind"] != kind or not item["eligibility"]["decision_grade"]
], anchor_series)
- def test_promotion_requires_exact_counts_and_routing_anchors(self) -> None:
+ def test_promotion_requires_exact_counts(self) -> None:
dataset = _promoted_dataset()
counts = _cohort_counts(dataset)
with mock.patch.object(
@@ -1818,33 +1789,6 @@ def test_promotion_requires_exact_counts_and_routing_anchors(self) -> None:
publisher._require_promotion_cohorts(
dataset["cohorts"], dataset["series"]
)
- routing = next(
- item for item in dataset["cohorts"] if item["kind"] == "routing"
- )
- eplb = next(
- item for item in dataset["series"]
- if item["series_id"] in routing["series_ids"]
- and item["workload"]["eplb"]
- )
- eplb["workload"]["eplb"] = False
- with self.assertRaisesRegex(publisher.PublisherError, "exact uniform"):
- publisher._require_promotion_cohorts(
- dataset["cohorts"], dataset["series"]
- )
-
- dataset = _promoted_dataset()
- routing = next(item for item in dataset["cohorts"] if item["kind"] == "routing")
- zipf = next(
- item for item in dataset["series"]
- if item["series_id"] in routing["series_ids"]
- and item["workload"]["routing"] == "zipf"
- and not item["workload"]["eplb"]
- )
- zipf["build"]["implementation_contract_sha256"] = "f" * 64
- with mock.patch.object(
- publisher, "REQUIRED_PROMOTION_COHORT_COUNTS", counts
- ), self.assertRaisesRegex(publisher.PublisherError, "identical off-EPLB"):
- publisher._require_promotion_cohorts(dataset["cohorts"], dataset["series"])
wrong_counts = {**counts, "library": counts["library"] + 1}
with mock.patch.object(
@@ -2024,11 +1968,10 @@ def test_workflow_matrix_and_catalog_digests_do_not_drift(self) -> None:
if item["disposition"] == "unsupported"
),
),
- (49, 748, 387, 361, 1740, 877, 863),
+ (49, 364, 191, 173, 1356, 681, 675),
)
library: dict[tuple, set[str]] = {}
system: dict[tuple, set[str]] = {}
- routing: dict[tuple, list[tuple[str, bool]]] = {}
for requested in matrix["requested_cases"]:
if requested["disposition"] != "runnable":
continue
@@ -2046,18 +1989,10 @@ def test_workflow_matrix_and_catalog_digests_do_not_drift(self) -> None:
)
else:
system.setdefault((shape, route), set()).add(requested["sku"])
- routing.setdefault(
- (requested["sku"], case["backend"], shape), []
- ).append(route)
- anchors = {("uniform", False), ("zipf", False), ("zipf", True)}
self.assertEqual(
{
"library": sum(len(variants) >= 2 for variants in library.values()),
"system": sum(len(variants) >= 2 for variants in system.values()),
- "routing": sum(
- len(variants) == 3 and set(variants) == anchors
- for variants in routing.values()
- ),
},
publisher.REQUIRED_PROMOTION_COHORT_COUNTS,
)
@@ -2360,37 +2295,6 @@ def test_routing_and_eplb_facts_must_match_across_repeats(self) -> None:
with self.assertRaisesRegex(publisher.PublisherError, "routing differs"):
publisher._exact_repeat_value([facts, changed], "routing")
- dataset = _promoted_dataset()
- dataset["promotion"]["status"] = "diagnostic"
- eplb = next(item for item in dataset["series"] if item["eplb"]["enabled"])
- eplb["points"][0]["routing"]["empty_expert_count"] = 280
- publisher.validate_public_dataset(dataset)
- eplb["points"][0]["routing"]["empty_expert_count"] = 288
- with self.assertRaisesRegex(publisher.PublisherError, "routing/load facts"):
- publisher.validate_public_dataset(dataset)
-
- for field, value in (
- ("mapping_sha256", "0" * 64),
- ("redundant_experts", 31),
- ("replicated_experts", 1),
- ("max_replicas", 2),
- ("replicated_experts", 257),
- ("max_replicas", 999),
- ("imbalance_after", 0.4),
- ("planner", "different-planner"),
- ("reference_tokens_per_rank", 1024),
- ):
- broken = _promoted_dataset()
- broken["promotion"]["status"] = "diagnostic"
- descriptor = next(
- item["eplb"] for item in broken["series"] if item["eplb"]["enabled"]
- )
- descriptor[field] = value
- with self.subTest(eplb_field=field), self.assertRaisesRegex(
- publisher.PublisherError, "EPLB descriptor"
- ):
- publisher.validate_public_dataset(broken)
-
def test_publisher_owns_stable_rankings_and_recommendations(self) -> None:
fast, fast_internal = _series("fast", "deepep", decision_grade=True)
slow, slow_internal = _series("slow", "uccl", decision_grade=True)
@@ -2426,101 +2330,6 @@ def test_publisher_owns_stable_rankings_and_recommendations(self) -> None:
for item in cohorts
))
- def test_routing_evidence_is_experimental_and_not_a_configuration_recommendation(self) -> None:
- dataset = _promoted_dataset()
- routing = next(item for item in dataset["cohorts"] if item["kind"] == "routing")
- members = [
- item for item in dataset["series"]
- if item["series_id"] in routing["series_ids"]
- ]
- self.assertEqual(
- {(item["workload"]["routing"], item["workload"]["eplb"]) for item in members},
- {("uniform", False), ("zipf", False), ("zipf", True)},
- )
- self.assertIn("implementation-static-build", routing["controlled_factors"])
- self.assertIn("resource", routing["controlled_factors"])
- self.assertEqual(
- routing["varying_factors"],
- ["workload.routing", "workload.eplb", "implementation-config"],
- )
- self.assertEqual(
- len({item["build"]["routing_control_sha256"] for item in members}),
- 1,
- )
- self.assertGreater(
- len({item["build"]["implementation_contract_sha256"] for item in members}),
- 1,
- )
- self.assertEqual(len({json.dumps(item["resource"], sort_keys=True) for item in members}), 1)
- self.assertEqual(routing["publication_tier"], "comparable-experimental")
- self.assertTrue(any(
- item["cohort_id"] == routing["cohort_id"] for item in dataset["rankings"]
- ))
- self.assertFalse(any(
- item["cohort_id"] == routing["cohort_id"] for item in dataset["recommendations"]
- ))
- self.assertTrue(all(
- item["publication_tier"] == "official"
- for item in dataset["recommendations"]
- ))
- self.assertFalse(any(
- dataset_cohort["publication_tier"] == "comparable-experimental"
- and item["cohort_id"] == dataset_cohort["cohort_id"]
- for item in dataset["recommendations"]
- for dataset_cohort in dataset["cohorts"]
- ))
- self.assertTrue(all(
- item["publication_tier"] == "comparable-experimental"
- for item in dataset["sensitivities"]
- if item["cohort_id"] == routing["cohort_id"]
- ))
-
- def test_routing_implementation_mismatch_blocks_all_decisions(self) -> None:
- dataset = _promoted_dataset()
- published = next(item for item in dataset["cohorts"] if item["kind"] == "routing")
- members = [
- item for item in dataset["series"]
- if item["series_id"] in published["series_ids"]
- ]
- zipf = next(
- item for item in members
- if item["workload"]["routing"] == "zipf" and not item["workload"]["eplb"]
- )
- zipf["build"]["implementation_contract_sha256"] = "f" * 64
- internals = {}
- for member in members:
- point = member["points"][0]
- roundtrip = point["components"]["roundtrip"]
- metrics = {
- "latency_us": {
- name: roundtrip["latency_us"][name] for name in ("p50", "p99")
- },
- **{
- field: {
- name: roundtrip[field][name] for name in ("p50", "p99")
- }
- for field in (
- "activation_data_rate_gbps_at_latency_percentile",
- "total_logical_data_rate_gbps_at_latency_percentile",
- )
- },
- }
- internals[member["series_id"]] = {
- "run_metrics": {
- str(run): {point["tokens_per_rank"]: metrics}
- for run in range(3)
- }
- }
- cohorts, rankings, recommendations, sensitivities = publisher.build_decisions(
- members, internals
- )
- routing = next(item for item in cohorts if item["kind"] == "routing")
- self.assertFalse(routing["eligibility"]["decision_grade"])
- self.assertIn(
- "implementation-config-mismatch", routing["eligibility"]["reasons"]
- )
- self.assertEqual((rankings, recommendations, sensitivities), ([], [], []))
-
def test_promoted_series_fields_are_bound_to_case_and_series_identities(self) -> None:
dataset = _promoted_dataset()
changed = copy.deepcopy(dataset)
diff --git a/experimental/CollectiveX/tests/test_qualification_planning.py b/experimental/CollectiveX/tests/test_qualification_planning.py
index bb3b745f00..af09bbfe53 100644
--- a/experimental/CollectiveX/tests/test_qualification_planning.py
+++ b/experimental/CollectiveX/tests/test_qualification_planning.py
@@ -227,7 +227,7 @@ def test_full_v1_matrix_emits_every_unsupported_terminal_outcome(self) -> None:
documents = [json.loads(path.read_text()) for path in written]
self.assertEqual(len(documents), len(expected))
- self.assertEqual(len(documents), 361)
+ self.assertEqual(len(documents), 173)
self.assertEqual(
{document["outcome"]["reason"] for document in documents},
{item["reason"] for item in expected},
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index a0f59407b6..451b223c75 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -165,7 +165,7 @@ def test_terminal_summary_uses_bound_sku_and_route(self) -> None:
"format": contracts.TERMINAL_FORMAT,
"case": {
"backend": "deepep", "phase": "prefill", "ep": 8,
- "suite": "ep-routing-v1", "routing": "zipf", "eplb": True,
+ "suite": "ep-precision-normal-v1", "routing": "uniform", "eplb": False,
"required_publication": "comparable-experimental",
},
"identity": {"case_factors": {"sku": "h100-dgxc"}},
@@ -173,7 +173,7 @@ def test_terminal_summary_uses_bound_sku_and_route(self) -> None:
self.assertEqual(
summarize._identity(terminal),
(
- "h100-dgxc", "ep-routing-v1", "zipf", "prefill", True,
+ "h100-dgxc", "ep-precision-normal-v1", "uniform", "prefill", False,
"comparable-experimental", 8,
),
)
@@ -209,7 +209,7 @@ def test_matrix_cases_and_shards_are_identity_bound(self) -> None:
sum(len(item["case"]["ladder"].split()) for item in runnable_cases),
sum(len(item["case"]["ladder"].split()) for item in unsupported_cases),
),
- (49, 748, 387, 361, 1740, 877, 863),
+ (49, 364, 191, 173, 1356, 681, 675),
)
b300_ep16 = [
item for item in unsupported_cases
@@ -276,7 +276,7 @@ def test_matrix_cases_and_shards_are_identity_bound(self) -> None:
self.assertIsNotNone(capability.topology_for("mi325x", 8))
self.assertEqual(
Counter(shard["n"] for shard in matrix["include"]),
- Counter({6: 29, 8: 6, 10: 1, 11: 1, 12: 12}),
+ Counter({2: 29, 8: 12, 4: 6, 7: 1, 6: 1}),
)
ll_cases = [
item for item in matrix["requested_cases"]
@@ -308,17 +308,6 @@ def test_matrix_cases_and_shards_are_identity_bound(self) -> None:
tuple(shard[field] for field in topology_fields),
expected_topologies[shard["sku"], ep],
)
- routing_points = {
- phase: {
- int(point)
- for item in matrix["requested_cases"]
- if item["case"]["suite"] == "ep-routing-v1"
- and item["case"]["phase"] == phase
- for point in item["case"]["ladder"].split()
- }
- for phase in ("decode", "prefill")
- }
- self.assertEqual(routing_points, {"decode": {128}, "prefill": {512}})
skus = sorted({shard["sku"] for shard in matrix["include"]})
self.assertEqual(
[shard["sku"] for shard in matrix["include"][:len(skus)]],
@@ -429,7 +418,7 @@ def test_matrix_yaml_and_config_validation_are_strict(self) -> None:
),
(
"integer EPLB",
- lambda s, _w: s["suites"]["ep-routing-v1"].update({"eplb": [0, 1]}),
+ lambda s, _w: s["suites"]["ep-core-v1"].update({"eplb": [0, 1]}),
),
(
"duplicate platform",
@@ -450,7 +439,7 @@ def test_matrix_yaml_and_config_validation_are_strict(self) -> None:
),
(
"unreachable phase ladder",
- lambda s, _w: s["suites"]["ep-routing-v1"].update({"phases": ["prefill"]}),
+ lambda s, _w: s["suites"]["ep-core-v1"].update({"phases": ["decode"]}),
),
)
for label, mutate in invalid:
@@ -2207,26 +2196,6 @@ def test_workloads_bind_generator_activation_and_trace(self) -> None:
self.assertEqual(member, manifest["workload_id"])
self.assertEqual(checksums, manifest["checksums"])
- def test_eplb_calibration_window_is_disjoint_and_identity_bound(self) -> None:
- evaluation = workload.canonical_member("zipf", 8, 2, 8, 2, 4, 67)
- calibration = workload.canonical_eplb_calibration_member(
- "zipf", 8, 2, 8, 2, 4, 67
- )
- self.assertNotEqual(evaluation[0], calibration[0])
- self.assertNotEqual(evaluation[1]["trace"], calibration[1]["trace"])
- self.assertGreater(
- workload.EPLB_CALIBRATION_TOKEN_OFFSET,
- 2 * 4,
- )
- repeated = workload.canonical_eplb_calibration_member(
- "zipf", 8, 2, 8, 2, 4, 67
- )
- self.assertEqual(calibration, repeated)
- with self.assertRaises(ValueError):
- workload.canonical_routing_rows(
- 8, 8, 2, "zipf", 67, token_offset=-1
- )
-
def test_canonical_members_are_bound_to_each_scheduled_row(self) -> None:
case = {
"routing": "uniform", "hidden": 8, "topk": 2, "experts": 4, "ep": 2,
@@ -2293,36 +2262,6 @@ def replace_member(document: dict, replacement: tuple[str, dict[str, str], str])
bad_proof, case, bad_rows, eplb_record
)
- def test_eplb_row_hash_is_bound_to_the_frozen_remap(self) -> None:
- case = {
- "routing": "zipf", "hidden": 8, "topk": 2, "experts": 4, "ep": 2,
- "mode": "normal",
- }
- physical = eplb.physical_count(4, 32, 2)
- plan = contracts._expected_eplb_plan("zipf", 2, 4, physical, 2, 67, 2048)
- eplb_record = {
- "enabled": True,
- "mapping_hash": eplb.mapping_hash(plan),
- "num_physical_experts": physical,
- }
- member, checksums, row_hash, _, _ = contracts._expected_canonical_trace(
- "zipf", 8, 2, 4, physical, 2, 1, 67, True, 2048
- )
- self.assertNotEqual(row_hash, checksums["trace"])
- workload_proof = {
- "manifest_checksums": {member: checksums},
- "members": [member],
- "workload_id": identity.workload_id({
- "members": [{"checksums": checksums, "workload_id": member}]
- }),
- }
- rows = [{"tokens_per_rank": 1, "routing": {"hash": row_hash}}]
- contracts._validate_canonical_workload(workload_proof, case, rows, eplb_record)
- with self.assertRaisesRegex(contracts.ContractError, "EPLB mapping"):
- contracts._validate_canonical_workload(
- workload_proof, case, rows, {**eplb_record, "mapping_hash": "0" * 64}
- )
-
def test_oracle_pass_cannot_ignore_combined_value_failure(self) -> None:
oracle = {
"atol": ep_harness.ORACLE_ATOL,
diff --git a/experimental/CollectiveX/tests/workload.py b/experimental/CollectiveX/tests/workload.py
index 74860be803..1b121e17f8 100644
--- a/experimental/CollectiveX/tests/workload.py
+++ b/experimental/CollectiveX/tests/workload.py
@@ -16,7 +16,6 @@
from __future__ import annotations
from array import array
-import bisect
import hashlib
import json
import os
@@ -67,21 +66,13 @@ def canonical_routing_rows(
token_offset: int = 0,
) -> tuple[list[list[int]], list[list[float]]]:
"""Generate a deterministic routing window from exact integer counters."""
- if routing not in {"uniform", "zipf"}:
- raise ValueError(f"unknown routing {routing!r} (uniform|zipf)")
+ if routing != "uniform":
+ raise ValueError(f"unknown routing {routing!r} (uniform)")
if global_tokens <= 0 or experts <= 0 or topk <= 0 or topk > experts:
raise ValueError("global_tokens/experts/topk must be positive and topk <= experts")
if type(token_offset) is not int or token_offset < 0:
raise ValueError("token_offset must be a non-negative integer")
- cumulative: list[int] | None = None
- if routing == "zipf":
- total = 0
- cumulative = []
- for expert in range(experts):
- total += (1 << 32) // (expert + 1)
- cumulative.append(total)
-
indices: list[list[int]] = []
weights: list[list[float]] = []
for local_token in range(global_tokens):
@@ -92,11 +83,7 @@ def canonical_routing_rows(
attempt = 0
while True:
value = _counter(seed, token, slot, attempt, 0)
- expert = (
- value % experts
- if cumulative is None
- else bisect.bisect_right(cumulative, value % cumulative[-1])
- )
+ expert = value % experts
if expert not in used:
used.add(expert)
selected.append(expert)
@@ -328,7 +315,7 @@ def verify_workload(manifest, idx_np, weights_np):
if (manifest["schema_version"] != WORKLOAD_SCHEMA_VERSION
or manifest["generator_version"] != GENERATOR_VERSION
or manifest["gate_weight_format"] != GATE_WEIGHT_FORMAT
- or manifest["routing_profile"] not in {"uniform", "zipf"}):
+ or manifest["routing_profile"] != "uniform"):
return False, "manifest version or generator is unsupported"
if (isinstance(manifest["seed"], bool) or not isinstance(manifest["seed"], int)
or not identity.is_typed_id(manifest["workload_id"], "workload")):
@@ -389,16 +376,16 @@ def verify_workload(manifest, idx_np, weights_np):
import sys
import tempfile
# (1) workload_id determinism + sensitivity — pure stdlib, always runs.
- a = compute_workload_id("zipf", 7168, 8, 256, 8, 4096, 67)
- b = compute_workload_id("zipf", 7168, 8, 256, 8, 4096, 67)
- c = compute_workload_id("uniform", 7168, 8, 256, 8, 4096, 67)
+ a = compute_workload_id("uniform", 7168, 8, 256, 8, 4096, 67)
+ b = compute_workload_id("uniform", 7168, 8, 256, 8, 4096, 67)
+ c = compute_workload_id("uniform", 7168, 8, 256, 8, 4096, 68)
assert a == b, "workload_id must be deterministic"
- assert a != c, "workload_id must depend on routing"
- print(f"workload_id determinism OK (zipf={a} uniform={c})")
+ assert a != c, "workload_id must depend on seed"
+ print(f"workload_id determinism OK (uniform={a})")
# (2) build/save/load/verify roundtrip + cross-build identity — needs torch+numpy.
try:
import numpy as np # noqa: F401
- idx, w, man = build_workload(7168, 8, 256, "zipf", 512, 67, 32)
+ idx, w, man = build_workload(7168, 8, 256, "uniform", 512, 67, 32)
with tempfile.TemporaryDirectory() as d:
wid = save_workload(d, idx, w, man)
idx2, w2, man2 = load_workload(os.path.join(d, f"{wid}.npz"), verify=True)
From 629e2eac88ad16e3f16bf24b3565c9dd8d21a586 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 7 Jul 2026 12:49:01 +0800
Subject: [PATCH 170/210] refactor(collectivex): pin EPLB off at the
schedulable edges
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
EPLB (expert replication/balanced placement) was the remedy for `zipf`
routing skew, which R1 removed. With v1 now uniform-only, EPLB is dead
weight at the producer edges — so pin `eplb=False` there and drop the
enabling surface:
- `ep_harness.add_common_args`: remove the `--eplb` flag.
- `run_ep.main`: pin `eplb=False` into `capability.resolve(...)` and
drop the pre-construction expert-count bump (logical -> physical).
- `contracts._terminal_case_from_environment`: pin `eplb=False`.
- `runtime/run_in_container.sh`: drop the now-dangling `--eplb`
emission (the flag no longer exists; `CX_EPLB` stays an inert,
always-empty env var so `common.sh` passthrough is unchanged).
Option B-lib: the EPLB *machinery* stays inert, not deleted —
`tests/eplb.py`, the frozen schema `eplb` fields, the `if enabled:`
contract branches, and the disabled `eplb_record` emission are all
untouched. Matrix already produced `eplb=False`-only after R1, so no
hash regen. 221 tests green.
为可调度边缘固定 EPLB 关闭。EPLB(专家复制/均衡放置)本是 `zipf`
路由倾斜的补救手段,而 R1 已移除该路由。v1 现为纯 uniform,EPLB 在
生产者边缘已成冗余:在此固定 `eplb=False` 并移除启用面——删除
`--eplb` 标志、专家数预扩展、以及 run_in_container.sh 中已失效的
`--eplb` 传递。采用 B-lib 方案:EPLB 机制(tests/eplb.py、冻结的
schema 字段、contract 分支、禁用的 eplb_record)保持惰性而非删除。
R1 后矩阵已仅产出 eplb=False,故无需重算哈希。221 项测试通过。
---
experimental/CollectiveX/contracts.py | 2 +-
.../CollectiveX/runtime/run_in_container.sh | 1 -
experimental/CollectiveX/tests/ep_harness.py | 5 -----
experimental/CollectiveX/tests/run_ep.py | 14 +-------------
4 files changed, 2 insertions(+), 20 deletions(-)
diff --git a/experimental/CollectiveX/contracts.py b/experimental/CollectiveX/contracts.py
index 23cac74586..a9d7e76065 100644
--- a/experimental/CollectiveX/contracts.py
+++ b/experimental/CollectiveX/contracts.py
@@ -2743,7 +2743,7 @@ def _terminal_case_from_environment(backend: str, phase: str) -> dict[str, Any]:
"routing": os.environ.get("CX_ROUTING", "uniform"),
"phase": phase,
"ep": ep,
- "eplb": _env_enabled("CX_EPLB"),
+ "eplb": False,
"hidden": _env_integer("CX_HIDDEN", 7168),
"topk": _env_integer("CX_TOPK", 8),
"experts": _env_integer("CX_EXPERTS", 256),
diff --git a/experimental/CollectiveX/runtime/run_in_container.sh b/experimental/CollectiveX/runtime/run_in_container.sh
index fc2bd5cb34..47c439d62d 100644
--- a/experimental/CollectiveX/runtime/run_in_container.sh
+++ b/experimental/CollectiveX/runtime/run_in_container.sh
@@ -93,7 +93,6 @@ run_ep_suite() {
--qualification-index "${CX_QUALIFICATION_INDEX:-1}"
--runner "$CX_RUNNER" --topology-class "$CX_TOPO" --transport "$CX_TRANSPORT"
--out "$out")
- cx_bool_enabled "${CX_EPLB:-0}" && EPARGS+=(--eplb)
[ -n "${CX_WORKLOAD_DIR:-}" ] && EPARGS+=(--workload-dir "$CX_WORKLOAD_DIR")
cx_write_runtime_stage execution || cx_die "cannot record runtime stage"
if timeout -k 30 "${CX_RUN_TIMEOUT:-900}" \
diff --git a/experimental/CollectiveX/tests/ep_harness.py b/experimental/CollectiveX/tests/ep_harness.py
index 18018ed4e8..4f7d1478ea 100644
--- a/experimental/CollectiveX/tests/ep_harness.py
+++ b/experimental/CollectiveX/tests/ep_harness.py
@@ -177,11 +177,6 @@ def add_common_args(ap: argparse.ArgumentParser) -> None:
ap.add_argument("--topk", type=int, default=8)
ap.add_argument("--experts", type=int, default=256, help="TOTAL experts (fixed across EP degrees)")
ap.add_argument("--routing", default="uniform", choices=["uniform"])
- # EPLB (Expert-Parallel Load Balancer): replicate hot experts onto redundant physical
- # slots + balanced-place so per-rank load equalizes. A pure routing-trace transform
- # (tests/eplb.py); experts becomes num_logical+redundant. The remedy for `zipf` skew.
- ap.add_argument("--eplb", action="store_true",
- help="apply EPLB expert replication/placement to the routing trace")
# Canonical workloads consume pre-generated trace bytes instead of the
# seeded runtime generator, so a result is provably the SAME workload as another machine's
# (checksum match). Points at a dir of .npz/.manifest.json (make_workloads.py).
diff --git a/experimental/CollectiveX/tests/run_ep.py b/experimental/CollectiveX/tests/run_ep.py
index 57ca820fc4..0c47cce456 100644
--- a/experimental/CollectiveX/tests/run_ep.py
+++ b/experimental/CollectiveX/tests/run_ep.py
@@ -406,7 +406,7 @@ def main() -> int:
ep=world_size,
nodes=observed_nodes,
routing=args.routing,
- eplb=args.eplb,
+ eplb=False,
mode=args.mode,
)
if not schedulable:
@@ -416,18 +416,6 @@ def main() -> int:
args.runtime_device_count = device_count
args.allocation_execution_id = os.environ.get("COLLECTIVEX_EXECUTION_ID")
- # EPLB bumps the expert count to PHYSICAL (logical + redundant) BEFORE backend construction
- # so the backend sizes its buffers for the replicated set; ep_harness builds the LOGICAL
- # routing trace and remaps it to the balanced physical placement (a pure routing transform,
- # tests/eplb.py — no adapter change). Deterministic, so every rank agrees on the count.
- if getattr(args, "eplb", False):
- import eplb
-
- args.num_logical_experts = args.experts
- args.experts = eplb.physical_count(
- args.experts, ep_harness.EPLB_REDUNDANT_EXPERTS, world_size
- )
-
# Reproduction provenance (recorded in the artifact). Rack launchers provide ranks directly
# through srun, while single-node launchers use torchrun; do not claim torchrun for both.
if os.environ.get("TORCHELASTIC_RUN_ID"):
From 52acabcea9d0fc5586de3a8bf84031eb11a982c9 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 7 Jul 2026 15:02:50 +0800
Subject: [PATCH 171/210] =?UTF-8?q?refactor(collectivex):=20drop=20logfmt1?=
=?UTF-8?q?0=20combine=20format=20(registry=2010=E2=86=928)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Remove the two low-latency logfmt10 precision profiles
(d-bf16.c-logfmt10-dynamic64 and d-fp8-e4m3fn-b128-f32-fused.c-logfmt10-dynamic64)
atomically across the identity registry, capability ledger, schema enums,
byte-provenance maps, and the DeepEP/UCCL supported-profile sets. The
logfmt10 wire format is no longer scheduled or publishable; the inert
use_logfmt native-API keyword wiring is kept because it pins the DeepEP/UCCL
low_latency_combine signature independently of any logfmt profile.
Regenerate the frozen digests and counts whose inputs shrank with the two
profiles gone: precision targets 94/59/35 → 62/33/29, the full-matrix case
tuple, shard-count Counter, low-latency case count 80 → 48, unsupported
terminal-outcome count 173 → 167, and the execution-plan / full-matrix /
case-catalog SHA-256 anchors. The publisher precision-cohort fixture moves
from the (now bf16-only) low-latency profiles to the normal-mode
dispatch×combine 2×2, preserving its cohort topology.
移除两个低延迟 logfmt10 精度档位，在身份注册表、能力账本、schema 枚举、
字节溯源映射以及 DeepEP/UCCL 支持档位集合中原子化删除。logfmt10 线格式不再
被排程或发布；保留 use_logfmt 原生 API 关键字接线（它独立于任何 logfmt 档位，
用于固定 DeepEP/UCCL low_latency_combine 签名）。重新生成随之收缩的冻结摘要与
计数：精度目标 94/59/35 → 62/33/29、全矩阵用例元组、分片计数 Counter、低延迟
用例数 80 → 48、不支持终态数 173 → 167，以及执行计划/全矩阵/用例目录 SHA-256。
发布器精度队列夹具从（现仅 bf16 的）低延迟档位改为普通模式 dispatch×combine
2×2，保持其队列拓扑不变。
---
experimental/CollectiveX/capability.py | 61 ++-----------------
experimental/CollectiveX/configs/suites.yaml | 2 -
experimental/CollectiveX/contracts.py | 5 +-
experimental/CollectiveX/identity.py | 27 --------
experimental/CollectiveX/publisher.py | 4 +-
.../schemas/public-dataset-v1.schema.json | 6 +-
.../schemas/raw-case-v1.schema.json | 10 ++-
.../schemas/terminal-outcome-v1.schema.json | 6 +-
experimental/CollectiveX/sweep_matrix.py | 6 +-
experimental/CollectiveX/tests/ep_deepep.py | 2 -
experimental/CollectiveX/tests/ep_harness.py | 3 +-
experimental/CollectiveX/tests/ep_uccl.py | 2 -
.../tests/test_ep_precision_adapters.py | 4 +-
.../tests/test_precision_scheduling.py | 19 ++----
.../CollectiveX/tests/test_publisher.py | 14 ++---
.../tests/test_qualification_planning.py | 2 +-
.../tests/test_sampling_contract.py | 18 ++----
17 files changed, 39 insertions(+), 152 deletions(-)
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
index 1220d800e5..d52b7b6f1b 100644
--- a/experimental/CollectiveX/capability.py
+++ b/experimental/CollectiveX/capability.py
@@ -232,10 +232,6 @@ def _precision_rule(
_NORMAL_E4M3FN_PROFILE = "d-fp8-e4m3fn-b128-f32-prequantized.c-bf16"
_NORMAL_E4M3FNUZ_PROFILE = "d-fp8-e4m3fnuz-b128-f32-prequantized.c-bf16"
_LL_FP8_PROFILE = "d-fp8-e4m3fn-b128-f32-fused.c-bf16"
-_LL_LOGFMT_PROFILE = "d-bf16.c-logfmt10-dynamic64"
-_LL_FP8_LOGFMT_PROFILE = (
- "d-fp8-e4m3fn-b128-f32-fused.c-logfmt10-dynamic64"
-)
_MORI_E4M3FN_DIRECT_PROFILE = "d-bf16.c-fp8-e4m3fn-direct-cast-noscale"
_MORI_E4M3FN_BOTH_PROFILE = (
"d-fp8-e4m3fn-b128-f32-prequantized.c-fp8-e4m3fn-direct-cast-noscale"
@@ -284,30 +280,6 @@ def _precision_rule(
basis="uccl-deepep-api-low-latency-fused-e4m3fn-block128-f32-scale",
),
),
- _LL_LOGFMT_PROFILE: (
- _precision_rule(
- backend="deepep", skus=_NVIDIA_SKUS, ep_degrees=(8, 16),
- mode="low-latency",
- basis="deepep-v1-low-latency-logfmt10-dynamic-per64-combine",
- ),
- _precision_rule(
- backend="uccl", skus=_HOPPER_UCCL_SKUS, ep_degrees=(8, 16),
- mode="low-latency",
- basis="uccl-deepep-api-low-latency-logfmt10-dynamic-per64-combine",
- ),
- ),
- _LL_FP8_LOGFMT_PROFILE: (
- _precision_rule(
- backend="deepep", skus=_NVIDIA_SKUS, ep_degrees=(8, 16),
- mode="low-latency",
- basis="deepep-v1-low-latency-fused-e4m3fn-dispatch-logfmt10-combine",
- ),
- _precision_rule(
- backend="uccl", skus=_HOPPER_UCCL_SKUS, ep_degrees=(8, 16),
- mode="low-latency",
- basis="uccl-deepep-api-low-latency-fused-e4m3fn-dispatch-logfmt10-combine",
- ),
- ),
_MORI_E4M3FN_DIRECT_PROFILE: (
_precision_rule(
backend="mori", skus=("mi355x",), ep_degrees=(8,), mode="normal",
@@ -347,8 +319,6 @@ def _precision_rule(
(_NORMAL_E4M3FN_PROFILE, "deepep-v2", "normal"),
(_NORMAL_E4M3FN_PROFILE, "deepep-hybrid", "normal"),
(_LL_FP8_PROFILE, "deepep", "low-latency"),
- (_LL_LOGFMT_PROFILE, "deepep", "low-latency"),
- (_LL_FP8_LOGFMT_PROFILE, "deepep", "low-latency"),
):
PRECISION_CELL_OVERRIDES[(_profile, _backend, "b300", 16, _mode)] = {
"disposition": "unsupported",
@@ -359,55 +329,38 @@ def _precision_rule(
# run, SKU, EP, backend, mode, profile, disposition, result
("28737315879", "b200-dgxc", 8, "deepep", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed"),
("28737315879", "b200-dgxc", 8, "deepep", "low-latency", _LL_FP8_PROFILE, "supported", "native-probe-passed"),
- ("28737315879", "b200-dgxc", 8, "deepep", "low-latency", _LL_LOGFMT_PROFILE, "supported", "native-probe-passed"),
- ("28737315879", "b200-dgxc", 8, "deepep", "low-latency", _LL_FP8_LOGFMT_PROFILE, "supported", "native-probe-passed"),
("28737315879", "b200-dgxc", 8, "deepep-hybrid", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "native-operation-failed"),
("28737315879", "b200-dgxc", 8, "deepep-v2", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "backend-construction-failed"),
("28745114766", "b200-dgxc", 16, "deepep", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "native-operation-timeout"),
- ("28746354426", "b200-dgxc", 16, "deepep", "low-latency", _LL_LOGFMT_PROFILE, "unsupported", "native-operation-timeout"),
("28747290376", "b200-dgxc", 16, "deepep", "low-latency", _LL_FP8_PROFILE, "unsupported", "native-operation-timeout"),
- ("28747292565", "b200-dgxc", 16, "deepep", "low-latency", _LL_FP8_LOGFMT_PROFILE, "unsupported", "native-operation-timeout"),
("28746910633", "b200-dgxc", 16, "deepep-v2", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "construction-consensus-accelerator-memory"),
("28748550531", "b200-dgxc", 16, "deepep-hybrid", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "backend-construction-failed"),
("28737422303", "h200-dgxc", 8, "deepep", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed"),
("28737422303", "h200-dgxc", 8, "deepep", "low-latency", _LL_FP8_PROFILE, "supported", "native-probe-passed"),
- ("28737422303", "h200-dgxc", 8, "deepep", "low-latency", _LL_LOGFMT_PROFILE, "supported", "native-probe-passed"),
- ("28737422303", "h200-dgxc", 8, "deepep", "low-latency", _LL_FP8_LOGFMT_PROFILE, "supported", "native-probe-passed"),
("28737422303", "h200-dgxc", 8, "deepep-hybrid", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "native-operation-failed"),
("28737422303", "h200-dgxc", 8, "deepep-v2", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "backend-construction-failed"),
*(("28737422303", "h200-dgxc", 8, "uccl", mode, profile, "supported", "native-probe-passed")
- for profile, mode in ((_NORMAL_E4M3FN_PROFILE, "normal"), (_LL_FP8_PROFILE, "low-latency"),
- (_LL_LOGFMT_PROFILE, "low-latency"), (_LL_FP8_LOGFMT_PROFILE, "low-latency"))),
+ for profile, mode in ((_NORMAL_E4M3FN_PROFILE, "normal"), (_LL_FP8_PROFILE, "low-latency"))),
*(("28737422902", "gb200", ep, "deepep", mode, profile, "supported", "native-probe-passed")
for ep in (8, 16)
- for profile, mode in ((_NORMAL_E4M3FN_PROFILE, "normal"), (_LL_FP8_PROFILE, "low-latency"),
- (_LL_LOGFMT_PROFILE, "low-latency"), (_LL_FP8_LOGFMT_PROFILE, "low-latency"))),
+ for profile, mode in ((_NORMAL_E4M3FN_PROFILE, "normal"), (_LL_FP8_PROFILE, "low-latency"))),
*(("28737422902", "gb200", ep, "deepep-v2", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed")
for ep in (8, 16)),
*(("28737422902", "gb200", ep, "deepep-hybrid", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "native-operation-failed")
for ep in (8, 16)),
("28738113606", "h100-dgxc", 8, "deepep", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed"),
("28738113606", "h100-dgxc", 8, "deepep", "low-latency", _LL_FP8_PROFILE, "supported", "native-probe-passed"),
- ("28738113606", "h100-dgxc", 8, "deepep", "low-latency", _LL_LOGFMT_PROFILE, "supported", "native-probe-passed"),
- ("28738113606", "h100-dgxc", 8, "deepep", "low-latency", _LL_FP8_LOGFMT_PROFILE, "supported", "native-probe-passed"),
("28738113606", "h100-dgxc", 8, "deepep-hybrid", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "native-operation-failed"),
("28738113606", "h100-dgxc", 8, "uccl", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed"),
- ("28738113606", "h100-dgxc", 8, "uccl", "low-latency", _LL_LOGFMT_PROFILE, "supported", "native-probe-passed"),
("28738113606", "h100-dgxc", 8, "uccl", "low-latency", _LL_FP8_PROFILE, "unsupported", "native-operation-timeout"),
- ("28738113606", "h100-dgxc", 8, "uccl", "low-latency", _LL_FP8_LOGFMT_PROFILE, "unsupported", "native-operation-timeout"),
("28745208954", "h100-dgxc", 16, "deepep", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed"),
("28745423523", "h100-dgxc", 16, "deepep", "low-latency", _LL_FP8_PROFILE, "supported", "native-probe-passed"),
- ("28745423523", "h100-dgxc", 16, "deepep", "low-latency", _LL_LOGFMT_PROFILE, "supported", "native-probe-passed"),
- ("28796549762", "h100-dgxc", 16, "deepep", "low-latency", _LL_FP8_LOGFMT_PROFILE, "unsupported", "native-operation-timeout"),
("28745423523", "h100-dgxc", 16, "deepep-hybrid", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "native-operation-failed"),
("28745423523", "h100-dgxc", 16, "uccl", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed"),
("28745423523", "h100-dgxc", 16, "uccl", "low-latency", _LL_FP8_PROFILE, "supported", "native-probe-passed"),
- ("28745423523", "h100-dgxc", 16, "uccl", "low-latency", _LL_LOGFMT_PROFILE, "supported", "native-probe-passed"),
- ("28745423523", "h100-dgxc", 16, "uccl", "low-latency", _LL_FP8_LOGFMT_PROFILE, "supported", "native-probe-passed"),
*(("28738445591", "gb300", ep, "deepep", mode, profile, "supported", "native-probe-passed")
for ep in (8, 16)
- for profile, mode in ((_NORMAL_E4M3FN_PROFILE, "normal"), (_LL_FP8_PROFILE, "low-latency"),
- (_LL_LOGFMT_PROFILE, "low-latency"), (_LL_FP8_LOGFMT_PROFILE, "low-latency"))),
+ for profile, mode in ((_NORMAL_E4M3FN_PROFILE, "normal"), (_LL_FP8_PROFILE, "low-latency"))),
*(("28738445591", "gb300", ep, "deepep-v2", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed")
for ep in (8, 16)),
*(("28738445591", "gb300", ep, "deepep-hybrid", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "native-operation-failed")
@@ -415,14 +368,10 @@ def _precision_rule(
("28738738793", "b300", 8, "deepep", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed"),
("28738738793", "b300", 8, "deepep-hybrid", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "native-operation-failed"),
("28739555164", "h200-dgxc", 16, "deepep", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed"),
- ("28739555164", "h200-dgxc", 16, "deepep", "low-latency", _LL_FP8_LOGFMT_PROFILE, "supported", "native-probe-passed"),
("28739555164", "h200-dgxc", 16, "deepep-hybrid", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "native-operation-failed"),
("28740154697", "h200-dgxc", 16, "deepep", "low-latency", _LL_FP8_PROFILE, "supported", "native-probe-passed"),
- ("28740154697", "h200-dgxc", 16, "deepep", "low-latency", _LL_LOGFMT_PROFILE, "supported", "native-probe-passed"),
- ("28740154697", "h200-dgxc", 16, "uccl", "low-latency", _LL_LOGFMT_PROFILE, "supported", "native-probe-passed"),
("28740074613", "b300", 8, "deepep-v2", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed"),
("28740154697", "h200-dgxc", 16, "uccl", "low-latency", _LL_FP8_PROFILE, "supported", "native-probe-passed"),
- ("28740154697", "h200-dgxc", 16, "uccl", "low-latency", _LL_FP8_LOGFMT_PROFILE, "supported", "native-probe-passed"),
("28740154697", "h200-dgxc", 16, "uccl", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed"),
("28750823474", "mi355x", 8, "mori", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "backend-construction-failed"),
("28750825814", "mi355x", 16, "mori", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "backend-construction-failed"),
@@ -433,9 +382,7 @@ def _precision_rule(
("28782309414", "mi300x", 8, "mori", "normal", _MORI_E4M3FNUZ_DIRECT_PROFILE, "unsupported", "backend-construction-failed"),
("28782309414", "mi300x", 8, "mori", "normal", _MORI_E4M3FNUZ_BOTH_PROFILE, "unsupported", "backend-construction-failed"),
("28788599713", "mi300x", 16, "mori", "normal", _NORMAL_E4M3FNUZ_PROFILE, "unsupported", "distributed-init-timeout"),
- *(("28743235213", "b300", 8, "deepep", "low-latency", profile,
- "supported", "native-probe-passed")
- for profile in (_LL_FP8_PROFILE, _LL_LOGFMT_PROFILE, _LL_FP8_LOGFMT_PROFILE)),
+ ("28743235213", "b300", 8, "deepep", "low-latency", _LL_FP8_PROFILE, "supported", "native-probe-passed"),
)
PRECISION_CELL_OVERRIDES.update({
(profile, backend, sku, ep, mode): {
diff --git a/experimental/CollectiveX/configs/suites.yaml b/experimental/CollectiveX/configs/suites.yaml
index 68a6a1502c..ab5fdc5e1e 100644
--- a/experimental/CollectiveX/configs/suites.yaml
+++ b/experimental/CollectiveX/configs/suites.yaml
@@ -54,7 +54,5 @@ suites:
token_points_decode: [128]
precision_profiles:
- d-fp8-e4m3fn-b128-f32-fused.c-bf16
- - d-bf16.c-logfmt10-dynamic64
- - d-fp8-e4m3fn-b128-f32-fused.c-logfmt10-dynamic64
provisional: false
required_publication: comparable-experimental
diff --git a/experimental/CollectiveX/contracts.py b/experimental/CollectiveX/contracts.py
index a9d7e76065..6a911faa05 100644
--- a/experimental/CollectiveX/contracts.py
+++ b/experimental/CollectiveX/contracts.py
@@ -860,11 +860,10 @@ def _precision_byte_provenance(
"bf16": 16,
"fp8-e4m3fn": 8,
"fp8-e4m3fnuz": 8,
- "logfmt10": 10,
}.get(axis["communication_format"])
if bits_per_value is None:
raise ContractError("unknown communication precision format")
- scale_size = {None: 0, "f32": 4, "implicit-logfmt10": 0}.get(axis["scale_dtype"])
+ scale_size = {None: 0, "f32": 4}.get(axis["scale_dtype"])
if scale_size is None:
raise ContractError("unknown communication scale dtype")
group_size = axis["scale_group_size"]
@@ -1553,7 +1552,7 @@ def _validate_precision_evidence(
if type(axis[field]) is not bool:
raise ContractError(f"{axis_path}.{field} must be boolean")
scale_dtype = communication_precision[direction]["scale_dtype"]
- expects_scales = scale_dtype not in (None, "implicit-logfmt10")
+ expects_scales = scale_dtype is not None
for field in ("scales_finite", "scales_positive"):
if expects_scales:
if type(axis[field]) is not bool:
diff --git a/experimental/CollectiveX/identity.py b/experimental/CollectiveX/identity.py
index d860180105..0fdacbaa6a 100644
--- a/experimental/CollectiveX/identity.py
+++ b/experimental/CollectiveX/identity.py
@@ -77,8 +77,6 @@
)
V1_LOW_LATENCY_PRECISION_PROFILE_IDS = (
"d-fp8-e4m3fn-b128-f32-fused.c-bf16",
- "d-bf16.c-logfmt10-dynamic64",
- "d-fp8-e4m3fn-b128-f32-fused.c-logfmt10-dynamic64",
)
@@ -157,18 +155,6 @@ def _communication_axis(
quantization_origin="backend-fused",
conversion_boundary="inside-dispatch-timing",
)
-_LOGFMT10_DYNAMIC64_COMBINE = _communication_axis(
- api_input_dtype="bf16",
- api_output_dtype="bf16",
- communication_format="logfmt10",
- scale_dtype="implicit-logfmt10",
- scale_layout="dynamic-per-64-values",
- scale_group_size=64,
- padding_contract="right-zero-pad-values-to-64",
- alignment_contract="value-block-64",
- quantization_origin="backend-internal",
- conversion_boundary="inside-combine-timing",
-)
_FP8_E4M3FN_DIRECT_CAST_COMBINE = _communication_axis(
api_input_dtype="bf16",
api_output_dtype="bf16",
@@ -210,16 +196,6 @@ def _communication_axis(
"dispatch": _FP8_E4M3FN_FUSED_DISPATCH,
"combine": _BF16_AXIS,
},
- "d-bf16.c-logfmt10-dynamic64": {
- "modes": ["low-latency"],
- "dispatch": _BF16_AXIS,
- "combine": _LOGFMT10_DYNAMIC64_COMBINE,
- },
- "d-fp8-e4m3fn-b128-f32-fused.c-logfmt10-dynamic64": {
- "modes": ["low-latency"],
- "dispatch": _FP8_E4M3FN_FUSED_DISPATCH,
- "combine": _LOGFMT10_DYNAMIC64_COMBINE,
- },
"d-bf16.c-fp8-e4m3fn-direct-cast-noscale": {
"modes": ["normal"],
"dispatch": _BF16_AXIS,
@@ -249,7 +225,6 @@ def _communication_axis(
V1_COMBINE_ORACLE_TOLERANCES = {
"bf16": {"atol": 2e-2, "rtol": 5e-2},
- "logfmt10": {"atol": 3e-2, "rtol": 6e-2},
"fp8-direct-cast": {"atol": 4e-2, "rtol": 8e-2},
}
@@ -260,8 +235,6 @@ def combine_oracle_tolerances(communication_precision: dict[str, Any]) -> dict[s
communication_format = combine["communication_format"]
if communication_format == "bf16":
key = "bf16"
- elif communication_format == "logfmt10":
- key = "logfmt10"
elif (
communication_format in {"fp8-e4m3fn", "fp8-e4m3fnuz"}
and combine["quantization_origin"] == "backend-internal-direct-cast"
diff --git a/experimental/CollectiveX/publisher.py b/experimental/CollectiveX/publisher.py
index cda268d66e..0eb212ff7a 100644
--- a/experimental/CollectiveX/publisher.py
+++ b/experimental/CollectiveX/publisher.py
@@ -46,10 +46,10 @@
)
REQUIRED_PROMOTION_COHORT_COUNTS = {"library": 24, "system": 4}
CANONICAL_FULL_V1_MATRIX_SHA256 = (
- "2108f920c4315a9f3292d57d81ac8873e0831b4ed97bc2f4604170d971c95396"
+ "ba09fb548bae7d4fcc1994b1461d15a9feeb241930366f027ca4ac5582f8e5eb"
)
CANONICAL_FULL_V1_CASE_CATALOG_SHA256 = (
- "78bc6969ee48a1c66bc732ef46d0fc367c5e4316758a0f9f482fe4935c9eef27"
+ "7bda7073a07c05952c0cabdea88161bd39c07552e6add49dd39c9d19e0d8d850"
)
P50_STABILITY_LIMIT = 1.10
P99_STABILITY_LIMIT = 1.25
diff --git a/experimental/CollectiveX/schemas/public-dataset-v1.schema.json b/experimental/CollectiveX/schemas/public-dataset-v1.schema.json
index cab8c5a80e..055a73b452 100644
--- a/experimental/CollectiveX/schemas/public-dataset-v1.schema.json
+++ b/experimental/CollectiveX/schemas/public-dataset-v1.schema.json
@@ -91,11 +91,11 @@
"alignment_contract":{"enum":["native-bf16-vector-alignment","hidden-block-128","native-fp8-vector-alignment","value-block-64"]},
"api_input_dtype":{"enum":["bf16","fp8-e4m3fn-with-f32-scale","fp8-e4m3fnuz-with-f32-scale"]},
"api_output_dtype":{"enum":["bf16","fp8-e4m3fn-with-f32-scale","fp8-e4m3fnuz-with-f32-scale"]},
- "communication_format":{"enum":["bf16","fp8-e4m3fn","fp8-e4m3fnuz","logfmt10"]},
+ "communication_format":{"enum":["bf16","fp8-e4m3fn","fp8-e4m3fnuz"]},
"conversion_boundary":{"enum":["none","before-dispatch-timing","inside-dispatch-timing","inside-combine-timing"]},
"padding_contract":{"enum":["none","right-zero-pad-hidden-to-128","right-zero-pad-values-to-64"]},
"quantization_origin":{"enum":["none","caller-prequantized","backend-fused","backend-internal","backend-internal-direct-cast"]},
- "scale_dtype":{"oneOf":[{"type":"null"},{"enum":["f32","implicit-logfmt10"]}]},
+ "scale_dtype":{"oneOf":[{"type":"null"},{"enum":["f32"]}]},
"scale_group_size":{"oneOf":[{"type":"null"},{"enum":[64,128]}]},
"scale_layout":{"enum":["none","per-token-hidden-block","dynamic-per-64-values"]}
}
@@ -105,8 +105,6 @@
"d-bf16.c-bf16",
"d-fp8-e4m3fn-b128-f32-prequantized.c-bf16",
"d-fp8-e4m3fn-b128-f32-fused.c-bf16",
- "d-bf16.c-logfmt10-dynamic64",
- "d-fp8-e4m3fn-b128-f32-fused.c-logfmt10-dynamic64",
"d-bf16.c-fp8-e4m3fn-direct-cast-noscale",
"d-fp8-e4m3fn-b128-f32-prequantized.c-fp8-e4m3fn-direct-cast-noscale",
"d-fp8-e4m3fnuz-b128-f32-prequantized.c-bf16",
diff --git a/experimental/CollectiveX/schemas/raw-case-v1.schema.json b/experimental/CollectiveX/schemas/raw-case-v1.schema.json
index 349e2fe997..4975157404 100644
--- a/experimental/CollectiveX/schemas/raw-case-v1.schema.json
+++ b/experimental/CollectiveX/schemas/raw-case-v1.schema.json
@@ -82,8 +82,6 @@
"d-bf16.c-bf16",
"d-fp8-e4m3fn-b128-f32-prequantized.c-bf16",
"d-fp8-e4m3fn-b128-f32-fused.c-bf16",
- "d-bf16.c-logfmt10-dynamic64",
- "d-fp8-e4m3fn-b128-f32-fused.c-logfmt10-dynamic64",
"d-bf16.c-fp8-e4m3fn-direct-cast-noscale",
"d-fp8-e4m3fn-b128-f32-prequantized.c-fp8-e4m3fn-direct-cast-noscale",
"d-fp8-e4m3fnuz-b128-f32-prequantized.c-bf16",
@@ -97,11 +95,11 @@
"alignment_contract": {"enum":["native-bf16-vector-alignment","hidden-block-128","native-fp8-vector-alignment","value-block-64"]},
"api_input_dtype": {"enum":["bf16","fp8-e4m3fn-with-f32-scale","fp8-e4m3fnuz-with-f32-scale"]},
"api_output_dtype": {"enum":["bf16","fp8-e4m3fn-with-f32-scale","fp8-e4m3fnuz-with-f32-scale"]},
- "communication_format": {"enum":["bf16","fp8-e4m3fn","fp8-e4m3fnuz","logfmt10"]},
+ "communication_format": {"enum":["bf16","fp8-e4m3fn","fp8-e4m3fnuz"]},
"conversion_boundary": {"enum":["none","before-dispatch-timing","inside-dispatch-timing","inside-combine-timing"]},
"padding_contract": {"enum":["none","right-zero-pad-hidden-to-128","right-zero-pad-values-to-64"]},
"quantization_origin": {"enum":["none","caller-prequantized","backend-fused","backend-internal","backend-internal-direct-cast"]},
- "scale_dtype": {"oneOf":[{"type":"null"},{"enum":["f32","implicit-logfmt10"]}]},
+ "scale_dtype": {"oneOf":[{"type":"null"},{"enum":["f32"]}]},
"scale_group_size": {"oneOf":[{"type":"null"},{"enum":[64,128]}]},
"scale_layout": {"enum":["none","per-token-hidden-block","dynamic-per-64-values"]}
},
@@ -252,7 +250,7 @@
"required": ["combine_values","counts","metadata","multiplicity","payload","source_set","weights"],
"type": "object"
},
- "atol": {"enum":[0.02,0.03,0.04]},
+ "atol": {"enum":[0.02,0.04]},
"combine_weight_semantics": {"enum":["unweighted-rank-sum","gate-weighted-sum"]},
"contract": {"enum":["expert-specific-transform-v1","expert-assignment-transform-v1"]},
"dispatch_sha256": {"$ref":"#/$defs/nullable_sha256"},
@@ -264,7 +262,7 @@
"ordering_contract": {"minLength":1,"type":"string"},
"passed": {"type":"boolean"},
"receive_count": {"minimum":0,"type":"integer"},
- "rtol": {"enum":[0.05,0.06,0.08]}
+ "rtol": {"enum":[0.05,0.08]}
},
"required": [
"atol",
diff --git a/experimental/CollectiveX/schemas/terminal-outcome-v1.schema.json b/experimental/CollectiveX/schemas/terminal-outcome-v1.schema.json
index c70d6b82a8..64babf7e9d 100644
--- a/experimental/CollectiveX/schemas/terminal-outcome-v1.schema.json
+++ b/experimental/CollectiveX/schemas/terminal-outcome-v1.schema.json
@@ -145,8 +145,6 @@
"d-bf16.c-bf16",
"d-fp8-e4m3fn-b128-f32-prequantized.c-bf16",
"d-fp8-e4m3fn-b128-f32-fused.c-bf16",
- "d-bf16.c-logfmt10-dynamic64",
- "d-fp8-e4m3fn-b128-f32-fused.c-logfmt10-dynamic64",
"d-bf16.c-fp8-e4m3fn-direct-cast-noscale",
"d-fp8-e4m3fn-b128-f32-prequantized.c-fp8-e4m3fn-direct-cast-noscale",
"d-fp8-e4m3fnuz-b128-f32-prequantized.c-bf16",
@@ -166,11 +164,11 @@
"alignment_contract":{"enum":["native-bf16-vector-alignment","hidden-block-128","native-fp8-vector-alignment","value-block-64"]},
"api_input_dtype":{"enum":["bf16","fp8-e4m3fn-with-f32-scale","fp8-e4m3fnuz-with-f32-scale"]},
"api_output_dtype":{"enum":["bf16","fp8-e4m3fn-with-f32-scale","fp8-e4m3fnuz-with-f32-scale"]},
- "communication_format":{"enum":["bf16","fp8-e4m3fn","fp8-e4m3fnuz","logfmt10"]},
+ "communication_format":{"enum":["bf16","fp8-e4m3fn","fp8-e4m3fnuz"]},
"conversion_boundary":{"enum":["none","before-dispatch-timing","inside-dispatch-timing","inside-combine-timing"]},
"padding_contract":{"enum":["none","right-zero-pad-hidden-to-128","right-zero-pad-values-to-64"]},
"quantization_origin":{"enum":["none","caller-prequantized","backend-fused","backend-internal","backend-internal-direct-cast"]},
- "scale_dtype":{"oneOf":[{"type":"null"},{"enum":["f32","implicit-logfmt10"]}]},
+ "scale_dtype":{"oneOf":[{"type":"null"},{"enum":["f32"]}]},
"scale_group_size":{"oneOf":[{"type":"null"},{"enum":[64,128]}]},
"scale_layout":{"enum":["none","per-token-hidden-block","dynamic-per-64-values"]}
}
diff --git a/experimental/CollectiveX/sweep_matrix.py b/experimental/CollectiveX/sweep_matrix.py
index 11dff01989..3fb39412c4 100644
--- a/experimental/CollectiveX/sweep_matrix.py
+++ b/experimental/CollectiveX/sweep_matrix.py
@@ -94,9 +94,9 @@
)
QUALIFICATION_INDICES = range(1, 4)
CANONICAL_V1_EXECUTION_PLAN_SHA256 = {
- 1: "cc6d65b4424630a80ca76fc792ca49142820315601d05979756a03f5e9d920f4",
- 2: "ce6856db2953a019414629f3130b58361feeae7f7e425f311dbf65b1aea7c1b0",
- 3: "c5826efac5d91bb5b333bbb721f534ed1cdf7a6c80185fa8c77b2d2c0fa2cf74",
+ 1: "7427de51c48e97c3283d8e239736343a314fb362a18d55ed7a2ab8002e252984",
+ 2: "648304f5f62ec8c85901ac9542c263dbb38b09ec01e859de8284c405ea932ff9",
+ 3: "b109a0c2a8fdf940f9885268cc3fbf3bd7bfc111e98b6e1c41317a96726a26c8",
}
diff --git a/experimental/CollectiveX/tests/ep_deepep.py b/experimental/CollectiveX/tests/ep_deepep.py
index 440d7de606..bbed6aaedd 100644
--- a/experimental/CollectiveX/tests/ep_deepep.py
+++ b/experimental/CollectiveX/tests/ep_deepep.py
@@ -94,8 +94,6 @@ def __init__(self, args, rank, world_size, local_rank, device):
"low-latency": {
"d-bf16.c-bf16",
"d-fp8-e4m3fn-b128-f32-fused.c-bf16",
- "d-bf16.c-logfmt10-dynamic64",
- "d-fp8-e4m3fn-b128-f32-fused.c-logfmt10-dynamic64",
},
}
self.precision_profile_id, self.communication_precision = (
diff --git a/experimental/CollectiveX/tests/ep_harness.py b/experimental/CollectiveX/tests/ep_harness.py
index 4f7d1478ea..ee23f72af5 100644
--- a/experimental/CollectiveX/tests/ep_harness.py
+++ b/experimental/CollectiveX/tests/ep_harness.py
@@ -128,12 +128,11 @@ def precision_byte_provenance(
"bf16": 16,
"fp8-e4m3fn": 8,
"fp8-e4m3fnuz": 8,
- "logfmt10": 10,
}.get(axis["communication_format"])
if bits_per_value is None:
raise ValueError(f"unknown communication format {axis['communication_format']!r}")
activation_data_bytes = logical_copies * math.ceil(hidden * bits_per_value / 8)
- scale_bytes_per_value = {None: 0, "f32": 4, "implicit-logfmt10": 0}.get(
+ scale_bytes_per_value = {None: 0, "f32": 4}.get(
axis["scale_dtype"]
)
if scale_bytes_per_value is None:
diff --git a/experimental/CollectiveX/tests/ep_uccl.py b/experimental/CollectiveX/tests/ep_uccl.py
index 8eb5147416..df1d4bdfba 100644
--- a/experimental/CollectiveX/tests/ep_uccl.py
+++ b/experimental/CollectiveX/tests/ep_uccl.py
@@ -207,8 +207,6 @@ def __init__(self, args, rank, world_size, local_rank, device):
"low-latency": {
"d-bf16.c-bf16",
"d-fp8-e4m3fn-b128-f32-fused.c-bf16",
- "d-bf16.c-logfmt10-dynamic64",
- "d-fp8-e4m3fn-b128-f32-fused.c-logfmt10-dynamic64",
},
}
self.precision_profile_id, self.communication_precision = (
diff --git a/experimental/CollectiveX/tests/test_ep_precision_adapters.py b/experimental/CollectiveX/tests/test_ep_precision_adapters.py
index 21fde94256..ad889e46a8 100644
--- a/experimental/CollectiveX/tests/test_ep_precision_adapters.py
+++ b/experimental/CollectiveX/tests/test_ep_precision_adapters.py
@@ -49,11 +49,11 @@ def test_profile_mode_mismatch_is_rejected(self):
with self.assertRaisesRegex(ep_precision.PrecisionError, "not valid in mode"):
ep_precision.resolve_precision(
SimpleNamespace(
- precision_profile="d-bf16.c-logfmt10-dynamic64"
+ precision_profile="d-fp8-e4m3fn-b128-f32-fused.c-bf16"
),
backend="deepep",
mode="normal",
- supported_profiles={"d-bf16.c-logfmt10-dynamic64"},
+ supported_profiles={"d-fp8-e4m3fn-b128-f32-fused.c-bf16"},
)
def test_required_native_keyword_is_checked(self):
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
index bc123de94c..7f0263a9c6 100644
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -206,23 +206,12 @@ def test_precision_profiles_bind_exact_formats_and_timing_boundaries(self) -> No
self.assertEqual(fused["conversion_boundary"], "inside-dispatch-timing")
self.assertEqual(prequantized["scale_group_size"], 128)
- logfmt = identity.precision_profile("d-bf16.c-logfmt10-dynamic64")["combine"]
- self.assertEqual(
- (logfmt["communication_format"], logfmt["scale_group_size"]),
- ("logfmt10", 64),
- )
self.assertEqual(
identity.combine_oracle_tolerances(
identity.precision_profile(identity.V1_CONTROL_PRECISION_PROFILE)
),
{"atol": 2e-2, "rtol": 5e-2},
)
- self.assertEqual(
- identity.combine_oracle_tolerances(
- identity.precision_profile("d-bf16.c-logfmt10-dynamic64")
- ),
- {"atol": 3e-2, "rtol": 6e-2},
- )
self.assertEqual(
identity.combine_oracle_tolerances(identity.precision_profile(
"d-bf16.c-fp8-e4m3fn-direct-cast-noscale"
@@ -250,13 +239,13 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
{"supported", "unsupported"},
)
self.assertEqual(
- len(targets), 94
+ len(targets), 62
)
self.assertEqual(
- sum(item["disposition"] == "supported" for item in targets), 59
+ sum(item["disposition"] == "supported" for item in targets), 33
)
self.assertEqual(
- sum(item["disposition"] == "unsupported" for item in targets), 35
+ sum(item["disposition"] == "unsupported" for item in targets), 29
)
self.assertEqual(len(capability.provisional_precision_targets()), 0)
keys = {
@@ -273,7 +262,7 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
normal = "d-fp8-e4m3fn-b128-f32-prequantized.c-bf16"
direct = "d-bf16.c-fp8-e4m3fn-direct-cast-noscale"
- low_latency = "d-bf16.c-logfmt10-dynamic64"
+ low_latency = "d-fp8-e4m3fn-b128-f32-fused.c-bf16"
cases = (
(("h200-dgxc", "deepep-v2", 8, "normal", normal), "unsupported"),
(("h100-dgxc", "deepep-v2", 8, "normal", normal), "not-applicable"),
diff --git a/experimental/CollectiveX/tests/test_publisher.py b/experimental/CollectiveX/tests/test_publisher.py
index 029db83c8c..9734ab5dd3 100644
--- a/experimental/CollectiveX/tests/test_publisher.py
+++ b/experimental/CollectiveX/tests/test_publisher.py
@@ -1968,7 +1968,7 @@ def test_workflow_matrix_and_catalog_digests_do_not_drift(self) -> None:
if item["disposition"] == "unsupported"
),
),
- (49, 364, 191, 173, 1356, 681, 675),
+ (49, 332, 165, 167, 1324, 655, 669),
)
library: dict[tuple, set[str]] = {}
system: dict[tuple, set[str]] = {}
@@ -2581,9 +2581,9 @@ def test_p99_tie_set_does_not_include_decisively_beaten_middle_series(self) -> N
def test_precision_cohorts_isolate_axes_and_never_recommend(self) -> None:
profiles = (
identity.V1_CONTROL_PRECISION_PROFILE,
- "d-fp8-e4m3fn-b128-f32-fused.c-bf16",
- "d-bf16.c-logfmt10-dynamic64",
- "d-fp8-e4m3fn-b128-f32-fused.c-logfmt10-dynamic64",
+ "d-fp8-e4m3fn-b128-f32-prequantized.c-bf16",
+ "d-bf16.c-fp8-e4m3fn-direct-cast-noscale",
+ "d-fp8-e4m3fn-b128-f32-prequantized.c-fp8-e4m3fn-direct-cast-noscale",
)
series = []
internals = {}
@@ -2593,11 +2593,11 @@ def test_precision_cohorts_isolate_axes_and_never_recommend(self) -> None:
)
precision = identity.precision_profile(profile_id)
item["suite"] = (
- "ep-low-latency-v1"
+ "ep-core-v1"
if index == 0
- else "ep-precision-low-latency-v1"
+ else "ep-precision-normal-v1"
)
- item["mode"] = "low-latency"
+ item["mode"] = "normal"
item["publication_tier"] = (
"official" if index == 0 else "comparable-experimental"
)
diff --git a/experimental/CollectiveX/tests/test_qualification_planning.py b/experimental/CollectiveX/tests/test_qualification_planning.py
index af09bbfe53..62b8ae66ff 100644
--- a/experimental/CollectiveX/tests/test_qualification_planning.py
+++ b/experimental/CollectiveX/tests/test_qualification_planning.py
@@ -227,7 +227,7 @@ def test_full_v1_matrix_emits_every_unsupported_terminal_outcome(self) -> None:
documents = [json.loads(path.read_text()) for path in written]
self.assertEqual(len(documents), len(expected))
- self.assertEqual(len(documents), 173)
+ self.assertEqual(len(documents), 167)
self.assertEqual(
{document["outcome"]["reason"] for document in documents},
{item["reason"] for item in expected},
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 451b223c75..9d80540554 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -209,7 +209,7 @@ def test_matrix_cases_and_shards_are_identity_bound(self) -> None:
sum(len(item["case"]["ladder"].split()) for item in runnable_cases),
sum(len(item["case"]["ladder"].split()) for item in unsupported_cases),
),
- (49, 364, 191, 173, 1356, 681, 675),
+ (49, 332, 165, 167, 1324, 655, 669),
)
b300_ep16 = [
item for item in unsupported_cases
@@ -276,13 +276,13 @@ def test_matrix_cases_and_shards_are_identity_bound(self) -> None:
self.assertIsNotNone(capability.topology_for("mi325x", 8))
self.assertEqual(
Counter(shard["n"] for shard in matrix["include"]),
- Counter({2: 29, 8: 12, 4: 6, 7: 1, 6: 1}),
+ Counter({2: 29, 6: 13, 4: 6, 5: 1}),
)
ll_cases = [
item for item in matrix["requested_cases"]
if item["case"]["mode"] == "low-latency"
]
- self.assertEqual(len(ll_cases), 80)
+ self.assertEqual(len(ll_cases), 48)
self.assertTrue(all(
item["case"]["backend"] in {"deepep", "uccl"}
and item["case"]["phase"] == "decode"
@@ -2298,7 +2298,6 @@ def test_oracle_pass_cannot_ignore_combined_value_failure(self) -> None:
contracts._validate_oracle(tampered, "oracle")
for profile_id, expected in (
- ("d-bf16.c-logfmt10-dynamic64", {"atol": 3e-2, "rtol": 6e-2}),
("d-bf16.c-fp8-e4m3fn-direct-cast-noscale", {"atol": 4e-2, "rtol": 8e-2}),
):
with self.subTest(profile_id=profile_id):
@@ -2349,21 +2348,14 @@ def test_precision_evidence_rejects_direct_cast_saturation(self) -> None:
saturated, profile_id, communication_precision, "precision"
)
- logfmt_profile_id = "d-bf16.c-logfmt10-dynamic64"
- logfmt_precision = identity.precision_profile(logfmt_profile_id)
- logfmt_evidence = copy.deepcopy(evidence)
- logfmt_evidence["profile_id"] = logfmt_profile_id
- contracts._validate_precision_evidence(
- logfmt_evidence, logfmt_profile_id, logfmt_precision, "precision"
- )
- invented_scales = copy.deepcopy(logfmt_evidence)
+ invented_scales = copy.deepcopy(evidence)
invented_scales["combine"].update({
"scales_finite": True,
"scales_positive": True,
})
with self.assertRaisesRegex(contracts.ContractError, "must be null"):
contracts._validate_precision_evidence(
- invented_scales, logfmt_profile_id, logfmt_precision, "precision"
+ invented_scales, profile_id, communication_precision, "precision"
)
def test_oracle_stability_canonicalizes_native_receive_order(self) -> None:
From 8aec0423ae9a738443e4723d8519c1c9232c8f74 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 7 Jul 2026 15:13:33 +0800
Subject: [PATCH 172/210] refactor(collectivex): trim normal precision sweep to
two profiles
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Reduce ep-precision-normal-v1 from six scheduled profiles to two
(d-fp8-e4m3fn-b128-f32-prequantized.c-bf16 and
d-fp8-e4m3fnuz-b128-f32-prequantized.c-fp8-e4m3fnuz-direct-cast-noscale),
spanning one bf16-combine and one fp8-combine codec across the e4m3fn and
e4m3fnuz families. The four dropped profiles stay defined in the
V1_PRECISION_PROFILES registry (defined-but-unscheduled) so their schema
enums and capability targets survive for republication of already
collected evidence.
- identity.V1_NORMAL_PRECISION_PROFILE_IDS trimmed to the two kept ids
(order-preserving); the frozen suite contract follows automatically.
- configs/suites.yaml ep-precision-normal-v1.precision_profiles matched
order-for-order to satisfy the frozen-catalog equality gate.
- test_precision_scheduling registry-equality relaxed to a subset check
plus an explicit defined-but-unscheduled set; expected-case counters
filtered to the scheduled profile ids.
- regenerated the execution-plan, full-matrix and case-catalog digests,
the unsupported terminal-outcome count (167 to 159) and the matrix
case/point tuples.
221 tests green.
将 ep-precision-normal-v1 的常规精度扫描从六个受调度配置削减为两个
（d-fp8-e4m3fn-b128-f32-prequantized.c-bf16 与
d-fp8-e4m3fnuz-b128-f32-prequantized.c-fp8-e4m3fnuz-direct-cast-noscale），
覆盖 e4m3fn 与 e4m3fnuz 两族各一个 bf16-combine 与 fp8-combine 编解码。
被移除的四个配置仍保留在 V1_PRECISION_PROFILES 注册表中（已定义但不调度），
使其 schema 枚举与能力目标得以留存，以便对已采集证据进行再发布。
- identity.V1_NORMAL_PRECISION_PROFILE_IDS 保序削减为保留的两个 id；
冻结的套件契约自动跟随。
- configs/suites.yaml 中对应字段按序对齐以通过冻结目录相等性校验。
- 放宽 test_precision_scheduling 的注册表相等断言为子集校验加显式的
已定义但不调度集合；预期用例计数按受调度配置 id 过滤。
- 重新生成执行计划、完整矩阵与用例目录摘要、不支持终态计数（167 至 159）
以及矩阵用例/采样点元组。
221 项测试通过。
---
experimental/CollectiveX/configs/suites.yaml | 4 ---
experimental/CollectiveX/identity.py | 4 ---
experimental/CollectiveX/publisher.py | 4 +--
experimental/CollectiveX/sweep_matrix.py | 6 ++---
.../tests/test_precision_scheduling.py | 26 ++++++++++++++++---
.../CollectiveX/tests/test_publisher.py | 2 +-
.../tests/test_qualification_planning.py | 2 +-
.../tests/test_sampling_contract.py | 4 +--
8 files changed, 31 insertions(+), 21 deletions(-)
diff --git a/experimental/CollectiveX/configs/suites.yaml b/experimental/CollectiveX/configs/suites.yaml
index ab5fdc5e1e..35a50bceaf 100644
--- a/experimental/CollectiveX/configs/suites.yaml
+++ b/experimental/CollectiveX/configs/suites.yaml
@@ -35,10 +35,6 @@ suites:
token_points_prefill: [512]
precision_profiles:
- d-fp8-e4m3fn-b128-f32-prequantized.c-bf16
- - d-bf16.c-fp8-e4m3fn-direct-cast-noscale
- - d-fp8-e4m3fn-b128-f32-prequantized.c-fp8-e4m3fn-direct-cast-noscale
- - d-fp8-e4m3fnuz-b128-f32-prequantized.c-bf16
- - d-bf16.c-fp8-e4m3fnuz-direct-cast-noscale
- d-fp8-e4m3fnuz-b128-f32-prequantized.c-fp8-e4m3fnuz-direct-cast-noscale
provisional: false
required_publication: comparable-experimental
diff --git a/experimental/CollectiveX/identity.py b/experimental/CollectiveX/identity.py
index 0fdacbaa6a..d574a1189e 100644
--- a/experimental/CollectiveX/identity.py
+++ b/experimental/CollectiveX/identity.py
@@ -69,10 +69,6 @@
V1_CONTROL_PRECISION_PROFILE = "d-bf16.c-bf16"
V1_NORMAL_PRECISION_PROFILE_IDS = (
"d-fp8-e4m3fn-b128-f32-prequantized.c-bf16",
- "d-bf16.c-fp8-e4m3fn-direct-cast-noscale",
- "d-fp8-e4m3fn-b128-f32-prequantized.c-fp8-e4m3fn-direct-cast-noscale",
- "d-fp8-e4m3fnuz-b128-f32-prequantized.c-bf16",
- "d-bf16.c-fp8-e4m3fnuz-direct-cast-noscale",
"d-fp8-e4m3fnuz-b128-f32-prequantized.c-fp8-e4m3fnuz-direct-cast-noscale",
)
V1_LOW_LATENCY_PRECISION_PROFILE_IDS = (
diff --git a/experimental/CollectiveX/publisher.py b/experimental/CollectiveX/publisher.py
index 0eb212ff7a..83b1c81940 100644
--- a/experimental/CollectiveX/publisher.py
+++ b/experimental/CollectiveX/publisher.py
@@ -46,10 +46,10 @@
)
REQUIRED_PROMOTION_COHORT_COUNTS = {"library": 24, "system": 4}
CANONICAL_FULL_V1_MATRIX_SHA256 = (
- "ba09fb548bae7d4fcc1994b1461d15a9feeb241930366f027ca4ac5582f8e5eb"
+ "1c8f9c27c4a3027e8a843478ff63c2c6e00f5c170436bb5c3fd039c886363111"
)
CANONICAL_FULL_V1_CASE_CATALOG_SHA256 = (
- "7bda7073a07c05952c0cabdea88161bd39c07552e6add49dd39c9d19e0d8d850"
+ "6bb6859307802bb3ea2baea5b20a9c517f6fb773c20fd539c4f4fcd06dd8ae98"
)
P50_STABILITY_LIMIT = 1.10
P99_STABILITY_LIMIT = 1.25
diff --git a/experimental/CollectiveX/sweep_matrix.py b/experimental/CollectiveX/sweep_matrix.py
index 3fb39412c4..974f6f6d79 100644
--- a/experimental/CollectiveX/sweep_matrix.py
+++ b/experimental/CollectiveX/sweep_matrix.py
@@ -94,9 +94,9 @@
)
QUALIFICATION_INDICES = range(1, 4)
CANONICAL_V1_EXECUTION_PLAN_SHA256 = {
- 1: "7427de51c48e97c3283d8e239736343a314fb362a18d55ed7a2ab8002e252984",
- 2: "648304f5f62ec8c85901ac9542c263dbb38b09ec01e859de8284c405ea932ff9",
- 3: "b109a0c2a8fdf940f9885268cc3fbf3bd7bfc111e98b6e1c41317a96726a26c8",
+ 1: "71faf69466bf3f35dac2defa04d4ea85434fbbde248631a9204fc8dcd8e2e37f",
+ 2: "8eae0bb33aacc56811e74e2ad80b24f3ea8c55e84bb2e50e76d4b23f3648f1c6",
+ 3: "9ac32155b789d3a9067d5c7fe2e23e8574431eed5041bb243a57ff886ae3e940",
}
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
index 7f0263a9c6..03fe13e326 100644
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -169,9 +169,24 @@ def test_precision_profiles_bind_exact_formats_and_timing_boundaries(self) -> No
scheduled = set(identity.V1_NORMAL_PRECISION_PROFILE_IDS) | set(
identity.V1_LOW_LATENCY_PRECISION_PROFILE_IDS
)
- self.assertEqual(
- set(identity.V1_PRECISION_PROFILES),
+ # The scheduled subset plus the control must be a subset of the frozen
+ # registry; the four normal profiles the v1 sweep no longer schedules
+ # stay defined-but-unscheduled so their schema enums and capability
+ # targets survive for republication of already-collected evidence.
+ self.assertLessEqual(
scheduled | {identity.V1_CONTROL_PRECISION_PROFILE},
+ set(identity.V1_PRECISION_PROFILES),
+ )
+ self.assertEqual(
+ set(identity.V1_PRECISION_PROFILES)
+ - scheduled
+ - {identity.V1_CONTROL_PRECISION_PROFILE},
+ {
+ "d-bf16.c-fp8-e4m3fn-direct-cast-noscale",
+ "d-fp8-e4m3fn-b128-f32-prequantized.c-fp8-e4m3fn-direct-cast-noscale",
+ "d-fp8-e4m3fnuz-b128-f32-prequantized.c-bf16",
+ "d-bf16.c-fp8-e4m3fnuz-direct-cast-noscale",
+ },
)
self.assertNotIn(identity.V1_CONTROL_PRECISION_PROFILE, scheduled)
self.assertNotIn("fp4", repr(identity.V1_PRECISION_PROFILES).lower())
@@ -456,13 +471,16 @@ def load_config(name: str) -> dict[str, object]:
raise AssertionError(name)
suite_names = "ep-precision-normal-v1,ep-precision-low-latency-v1"
+ scheduled_ids = list(identity.V1_NORMAL_PRECISION_PROFILE_IDS) + list(
+ identity.V1_LOW_LATENCY_PRECISION_PROFILE_IDS
+ )
expected_cases = sum(
len(resolved_suites["suites"][
"ep-precision-normal-v1"
if target["mode"] == "normal"
else "ep-precision-low-latency-v1"
]["phases"])
- for target in capability.precision_targets()
+ for target in capability.precision_targets(scheduled_ids)
)
with mock.patch.object(capability, "PRECISION_CAPABILITIES", promoted):
sweep_matrix.validate_config_documents(resolved_suites, workloads)
@@ -476,7 +494,7 @@ def load_config(name: str) -> dict[str, object]:
if target["mode"] == "normal"
else "ep-precision-low-latency-v1"
]["phases"])
- for target in capability.precision_targets()
+ for target in capability.precision_targets(scheduled_ids)
if capability.resolve_disposition(
target["sku"], target["backend"], ep=target["ep"],
nodes=capability.topology_for(target["sku"], target["ep"])["nodes"],
diff --git a/experimental/CollectiveX/tests/test_publisher.py b/experimental/CollectiveX/tests/test_publisher.py
index 9734ab5dd3..c84a5d17f6 100644
--- a/experimental/CollectiveX/tests/test_publisher.py
+++ b/experimental/CollectiveX/tests/test_publisher.py
@@ -1968,7 +1968,7 @@ def test_workflow_matrix_and_catalog_digests_do_not_drift(self) -> None:
if item["disposition"] == "unsupported"
),
),
- (49, 332, 165, 167, 1324, 655, 669),
+ (49, 322, 163, 159, 1314, 653, 661),
)
library: dict[tuple, set[str]] = {}
system: dict[tuple, set[str]] = {}
diff --git a/experimental/CollectiveX/tests/test_qualification_planning.py b/experimental/CollectiveX/tests/test_qualification_planning.py
index 62b8ae66ff..4614aa2d77 100644
--- a/experimental/CollectiveX/tests/test_qualification_planning.py
+++ b/experimental/CollectiveX/tests/test_qualification_planning.py
@@ -227,7 +227,7 @@ def test_full_v1_matrix_emits_every_unsupported_terminal_outcome(self) -> None:
documents = [json.loads(path.read_text()) for path in written]
self.assertEqual(len(documents), len(expected))
- self.assertEqual(len(documents), 167)
+ self.assertEqual(len(documents), 159)
self.assertEqual(
{document["outcome"]["reason"] for document in documents},
{item["reason"] for item in expected},
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 9d80540554..255de0a829 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -209,7 +209,7 @@ def test_matrix_cases_and_shards_are_identity_bound(self) -> None:
sum(len(item["case"]["ladder"].split()) for item in runnable_cases),
sum(len(item["case"]["ladder"].split()) for item in unsupported_cases),
),
- (49, 332, 165, 167, 1324, 655, 669),
+ (49, 322, 163, 159, 1314, 653, 661),
)
b300_ep16 = [
item for item in unsupported_cases
@@ -276,7 +276,7 @@ def test_matrix_cases_and_shards_are_identity_bound(self) -> None:
self.assertIsNotNone(capability.topology_for("mi325x", 8))
self.assertEqual(
Counter(shard["n"] for shard in matrix["include"]),
- Counter({2: 29, 6: 13, 4: 6, 5: 1}),
+ Counter({2: 30, 6: 13, 4: 5, 5: 1}),
)
ll_cases = [
item for item in matrix["requested_cases"]
From eee812804f507ef9121601dd0ce4243d4f04aa14 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 7 Jul 2026 15:35:05 +0800
Subject: [PATCH 173/210] refactor(collectivex): add inert EPBackend ABC
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Introduce tests/ep_backend.py holding the EPBackend abstract base class
plus the WorkloadSpec / RankInputs dataclasses. This is the target shape
for the EP dispatch/combine benchmark: the base owns input generation
(make_inputs), problem encoding (make_problem), and the two Pass-2 timing
branch rules (benchmark_dispatch/benchmark_combine), while subclasses
supply only the transport (create_buffer/dispatch/stage/combine).
make_inputs runs before create_buffer, resolving the historical inversion
where buffers were sized in __init__ before any input existed: buffer
sizing needs the ladder numbers, not the input tensors.
The module has zero consumers yet — nothing imports it at runtime, so the
221-test baseline is unchanged. torch and routing are imported lazily
inside device-touching methods (mirroring ep_harness) so the module
imports under the CPU-only test environment.
新增惰性的 EPBackend 抽象基类
引入 tests/ep_backend.py，包含 EPBackend 抽象基类以及 WorkloadSpec /
RankInputs 数据类。这是 EP dispatch/combine 基准测试的目标结构：基类负责
输入生成（make_inputs）、问题编码（make_problem）以及两条 Pass-2 计时分支
规则（benchmark_dispatch/benchmark_combine），子类只需提供传输实现
（create_buffer/dispatch/stage/combine）。
make_inputs 在 create_buffer 之前运行，纠正了以往在任何输入存在之前就于
__init__ 中构建缓冲区的顺序倒置：缓冲区大小只需要阶梯数值，而非输入张量。
该模块目前没有任何调用方——运行时不会被导入，因此 221 项测试基线保持不变。
torch 与 routing 在触及设备的方法内惰性导入（与 ep_harness 一致），使该模块
可在仅 CPU 的测试环境中导入。
---
experimental/CollectiveX/tests/ep_backend.py | 460 +++++++++++++++++++
1 file changed, 460 insertions(+)
create mode 100644 experimental/CollectiveX/tests/ep_backend.py
diff --git a/experimental/CollectiveX/tests/ep_backend.py b/experimental/CollectiveX/tests/ep_backend.py
new file mode 100644
index 0000000000..76e96cf34d
--- /dev/null
+++ b/experimental/CollectiveX/tests/ep_backend.py
@@ -0,0 +1,460 @@
+#!/usr/bin/env python3
+"""Abstract base class for CollectiveX EP dispatch/combine backends.
+
+The benchmark drives every backend through one fixed lifecycle, modelled on the
+DeepEP (`tests/legacy/test_internode.py`) and ROCm/mori
+(`examples/ops/dispatch_combine/test_dispatch_combine_internode.py`) reference
+harnesses, but wrapped in a real base class so the shared plumbing lives in one
+place and each adapter supplies only the transport-specific pieces:
+
+ spec = backend.make_inputs(args) # base: ladder + per-rank routing/activations
+ backend.create_buffer(spec) # subclass: size the communicator from `spec`
+ problem = backend.make_problem(...) # base: dtype coercion + dispatch encoding
+ backend.benchmark_dispatch(...) # base template: warm + time the raw op
+ backend.benchmark_combine(...) # base template: warm + time the raw op
+
+`make_inputs` computes the sweep's numeric shape (token ladder, max tokens/rank)
+and materialises the per-rank inputs; `create_buffer` consumes those numbers to
+size the communicator. This resolves the historical inversion where buffers were
+built in ``__init__`` before any input existed.
+
+The base class carries the contract flags the driver reads (``stage_device_work``,
+``combine_needs_redispatch``, ``dispatch_needs_combine_cleanup``,
+``combine_weight_semantics``, ``oracle_layout``/``payload_unit``, ``roundtrip_only``)
+with defaults matching the common (non-fp8, normal-mode) case; subclasses and the
+low-latency path override them. The two Pass-2 timing branch rules — untimed
+stage+combine cleanup after a timed dispatch (MoRI), and per-iter re-dispatch for
+stateful combine (MoRI + DeepEP low-latency) — are encoded once in
+``benchmark_dispatch``/``benchmark_combine`` so subclasses supply only the raw op.
+
+``torch`` and ``routing`` are imported lazily inside the methods that touch a
+device, mirroring ``ep_harness`` — so this module byte-compiles and imports under
+the CPU-only test environment, while ``run_sweep`` imports it lazily on the GPU
+image to avoid an import cycle.
+"""
+from __future__ import annotations
+
+import abc
+import os
+import types
+from dataclasses import dataclass, field
+
+import ep_precision
+from ep_harness import (
+ CONDITIONING_LADDERS,
+ CONDITIONING_ROUNDS_PER_SHAPE,
+ time_us,
+ token_ladder,
+)
+
+
+@dataclass
+class RankInputs:
+ """Inputs for one token-ladder shape at ``tokens_per_rank`` tokens on this rank.
+
+ ``topk_idx``/``topk_weights`` are this rank's contiguous slice of the global
+ routing trace (host tensors; moved to device at ``make_problem`` time);
+ ``activations`` are the rank's token activations (already on device). The
+ global trace is retained for measured ladder shapes so Pass 1 can compute
+ routing statistics and input snapshots; warm-only conditioning shapes leave
+ it ``None``.
+ """
+
+ tokens_per_rank: int
+ topk_idx: "torch.Tensor"
+ topk_weights: "torch.Tensor"
+ activations: "torch.Tensor"
+ global_tokens: int = 0
+ global_idx: "torch.Tensor | None" = None
+ global_weights: "torch.Tensor | None" = None
+ workload_id: "str | None" = None
+ checksums: "dict | None" = None
+
+
+@dataclass
+class WorkloadSpec:
+ """Numeric shape + materialised inputs for one fully-specified sweep line.
+
+ Fully default-constructible so ``make_inputs`` can early-return a tensor-free
+ spec (``ok=False`` + ``rc``) on an empty ladder or a buffer cap too small for
+ the conditioning ladder; the driver prints ``message`` and returns ``rc``.
+ """
+
+ ok: bool = True
+ rc: int = 0
+ message: str = ""
+ phase: str = ""
+ routing: str = "uniform"
+ seed: int = 0
+ hidden: int = 0
+ topk: int = 0
+ experts: int = 0
+ num_logical: int = 0
+ ep_size: int = 0
+ experts_per_rank: int = 0
+ cap: "int | None" = None
+ dropped: list = field(default_factory=list)
+ max_tokens_per_rank: int = 0
+ ladder: list = field(default_factory=list)
+ conditioning_ladder: list = field(default_factory=list)
+ points: dict = field(default_factory=dict)
+ conditioning_points: dict = field(default_factory=dict)
+ loaded_workload_ids: list = field(default_factory=list)
+ loaded_checksums: dict = field(default_factory=dict)
+
+
+class EPBackend(abc.ABC):
+ """One expert-parallel dispatch/combine transport under a fixed benchmark contract.
+
+ Subclasses implement the transport (``create_buffer``, ``dispatch``, ``stage``,
+ ``combine``, ``recv_tokens``, ``inspect_dispatch``, ``combine_transformed``,
+ and the ``supported_profiles`` set); everything the driver and the oracles
+ need beyond that is provided here.
+ """
+
+ # ---- Contract flags (class defaults; subclasses / low-latency override) ----
+ name: str = ""
+ SUPPORTED_MODES: tuple = ("normal",)
+ stage_device_work = False
+ combine_needs_redispatch = False
+ dispatch_needs_combine_cleanup = False
+ # Adapters that reduce activations and top-k weights independently must carry
+ # the complete local weighted expert sum in the activation tensor.
+ combine_weight_semantics = "unweighted-rank-sum"
+ oracle_layout = "token-rank"
+ payload_unit = "token-rank"
+ roundtrip_only = False
+ # Precision-evidence knob: adapters that hand scales back as uint8-viewed
+ # storage (deepep-hybrid) flip this so the evidence pass reads them correctly.
+ _uint8_scale_storage = False
+
+ def __init_subclass__(cls, **kwargs):
+ super().__init_subclass__(**kwargs)
+ if not getattr(cls, "name", ""):
+ raise TypeError(
+ f"{cls.__name__} must declare a non-empty class-level `name`"
+ )
+
+ def __init__(self, args, rank, world_size, local_rank, device):
+ self.args = args
+ self.rank = rank
+ self.world_size = world_size
+ self.local_rank = local_rank
+ self.device = device
+ self.mode = getattr(args, "mode", "normal")
+ if self.mode not in self.SUPPORTED_MODES:
+ raise ValueError(f"{self.name} does not support mode {self.mode!r}")
+
+ # ---- Abstract transport contract -------------------------------------------------
+
+ @property
+ @abc.abstractmethod
+ def supported_profiles(self):
+ """Precision-profile ids this backend realizes for the current ``self.mode``."""
+
+ @abc.abstractmethod
+ def create_buffer(self, spec: WorkloadSpec):
+ """Size the communicator from ``spec``; must leave ``backend_provenance`` complete."""
+
+ @abc.abstractmethod
+ def dispatch(self, problem):
+ """Scatter tokens to their experts; return an opaque per-call handle."""
+
+ @abc.abstractmethod
+ def stage(self, problem, handle):
+ """Prepare the combine input on ``handle`` (dequantize / copy into place)."""
+
+ @abc.abstractmethod
+ def combine(self, problem, handle):
+ """Gather the staged tokens back to their source rank; return combined activations."""
+
+ @abc.abstractmethod
+ def recv_tokens(self, handle):
+ """Number of tokens this rank received in dispatch (stable for a fixed trace)."""
+
+ @abc.abstractmethod
+ def inspect_dispatch(self, problem, handle):
+ """Normalized post-dispatch view for the token-rank correctness oracle."""
+
+ @abc.abstractmethod
+ def combine_transformed(self, problem, handle, transformed):
+ """Combine an oracle-transformed payload in place of the staged input."""
+
+ # ---- Input generation (shared) ---------------------------------------------------
+
+ def buffer_cap(self, args):
+ """Max tokens/rank the communicator can serve, or ``None`` when unbounded."""
+ return None
+
+ def make_inputs(self, args) -> WorkloadSpec:
+ """Resolve the token ladder and materialise per-rank inputs for the sweep.
+
+ Buffer sizing needs the ladder *numbers* (not the input tensors), so this
+ runs before ``create_buffer``. Returns a tensor-free spec with ``ok=False``
+ when the ladder is empty or the cap cannot serve the conditioning ladder.
+ """
+ ep_size = self.world_size
+ num_logical = getattr(args, "num_logical_experts", args.experts)
+ experts_per_rank = args.experts // ep_size
+ conditioning_ladder = list(CONDITIONING_LADDERS[args.phase])
+ cap = self.buffer_cap(args)
+ if cap is not None and cap < conditioning_ladder[-1]:
+ return WorkloadSpec(
+ ok=False, rc=2, phase=args.phase,
+ message=(
+ f"{self.name} buffer cap {cap} cannot run the v1 conditioning ladder"
+ ),
+ )
+ ladder, dropped = token_ladder(args.tokens_ladder, args.phase, cap)
+ if not ladder:
+ return WorkloadSpec(
+ ok=False, rc=2, phase=args.phase,
+ message=f"empty token ladder (phase={args.phase}, cap={cap})",
+ )
+ spec = WorkloadSpec(
+ phase=args.phase,
+ routing=args.routing,
+ seed=args.seed,
+ hidden=args.hidden,
+ topk=args.topk,
+ experts=args.experts,
+ num_logical=num_logical,
+ ep_size=ep_size,
+ experts_per_rank=experts_per_rank,
+ cap=cap,
+ dropped=list(dropped),
+ max_tokens_per_rank=max(list(ladder) + conditioning_ladder),
+ ladder=list(ladder),
+ conditioning_ladder=conditioning_ladder,
+ )
+ # Warm-only conditioning shapes never need canonical manifests or the
+ # global trace: they are never measured or emitted.
+ for wt in conditioning_ladder:
+ spec.conditioning_points[wt] = self._build_rank_inputs(
+ args, wt, canonical=False, retain_global=False
+ )
+ canonical = bool(getattr(args, "workload_dir", ""))
+ for tokens_per_rank in ladder:
+ point = self._build_rank_inputs(
+ args, tokens_per_rank, canonical=canonical, retain_global=True
+ )
+ spec.points[tokens_per_rank] = point
+ if point.workload_id is not None and point.workload_id not in spec.loaded_workload_ids:
+ spec.loaded_workload_ids.append(point.workload_id)
+ spec.loaded_checksums[point.workload_id] = point.checksums
+ return spec
+
+ def _build_rank_inputs(self, args, tokens_per_rank, *, canonical, retain_global) -> RankInputs:
+ """Build one rank's inputs for a given tokens-per-rank shape.
+
+ canonical: load pre-serialized trace bytes (checksum-verified) so this run
+ is provably the SAME workload as any other consuming the same files;
+ otherwise seeded generation. (EPLB is off, so no logical->physical remap.)
+ """
+ import torch
+ import routing
+
+ ep_size = self.world_size
+ num_logical = getattr(args, "num_logical_experts", args.experts)
+ global_tokens = tokens_per_rank * ep_size
+ workload_id = None
+ checksums = None
+ if canonical:
+ import workload as _wl
+
+ workload_id = _wl.compute_workload_id(
+ args.routing, args.hidden, args.topk, num_logical, ep_size,
+ global_tokens, args.seed,
+ )
+ idx_np, w_np, manifest = _wl.load_workload(
+ os.path.join(args.workload_dir, f"{workload_id}.npz"), verify=True
+ )
+ idx_g = torch.from_numpy(idx_np).to(torch.int64)
+ w_g = torch.from_numpy(w_np).to(torch.float32)
+ checksums = manifest.get("checksums")
+ else:
+ idx_g, w_g = routing.build_global_routing(
+ global_tokens, num_logical, args.topk, args.routing, args.seed
+ )
+ idx_s, w_s = routing.rank_slice(idx_g, w_g, self.rank, tokens_per_rank)
+ activations = routing.rank_activations(
+ tokens_per_rank, args.hidden, args.seed, self.rank, self.device, torch.bfloat16
+ )
+ return RankInputs(
+ tokens_per_rank=tokens_per_rank,
+ topk_idx=idx_s.contiguous(),
+ topk_weights=w_s.contiguous(),
+ activations=activations,
+ global_tokens=global_tokens,
+ global_idx=idx_g if retain_global else None,
+ global_weights=w_g if retain_global else None,
+ workload_id=workload_id,
+ checksums=checksums,
+ )
+
+ def make_problem(self, T, idx, weights, x):
+ """Encode the dispatch payload and assemble the per-shape problem namespace."""
+ import torch
+
+ encoding = ep_precision.encode_dispatch(torch, x, self.communication_precision)
+ problem = types.SimpleNamespace(
+ T=T,
+ x=x,
+ dispatch_x=encoding.native_input,
+ oracle_x=encoding.semantic,
+ dispatch_precision_evidence=encoding.evidence,
+ topk_idx=idx.to(self._topk_idx_dtype()),
+ topk_weights=weights.to(torch.float32),
+ )
+ self._extend_problem(problem, encoding)
+ return problem
+
+ def _topk_idx_dtype(self):
+ """Integer dtype the backend's kernels expect for top-k routing indices."""
+ import torch
+
+ return torch.int64
+
+ def _extend_problem(self, problem, encoding):
+ """Hook: subclasses attach backend-specific problem fields (scales, layout, ...)."""
+
+ # ---- Timing template methods -----------------------------------------------------
+
+ def timed_components(self):
+ """Components measured for this backend: roundtrip always; the rest unless
+ the backend exposes only a stateful paired round trip."""
+ components = ["roundtrip"]
+ if not self.roundtrip_only:
+ components.extend(["dispatch", "combine"])
+ if self.stage_device_work:
+ components.append("stage")
+ return components
+
+ def warm(self, problem, count):
+ """Untimed synchronized full round trips (fabric/clock warm-up; cold-jump-safe).
+
+ Caches the dynamic receive cardinality once so adapters never read a device
+ scalar during a timed trial (the count is stable for a fixed routing trace).
+ """
+ import torch
+
+ for _ in range(count):
+ handle = self.dispatch(problem)
+ if not hasattr(problem, "recv_tokens"):
+ problem.recv_tokens = self.recv_tokens(handle)
+ self.stage(problem, handle)
+ self.combine(problem, handle)
+ torch.cuda.synchronize()
+
+ def run_roundtrip(self, problem):
+ """One full dispatch -> stage -> combine round trip; returns combined activations."""
+ handle = self.dispatch(problem)
+ self.stage(problem, handle)
+ return self.combine(problem, handle)
+
+ def benchmark_component(self, component, problem, warmup, iters):
+ """Measure one named component; every component gets the same warm-up first."""
+ if component == "roundtrip":
+ return self.benchmark_roundtrip(problem, warmup, iters)
+ if component == "dispatch":
+ return self.benchmark_dispatch(problem, warmup, iters)
+ if component == "stage":
+ return self.benchmark_stage(problem, warmup, iters)
+ if component == "combine":
+ return self.benchmark_combine(problem, warmup, iters)
+ raise RuntimeError(f"unknown timed component {component!r}")
+
+ def benchmark_roundtrip(self, problem, warmup, iters):
+ import torch
+
+ self.warm(problem, warmup)
+ return time_us(torch, lambda p=problem: self.run_roundtrip(p), 0, iters)
+
+ def benchmark_dispatch(self, problem, warmup, iters):
+ import torch
+
+ self.warm(problem, warmup)
+
+ def finish_dispatch(hh, p=problem):
+ self.stage(p, hh)
+ self.combine(p, hh)
+
+ dispatch_needs_cleanup = self.dispatch_needs_combine_cleanup
+ return time_us(
+ torch, lambda p=problem: self.dispatch(p), 0, iters,
+ post=finish_dispatch if dispatch_needs_cleanup else None,
+ )
+
+ def benchmark_stage(self, problem, warmup, iters):
+ import torch
+
+ self.warm(problem, warmup)
+
+ def prep_stage(p=problem):
+ return self.dispatch(p)
+
+ return time_us(
+ torch, lambda hh, p=problem: self.stage(p, hh), 0, iters, pre=prep_stage,
+ )
+
+ def benchmark_combine(self, problem, warmup, iters):
+ import torch
+
+ self.warm(problem, warmup)
+
+ def prep_combine(p=problem):
+ hh = self.dispatch(p)
+ self.stage(p, hh)
+ return hh
+
+ if self.combine_needs_redispatch:
+ return time_us(
+ torch, lambda hh, p=problem: self.combine(p, hh), 0, iters, pre=prep_combine,
+ )
+ hh = prep_combine()
+ torch.cuda.synchronize()
+ return time_us(torch, lambda p=problem, hx=hh: self.combine(p, hx), 0, iters)
+
+ # ---- Correctness / precision hooks (shared) --------------------------------------
+
+ def inspect_expert_dispatch(self, problem, handle):
+ """Expert-packed post-dispatch view; only low-latency backends provide one."""
+ raise RuntimeError("expert-packed inspection requires low-latency mode")
+
+ def oracle_dispatch_payload(self, payload):
+ """Project an oracle-built payload through the dispatch encoding's semantics."""
+ import torch
+
+ return ep_precision.encode_dispatch(
+ torch, payload, self.communication_precision
+ ).semantic
+
+ def precision_evidence(self, problem, view=None):
+ """Per-direction precision evidence for the raw document."""
+ import torch
+
+ return ep_precision.precision_evidence(
+ torch,
+ profile_id=self.precision_profile_id,
+ profile=self.communication_precision,
+ problem=problem,
+ view=view,
+ uint8_storage=self._uint8_scale_storage,
+ )
+
+ def capture_deferred_provenance(self):
+ """Resolve provenance materialized only after conditioning (e.g. JIT CUBINs).
+
+ No-op by default; JIT backends override to record and cross-rank-check it.
+ """
+
+ def finalize(self, rc):
+ """Barrier and tear down the process group; returns ``rc``."""
+ import torch.distributed as dist
+
+ try:
+ dist.barrier()
+ dist.destroy_process_group()
+ except Exception:
+ pass
+ return rc
From 826e81c72ee3b65d6d101616f22daf71332422eb Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 7 Jul 2026 16:03:56 +0800
Subject: [PATCH 174/210] refactor(collectivex): cut EP adapters over to the
EPBackend ABC
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Restructure the EP dispatch/combine benchmark so every backend subclasses
the EPBackend ABC and the driver drives a real lifecycle instead of a
duck-typed protocol. Buffer construction moves out of __init__ into
create_buffer(spec), which consumes what make_inputs(args) produces —
resolving the historical inversion where buffers were sized before inputs
existed. run_sweep now calls make_inputs -> create_buffer and reads all
provenance/contract flags off the backend after the buffer is built.
All six adapters (nccl, deepep, deepep-v2, mori, deepep-hybrid, uccl)
subclass EPBackend via super().__init__, which sets self.mode and validates
SUPPORTED_MODES. Buffer bodies are moved verbatim behind a local-alias line
with an `assert spec.max_tokens_per_rank <= cap` guard. probe_precision, the
second protocol consumer, calls create_buffer(make_inputs(args)) after
construction. Drop the unused abstract supported_profiles hook — each adapter
resolves its own profile set inline via ep_precision.resolve_precision.
Scrape-anchor tests updated in lockstep: the ElasticBuffer/JIT/cache-dir and
HybridEPBuffer ordering pins now walk create_buffer, and the combine-timing
and post=finish_dispatch fragments target ep_backend.py.
将 EP dispatch/combine 基准重构为每个后端都继承 EPBackend 抽象基类，驱动器执行
真正的生命周期，而非鸭子类型协议。缓冲区构造从 __init__ 移到 create_buffer(spec)，
由 make_inputs(args) 的产物驱动，纠正了过去"先建缓冲区后有输入"的顺序倒置。
run_sweep 现按 make_inputs -> create_buffer 顺序调用，并在缓冲区建成后从后端读取
全部 provenance/契约标志。
六个适配器（nccl、deepep、deepep-v2、mori、deepep-hybrid、uccl）均通过
super().__init__ 继承 EPBackend，由基类设置 self.mode 并校验 SUPPORTED_MODES。
缓冲区代码体在局部别名行后原样移动，并加 `assert spec.max_tokens_per_rank <= cap`
守卫。第二个协议消费方 probe_precision 在构造后调用 create_buffer(make_inputs(args))。
移除未实现的抽象 supported_profiles 钩子——各适配器在 __init__ 中经
ep_precision.resolve_precision 内联解析各自的 profile 集合。
同步更新源码抓取测试：ElasticBuffer/JIT/缓存目录 及 HybridEPBuffer 的顺序断言
现遍历 create_buffer；combine 计时与 post=finish_dispatch 片段改指向 ep_backend.py。
---
experimental/CollectiveX/tests/ep_backend.py | 13 +-
experimental/CollectiveX/tests/ep_deepep.py | 40 +--
.../CollectiveX/tests/ep_deepep_hybrid.py | 47 ++--
.../CollectiveX/tests/ep_deepep_v2.py | 25 +-
experimental/CollectiveX/tests/ep_harness.py | 231 ++++--------------
experimental/CollectiveX/tests/ep_mori.py | 22 +-
experimental/CollectiveX/tests/ep_nccl.py | 18 +-
experimental/CollectiveX/tests/ep_uccl.py | 40 +--
.../CollectiveX/tests/probe_precision.py | 1 +
.../tests/test_deepep_v2_contract.py | 16 +-
.../tests/test_ep_precision_adapters.py | 4 +-
.../tests/test_sampling_contract.py | 7 +-
12 files changed, 182 insertions(+), 282 deletions(-)
diff --git a/experimental/CollectiveX/tests/ep_backend.py b/experimental/CollectiveX/tests/ep_backend.py
index 76e96cf34d..6dd879c8e8 100644
--- a/experimental/CollectiveX/tests/ep_backend.py
+++ b/experimental/CollectiveX/tests/ep_backend.py
@@ -107,9 +107,11 @@ class EPBackend(abc.ABC):
"""One expert-parallel dispatch/combine transport under a fixed benchmark contract.
Subclasses implement the transport (``create_buffer``, ``dispatch``, ``stage``,
- ``combine``, ``recv_tokens``, ``inspect_dispatch``, ``combine_transformed``,
- and the ``supported_profiles`` set); everything the driver and the oracles
- need beyond that is provided here.
+ ``combine``, ``recv_tokens``, ``inspect_dispatch``, ``combine_transformed``);
+ everything the driver and the oracles need beyond that is provided here. Each
+ adapter resolves its own precision profile in ``__init__`` (passing the profile
+ set inline to ``ep_precision.resolve_precision``), so there is no abstract
+ ``supported_profiles`` hook.
"""
# ---- Contract flags (class defaults; subclasses / low-latency override) ----
@@ -147,11 +149,6 @@ def __init__(self, args, rank, world_size, local_rank, device):
# ---- Abstract transport contract -------------------------------------------------
- @property
- @abc.abstractmethod
- def supported_profiles(self):
- """Precision-profile ids this backend realizes for the current ``self.mode``."""
-
@abc.abstractmethod
def create_buffer(self, spec: WorkloadSpec):
"""Size the communicator from ``spec``; must leave ``backend_provenance`` complete."""
diff --git a/experimental/CollectiveX/tests/ep_deepep.py b/experimental/CollectiveX/tests/ep_deepep.py
index bbed6aaedd..931d4c3ccf 100644
--- a/experimental/CollectiveX/tests/ep_deepep.py
+++ b/experimental/CollectiveX/tests/ep_deepep.py
@@ -11,6 +11,7 @@
import torch.distributed as dist
import contracts
import ep_precision
+from ep_backend import EPBackend
try:
import deep_ep
@@ -68,8 +69,9 @@ def _normal_buffer_sizes(hidden: int, world_size: int) -> tuple[int, int]:
return num_nvl_bytes, num_rdma_bytes
-class DeepEPBackend:
+class DeepEPBackend(EPBackend):
name = "deepep"
+ SUPPORTED_MODES = ("normal", "low-latency")
stage_device_work = False
combine_needs_redispatch = False
# DeepEP reduces activations and top-k weights independently. The activation
@@ -79,13 +81,8 @@ class DeepEPBackend:
payload_unit = "token-rank"
def __init__(self, args, rank, world_size, local_rank, device):
- self.args = args
- self.rank = rank
- self.world_size = world_size
- self.device = device
- self.mode = getattr(args, "mode", "normal")
- if self.mode not in {"normal", "low-latency"}:
- raise ValueError(f"unsupported DeepEP mode {self.mode!r}")
+ # Base validates mode against SUPPORTED_MODES (normal / low-latency).
+ super().__init__(args, rank, world_size, local_rank, device)
supported_profiles = {
"normal": {
"d-bf16.c-bf16",
@@ -111,11 +108,27 @@ def __init__(self, args, rank, world_size, local_rank, device):
self.communication_precision
)
self.stage_device_work = self._fp8_dispatch
-
self.group = dist.group.WORLD
+ # Low-latency flips the contract flags and fixes the per-rank cap so
+ # buffer_cap can report it to make_inputs before create_buffer runs.
+ if self.mode == "low-latency":
+ if args.phase != "decode":
+ raise ValueError("DeepEP low-latency mode only supports the decode ladder")
+ if args.experts % world_size:
+ raise ValueError("DeepEP low-latency experts must divide the EP group")
+ self.combine_needs_redispatch = True
+ self.combine_weight_semantics = "gate-weighted-sum"
+ self.oracle_layout = "expert-packed"
+ self.payload_unit = "token-expert"
+ self.max_tokens_per_rank = 128
+
+ def create_buffer(self, spec):
+ # Local aliases keep the moved buffer-construction body byte-verbatim.
+ args, world_size, device = self.args, self.world_size, self.device
device_sms = torch.cuda.get_device_properties(device).multi_processor_count
mnnvl_kwargs, mnnvl_comm = _mnnvl_buffer_configuration()
if self.mode == "low-latency":
+ assert spec.max_tokens_per_rank <= 128
ep_precision.require_keyword(
Buffer.low_latency_dispatch,
"use_fp8",
@@ -126,15 +139,6 @@ def __init__(self, args, rank, world_size, local_rank, device):
"use_logfmt",
api="deep_ep.Buffer.low_latency_combine",
)
- if args.phase != "decode":
- raise ValueError("DeepEP low-latency mode only supports the decode ladder")
- if args.experts % world_size:
- raise ValueError("DeepEP low-latency experts must divide the EP group")
- self.combine_needs_redispatch = True
- self.combine_weight_semantics = "gate-weighted-sum"
- self.oracle_layout = "expert-packed"
- self.payload_unit = "token-expert"
- self.max_tokens_per_rank = 128
num_qps_per_rank = args.experts // world_size
num_rdma_bytes = Buffer.get_low_latency_rdma_size_hint(
self.max_tokens_per_rank, args.hidden, world_size, args.experts
diff --git a/experimental/CollectiveX/tests/ep_deepep_hybrid.py b/experimental/CollectiveX/tests/ep_deepep_hybrid.py
index f48005e4c0..666a428e5b 100644
--- a/experimental/CollectiveX/tests/ep_deepep_hybrid.py
+++ b/experimental/CollectiveX/tests/ep_deepep_hybrid.py
@@ -40,6 +40,7 @@
import torch.distributed as dist
import contracts
import ep_precision
+from ep_backend import EPBackend
try:
import deep_ep
@@ -190,20 +191,20 @@ def _hybrid_topology(args, world_size: int) -> dict[str, int | str]:
}
-class DeepEPHybridBackend:
+class DeepEPHybridBackend(EPBackend):
name = "deepep-hybrid"
stage_device_work = False
# HybridEPBuffer.combine consumes the recv payload + the dispatch handle (no re-dispatch needed
# before a timed combine); the harness times dispatch and combine separately (like ep_deepep).
combine_needs_redispatch = False
combine_weight_semantics = "unweighted-rank-sum"
+ # Hybrid hands scales back as uint8-viewed storage; the precision-evidence pass
+ # must read them that way.
+ _uint8_scale_storage = True
def __init__(self, args, rank, world_size, local_rank, device):
- self.args = args
- self.rank = rank
- self.world_size = world_size
- self.device = device
- self.mode = "normal"
+ # deepep-hybrid is normal-mode only; base SUPPORTED_MODES=("normal",) enforces it.
+ super().__init__(args, rank, world_size, local_rank, device)
self.precision_profile_id, self.communication_precision = (
ep_precision.resolve_precision(
args,
@@ -255,6 +256,15 @@ def __init__(self, args, rank, world_size, local_rank, device):
raise RuntimeError("DeepEP Hybrid MNNVL runtime enablement is incomplete")
# Token cap (per rank) for the symmetric buffer; the sweep is capped here (buffer_cap).
self.max_tokens = 4096
+ # Stash the topology so the moved create_buffer provenance can read it back.
+ self._topology = topology
+
+ def create_buffer(self, spec):
+ # Local aliases re-expose the __init__ names so the moved tempdir / buffer /
+ # geometry / monkeypatch / provenance body below stays byte-verbatim.
+ args, world_size, rank, device = self.args, self.world_size, self.rank, self.device
+ topology = self._topology
+ assert spec.max_tokens_per_rank <= self.max_tokens
dev_sms = torch.cuda.get_device_properties(device).multi_processor_count
ver = _deepep_hybrid_version()
loaded_libraries = _hybrid_build_evidence()
@@ -537,15 +547,18 @@ def finalize(self, rc):
dist.destroy_process_group()
except Exception:
pass
- shutil.rmtree(self._jit_cache_dir, ignore_errors=True)
- if self._previous_jit_cache_dir is None:
- os.environ.pop("HYBRID_EP_CACHE_DIR", None)
- else:
- os.environ["HYBRID_EP_CACHE_DIR"] = self._previous_jit_cache_dir
- if self._previous_domain_ranks is None:
- os.environ.pop("NUM_OF_HYBRID_EP_RANKS_PER_NVLINK_DOMAIN", None)
- else:
- os.environ[
- "NUM_OF_HYBRID_EP_RANKS_PER_NVLINK_DOMAIN"
- ] = self._previous_domain_ranks
+ # create_buffer may not have run (e.g. make_inputs early-returned before it),
+ # so the JIT tempdir/env state only needs unwinding when it was established.
+ if hasattr(self, "_jit_cache_dir"):
+ shutil.rmtree(self._jit_cache_dir, ignore_errors=True)
+ if self._previous_jit_cache_dir is None:
+ os.environ.pop("HYBRID_EP_CACHE_DIR", None)
+ else:
+ os.environ["HYBRID_EP_CACHE_DIR"] = self._previous_jit_cache_dir
+ if self._previous_domain_ranks is None:
+ os.environ.pop("NUM_OF_HYBRID_EP_RANKS_PER_NVLINK_DOMAIN", None)
+ else:
+ os.environ[
+ "NUM_OF_HYBRID_EP_RANKS_PER_NVLINK_DOMAIN"
+ ] = self._previous_domain_ranks
return rc
diff --git a/experimental/CollectiveX/tests/ep_deepep_v2.py b/experimental/CollectiveX/tests/ep_deepep_v2.py
index bfb11c6cde..cf2369d61f 100644
--- a/experimental/CollectiveX/tests/ep_deepep_v2.py
+++ b/experimental/CollectiveX/tests/ep_deepep_v2.py
@@ -19,6 +19,7 @@
import contracts
import ep_harness
import ep_precision
+from ep_backend import EPBackend
try:
import deep_ep
@@ -328,18 +329,15 @@ def _require_runtime() -> tuple[str, str]:
return torch_version, nccl_runtime_version
-class DeepEPV2Backend:
+class DeepEPV2Backend(EPBackend):
name = "deepep-v2"
stage_device_work = False
combine_needs_redispatch = False
combine_weight_semantics = "unweighted-rank-sum"
def __init__(self, args, rank, world_size, local_rank, device):
- self.args = args
- self.rank = rank
- self.world_size = world_size
- self.device = device
- self.mode = "normal"
+ # deepep-v2 is normal-mode only; base SUPPORTED_MODES=("normal",) enforces it.
+ super().__init__(args, rank, world_size, local_rank, device)
self.precision_profile_id, self.communication_precision = (
ep_precision.resolve_precision(
args,
@@ -361,10 +359,15 @@ def __init__(self, args, rank, world_size, local_rank, device):
api="deep_ep.ElasticBuffer.__init__",
)
self.group = dist.group.WORLD
+ self._deferred_jit_snapshot = None
+
+ def create_buffer(self, spec):
+ # Local aliases keep the moved buffer-construction body byte-verbatim. The cap
+ # equals the value the deleted __init__ ladder-peek computed
+ # (token_ladder(.., None) + conditioning), so the jit_cache_key stays stable.
+ args, world_size, device = self.args, self.world_size, self.device
+ self.max_tokens = spec.max_tokens_per_rank
torch_version, nccl_runtime_version = _require_runtime()
- ladder, _ = ep_harness.token_ladder(args.tokens_ladder, args.phase, None)
- conditioning = ep_harness.CONDITIONING_LADDERS[args.phase]
- self.max_tokens = max([*ladder, *conditioning])
jit_root = Path(os.environ["EP_JIT_CACHE_DIR"])
scale_up_domain = int(
getattr(args, "scale_up_domain", None)
@@ -374,7 +377,6 @@ def __init__(self, args, rank, world_size, local_rank, device):
allow_hybrid_mode = _configure_gin_mode(args, world_size)
gin_enabled = allow_hybrid_mode
communication_backend = "nccl-gin" if gin_enabled else "nccl-device-lsa"
- self._deferred_jit_snapshot = None
self.buffer = ElasticBuffer(
self.group,
num_max_tokens_per_rank=self.max_tokens,
@@ -498,9 +500,6 @@ def __init__(self, args, rank, world_size, local_rank, device):
"physical_nvlink_ranks": int(self.buffer.num_nvlink_ranks),
}
- def buffer_cap(self, args):
- return self.max_tokens
-
def make_problem(self, T, idx, weights, x):
encoding = ep_precision.encode_dispatch(
torch, x, self.communication_precision
diff --git a/experimental/CollectiveX/tests/ep_harness.py b/experimental/CollectiveX/tests/ep_harness.py
index ee23f72af5..fd1f3a29a7 100644
--- a/experimental/CollectiveX/tests/ep_harness.py
+++ b/experimental/CollectiveX/tests/ep_harness.py
@@ -1233,11 +1233,6 @@ def run_sweep(args, backend, torch, dist, device, rank: int, world_size: int) ->
import eplb # stdlib planner + torch remap (the EPLB transform)
ep_size = world_size
- # EPLB (if on): run_ep.py already bumped args.experts to the PHYSICAL count and stashed the
- # logical count, so experts_per_rank below is physical. The trace is built over LOGICAL
- # experts then remapped to physical (build_trace), so the whole sweep runs over the
- # balanced physical placement with no adapter change.
- eplb_on = getattr(args, "eplb", False)
num_logical = getattr(args, "num_logical_experts", args.experts)
if args.experts % ep_size != 0:
if rank == 0:
@@ -1275,120 +1270,52 @@ def run_sweep(args, backend, torch, dist, device, rank: int, world_size: int) ->
print("ERROR: low-latency requires decode expert-packed token-expert execution")
return 2
- cap = backend.buffer_cap(args)
- conditioning_ladder = CONDITIONING_LADDERS[args.phase]
- if cap is not None and cap < conditioning_ladder[-1]:
+ spec = backend.make_inputs(args)
+ if not spec.ok:
if rank == 0:
- print(f"ERROR: {backend.name} buffer cap {cap} cannot run the v1 conditioning ladder")
- return 2
- ladder, dropped = token_ladder(args.tokens_ladder, args.phase, cap)
+ print(f"ERROR: {spec.message}")
+ return spec.rc
+ cap = spec.cap
+ conditioning_ladder = spec.conditioning_ladder
+ ladder, dropped = spec.ladder, spec.dropped
if rank == 0 and dropped:
print(f"NOTE: dropped tokens/rank {dropped} — exceed {backend.name} buffer cap {cap} "
f"(hidden={args.hidden}); not silently truncated.")
- if not ladder:
- if rank == 0:
- print(f"ERROR: empty token ladder (phase={args.phase}, cap={cap})")
- return 2
+ loaded_workload_ids = spec.loaded_workload_ids
+ loaded_checksums = spec.loaded_checksums
+ canonical = bool(getattr(args, "workload_dir", ""))
MAX, MIN, SUM = dist.ReduceOp.MAX, dist.ReduceOp.MIN, dist.ReduceOp.SUM
- # EPLB plan (once): estimate logical load from the global logical trace at the largest
- # ladder T (most samples), then replicate+place. Held fixed across all T (as real EPLB
- # plans from an observed load estimate). build_trace builds the LOGICAL trace and remaps
- # to physical when the plan is present; otherwise it's the identity (logical == physical).
+ # EPLB is off in v1 (uniform placement): the plan/calibration stay None so the
+ # disabled eplb_record and the identity trace-remap paths downstream take their
+ # inert branch. `eplb` remains imported for those None-guarded call sites.
eplb_plan = None
eplb_calibration = None
- if eplb_on:
- calibration_id, calibration_checksums, calibration_rows, _ = (
- workload_contract.canonical_eplb_calibration_member(
- args.routing,
- args.hidden,
- args.topk,
- num_logical,
- ep_size,
- EPLB_REFERENCE_TOKENS_PER_RANK,
- args.seed,
- )
- )
- ref_idx = torch.tensor(
- calibration_rows,
- dtype=torch.int64,
- )
- eplb_calibration = {
- "token_offset": workload_contract.EPLB_CALIBRATION_TOKEN_OFFSET,
- "trace_sha256": calibration_checksums["trace"],
- "window": workload_contract.EPLB_CALIBRATION_WINDOW,
- "workload_id": calibration_id,
- }
- if ref_idx.shape != (
- EPLB_REFERENCE_TOKENS_PER_RANK * ep_size,
- args.topk,
- ):
- raise RuntimeError("EPLB calibration trace dimensions differ from the contract")
- load = torch.bincount(ref_idx.reshape(-1), minlength=num_logical).float().tolist()
- eplb_plan = eplb.build_plan(load, args.experts, ep_size)
- if rank == 0:
- print(f"NOTE: EPLB {num_logical}->{args.experts} experts ({ep_size}x{experts_per_rank}); "
- f"per-rank load imbalance {eplb_plan['imbalance_before']:.2f}x -> "
- f"{eplb_plan['imbalance_after']:.2f}x; {eplb_plan['replicated_experts']} experts "
- f"replicated (hottest {eplb_plan['max_replicas']}x)")
- canonical = bool(getattr(args, "workload_dir", ""))
- loaded_workload_ids, loaded_checksums = [], {}
- if canonical:
- import workload as _wl
-
- def build_trace(gt):
- # canonical: load pre-serialized trace bytes (verified by checksum) so this run is
- # provably the SAME workload as any other consuming the same files. else: seeded gen.
- if canonical:
- wid = _wl.compute_workload_id(
- args.routing, args.hidden, args.topk, num_logical, ep_size, gt, args.seed
- )
- idx_np, w_np, man = _wl.load_workload(os.path.join(args.workload_dir, f"{wid}.npz"), verify=True)
- idx_l = torch.from_numpy(idx_np).to(torch.int64)
- w = torch.from_numpy(w_np).to(torch.float32)
- if wid not in loaded_workload_ids:
- loaded_workload_ids.append(wid)
- loaded_checksums[wid] = man.get("checksums")
- else:
- idx_l, w = routing.build_global_routing(
- gt, num_logical, args.topk, args.routing, args.seed
- )
- return (eplb.remap_idx(idx_l, eplb_plan) if eplb_plan is not None else idx_l), w
-
- # Fabric/clock warm-up BEFORE any timed point (review: H200 had an anomalous cold
- # first point and a 40% decode-vs-prefill mismatch at the shared T=128). Gradually
- # ramp through the small ladder shapes untimed — warms clocks/fabric for everyone
- # and is also cold-jump-safe for MoRI.
- def warm_roundtrips(problem, count):
- for _ in range(count):
- handle = backend.dispatch(problem)
- if not hasattr(problem, "recv_tokens"):
- # Dynamic receive cardinality is stable for this fixed routing trace. Cache it
- # during untimed conditioning so adapters never read a device scalar in timing.
- problem.recv_tokens = backend.recv_tokens(handle)
- backend.stage(problem, handle)
- backend.combine(problem, handle)
- torch.cuda.synchronize()
+ # Size the communicator from the resolved numeric shape. Buffer sizing needs the
+ # ladder numbers (not the input tensors), so it runs after make_inputs rather than
+ # in __init__ — resolving the historical build-buffers-before-inputs inversion.
+ # make_inputs already materialized the per-rank routing/activation inputs (canonical
+ # bytes when a workload_dir was staged, else seeded generation) on `spec`.
+ backend.create_buffer(spec)
for wt in conditioning_ladder:
- # Warm-only shapes need not have canonical manifests: they are never measured or emitted.
- wi, ww = routing.build_global_routing(
- wt * ep_size, num_logical, args.topk, args.routing, args.seed,
+ # Fabric/clock warm-up BEFORE any timed point (review: H200 had an anomalous
+ # cold first point and a 40% decode-vs-prefill mismatch at the shared T=128).
+ # make_inputs already materialized these warm-only per-rank inputs; ramping
+ # through the small ladder shapes untimed warms clocks/fabric for everyone and
+ # is cold-jump-safe for MoRI. Warm-only shapes carry no canonical manifest:
+ # they are never measured or emitted.
+ cwp = spec.conditioning_points[wt]
+ wp = backend.make_problem(
+ wt, cwp.topk_idx.to(device), cwp.topk_weights.to(device), cwp.activations
)
- if eplb_plan is not None:
- wi = eplb.remap_idx(wi, eplb_plan)
- wsi, wsw = routing.rank_slice(wi, ww, rank, wt)
- wx = routing.rank_activations(wt, args.hidden, args.seed, rank, device, torch.bfloat16)
- wp = backend.make_problem(wt, wsi.to(device), wsw.to(device), wx)
- warm_roundtrips(wp, CONDITIONING_ROUNDS_PER_SHAPE)
+ backend.warm(wp, CONDITIONING_ROUNDS_PER_SHAPE)
torch.cuda.synchronize()
dist.barrier()
# Setup may materialize deferred provenance such as DeepEP V2 JIT CUBINs.
# Resolve it after conditioning but before correctness or timed measurements.
- capture_deferred_provenance = getattr(backend, "capture_deferred_provenance", None)
- if capture_deferred_provenance is not None:
- capture_deferred_provenance()
+ backend.capture_deferred_provenance()
provenance_issues = contracts.backend_provenance_issues(
backend.name, backend.backend_provenance
)
@@ -1406,18 +1333,18 @@ def warm_roundtrips(problem, count):
counts = [T] * ep_size
gt = T * ep_size
gts[T] = gt
- idx_g, w_g = build_trace(gt)
+ point = spec.points[T]
+ idx_g, w_g = point.global_idx, point.global_weights
rstats = routing.routing_stats(idx_g, args.experts, experts_per_rank, weights=w_g)
gpn = args.gpus_per_node or ep_size
rstats["locality"] = routing.routing_locality(idx_g, experts_per_rank, ep_size, max(1, T),
gpn, args.scale_up_domain or None)
rstats["source_token_stats"] = _stats_vec(counts)
routing_hashes.add(rstats["routing_hash"])
- my_off, my_cnt = rank * T, T
- idx_s = idx_g[my_off:my_off + my_cnt].contiguous()
- w_s = w_g[my_off:my_off + my_cnt].contiguous()
- x = routing.rank_activations(my_cnt, args.hidden, args.seed, rank, device, torch.bfloat16)
- problem = backend.make_problem(my_cnt, idx_s.to(device), w_s.to(device), x)
+ my_cnt = T
+ problem = backend.make_problem(
+ my_cnt, point.topk_idx.to(device), point.topk_weights.to(device), point.activations
+ )
input_snapshots[T] = (
problem.x.clone(), problem.topk_idx.clone(), problem.topk_weights.clone()
)
@@ -1454,12 +1381,6 @@ def warm_roundtrips(problem, count):
stage_trials = {T: [] for T in ladder}
comb_trials = {T: [] for T in ladder}
rt_trials = {T: [] for T in ladder}
- stage_device_work = getattr(backend, "stage_device_work", False)
- if type(stage_device_work) is not bool:
- raise ValueError("backend.stage_device_work must be a boolean")
- dispatch_needs_cleanup = getattr(backend, "dispatch_needs_combine_cleanup", False)
- if type(dispatch_needs_cleanup) is not bool:
- raise ValueError("backend.dispatch_needs_combine_cleanup must be a boolean")
order_identity = args.case_id or _sha256_json({
"backend": backend.name,
"ep_size": ep_size,
@@ -1477,22 +1398,10 @@ def warm_roundtrips(problem, count):
)
for T in order:
problem = problems[T]
- # Stateful paired APIs may expose only a measured round trip.
- # Do not synthesize component latency from that measurement.
- roundtrip_only = getattr(backend, "roundtrip_only", False)
-
- def rt_once(p=problem):
- hh = backend.dispatch(p)
- backend.stage(p, hh)
- return backend.combine(p, hh)
-
- available_components = ["roundtrip"]
- if not roundtrip_only:
- available_components.extend(["dispatch", "combine"])
- if stage_device_work:
- available_components.append("stage")
+ # timed_components() encodes the roundtrip-only vs full-component contract
+ # (and whether stage launches device work) once, in the base class.
component_order = qualification_order(
- available_components,
+ backend.timed_components(),
args.qualification_index,
trial_index,
identity_key=f"{order_identity}:components:{T}",
@@ -1503,60 +1412,13 @@ def rt_once(p=problem):
"trial_index": trial_index,
})
measured = {name: [] for name in ("dispatch", "stage", "combine", "roundtrip")}
-
- def prep_stage(p=problem):
- return backend.dispatch(p)
-
- def prep_combine(p=problem):
- hh = backend.dispatch(p)
- backend.stage(p, hh)
- return hh
-
- def finish_dispatch(hh, p=problem):
- backend.stage(p, hh)
- backend.combine(p, hh)
-
for component_name in component_order:
- # Every measured component receives the same 32 synchronized full-roundtrip
- # warmups immediately before its timed trial.
- warm_roundtrips(problem, args.warmup)
- if component_name == "roundtrip":
- measured[component_name] = time_us(
- torch, lambda p=problem: rt_once(p), 0, args.iters
- )
- elif component_name == "dispatch":
- measured[component_name] = time_us(
- torch, lambda p=problem: backend.dispatch(p), 0, args.iters,
- post=finish_dispatch if dispatch_needs_cleanup else None,
- )
- elif component_name == "stage":
- measured[component_name] = time_us(
- torch,
- lambda hh, p=problem: backend.stage(p, hh),
- 0,
- args.iters,
- pre=prep_stage,
- )
- elif component_name == "combine":
- if backend.combine_needs_redispatch:
- measured[component_name] = time_us(
- torch,
- lambda hh, p=problem: backend.combine(p, hh),
- 0,
- args.iters,
- pre=prep_combine,
- )
- else:
- hh = prep_combine()
- torch.cuda.synchronize()
- measured[component_name] = time_us(
- torch,
- lambda p=problem, hx=hh: backend.combine(p, hx),
- 0,
- args.iters,
- )
- else: # pragma: no cover - generated from the fixed list above
- raise RuntimeError(f"unknown timed component {component_name!r}")
+ # The base template gives every component the same synchronized
+ # full-roundtrip warm-up before its timed trial and encodes the two
+ # branch rules (dispatch cleanup, combine re-dispatch) internally.
+ measured[component_name] = backend.benchmark_component(
+ component_name, problem, args.warmup, args.iters
+ )
disp_iters = measured["dispatch"]
stage_iters = measured["stage"]
comb_iters = measured["combine"]
@@ -1791,8 +1653,7 @@ def component(percentiles, count, *, derived=False):
# Capture again after correctness and timing so no lazily generated kernel can escape
# the implementation identity recorded in the artifact.
- if capture_deferred_provenance is not None:
- capture_deferred_provenance()
+ backend.capture_deferred_provenance()
# status=valid requires correctness AND a proven-identical routing trace across ranks.
all_ok = bool(rows) and all(r["correctness"]["passed"] for r in rows) and routing_consistent
diff --git a/experimental/CollectiveX/tests/ep_mori.py b/experimental/CollectiveX/tests/ep_mori.py
index 3e17b9d94b..a304f8ceba 100644
--- a/experimental/CollectiveX/tests/ep_mori.py
+++ b/experimental/CollectiveX/tests/ep_mori.py
@@ -15,6 +15,7 @@
import torch
import torch.distributed as dist
import ep_precision
+from ep_backend import EPBackend
try:
import mori # type: ignore
@@ -60,7 +61,7 @@ def _mori_input_capacity(args, world_size: int) -> int:
return max(base, largest * world_size)
-class MoRIBackend:
+class MoRIBackend(EPBackend):
name = "mori"
stage_device_work = False
combine_needs_redispatch = True
@@ -68,11 +69,8 @@ class MoRIBackend:
combine_weight_semantics = "unweighted-rank-sum"
def __init__(self, args, rank, world_size, local_rank, device):
- self.args = args
- self.rank = rank
- self.world_size = world_size
- self.device = device
- self.mode = "normal"
+ # mori is normal-mode only; base SUPPORTED_MODES=("normal",) enforces it.
+ super().__init__(args, rank, world_size, local_rank, device)
runner = str(getattr(args, "runner", ""))
if runner.startswith("mi355x"):
fp8_format = "fp8-e4m3fn"
@@ -234,6 +232,17 @@ def __init__(self, args, rank, world_size, local_rank, device):
# Registered-input MoRI copies expert output into a device-side symmetric buffer. External
# input kernels consume the dispatch output directly, so their stage is not applicable.
self.stage_device_work = self._fp8_dispatch or not self._external_input
+ # Stash the __init__-only locals the moved create_buffer body reads back.
+ self._fp8_format = fp8_format
+ self._gpus_per_node = gpus_per_node
+
+ def create_buffer(self, spec):
+ # Local aliases re-expose the __init__ names so the moved shmem-init /
+ # config / op / provenance body below stays byte-verbatim.
+ args, world_size, rank, device = self.args, self.world_size, self.rank, self.device
+ fp8_format = self._fp8_format
+ gpus_per_node = self._gpus_per_node
+ device_cus = torch.cuda.get_device_properties(device).multi_processor_count
world_group = torch.distributed.group.WORLD
torch._C._distributed_c10d._register_process_group("default", world_group)
@@ -245,6 +254,7 @@ def __init__(self, args, rank, world_size, local_rank, device):
)
self._cap = self.buffer_cap(args)
+ assert spec.max_tokens_per_rank <= self._cap
dispatch_dtype = (
getattr(
torch,
diff --git a/experimental/CollectiveX/tests/ep_nccl.py b/experimental/CollectiveX/tests/ep_nccl.py
index ee8143aeee..502a798b5b 100644
--- a/experimental/CollectiveX/tests/ep_nccl.py
+++ b/experimental/CollectiveX/tests/ep_nccl.py
@@ -21,6 +21,7 @@
import torch.distributed as dist
import contracts
import ep_precision
+from ep_backend import EPBackend
def _runtime_collective(args, torch_module) -> tuple[str, str]:
@@ -37,21 +38,17 @@ def _runtime_collective(args, torch_module) -> tuple[str, str]:
return expected, collective["version"]
-class NCCLBackend:
+class NCCLBackend(EPBackend):
name = "nccl-ep"
stage_device_work = False
combine_needs_redispatch = False # dispatch saves the permutation + splits
combine_weight_semantics = "unweighted-rank-sum"
def __init__(self, args, rank, world_size, local_rank, device):
- self.args = args
- self.rank = rank
- self.world_size = world_size
- self.device = device
+ # Base validates mode against SUPPORTED_MODES=("normal",) — this backend is
+ # bf16 normal only, so an unsupported mode raises there.
+ super().__init__(args, rank, world_size, local_rank, device)
self.experts = args.experts
- self.mode = getattr(args, "mode", "normal")
- if self.mode != "normal":
- raise ep_precision.PrecisionError("NCCL/RCCL EP supports normal mode only")
self.precision_profile_id, self.communication_precision = (
ep_precision.resolve_precision(
args,
@@ -106,6 +103,11 @@ def __init__(self, args, rank, world_size, local_rank, device):
"combine_dtype": "bf16",
}
+ def create_buffer(self, spec):
+ # No fixed pre-allocated communicator: all-to-all sizes itself per step, and
+ # all provenance is resolved in __init__. Nothing to size from `spec`.
+ return None
+
def buffer_cap(self, args):
return None # no fixed pre-allocated buffer; all-to-all sizes itself per step
diff --git a/experimental/CollectiveX/tests/ep_uccl.py b/experimental/CollectiveX/tests/ep_uccl.py
index df1d4bdfba..aca9b06595 100644
--- a/experimental/CollectiveX/tests/ep_uccl.py
+++ b/experimental/CollectiveX/tests/ep_uccl.py
@@ -14,6 +14,7 @@
import torch.distributed as dist
import contracts
import ep_precision
+from ep_backend import EPBackend
try:
import uccl
@@ -183,8 +184,9 @@ def _normal_buffer_sizes(hidden: int, world_size: int) -> tuple[int, int]:
return num_nvl_bytes, num_rdma_bytes
-class UCCLBackend:
+class UCCLBackend(EPBackend):
name = "uccl"
+ SUPPORTED_MODES = ("normal", "low-latency")
stage_device_work = False
combine_needs_redispatch = False
combine_weight_semantics = "unweighted-rank-sum"
@@ -192,13 +194,8 @@ class UCCLBackend:
payload_unit = "token-rank"
def __init__(self, args, rank, world_size, local_rank, device):
- self.args = args
- self.rank = rank
- self.world_size = world_size
- self.device = device
- self.mode = getattr(args, "mode", "normal")
- if self.mode not in {"normal", "low-latency"}:
- raise ValueError(f"unsupported UCCL mode {self.mode!r}")
+ # Base validates mode against SUPPORTED_MODES (normal / low-latency).
+ super().__init__(args, rank, world_size, local_rank, device)
supported_profiles = {
"normal": {
"d-bf16.c-bf16",
@@ -224,10 +221,26 @@ def __init__(self, args, rank, world_size, local_rank, device):
self.communication_precision
)
self.stage_device_work = self._fp8_dispatch
-
self.group = dist.group.WORLD
+ # Low-latency flips the contract flags and fixes the per-rank cap so
+ # buffer_cap can report it to make_inputs before create_buffer runs.
+ if self.mode == "low-latency":
+ if args.phase != "decode":
+ raise ValueError("UCCL low-latency mode only supports the decode ladder")
+ if args.experts % world_size:
+ raise ValueError("UCCL low-latency experts must divide the EP group")
+ self.combine_needs_redispatch = True
+ self.combine_weight_semantics = "gate-weighted-sum"
+ self.oracle_layout = "expert-packed"
+ self.payload_unit = "token-expert"
+ self.max_tokens_per_rank = 128
+
+ def create_buffer(self, spec):
+ # Local aliases keep the moved buffer-construction body byte-verbatim.
+ args, world_size, device = self.args, self.world_size, self.device
device_sms = torch.cuda.get_device_properties(device).multi_processor_count
if self.mode == "low-latency":
+ assert spec.max_tokens_per_rank <= 128
ep_precision.require_keyword(
Buffer.low_latency_dispatch,
"use_fp8",
@@ -238,15 +251,6 @@ def __init__(self, args, rank, world_size, local_rank, device):
"use_logfmt",
api="uccl_deepep.Buffer.low_latency_combine",
)
- if args.phase != "decode":
- raise ValueError("UCCL low-latency mode only supports the decode ladder")
- if args.experts % world_size:
- raise ValueError("UCCL low-latency experts must divide the EP group")
- self.combine_needs_redispatch = True
- self.combine_weight_semantics = "gate-weighted-sum"
- self.oracle_layout = "expert-packed"
- self.payload_unit = "token-expert"
- self.max_tokens_per_rank = 128
num_qps_per_rank = args.experts // world_size
num_rdma_bytes = Buffer.get_low_latency_rdma_size_hint(
self.max_tokens_per_rank, args.hidden, world_size, args.experts
diff --git a/experimental/CollectiveX/tests/probe_precision.py b/experimental/CollectiveX/tests/probe_precision.py
index e1de5f4ff5..25060032a3 100644
--- a/experimental/CollectiveX/tests/probe_precision.py
+++ b/experimental/CollectiveX/tests/probe_precision.py
@@ -1071,6 +1071,7 @@ def stage(name: str) -> None:
backend = _backend_class(target["backend"])(
args, rank, world_size, local_rank, device
)
+ backend.create_buffer(backend.make_inputs(args))
construction = {"ok": True}
except Exception:
construction = {"ok": False, "reason": "backend-construction-failed"}
diff --git a/experimental/CollectiveX/tests/test_deepep_v2_contract.py b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
index 2a082a6aea..5fd5fb522e 100644
--- a/experimental/CollectiveX/tests/test_deepep_v2_contract.py
+++ b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
@@ -284,25 +284,27 @@ def test_adapter_ast_pins_elastic_api_and_weight_semantics(self) -> None:
key(baseline, 8, 128, False, {**realized, field: changed}),
field,
)
- init = next(
+ # Buffer construction now lives in create_buffer (it consumes make_inputs),
+ # so the ElasticBuffer/JIT-config/cache-dir ordering is pinned there.
+ build = next(
node for node in backend.body
- if isinstance(node, ast.FunctionDef) and node.name == "__init__"
+ if isinstance(node, ast.FunctionDef) and node.name == "create_buffer"
)
buffer_call = next(
- node for node in ast.walk(init)
+ node for node in ast.walk(build)
if isinstance(node, ast.Call)
and isinstance(node.func, ast.Name)
and node.func.id == "ElasticBuffer"
)
jit_config_check = next(
- node for node in ast.walk(init)
+ node for node in ast.walk(build)
if isinstance(node, ast.Call)
and isinstance(node.func, ast.Name)
and node.func.id == "_require_cross_rank_equal"
and ast.literal_eval(node.args[1]) == "JIT configuration"
)
cache_assignment = next(
- node for node in ast.walk(init)
+ node for node in ast.walk(build)
if isinstance(node, ast.Assign)
and isinstance(node.targets[0], ast.Subscript)
and ast.unparse(node.targets[0].value) == "os.environ"
@@ -986,7 +988,9 @@ def test_hybrid_deferred_provenance_wraps_before_conditioning_and_recaptures(sel
)
methods = {node.name for node in backend.body if isinstance(node, ast.FunctionDef)}
self.assertIn("capture_deferred_provenance", methods)
- constructor = next(node for node in backend.body if isinstance(node, ast.FunctionDef) and node.name == "__init__")
+ # Buffer construction moved into create_buffer (consumes make_inputs); the
+ # cache-dir → HybridEPBuffer → wrapper-install ordering is pinned there now.
+ constructor = next(node for node in backend.body if isinstance(node, ast.FunctionDef) and node.name == "create_buffer")
buffer_call = next(
node for node in ast.walk(constructor)
if isinstance(node, ast.Call) and isinstance(node.func, ast.Name)
diff --git a/experimental/CollectiveX/tests/test_ep_precision_adapters.py b/experimental/CollectiveX/tests/test_ep_precision_adapters.py
index ad889e46a8..d268d788ae 100644
--- a/experimental/CollectiveX/tests/test_ep_precision_adapters.py
+++ b/experimental/CollectiveX/tests/test_ep_precision_adapters.py
@@ -122,12 +122,14 @@ def test_elastic_and_hybrid_constructors_enable_native_fp8(self):
def test_mori_wires_dispatch_scales_and_direct_cast_config(self):
source = (TESTS / "ep_mori.py").read_text(encoding="utf-8")
harness = (TESTS / "ep_harness.py").read_text(encoding="utf-8")
+ backend = (TESTS / "ep_backend.py").read_text(encoding="utf-8")
precision = (TESTS / "ep_precision.py").read_text(encoding="utf-8")
self.assertIn('"fp8_direct_cast" if self._direct_cast_combine', source)
self.assertIn("p.scales,", source)
self.assertIn("dispatch_scales=_scales", source)
self.assertIn("dispatch_needs_combine_cleanup = True", source)
- self.assertIn("post=finish_dispatch if dispatch_needs_cleanup else None", harness)
+ # The dispatch-cleanup branch rule is encoded once in the base timing template.
+ self.assertIn("post=finish_dispatch if dispatch_needs_cleanup else None", backend)
self.assertEqual(harness.count("view.combine_input = transformed"), 2)
self.assertIn("transformed = view.combine_input.float()", precision)
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 255de0a829..a1d41c582f 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -971,9 +971,12 @@ def test_image_pinned_deepep_and_input_integrity_order_are_explicit(self) -> Non
pass_one.index("oracle = _run_expert_oracle"),
)
self.assertIn("pre_input_unchanged", pass_one)
+ # The combine timing template reuses one dispatched+staged handle (no per-iter
+ # re-dispatch) for stateless-combine backends; it lives in the base class now.
+ backend = (HERE / "ep_backend.py").read_text()
self.assertIn(
- "hh = prep_combine()\n torch.cuda.synchronize()",
- harness,
+ "hh = prep_combine()\n torch.cuda.synchronize()",
+ backend,
)
def test_squash_imports_are_reproducible_and_use_a_fresh_cache_key(self) -> None:
From d9651f38f9212e5993390abe91a2e53c5546c1c2 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 7 Jul 2026 16:08:23 +0800
Subject: [PATCH 175/210] refactor(collectivex): drop adapter methods now
covered by EPBackend
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Pure dedup after the ABC cutover: delete per-adapter methods whose bodies the
base class now reproduces exactly.
- oracle_dispatch_payload: deleted from deepep, deepep-v2, mori, hybrid, uccl
(all were the base's encode_dispatch(...).semantic). nccl keeps its identity
override (the wire payload is what the oracle built).
- precision_evidence: deleted from every adapter. The base threads
uint8_storage=self._uint8_scale_storage, which is False by default and True
for hybrid (via its class flag) — matching each adapter's former literal.
- make_problem: deleted from nccl, deepep, deepep-v2, uccl (base reproduces
them; deepep-v2 keeps its int-dtype via a new _topk_idx_dtype override
returning deep_ep.topk_idx_t). mori/hybrid keep make_problem (extra
scales/indices fields and conditional dispatch_x).
- buffer_cap/finalize: deleted from nccl (both matched the base). deepep/uccl
keep buffer_cap (low-latency cap) and finalize (LL teardown / os._exit);
mori keeps finalize (os._exit); deepep-v2/hybrid keep finalize (rc=1 / JIT
teardown).
No behavioral change: every deletion was verified equal to the base, and the
dead problem.layout field (never read) is dropped with nccl.make_problem.
ABC 切换后的纯去重：删除那些基类已逐字复现的各适配器方法。
- oracle_dispatch_payload：从 deepep、deepep-v2、mori、hybrid、uccl 删除（均为基类的
encode_dispatch(...).semantic）。nccl 保留其恒等重写（线上载荷即 oracle 所构造）。
- precision_evidence：从所有适配器删除。基类透传
uint8_storage=self._uint8_scale_storage，默认 False、hybrid 经类标志为 True，
与各适配器原有字面量一致。
- make_problem：从 nccl、deepep、deepep-v2、uccl 删除（基类可复现；deepep-v2 通过新增
_topk_idx_dtype 重写返回 deep_ep.topk_idx_t 保留其整型 dtype）。mori/hybrid 保留
make_problem（含额外 scales/indices 字段与条件式 dispatch_x）。
- buffer_cap/finalize：从 nccl 删除（均与基类一致）。deepep/uccl 保留 buffer_cap
（低延迟容量）与 finalize（LL 拆除 / os._exit）；mori 保留 finalize（os._exit）；
deepep-v2/hybrid 保留 finalize（rc=1 / JIT 拆除）。
无行为变化：每处删除均已核验与基类等价；随 nccl.make_problem 一并移除从未被读取的
无用 problem.layout 字段。
---
experimental/CollectiveX/tests/ep_deepep.py | 28 --------------
.../CollectiveX/tests/ep_deepep_hybrid.py | 15 --------
.../CollectiveX/tests/ep_deepep_v2.py | 30 ++-------------
experimental/CollectiveX/tests/ep_mori.py | 14 -------
experimental/CollectiveX/tests/ep_nccl.py | 37 +------------------
experimental/CollectiveX/tests/ep_uccl.py | 28 --------------
6 files changed, 5 insertions(+), 147 deletions(-)
diff --git a/experimental/CollectiveX/tests/ep_deepep.py b/experimental/CollectiveX/tests/ep_deepep.py
index 931d4c3ccf..bfa69eede8 100644
--- a/experimental/CollectiveX/tests/ep_deepep.py
+++ b/experimental/CollectiveX/tests/ep_deepep.py
@@ -228,20 +228,6 @@ def create_buffer(self, spec):
def buffer_cap(self, args):
return self.max_tokens_per_rank if self.mode == "low-latency" else None
- def make_problem(self, T, idx, weights, x):
- encoding = ep_precision.encode_dispatch(
- torch, x, self.communication_precision
- )
- return types.SimpleNamespace(
- T=T,
- x=x,
- dispatch_x=encoding.native_input,
- oracle_x=encoding.semantic,
- dispatch_precision_evidence=encoding.evidence,
- topk_idx=idx.to(torch.int64),
- topk_weights=weights.to(torch.float32),
- )
-
def dispatch(self, p):
if self.mode == "low-latency":
recv_x, recv_counts, handle, _, _ = self.buffer.low_latency_dispatch(
@@ -404,20 +390,6 @@ def _semantic_recv(self, h, problem=None):
)
return h.recv_semantic
- def oracle_dispatch_payload(self, payload):
- return ep_precision.encode_dispatch(
- torch, payload, self.communication_precision
- ).semantic
-
- def precision_evidence(self, problem, view=None):
- return ep_precision.precision_evidence(
- torch,
- profile_id=self.precision_profile_id,
- profile=self.communication_precision,
- problem=problem,
- view=view,
- )
-
def finalize(self, rc):
try:
dist.barrier()
diff --git a/experimental/CollectiveX/tests/ep_deepep_hybrid.py b/experimental/CollectiveX/tests/ep_deepep_hybrid.py
index 666a428e5b..7e1a688bd8 100644
--- a/experimental/CollectiveX/tests/ep_deepep_hybrid.py
+++ b/experimental/CollectiveX/tests/ep_deepep_hybrid.py
@@ -526,21 +526,6 @@ def _semantic_recv(self, h, rows):
raise RuntimeError("DeepEP Hybrid receive count changed for one dispatch handle")
return h.recv_semantic
- def oracle_dispatch_payload(self, payload):
- return ep_precision.encode_dispatch(
- torch, payload, self.communication_precision
- ).semantic
-
- def precision_evidence(self, problem, view=None):
- return ep_precision.precision_evidence(
- torch,
- profile_id=self.precision_profile_id,
- profile=self.communication_precision,
- problem=problem,
- view=view,
- uint8_storage=True,
- )
-
def finalize(self, rc):
try:
dist.barrier()
diff --git a/experimental/CollectiveX/tests/ep_deepep_v2.py b/experimental/CollectiveX/tests/ep_deepep_v2.py
index cf2369d61f..6d366370bd 100644
--- a/experimental/CollectiveX/tests/ep_deepep_v2.py
+++ b/experimental/CollectiveX/tests/ep_deepep_v2.py
@@ -500,19 +500,9 @@ def create_buffer(self, spec):
"physical_nvlink_ranks": int(self.buffer.num_nvlink_ranks),
}
- def make_problem(self, T, idx, weights, x):
- encoding = ep_precision.encode_dispatch(
- torch, x, self.communication_precision
- )
- return types.SimpleNamespace(
- T=T,
- x=x,
- dispatch_x=encoding.native_input,
- oracle_x=encoding.semantic,
- dispatch_precision_evidence=encoding.evidence,
- topk_idx=idx.to(deep_ep.topk_idx_t),
- topk_weights=weights.to(torch.float32),
- )
+ def _topk_idx_dtype(self):
+ # DeepEP V2's kernels key routing indices on deep_ep.topk_idx_t, not int64.
+ return deep_ep.topk_idx_t
def dispatch(self, p):
recv_x, recv_topk_idx, recv_topk_weights, handle, _ = self.buffer.dispatch(
@@ -632,20 +622,6 @@ def _semantic_recv(self, h, rows):
raise RuntimeError("DeepEP V2 receive count changed for one dispatch handle")
return h.recv_semantic
- def oracle_dispatch_payload(self, payload):
- return ep_precision.encode_dispatch(
- torch, payload, self.communication_precision
- ).semantic
-
- def precision_evidence(self, problem, view=None):
- return ep_precision.precision_evidence(
- torch,
- profile_id=self.precision_profile_id,
- profile=self.communication_precision,
- problem=problem,
- view=view,
- )
-
def finalize(self, rc):
try:
dist.barrier()
diff --git a/experimental/CollectiveX/tests/ep_mori.py b/experimental/CollectiveX/tests/ep_mori.py
index a304f8ceba..30e0d264e2 100644
--- a/experimental/CollectiveX/tests/ep_mori.py
+++ b/experimental/CollectiveX/tests/ep_mori.py
@@ -524,20 +524,6 @@ def _semantic_recv(self, h, rows):
raise RuntimeError("MoRI receive count changed for one dispatch handle")
return h.recv_semantic
- def oracle_dispatch_payload(self, payload):
- return ep_precision.encode_dispatch(
- torch, payload, self.communication_precision
- ).semantic
-
- def precision_evidence(self, problem, view=None):
- return ep_precision.precision_evidence(
- torch,
- profile_id=self.precision_profile_id,
- profile=self.communication_precision,
- problem=problem,
- view=view,
- )
-
def finalize(self, rc):
try:
dist.barrier()
diff --git a/experimental/CollectiveX/tests/ep_nccl.py b/experimental/CollectiveX/tests/ep_nccl.py
index 502a798b5b..0b472aec77 100644
--- a/experimental/CollectiveX/tests/ep_nccl.py
+++ b/experimental/CollectiveX/tests/ep_nccl.py
@@ -108,24 +108,6 @@ def create_buffer(self, spec):
# all provenance is resolved in __init__. Nothing to size from `spec`.
return None
- def buffer_cap(self, args):
- return None # no fixed pre-allocated buffer; all-to-all sizes itself per step
-
- def make_problem(self, T, idx, weights, x):
- encoding = ep_precision.encode_dispatch(
- torch, x, self.communication_precision
- )
- # idx[T,topk] int64, weights[T,topk] f32, x[T,hidden] bf16 — the shared routing-trace slice.
- return types.SimpleNamespace(
- T=T,
- x=x,
- oracle_x=encoding.semantic,
- dispatch_precision_evidence=encoding.evidence,
- topk_idx=idx.to(torch.int64),
- topk_weights=weights.to(torch.float32),
- layout=None,
- )
-
def dispatch(self, p):
ws = self.world_size
x = p.x # [T, H] bf16
@@ -221,21 +203,6 @@ def recv_tokens(self, h):
return int(h.total_recv)
def oracle_dispatch_payload(self, payload):
+ # NCCL is the identity reference: the wire payload is exactly what the oracle
+ # built, so no dispatch-encoding projection (unlike the base) applies.
return payload
-
- def precision_evidence(self, problem, view=None):
- return ep_precision.precision_evidence(
- torch,
- profile_id=self.precision_profile_id,
- profile=self.communication_precision,
- problem=problem,
- view=view,
- )
-
- def finalize(self, rc):
- try:
- dist.barrier()
- dist.destroy_process_group()
- except Exception:
- pass
- return rc
diff --git a/experimental/CollectiveX/tests/ep_uccl.py b/experimental/CollectiveX/tests/ep_uccl.py
index aca9b06595..4af636193e 100644
--- a/experimental/CollectiveX/tests/ep_uccl.py
+++ b/experimental/CollectiveX/tests/ep_uccl.py
@@ -336,20 +336,6 @@ def create_buffer(self, spec):
def buffer_cap(self, args):
return self.max_tokens_per_rank if self.mode == "low-latency" else None
- def make_problem(self, T, idx, weights, x):
- encoding = ep_precision.encode_dispatch(
- torch, x, self.communication_precision
- )
- return types.SimpleNamespace(
- T=T,
- x=x,
- dispatch_x=encoding.native_input,
- oracle_x=encoding.semantic,
- dispatch_precision_evidence=encoding.evidence,
- topk_idx=idx.to(torch.int64),
- topk_weights=weights.to(torch.float32),
- )
-
def dispatch(self, p):
if self.mode == "low-latency":
recv_x, recv_counts, handle, _, _ = self.buffer.low_latency_dispatch(
@@ -512,20 +498,6 @@ def _semantic_recv(self, h, problem=None):
)
return h.recv_semantic
- def oracle_dispatch_payload(self, payload):
- return ep_precision.encode_dispatch(
- torch, payload, self.communication_precision
- ).semantic
-
- def precision_evidence(self, problem, view=None):
- return ep_precision.precision_evidence(
- torch,
- profile_id=self.precision_profile_id,
- profile=self.communication_precision,
- problem=problem,
- view=view,
- )
-
def finalize(self, rc):
# UCCL's proxy teardown can crash after results are written; preserve the real rc.
try:
From 78c3ba9673230c7d75356cc2a630418598bdf394 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 7 Jul 2026 16:10:35 +0800
Subject: [PATCH 176/210] test(collectivex): unit-test the EPBackend ABC
lifecycle
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Add torch-free unit tests for the shared base-class plumbing the driver relies
on, using a minimal in-process fake backend:
- make_inputs: rc=2 early-return when the buffer cap can't serve the conditioning
ladder and when the token ladder is empty; on success, ladder/conditioning
sizing, max-tokens-per-rank span, and canonical-measured workload registration
(conditioning shapes never register). _build_rank_inputs is overridden so the
path needs no routing/activation tensors.
- timed_components: the roundtrip / +dispatch,combine / +stage / roundtrip-only
matrix, including roundtrip_only suppressing the stage flag.
- benchmark_dispatch/combine branch selection: post=cleanup only when
dispatch_needs_combine_cleanup; combine supplies a per-iter re-dispatch `pre`
when combine_needs_redispatch, else dispatches once and reuses the handle.
time_us is stubbed to capture pre/post without running a timing loop.
- __init__ mode validation (SUPPORTED_MODES) and __init_subclass__ empty-name
enforcement.
为驱动器所依赖的基类公共管线新增无需 torch 的单元测试，使用进程内最小假后端：
- make_inputs：当缓冲区容量无法承载 conditioning 阶梯、以及 token 阶梯为空时的
rc=2 提前返回；成功路径下的阶梯/conditioning 定尺、max-tokens-per-rank 跨度，
以及仅对 canonical 实测点登记 workload（conditioning 形状不登记）。重写
_build_rank_inputs 使该路径无需 routing/activation 张量。
- timed_components：roundtrip / +dispatch,combine / +stage / 仅 roundtrip 矩阵，
含 roundtrip_only 抑制 stage 标志。
- benchmark_dispatch/combine 分支选择：仅当 dispatch_needs_combine_cleanup 时
post=清理；combine 在 combine_needs_redispatch 时提供逐迭代重派发的 pre，
否则派发一次并复用句柄。time_us 被打桩以捕获 pre/post 而不运行计时循环。
- __init__ 模式校验（SUPPORTED_MODES）与 __init_subclass__ 空 name 强制。
---
.../CollectiveX/tests/test_ep_backend.py | 303 ++++++++++++++++++
1 file changed, 303 insertions(+)
create mode 100644 experimental/CollectiveX/tests/test_ep_backend.py
diff --git a/experimental/CollectiveX/tests/test_ep_backend.py b/experimental/CollectiveX/tests/test_ep_backend.py
new file mode 100644
index 0000000000..97e253da2e
--- /dev/null
+++ b/experimental/CollectiveX/tests/test_ep_backend.py
@@ -0,0 +1,303 @@
+#!/usr/bin/env python3
+"""Unit tests for the EPBackend abstract base class (torch-free).
+
+These exercise the shared lifecycle/plumbing the driver relies on — make_inputs
+sizing and early-returns, the timed_components matrix, the two Pass-2 timing
+branch rules, mode validation, and __init_subclass__ name enforcement — against a
+minimal in-process fake backend.
+
+torch is never installed in this environment, so the few base methods that
+``import torch`` locally are driven with a stub module injected into
+``sys.modules``; make_inputs is kept torch-free by overriding
+``_build_rank_inputs`` on the fake so no real routing/activation tensors are
+materialised.
+"""
+from __future__ import annotations
+
+import sys
+import types
+import unittest
+from pathlib import Path
+from unittest import mock
+
+ROOT = Path(__file__).resolve().parents[1]
+TESTS = ROOT / "tests"
+sys.path.insert(0, str(ROOT))
+sys.path.insert(0, str(TESTS))
+
+import ep_backend # noqa: E402
+from ep_backend import EPBackend, RankInputs # noqa: E402
+
+
+def _args(**over):
+ """A minimal args namespace with the fields make_inputs / __init__ read."""
+ base = dict(
+ experts=8,
+ phase="decode",
+ tokens_ladder="",
+ routing="uniform",
+ seed=0,
+ hidden=16,
+ topk=2,
+ mode="normal",
+ workload_dir="",
+ )
+ base.update(over)
+ return types.SimpleNamespace(**base)
+
+
+class FakeBackend(EPBackend):
+ """Concrete no-op backend: records transport calls, never touches a device."""
+
+ name = "fake"
+
+ def __init__(self, args, rank=0, world_size=1, local_rank=0, device="cpu", cap=None):
+ super().__init__(args, rank, world_size, local_rank, device)
+ self._cap = cap
+ self.calls = []
+ self.built = []
+
+ # ---- transport contract: recording no-ops -----------------------------------
+ def create_buffer(self, spec):
+ self.calls.append("create_buffer")
+ return None
+
+ def dispatch(self, p):
+ self.calls.append("dispatch")
+ return types.SimpleNamespace()
+
+ def stage(self, p, h):
+ self.calls.append("stage")
+ return None
+
+ def combine(self, p, h):
+ self.calls.append("combine")
+ return None
+
+ def recv_tokens(self, h):
+ self.calls.append("recv_tokens")
+ return 0
+
+ def inspect_dispatch(self, p, h):
+ raise NotImplementedError
+
+ def combine_transformed(self, p, h, transformed):
+ raise NotImplementedError
+
+ # ---- keep make_inputs torch-free --------------------------------------------
+ def buffer_cap(self, args):
+ return self._cap
+
+ def _build_rank_inputs(self, args, tokens_per_rank, *, canonical, retain_global):
+ self.built.append((tokens_per_rank, canonical, retain_global))
+ workload_id = f"wl-{tokens_per_rank}" if canonical else None
+ return RankInputs(
+ tokens_per_rank=tokens_per_rank,
+ topk_idx=None,
+ topk_weights=None,
+ activations=None,
+ workload_id=workload_id,
+ checksums={"c": tokens_per_rank},
+ )
+
+
+class MakeInputsTests(unittest.TestCase):
+ def test_cap_below_conditioning_ladder_early_returns_rc2(self):
+ # decode conditioning ladder tops out at 128; a cap of 64 cannot serve it.
+ backend = FakeBackend(_args(phase="decode"), cap=64)
+ spec = backend.make_inputs(backend.args)
+ self.assertFalse(spec.ok)
+ self.assertEqual(spec.rc, 2)
+ self.assertIn("buffer cap 64", spec.message)
+ self.assertEqual(backend.built, []) # no inputs materialised on the reject path
+
+ def test_empty_ladder_early_returns_rc2(self):
+ # "0" filters to no positive token counts -> empty ladder (cap is unbounded).
+ backend = FakeBackend(_args(tokens_ladder="0"), cap=None)
+ spec = backend.make_inputs(backend.args)
+ self.assertFalse(spec.ok)
+ self.assertEqual(spec.rc, 2)
+ self.assertIn("empty token ladder", spec.message)
+ self.assertEqual(backend.built, [])
+
+ def test_success_spec_sizes_and_materialises(self):
+ backend = FakeBackend(
+ _args(tokens_ladder="8 16", workload_dir="/w"), world_size=2, cap=None
+ )
+ spec = backend.make_inputs(backend.args)
+ self.assertTrue(spec.ok)
+ self.assertEqual(spec.ladder, [8, 16])
+ self.assertEqual(spec.conditioning_ladder, [1, 2, 4, 8, 16, 32, 64, 128])
+ # max tokens/rank spans measured + conditioning shapes (drives buffer sizing).
+ self.assertEqual(spec.max_tokens_per_rank, 128)
+ self.assertEqual(spec.ep_size, 2)
+ self.assertEqual(spec.experts_per_rank, 4)
+ self.assertEqual(sorted(spec.points), [8, 16])
+ self.assertEqual(sorted(spec.conditioning_points), [1, 2, 4, 8, 16, 32, 64, 128])
+
+ def test_success_registers_canonical_measured_workloads_only(self):
+ backend = FakeBackend(
+ _args(tokens_ladder="8 16", workload_dir="/w"), world_size=2, cap=None
+ )
+ spec = backend.make_inputs(backend.args)
+ # canonical measured points register ids/checksums; conditioning shapes
+ # (canonical=False -> workload_id None) never do.
+ self.assertEqual(spec.loaded_workload_ids, ["wl-8", "wl-16"])
+ self.assertEqual(spec.loaded_checksums["wl-8"], {"c": 8})
+ # measured points are canonical + retain the global trace; conditioning is neither.
+ self.assertIn((8, True, True), backend.built)
+ self.assertIn((1, False, False), backend.built)
+
+ def test_non_canonical_run_registers_no_workload_ids(self):
+ backend = FakeBackend(_args(tokens_ladder="8 16"), world_size=2, cap=None)
+ spec = backend.make_inputs(backend.args) # workload_dir "" -> non-canonical
+ self.assertEqual(spec.loaded_workload_ids, [])
+ self.assertIn((8, False, True), backend.built)
+
+
+class TimedComponentsTests(unittest.TestCase):
+ def _backend(self, **flags):
+ backend = FakeBackend(_args())
+ for key, value in flags.items():
+ setattr(backend, key, value)
+ return backend
+
+ def test_default_measures_roundtrip_dispatch_combine(self):
+ self.assertEqual(
+ self._backend().timed_components(), ["roundtrip", "dispatch", "combine"]
+ )
+
+ def test_stage_device_work_adds_stage(self):
+ self.assertEqual(
+ self._backend(stage_device_work=True).timed_components(),
+ ["roundtrip", "dispatch", "combine", "stage"],
+ )
+
+ def test_roundtrip_only_measures_roundtrip_alone(self):
+ self.assertEqual(self._backend(roundtrip_only=True).timed_components(), ["roundtrip"])
+
+ def test_roundtrip_only_suppresses_stage_flag(self):
+ backend = self._backend(roundtrip_only=True, stage_device_work=True)
+ self.assertEqual(backend.timed_components(), ["roundtrip"])
+
+
+class BenchmarkBranchTests(unittest.TestCase):
+ """The two Pass-2 branch rules encoded in benchmark_dispatch/combine.
+
+ time_us is replaced with a recorder that captures its pre/post kwargs without
+ invoking the timed callables, so we observe branch *selection* rather than run
+ a device timing loop.
+ """
+
+ def setUp(self):
+ self.records = []
+
+ def recorder(torch_module, fn, warmup, iters, pre=None, post=None):
+ self.records.append(
+ types.SimpleNamespace(fn=fn, warmup=warmup, iters=iters, pre=pre, post=post)
+ )
+ return [1.0]
+
+ self._patch_time_us = mock.patch.object(ep_backend, "time_us", recorder)
+ self._patch_time_us.start()
+ self.addCleanup(self._patch_time_us.stop)
+
+ fake_torch = types.SimpleNamespace(
+ cuda=types.SimpleNamespace(synchronize=lambda: None)
+ )
+ self._patch_torch = mock.patch.dict(sys.modules, {"torch": fake_torch})
+ self._patch_torch.start()
+ self.addCleanup(self._patch_torch.stop)
+
+ def test_dispatch_without_cleanup_passes_no_post(self):
+ backend = FakeBackend(_args())
+ backend.benchmark_dispatch(types.SimpleNamespace(), warmup=0, iters=3)
+ self.assertEqual(len(self.records), 1)
+ self.assertIsNone(self.records[0].post)
+ self.assertEqual(self.records[0].iters, 3)
+
+ def test_dispatch_with_cleanup_attaches_untimed_stage_combine(self):
+ backend = FakeBackend(_args())
+ backend.dispatch_needs_combine_cleanup = True
+ backend.benchmark_dispatch(types.SimpleNamespace(), warmup=0, iters=3)
+ self.assertTrue(callable(self.records[0].post))
+ # the post hook runs stage then combine (untimed cleanup after a timed dispatch).
+ backend.calls.clear()
+ self.records[0].post(types.SimpleNamespace())
+ self.assertEqual(backend.calls, ["stage", "combine"])
+
+ def test_combine_redispatch_supplies_pre_and_defers_it(self):
+ backend = FakeBackend(_args())
+ backend.combine_needs_redispatch = True
+ backend.calls.clear()
+ backend.benchmark_combine(types.SimpleNamespace(), warmup=0, iters=3)
+ # pre is handed to time_us but not executed here (recorder never calls it).
+ self.assertTrue(callable(self.records[0].pre))
+ self.assertEqual(backend.calls, [])
+ # invoking pre performs exactly one untimed dispatch + stage per iteration.
+ backend.calls.clear()
+ self.records[0].pre()
+ self.assertEqual(backend.calls, ["dispatch", "stage"])
+
+ def test_combine_stateless_dispatches_once_and_reuses_handle(self):
+ backend = FakeBackend(_args()) # combine_needs_redispatch stays False
+ backend.calls.clear()
+ backend.benchmark_combine(types.SimpleNamespace(), warmup=0, iters=3)
+ self.assertIsNone(self.records[0].pre)
+ # one dispatch+stage happens before timing; the handle is reused across iters.
+ self.assertEqual(backend.calls, ["dispatch", "stage"])
+
+
+class ContractEnforcementTests(unittest.TestCase):
+ def test_unsupported_mode_raises_valueerror(self):
+ with self.assertRaises(ValueError):
+ FakeBackend(_args(mode="low-latency"))
+
+ def test_supported_mode_constructs(self):
+ backend = FakeBackend(_args(mode="normal"))
+ self.assertEqual(backend.mode, "normal")
+
+ def test_multi_mode_backend_accepts_declared_mode(self):
+ class MultiMode(FakeBackend):
+ name = "multi"
+ SUPPORTED_MODES = ("normal", "low-latency")
+
+ backend = MultiMode(_args(mode="low-latency"))
+ self.assertEqual(backend.mode, "low-latency")
+
+ def test_missing_name_raises_at_class_definition(self):
+ with self.assertRaises(TypeError):
+
+ class NoName(EPBackend):
+ pass
+
+ def test_named_subclass_defines_cleanly(self):
+ class Named(EPBackend):
+ name = "named"
+
+ def create_buffer(self, spec):
+ return None
+
+ def dispatch(self, p):
+ return None
+
+ def stage(self, p, h):
+ return None
+
+ def combine(self, p, h):
+ return None
+
+ def recv_tokens(self, h):
+ return 0
+
+ def inspect_dispatch(self, p, h):
+ return None
+
+ def combine_transformed(self, p, h, transformed):
+ return None
+
+ self.assertEqual(Named.name, "named")
+
+
+if __name__ == "__main__":
+ unittest.main()
From 7c8fd531fca6f4655e77bff7f1444cae088b8c69 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 7 Jul 2026 16:49:16 +0800
Subject: [PATCH 177/210] refactor(collectivex): share DeepEP/UCCL adapter
surface via family base
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
UCCL's uccl_deepep.Buffer is a DeepEP-API clone, so ep_deepep.py and
ep_uccl.py carried a byte-identical ~250-line operational surface (mode
handling, dispatch/combine, expert-packed inspection, fp8 dequantization).
Extract it into a new deep_ep-free DeepEPFamilyBackend base; each concrete
adapter keeps only its native buffer import, create_buffer provenance, and
teardown. The base imports no vendor module, so importing ep_uccl never
transitively requires deep_ep (absent on the UCCL image).
Re-target the three adapter-genuineness scrape-tests to follow the moved
code into the base file rather than deleting them, preserving all CPU-side
coverage of the un-runnable native low-latency calls. Suite stays 239 green.
重构：通过 family 基类共享 DeepEP/UCCL 适配器公共代码
UCCL 的 uccl_deepep.Buffer 是 DeepEP API 的克隆，因此 ep_deepep.py 与
ep_uccl.py 存在约 250 行逐字节相同的操作代码（模式处理、dispatch/combine、
expert-packed 检查、fp8 反量化）。将其提取到不依赖 deep_ep 的
DeepEPFamilyBackend 基类；各具体适配器仅保留各自的原生 buffer 导入、
create_buffer 溯源与清理逻辑。基类不导入任何厂商模块，因此导入 ep_uccl
不会间接引入 deep_ep（UCCL 镜像中并不存在）。
将三个适配器真实性源码扫描测试重定向到基类文件以跟随迁移后的代码，而非删除，
从而保留对不可运行的原生 low-latency 调用的全部 CPU 侧覆盖。测试保持 239 通过。
---
experimental/CollectiveX/tests/ep_deepep.py | 222 +---------------
.../CollectiveX/tests/ep_deepep_family.py | 250 ++++++++++++++++++
experimental/CollectiveX/tests/ep_uccl.py | 222 +---------------
.../tests/test_deepep_v2_contract.py | 34 ++-
.../tests/test_ep_precision_adapters.py | 13 +-
5 files changed, 288 insertions(+), 453 deletions(-)
create mode 100644 experimental/CollectiveX/tests/ep_deepep_family.py
diff --git a/experimental/CollectiveX/tests/ep_deepep.py b/experimental/CollectiveX/tests/ep_deepep.py
index bfa69eede8..270ece9600 100644
--- a/experimental/CollectiveX/tests/ep_deepep.py
+++ b/experimental/CollectiveX/tests/ep_deepep.py
@@ -5,13 +5,12 @@
import inspect
import os
import sys
-import types
import torch
import torch.distributed as dist
import contracts
import ep_precision
-from ep_backend import EPBackend
+from ep_deepep_family import DeepEPFamilyBackend
try:
import deep_ep
@@ -69,58 +68,10 @@ def _normal_buffer_sizes(hidden: int, world_size: int) -> tuple[int, int]:
return num_nvl_bytes, num_rdma_bytes
-class DeepEPBackend(EPBackend):
+class DeepEPBackend(DeepEPFamilyBackend):
+ # Mode handling, dispatch/combine, and fp8 dequantization are shared with UCCL in
+ # DeepEPFamilyBackend; only the native deep_ep buffer construction and teardown live here.
name = "deepep"
- SUPPORTED_MODES = ("normal", "low-latency")
- stage_device_work = False
- combine_needs_redispatch = False
- # DeepEP reduces activations and top-k weights independently. The activation
- # tensor must therefore carry the complete local weighted expert sum.
- combine_weight_semantics = "unweighted-rank-sum"
- oracle_layout = "token-rank"
- payload_unit = "token-rank"
-
- def __init__(self, args, rank, world_size, local_rank, device):
- # Base validates mode against SUPPORTED_MODES (normal / low-latency).
- super().__init__(args, rank, world_size, local_rank, device)
- supported_profiles = {
- "normal": {
- "d-bf16.c-bf16",
- "d-fp8-e4m3fn-b128-f32-prequantized.c-bf16",
- },
- "low-latency": {
- "d-bf16.c-bf16",
- "d-fp8-e4m3fn-b128-f32-fused.c-bf16",
- },
- }
- self.precision_profile_id, self.communication_precision = (
- ep_precision.resolve_precision(
- args,
- backend=self.name,
- mode=self.mode,
- supported_profiles=supported_profiles[self.mode],
- )
- )
- self._fp8_dispatch = ep_precision.is_low_precision_dispatch(
- self.communication_precision
- )
- self._use_logfmt = ep_precision.uses_logfmt_combine(
- self.communication_precision
- )
- self.stage_device_work = self._fp8_dispatch
- self.group = dist.group.WORLD
- # Low-latency flips the contract flags and fixes the per-rank cap so
- # buffer_cap can report it to make_inputs before create_buffer runs.
- if self.mode == "low-latency":
- if args.phase != "decode":
- raise ValueError("DeepEP low-latency mode only supports the decode ladder")
- if args.experts % world_size:
- raise ValueError("DeepEP low-latency experts must divide the EP group")
- self.combine_needs_redispatch = True
- self.combine_weight_semantics = "gate-weighted-sum"
- self.oracle_layout = "expert-packed"
- self.payload_unit = "token-expert"
- self.max_tokens_per_rank = 128
def create_buffer(self, spec):
# Local aliases keep the moved buffer-construction body byte-verbatim.
@@ -225,171 +176,6 @@ def create_buffer(self, spec):
**resource_provenance,
}
- def buffer_cap(self, args):
- return self.max_tokens_per_rank if self.mode == "low-latency" else None
-
- def dispatch(self, p):
- if self.mode == "low-latency":
- recv_x, recv_counts, handle, _, _ = self.buffer.low_latency_dispatch(
- p.x,
- p.topk_idx,
- self.max_tokens_per_rank,
- self.args.experts,
- use_fp8=self._fp8_dispatch, # BF16 control realizes use_fp8=False.
- async_finish=False,
- return_recv_hook=False,
- )
- return types.SimpleNamespace(
- recv_x=recv_x,
- recv_counts=recv_counts,
- handle=handle,
- )
- (
- num_tokens_per_rank,
- num_tokens_per_rdma_rank,
- num_tokens_per_expert,
- is_token_in_rank,
- _,
- ) = self.buffer.get_dispatch_layout(p.topk_idx, self.args.experts)
- recv_x, recv_topk_idx, recv_topk_weights, recv_counts, handle, _ = self.buffer.dispatch(
- p.dispatch_x,
- topk_idx=p.topk_idx,
- topk_weights=p.topk_weights,
- num_tokens_per_rank=num_tokens_per_rank,
- num_tokens_per_rdma_rank=num_tokens_per_rdma_rank,
- is_token_in_rank=is_token_in_rank,
- num_tokens_per_expert=num_tokens_per_expert,
- async_finish=False,
- )
- return types.SimpleNamespace(
- recv_x=recv_x,
- recv_topk_idx=recv_topk_idx,
- recv_topk_weights=recv_topk_weights,
- recv_counts=recv_counts,
- handle=handle,
- )
-
- def stage(self, p, h):
- h.combine_input = self._semantic_recv(h, p)
-
- def combine(self, p, h):
- if self.mode == "low-latency":
- combined_x, _, _ = self.buffer.low_latency_combine(
- h.combine_input,
- p.topk_idx,
- p.topk_weights,
- h.handle,
- use_logfmt=self._use_logfmt,
- async_finish=False,
- return_recv_hook=False,
- )
- return combined_x
- combined_x, _, _ = self.buffer.combine(
- h.combine_input, h.handle, async_finish=False
- )
- return combined_x
-
- def inspect_dispatch(self, p, h):
- valid = h.recv_topk_idx >= 0
- expert_ids = torch.where(
- valid,
- h.recv_topk_idx + self.rank * (self.args.experts // self.world_size),
- h.recv_topk_idx,
- )
- return types.SimpleNamespace(
- payload=self._semantic_recv(h, p),
- encoded_payload=self._encoded_recv(h),
- scales=self._recv_scales(h),
- expert_ids=expert_ids,
- weights=h.recv_topk_weights.masked_fill(~valid, 0),
- local_expert_counts=torch.tensor(h.recv_counts, device=self.device, dtype=torch.int64),
- ordering_contract="source-rank-major-stable-v1",
- )
-
- def inspect_expert_dispatch(self, p, h):
- if self.mode != "low-latency":
- raise RuntimeError("expert-packed inspection requires low-latency mode")
- p.recv_counts = tuple(int(value) for value in h.recv_counts.tolist())
- return types.SimpleNamespace(
- payload=self._semantic_recv(h, p),
- encoded_payload=self._encoded_recv(h),
- scales=self._recv_scales(h),
- local_expert_counts=h.recv_counts,
- source_info=h.handle[0],
- layout_range=h.handle[1],
- )
-
- def combine_transformed(self, p, h, transformed):
- if self.mode == "low-latency":
- packed = torch.zeros(
- self._encoded_recv(h).shape,
- dtype=torch.bfloat16,
- device=self._encoded_recv(h).device,
- )
- packed[h.oracle_local_expert_slots, h.oracle_packed_positions] = transformed.to(
- packed.dtype
- )
- combined, _, _ = self.buffer.low_latency_combine(
- packed,
- p.topk_idx,
- p.topk_weights,
- h.handle,
- use_logfmt=self._use_logfmt,
- async_finish=False,
- return_recv_hook=False,
- )
- return combined
- semantic = self._semantic_recv(h, p)
- combined, _, _ = self.buffer.combine(
- transformed.to(semantic.dtype), h.handle, async_finish=False
- )
- return combined
-
- def recv_tokens(self, h):
- if self.mode == "low-latency":
- return int(h.recv_counts.to(torch.int64).sum().item())
- return int(self._encoded_recv(h).shape[0])
-
- def _encoded_recv(self, h):
- return h.recv_x[0] if isinstance(h.recv_x, tuple) else h.recv_x
-
- def _recv_scales(self, h):
- return h.recv_x[1] if isinstance(h.recv_x, tuple) else None
-
- def _semantic_recv(self, h, problem=None):
- if not self._fp8_dispatch:
- return h.recv_x
- if not hasattr(h, "recv_semantic"):
- if self.mode == "low-latency":
- counts = getattr(problem, "recv_counts", None)
- if counts is None:
- counts = tuple(int(value) for value in h.recv_counts.tolist())
- if problem is not None:
- problem.recv_counts = counts
- workspace = getattr(self, "_ll_semantic_workspace", None)
- if workspace is None:
- encoded = self._encoded_recv(h)
- workspace = torch.empty(
- encoded.shape, dtype=torch.bfloat16, device=encoded.device
- )
- self._ll_semantic_workspace = workspace
- h.recv_semantic = ep_precision.dequantize_expert_prefixes(
- torch,
- self._encoded_recv(h),
- self._recv_scales(h),
- self.communication_precision["dispatch"],
- counts,
- workspace,
- )
- else:
- h.recv_semantic = ep_precision.dequantize_dispatch(
- torch,
- self._encoded_recv(h),
- self._recv_scales(h),
- self.communication_precision["dispatch"],
- )
- return h.recv_semantic
-
def finalize(self, rc):
try:
dist.barrier()
diff --git a/experimental/CollectiveX/tests/ep_deepep_family.py b/experimental/CollectiveX/tests/ep_deepep_family.py
new file mode 100644
index 0000000000..1e069d8b3b
--- /dev/null
+++ b/experimental/CollectiveX/tests/ep_deepep_family.py
@@ -0,0 +1,250 @@
+#!/usr/bin/env python3
+"""Shared DeepEP-API dispatch/combine surface for the DeepEP and UCCL adapters.
+
+UCCL's ``uccl_deepep.Buffer`` is a drop-in clone of DeepEP's ``deep_ep.Buffer`` low-latency and
+normal API, so both adapters run byte-identical mode handling, dispatch/combine, expert-packed
+inspection, and fp8 dequantization. That shared operation lives here; each concrete backend keeps
+only what is genuinely vendor-specific: its native buffer import, ``create_buffer`` provenance, and
+process teardown.
+
+This base is deliberately free of any ``deep_ep``/``uccl`` import. The UCCL benchmark image installs
+uccl WITHOUT deep_ep, so importing ``ep_uccl`` must never transitively require ``deep_ep`` — an
+inherited method resolves module globals from where it is *defined*, so keeping this file
+vendor-agnostic is what makes the shared base safe for both images.
+"""
+from __future__ import annotations
+
+import types
+
+import torch
+import torch.distributed as dist
+import ep_precision
+from ep_backend import EPBackend
+
+
+class DeepEPFamilyBackend(EPBackend):
+ # Abstract intermediate: never instantiated or registered (capability.BACKENDS is an
+ # explicit dict, and nothing enumerates EPBackend subclasses). The non-empty name only
+ # satisfies EPBackend.__init_subclass__; concrete subclasses override it. create_buffer
+ # stays abstract here, so this class cannot itself be constructed.
+ name = "deepep-family"
+ _vendor = "DeepEP"
+ SUPPORTED_MODES = ("normal", "low-latency")
+ stage_device_work = False
+ combine_needs_redispatch = False
+ # DeepEP reduces activations and top-k weights independently. The activation
+ # tensor must therefore carry the complete local weighted expert sum.
+ combine_weight_semantics = "unweighted-rank-sum"
+ oracle_layout = "token-rank"
+ payload_unit = "token-rank"
+
+ def __init__(self, args, rank, world_size, local_rank, device):
+ # Base validates mode against SUPPORTED_MODES (normal / low-latency).
+ super().__init__(args, rank, world_size, local_rank, device)
+ supported_profiles = {
+ "normal": {
+ "d-bf16.c-bf16",
+ "d-fp8-e4m3fn-b128-f32-prequantized.c-bf16",
+ },
+ "low-latency": {
+ "d-bf16.c-bf16",
+ "d-fp8-e4m3fn-b128-f32-fused.c-bf16",
+ },
+ }
+ self.precision_profile_id, self.communication_precision = (
+ ep_precision.resolve_precision(
+ args,
+ backend=self.name,
+ mode=self.mode,
+ supported_profiles=supported_profiles[self.mode],
+ )
+ )
+ self._fp8_dispatch = ep_precision.is_low_precision_dispatch(
+ self.communication_precision
+ )
+ self._use_logfmt = ep_precision.uses_logfmt_combine(
+ self.communication_precision
+ )
+ self.stage_device_work = self._fp8_dispatch
+ self.group = dist.group.WORLD
+ # Low-latency flips the contract flags and fixes the per-rank cap so
+ # buffer_cap can report it to make_inputs before create_buffer runs.
+ if self.mode == "low-latency":
+ if args.phase != "decode":
+ raise ValueError(
+ f"{self._vendor} low-latency mode only supports the decode ladder"
+ )
+ if args.experts % world_size:
+ raise ValueError(
+ f"{self._vendor} low-latency experts must divide the EP group"
+ )
+ self.combine_needs_redispatch = True
+ self.combine_weight_semantics = "gate-weighted-sum"
+ self.oracle_layout = "expert-packed"
+ self.payload_unit = "token-expert"
+ self.max_tokens_per_rank = 128
+
+ def buffer_cap(self, args):
+ return self.max_tokens_per_rank if self.mode == "low-latency" else None
+
+ def dispatch(self, p):
+ if self.mode == "low-latency":
+ recv_x, recv_counts, handle, _, _ = self.buffer.low_latency_dispatch(
+ p.x,
+ p.topk_idx,
+ self.max_tokens_per_rank,
+ self.args.experts,
+ use_fp8=self._fp8_dispatch, # BF16 control realizes use_fp8=False.
+ async_finish=False,
+ return_recv_hook=False,
+ )
+ return types.SimpleNamespace(
+ recv_x=recv_x,
+ recv_counts=recv_counts,
+ handle=handle,
+ )
+ (
+ num_tokens_per_rank,
+ num_tokens_per_rdma_rank,
+ num_tokens_per_expert,
+ is_token_in_rank,
+ _,
+ ) = self.buffer.get_dispatch_layout(p.topk_idx, self.args.experts)
+ recv_x, recv_topk_idx, recv_topk_weights, recv_counts, handle, _ = self.buffer.dispatch(
+ p.dispatch_x,
+ topk_idx=p.topk_idx,
+ topk_weights=p.topk_weights,
+ num_tokens_per_rank=num_tokens_per_rank,
+ num_tokens_per_rdma_rank=num_tokens_per_rdma_rank,
+ is_token_in_rank=is_token_in_rank,
+ num_tokens_per_expert=num_tokens_per_expert,
+ async_finish=False,
+ )
+ return types.SimpleNamespace(
+ recv_x=recv_x,
+ recv_topk_idx=recv_topk_idx,
+ recv_topk_weights=recv_topk_weights,
+ recv_counts=recv_counts,
+ handle=handle,
+ )
+
+ def stage(self, p, h):
+ h.combine_input = self._semantic_recv(h, p)
+
+ def combine(self, p, h):
+ if self.mode == "low-latency":
+ combined_x, _, _ = self.buffer.low_latency_combine(
+ h.combine_input,
+ p.topk_idx,
+ p.topk_weights,
+ h.handle,
+ use_logfmt=self._use_logfmt,
+ async_finish=False,
+ return_recv_hook=False,
+ )
+ return combined_x
+ combined_x, _, _ = self.buffer.combine(
+ h.combine_input, h.handle, async_finish=False
+ )
+ return combined_x
+
+ def inspect_dispatch(self, p, h):
+ valid = h.recv_topk_idx >= 0
+ expert_ids = torch.where(
+ valid,
+ h.recv_topk_idx + self.rank * (self.args.experts // self.world_size),
+ h.recv_topk_idx,
+ )
+ return types.SimpleNamespace(
+ payload=self._semantic_recv(h, p),
+ encoded_payload=self._encoded_recv(h),
+ scales=self._recv_scales(h),
+ expert_ids=expert_ids,
+ weights=h.recv_topk_weights.masked_fill(~valid, 0),
+ local_expert_counts=torch.tensor(h.recv_counts, device=self.device, dtype=torch.int64),
+ ordering_contract="source-rank-major-stable-v1",
+ )
+
+ def inspect_expert_dispatch(self, p, h):
+ if self.mode != "low-latency":
+ raise RuntimeError("expert-packed inspection requires low-latency mode")
+ p.recv_counts = tuple(int(value) for value in h.recv_counts.tolist())
+ return types.SimpleNamespace(
+ payload=self._semantic_recv(h, p),
+ encoded_payload=self._encoded_recv(h),
+ scales=self._recv_scales(h),
+ local_expert_counts=h.recv_counts,
+ source_info=h.handle[0],
+ layout_range=h.handle[1],
+ )
+
+ def combine_transformed(self, p, h, transformed):
+ if self.mode == "low-latency":
+ packed = torch.zeros(
+ self._encoded_recv(h).shape,
+ dtype=torch.bfloat16,
+ device=self._encoded_recv(h).device,
+ )
+ packed[h.oracle_local_expert_slots, h.oracle_packed_positions] = transformed.to(
+ packed.dtype
+ )
+ combined, _, _ = self.buffer.low_latency_combine(
+ packed,
+ p.topk_idx,
+ p.topk_weights,
+ h.handle,
+ use_logfmt=self._use_logfmt,
+ async_finish=False,
+ return_recv_hook=False,
+ )
+ return combined
+ semantic = self._semantic_recv(h, p)
+ combined, _, _ = self.buffer.combine(
+ transformed.to(semantic.dtype), h.handle, async_finish=False
+ )
+ return combined
+
+ def recv_tokens(self, h):
+ if self.mode == "low-latency":
+ return int(h.recv_counts.to(torch.int64).sum().item())
+ return int(self._encoded_recv(h).shape[0])
+
+ def _encoded_recv(self, h):
+ return h.recv_x[0] if isinstance(h.recv_x, tuple) else h.recv_x
+
+ def _recv_scales(self, h):
+ return h.recv_x[1] if isinstance(h.recv_x, tuple) else None
+
+ def _semantic_recv(self, h, problem=None):
+ if not self._fp8_dispatch:
+ return h.recv_x
+ if not hasattr(h, "recv_semantic"):
+ if self.mode == "low-latency":
+ counts = getattr(problem, "recv_counts", None)
+ if counts is None:
+ counts = tuple(int(value) for value in h.recv_counts.tolist())
+ if problem is not None:
+ problem.recv_counts = counts
+ workspace = getattr(self, "_ll_semantic_workspace", None)
+ if workspace is None:
+ encoded = self._encoded_recv(h)
+ workspace = torch.empty(
+ encoded.shape, dtype=torch.bfloat16, device=encoded.device
+ )
+ self._ll_semantic_workspace = workspace
+ h.recv_semantic = ep_precision.dequantize_expert_prefixes(
+ torch,
+ self._encoded_recv(h),
+ self._recv_scales(h),
+ self.communication_precision["dispatch"],
+ counts,
+ workspace,
+ )
+ else:
+ h.recv_semantic = ep_precision.dequantize_dispatch(
+ torch,
+ self._encoded_recv(h),
+ self._recv_scales(h),
+ self.communication_precision["dispatch"],
+ )
+ return h.recv_semantic
diff --git a/experimental/CollectiveX/tests/ep_uccl.py b/experimental/CollectiveX/tests/ep_uccl.py
index 4af636193e..06b17a3b32 100644
--- a/experimental/CollectiveX/tests/ep_uccl.py
+++ b/experimental/CollectiveX/tests/ep_uccl.py
@@ -8,13 +8,12 @@
from pathlib import Path
from pathlib import PurePosixPath
import sys
-import types
import torch
import torch.distributed as dist
import contracts
import ep_precision
-from ep_backend import EPBackend
+from ep_deepep_family import DeepEPFamilyBackend
try:
import uccl
@@ -184,56 +183,12 @@ def _normal_buffer_sizes(hidden: int, world_size: int) -> tuple[int, int]:
return num_nvl_bytes, num_rdma_bytes
-class UCCLBackend(EPBackend):
+class UCCLBackend(DeepEPFamilyBackend):
+ # uccl_deepep.Buffer is a DeepEP-API clone, so mode handling, dispatch/combine, and fp8
+ # dequantization are shared in DeepEPFamilyBackend; only the native uccl_deepep buffer
+ # construction, its content-provenance, and process teardown are UCCL-specific.
name = "uccl"
- SUPPORTED_MODES = ("normal", "low-latency")
- stage_device_work = False
- combine_needs_redispatch = False
- combine_weight_semantics = "unweighted-rank-sum"
- oracle_layout = "token-rank"
- payload_unit = "token-rank"
-
- def __init__(self, args, rank, world_size, local_rank, device):
- # Base validates mode against SUPPORTED_MODES (normal / low-latency).
- super().__init__(args, rank, world_size, local_rank, device)
- supported_profiles = {
- "normal": {
- "d-bf16.c-bf16",
- "d-fp8-e4m3fn-b128-f32-prequantized.c-bf16",
- },
- "low-latency": {
- "d-bf16.c-bf16",
- "d-fp8-e4m3fn-b128-f32-fused.c-bf16",
- },
- }
- self.precision_profile_id, self.communication_precision = (
- ep_precision.resolve_precision(
- args,
- backend=self.name,
- mode=self.mode,
- supported_profiles=supported_profiles[self.mode],
- )
- )
- self._fp8_dispatch = ep_precision.is_low_precision_dispatch(
- self.communication_precision
- )
- self._use_logfmt = ep_precision.uses_logfmt_combine(
- self.communication_precision
- )
- self.stage_device_work = self._fp8_dispatch
- self.group = dist.group.WORLD
- # Low-latency flips the contract flags and fixes the per-rank cap so
- # buffer_cap can report it to make_inputs before create_buffer runs.
- if self.mode == "low-latency":
- if args.phase != "decode":
- raise ValueError("UCCL low-latency mode only supports the decode ladder")
- if args.experts % world_size:
- raise ValueError("UCCL low-latency experts must divide the EP group")
- self.combine_needs_redispatch = True
- self.combine_weight_semantics = "gate-weighted-sum"
- self.oracle_layout = "expert-packed"
- self.payload_unit = "token-expert"
- self.max_tokens_per_rank = 128
+ _vendor = "UCCL"
def create_buffer(self, spec):
# Local aliases keep the moved buffer-construction body byte-verbatim.
@@ -333,171 +288,6 @@ def create_buffer(self, spec):
**resource_provenance,
}
- def buffer_cap(self, args):
- return self.max_tokens_per_rank if self.mode == "low-latency" else None
-
- def dispatch(self, p):
- if self.mode == "low-latency":
- recv_x, recv_counts, handle, _, _ = self.buffer.low_latency_dispatch(
- p.x,
- p.topk_idx,
- self.max_tokens_per_rank,
- self.args.experts,
- use_fp8=self._fp8_dispatch, # BF16 control realizes use_fp8=False.
- async_finish=False,
- return_recv_hook=False,
- )
- return types.SimpleNamespace(
- recv_x=recv_x,
- recv_counts=recv_counts,
- handle=handle,
- )
- (
- num_tokens_per_rank,
- num_tokens_per_rdma_rank,
- num_tokens_per_expert,
- is_token_in_rank,
- _,
- ) = self.buffer.get_dispatch_layout(p.topk_idx, self.args.experts)
- recv_x, recv_topk_idx, recv_topk_weights, recv_counts, handle, _ = self.buffer.dispatch(
- p.dispatch_x,
- topk_idx=p.topk_idx,
- topk_weights=p.topk_weights,
- num_tokens_per_rank=num_tokens_per_rank,
- num_tokens_per_rdma_rank=num_tokens_per_rdma_rank,
- is_token_in_rank=is_token_in_rank,
- num_tokens_per_expert=num_tokens_per_expert,
- async_finish=False,
- )
- return types.SimpleNamespace(
- recv_x=recv_x,
- recv_topk_idx=recv_topk_idx,
- recv_topk_weights=recv_topk_weights,
- recv_counts=recv_counts,
- handle=handle,
- )
-
- def stage(self, p, h):
- h.combine_input = self._semantic_recv(h, p)
-
- def combine(self, p, h):
- if self.mode == "low-latency":
- combined_x, _, _ = self.buffer.low_latency_combine(
- h.combine_input,
- p.topk_idx,
- p.topk_weights,
- h.handle,
- use_logfmt=self._use_logfmt,
- async_finish=False,
- return_recv_hook=False,
- )
- return combined_x
- combined_x, _, _ = self.buffer.combine(
- h.combine_input, h.handle, async_finish=False
- )
- return combined_x
-
- def inspect_dispatch(self, p, h):
- valid = h.recv_topk_idx >= 0
- expert_ids = torch.where(
- valid,
- h.recv_topk_idx + self.rank * (self.args.experts // self.world_size),
- h.recv_topk_idx,
- )
- return types.SimpleNamespace(
- payload=self._semantic_recv(h, p),
- encoded_payload=self._encoded_recv(h),
- scales=self._recv_scales(h),
- expert_ids=expert_ids,
- weights=h.recv_topk_weights.masked_fill(~valid, 0),
- local_expert_counts=torch.tensor(h.recv_counts, device=self.device, dtype=torch.int64),
- ordering_contract="source-rank-major-stable-v1",
- )
-
- def inspect_expert_dispatch(self, p, h):
- if self.mode != "low-latency":
- raise RuntimeError("expert-packed inspection requires low-latency mode")
- p.recv_counts = tuple(int(value) for value in h.recv_counts.tolist())
- return types.SimpleNamespace(
- payload=self._semantic_recv(h, p),
- encoded_payload=self._encoded_recv(h),
- scales=self._recv_scales(h),
- local_expert_counts=h.recv_counts,
- source_info=h.handle[0],
- layout_range=h.handle[1],
- )
-
- def combine_transformed(self, p, h, transformed):
- if self.mode == "low-latency":
- packed = torch.zeros(
- self._encoded_recv(h).shape,
- dtype=torch.bfloat16,
- device=self._encoded_recv(h).device,
- )
- packed[h.oracle_local_expert_slots, h.oracle_packed_positions] = transformed.to(
- packed.dtype
- )
- combined, _, _ = self.buffer.low_latency_combine(
- packed,
- p.topk_idx,
- p.topk_weights,
- h.handle,
- use_logfmt=self._use_logfmt,
- async_finish=False,
- return_recv_hook=False,
- )
- return combined
- semantic = self._semantic_recv(h, p)
- combined, _, _ = self.buffer.combine(
- transformed.to(semantic.dtype), h.handle, async_finish=False
- )
- return combined
-
- def recv_tokens(self, h):
- if self.mode == "low-latency":
- return int(h.recv_counts.to(torch.int64).sum().item())
- return int(self._encoded_recv(h).shape[0])
-
- def _encoded_recv(self, h):
- return h.recv_x[0] if isinstance(h.recv_x, tuple) else h.recv_x
-
- def _recv_scales(self, h):
- return h.recv_x[1] if isinstance(h.recv_x, tuple) else None
-
- def _semantic_recv(self, h, problem=None):
- if not self._fp8_dispatch:
- return h.recv_x
- if not hasattr(h, "recv_semantic"):
- if self.mode == "low-latency":
- counts = getattr(problem, "recv_counts", None)
- if counts is None:
- counts = tuple(int(value) for value in h.recv_counts.tolist())
- if problem is not None:
- problem.recv_counts = counts
- workspace = getattr(self, "_ll_semantic_workspace", None)
- if workspace is None:
- encoded = self._encoded_recv(h)
- workspace = torch.empty(
- encoded.shape, dtype=torch.bfloat16, device=encoded.device
- )
- self._ll_semantic_workspace = workspace
- h.recv_semantic = ep_precision.dequantize_expert_prefixes(
- torch,
- self._encoded_recv(h),
- self._recv_scales(h),
- self.communication_precision["dispatch"],
- counts,
- workspace,
- )
- else:
- h.recv_semantic = ep_precision.dequantize_dispatch(
- torch,
- self._encoded_recv(h),
- self._recv_scales(h),
- self.communication_precision["dispatch"],
- )
- return h.recv_semantic
-
def finalize(self, rc):
# UCCL's proxy teardown can crash after results are written; preserve the real rc.
try:
diff --git a/experimental/CollectiveX/tests/test_deepep_v2_contract.py b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
index 5fd5fb522e..754c0083df 100644
--- a/experimental/CollectiveX/tests/test_deepep_v2_contract.py
+++ b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
@@ -474,13 +474,14 @@ def test_v2_gin_mode_uses_the_scale_up_domain_and_safe_fallbacks(self) -> None:
os.environ["EP_DISABLE_GIN"] = original
def test_ep_adapters_declare_unweighted_rank_sum(self) -> None:
+ # DeepEP and UCCL share their combine semantics and dispatch/combine methods in the
+ # family base, so it stands in for both here; the other adapters declare their own.
adapters = {
- "ep_deepep.py": "DeepEPBackend",
+ "ep_deepep_family.py": "DeepEPFamilyBackend",
"ep_deepep_v2.py": "DeepEPV2Backend",
"ep_deepep_hybrid.py": "DeepEPHybridBackend",
"ep_mori.py": "MoRIBackend",
"ep_nccl.py": "NCCLBackend",
- "ep_uccl.py": "UCCLBackend",
}
for filename, class_name in adapters.items():
with self.subTest(adapter=filename):
@@ -504,7 +505,7 @@ def test_ep_adapters_declare_unweighted_rank_sum(self) -> None:
self.assertEqual(len(combine_methods), 2)
for method in combine_methods:
source = ast.unparse(method)
- if filename in {"ep_deepep.py", "ep_uccl.py"}:
+ if filename == "ep_deepep_family.py":
self.assertIn("self.mode == 'low-latency'", source)
else:
self.assertNotIn("topk_weights", source)
@@ -565,11 +566,9 @@ def test_expert_packed_slot_map_reconstructs_exact_sources(self) -> None:
)
def test_deepep_and_uccl_expose_genuine_low_latency_calls(self) -> None:
- required_fragments = (
- "Buffer.get_low_latency_rdma_size_hint(",
- "low_latency_mode=True",
- "num_qps_per_rank=num_qps_per_rank",
- "self.buffer.clean_low_latency_buffer(",
+ # The operational low-latency surface is shared by DeepEP and UCCL in the family base;
+ # each concrete adapter keeps only its own native buffer construction.
+ shared_fragments = (
"self.buffer.low_latency_dispatch(",
"use_fp8=False",
"self.buffer.low_latency_combine(",
@@ -577,15 +576,26 @@ def test_deepep_and_uccl_expose_genuine_low_latency_calls(self) -> None:
'self.combine_weight_semantics = "gate-weighted-sum"',
"self.combine_needs_redispatch = True",
"def inspect_expert_dispatch(",
+ "self.max_tokens_per_rank = 128",
+ "async_finish=False",
+ "return_recv_hook=False",
+ )
+ family = (HERE / "ep_deepep_family.py").read_text()
+ for fragment in shared_fragments:
+ with self.subTest(fragment=fragment):
+ self.assertIn(fragment, family)
+
+ buffer_fragments = (
+ "Buffer.get_low_latency_rdma_size_hint(",
+ "low_latency_mode=True",
+ "num_qps_per_rank=num_qps_per_rank",
+ "self.buffer.clean_low_latency_buffer(",
)
for filename in ("ep_deepep.py", "ep_uccl.py"):
source = (HERE / filename).read_text()
with self.subTest(adapter=filename):
- for fragment in required_fragments:
+ for fragment in buffer_fragments:
self.assertIn(fragment, source)
- self.assertIn("self.max_tokens_per_rank = 128", source)
- self.assertIn("async_finish=False", source)
- self.assertIn("return_recv_hook=False", source)
run_ep_source = (HERE / "run_ep.py").read_text()
self.assertIn('args.backend not in {"deepep", "uccl"}', run_ep_source)
diff --git a/experimental/CollectiveX/tests/test_ep_precision_adapters.py b/experimental/CollectiveX/tests/test_ep_precision_adapters.py
index d268d788ae..d68b1e2571 100644
--- a/experimental/CollectiveX/tests/test_ep_precision_adapters.py
+++ b/experimental/CollectiveX/tests/test_ep_precision_adapters.py
@@ -98,13 +98,12 @@ def _keywords(tree: ast.AST, attribute: str) -> list[set[str]]:
return result
def test_deepep_and_uccl_wire_native_low_latency_controls(self):
- for filename in ("ep_deepep.py", "ep_uccl.py"):
- with self.subTest(filename=filename):
- tree = self._tree(filename)
- dispatch_calls = self._keywords(tree, "low_latency_dispatch")
- combine_calls = self._keywords(tree, "low_latency_combine")
- self.assertTrue(any("use_fp8" in call for call in dispatch_calls))
- self.assertTrue(any("use_logfmt" in call for call in combine_calls))
+ # DeepEP and UCCL share the low-latency dispatch/combine surface in the family base.
+ tree = self._tree("ep_deepep_family.py")
+ dispatch_calls = self._keywords(tree, "low_latency_dispatch")
+ combine_calls = self._keywords(tree, "low_latency_combine")
+ self.assertTrue(any("use_fp8" in call for call in dispatch_calls))
+ self.assertTrue(any("use_logfmt" in call for call in combine_calls))
def test_elastic_and_hybrid_constructors_enable_native_fp8(self):
v2 = self._tree("ep_deepep_v2.py")
From 12afb0c2c3453ebdc2f1c3d2549726934ee23ee4 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 7 Jul 2026 18:25:31 +0800
Subject: [PATCH 178/210] feat(collectivex): enable B300 EP16 cross-node over
RoCE
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Lift the B300 EP16 topology wall now that cross-node dispatch/combine is
proven over the 400G RoCE GPU-fabric rails (world=16 all-reduce PASS on
adjacent B300 nodes); the prior wall tested only the storage-IB rails.
- capability.py: clear TOPOLOGY_CELL_OVERRIDES (B300 EP16 -> runnable for
deepep, deepep-v2, deepep-hybrid, nccl-ep; uccl/mori stay unsupported)
- collectivex-sweep.yml: add B300 network overlay slot (RoCE selector kept
in the write-only per-SKU secret, never in-tree)
- publisher.py: recompute CANONICAL_FULL_V1 matrix + case-catalog digests
- tests: update matrix count contract (49->53 shards, 172 runnable) and
B300 EP16 disposition expectations
启用 B300 EP16 跨节点（经 RoCE 网络）
已验证跨节点 dispatch/combine 可经 400G RoCE GPU 网络链路运行（相邻 B300
节点 world=16 all-reduce 通过）；此前的限制仅测试了存储 IB 链路。
- capability.py：清空 TOPOLOGY_CELL_OVERRIDES（B300 EP16 对 deepep、
deepep-v2、deepep-hybrid、nccl-ep 变为可运行；uccl/mori 仍不支持）
- collectivex-sweep.yml：新增 B300 网络覆盖槽位（RoCE 选择器仅存于只写的
分 SKU secret，绝不入库）
- publisher.py：重新计算 CANONICAL_FULL_V1 矩阵与用例目录摘要
- tests：更新矩阵计数契约（49->53 个分片，172 个可运行）及 B300 EP16
处置预期
---
.github/workflows/collectivex-sweep.yml | 8 +++++++-
experimental/CollectiveX/capability.py | 4 +---
experimental/CollectiveX/publisher.py | 4 ++--
.../CollectiveX/tests/test_precision_scheduling.py | 6 ++----
experimental/CollectiveX/tests/test_publisher.py | 2 +-
.../CollectiveX/tests/test_sampling_contract.py | 12 ++++++------
6 files changed, 19 insertions(+), 17 deletions(-)
diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml
index 450950c71e..8f5673239f 100644
--- a/.github/workflows/collectivex-sweep.yml
+++ b/.github/workflows/collectivex-sweep.yml
@@ -491,6 +491,7 @@ jobs:
COLLECTIVEX_OPERATOR_CONFIG_CONTENT: ${{ secrets.COLLECTIVEX_OPERATOR_CONFIG_V1 }}
COLLECTIVEX_NETWORK_CONFIG_CONTENT: ${{ secrets.COLLECTIVEX_NETWORK_CONFIG_V1 }}
COLLECTIVEX_H100_CONFIG_CONTENT: ${{ secrets.COLLECTIVEX_H100_CONFIG_V1 }}
+ COLLECTIVEX_B300_CONFIG_CONTENT: ${{ secrets.COLLECTIVEX_B300_CONFIG_V1 }}
COLLECTIVEX_B200_CONFIG_CONTENT: ${{ secrets.COLLECTIVEX_B200_CONFIG_V1 }}
COLLECTIVEX_MI300_CONFIG_CONTENT: ${{ secrets.COLLECTIVEX_MI300_CONFIG_V1 }}
COLLECTIVEX_MI355_CONFIG_CONTENT: ${{ secrets.COLLECTIVEX_MI355_CONFIG_V1 }}
@@ -523,6 +524,7 @@ jobs:
base = load("COLLECTIVEX_OPERATOR_CONFIG_CONTENT")
for overlay_name in (
"COLLECTIVEX_NETWORK_CONFIG_CONTENT", "COLLECTIVEX_H100_CONFIG_CONTENT",
+ "COLLECTIVEX_B300_CONFIG_CONTENT",
"COLLECTIVEX_B200_CONFIG_CONTENT", "COLLECTIVEX_MI300_CONFIG_CONTENT",
"COLLECTIVEX_MI355_CONFIG_CONTENT",
):
@@ -546,6 +548,10 @@ jobs:
if set(overlay["runners"]) != {"h100-dgxc"}:
raise ValueError("invalid H100 overlay runner")
private_fields.add("exclude_nodes")
+ elif overlay_name == "COLLECTIVEX_B300_CONFIG_CONTENT":
+ if set(overlay["runners"]) != {"b300"}:
+ raise ValueError("invalid B300 overlay runner")
+ private_fields.add("exclude_nodes")
elif overlay_name == "COLLECTIVEX_MI355_CONFIG_CONTENT":
if set(overlay["runners"]) != {"mi355x"}:
raise ValueError("invalid MI355 overlay runner")
@@ -575,7 +581,7 @@ jobs:
stream.write(payload)
PY
unset COLLECTIVEX_OPERATOR_CONFIG_CONTENT COLLECTIVEX_NETWORK_CONFIG_CONTENT
- unset COLLECTIVEX_H100_CONFIG_CONTENT
+ unset COLLECTIVEX_H100_CONFIG_CONTENT COLLECTIVEX_B300_CONFIG_CONTENT
unset COLLECTIVEX_B200_CONFIG_CONTENT
unset COLLECTIVEX_MI300_CONFIG_CONTENT COLLECTIVEX_MI355_CONFIG_CONTENT
unset COLLECTIVEX_OPERATOR_CONFIG_REQUIRED
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
index d52b7b6f1b..2e809a2fd6 100644
--- a/experimental/CollectiveX/capability.py
+++ b/experimental/CollectiveX/capability.py
@@ -173,9 +173,7 @@ def _platform(
# Publication-quality topology exceptions apply after ordinary backend support
# checks. They describe the currently usable benchmark fabric, not a library
# implementation limit, and can be removed when the named topology is repaired.
-TOPOLOGY_CELL_OVERRIDES: dict[tuple[str, int], str] = {
- ("b300", 16): "v1 publication fabric unavailable for B300 EP16",
-}
+TOPOLOGY_CELL_OVERRIDES: dict[tuple[str, int], str] = {}
# Backend-specific topology limits require repeated native execution evidence.
# Keep these narrower than platform overrides so working reference paths remain
diff --git a/experimental/CollectiveX/publisher.py b/experimental/CollectiveX/publisher.py
index 83b1c81940..1055757570 100644
--- a/experimental/CollectiveX/publisher.py
+++ b/experimental/CollectiveX/publisher.py
@@ -46,10 +46,10 @@
)
REQUIRED_PROMOTION_COHORT_COUNTS = {"library": 24, "system": 4}
CANONICAL_FULL_V1_MATRIX_SHA256 = (
- "1c8f9c27c4a3027e8a843478ff63c2c6e00f5c170436bb5c3fd039c886363111"
+ "cd2241e5417249f304ad7a34a0448c63d5e2c36198f7beee6cb86fec08a6952d"
)
CANONICAL_FULL_V1_CASE_CATALOG_SHA256 = (
- "6bb6859307802bb3ea2baea5b20a9c517f6fb773c20fd539c4f4fcd06dd8ae98"
+ "c0580ae747f4ecf347166a24c19db4f45943b37ae9caa197fcdf2fe39ee8f414"
)
P50_STABILITY_LIMIT = 1.10
P99_STABILITY_LIMIT = 1.25
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
index 03fe13e326..96d9216f63 100644
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -323,10 +323,8 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
"b300", backend, ep=16, nodes=2,
precision_profile=identity.V1_CONTROL_PRECISION_PROFILE,
)
- self.assertEqual(disposition, "unsupported")
- self.assertEqual(
- reason, "v1 publication fabric unavailable for B300 EP16"
- )
+ self.assertEqual(disposition, "supported")
+ self.assertEqual(reason, "ok")
b300_ep8, reason = capability.resolve_disposition(
"b300", "nccl-ep", ep=8, nodes=1,
precision_profile=identity.V1_CONTROL_PRECISION_PROFILE,
diff --git a/experimental/CollectiveX/tests/test_publisher.py b/experimental/CollectiveX/tests/test_publisher.py
index c84a5d17f6..95bdb628f9 100644
--- a/experimental/CollectiveX/tests/test_publisher.py
+++ b/experimental/CollectiveX/tests/test_publisher.py
@@ -1968,7 +1968,7 @@ def test_workflow_matrix_and_catalog_digests_do_not_drift(self) -> None:
if item["disposition"] == "unsupported"
),
),
- (49, 322, 163, 159, 1314, 653, 661),
+ (53, 322, 172, 150, 1314, 701, 613),
)
library: dict[tuple, set[str]] = {}
system: dict[tuple, set[str]] = {}
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index a1d41c582f..f6062d2e09 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -209,20 +209,20 @@ def test_matrix_cases_and_shards_are_identity_bound(self) -> None:
sum(len(item["case"]["ladder"].split()) for item in runnable_cases),
sum(len(item["case"]["ladder"].split()) for item in unsupported_cases),
),
- (49, 322, 163, 159, 1314, 653, 661),
+ (53, 322, 172, 150, 1314, 701, 613),
)
b300_ep16 = [
- item for item in unsupported_cases
+ item for item in runnable_cases
if item["sku"] == "b300" and item["case"]["ep"] == 16
and item["case"]["backend"] in {
"deepep", "deepep-v2", "deepep-hybrid", "nccl-ep",
}
]
self.assertTrue(b300_ep16)
- self.assertTrue(all(
- item["detail"] == "v1 publication fabric unavailable for B300 EP16"
- for item in b300_ep16
- ))
+ self.assertEqual(
+ {item["case"]["backend"] for item in b300_ep16},
+ {"deepep", "deepep-v2", "deepep-hybrid", "nccl-ep"},
+ )
expected_topologies = {}
for sku, product in (
("h100-dgxc", "h100"), ("h200-dgxc", "h200"),
From 7a081dc062552647455f321d838fa85a1451dc7e Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 7 Jul 2026 20:01:13 +0800
Subject: [PATCH 179/210] fix(collectivex): wall B300 EP16 DeepEP V1/V2 (no
GDRCopy /dev/gdrdrv)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
B300 EP16 runs over the RoCE GPU-fabric: nccl-ep and the DeepEP Hybrid
control path (native DOCA transport) execute there and stay runnable.
DeepEP V1/V2 cannot -- their internode path is NVSHMEM-IBGDA, which needs
GDRCopy /dev/gdrdrv, and that char device is unprovisioned on B300 hosts
(module loaded, no device node; gdr_open returns NULL). Wall (b300,
deepep|deepep-v2, 16) at the topology layer and keep their precision cells
unsupported. Cascade the frozen digests/counts: full V1 matrix is now 51
shards / 322 cases / 167 runnable / 155 unsupported; refresh the publisher
canonical matrix + case-catalog digests, the qualification execution-plan
digests, and the pinned CPU-test expectations.
B300 EP16 在 RoCE GPU 光纤上运行:nccl-ep 与 DeepEP Hybrid 控制路径
(原生 DOCA 传输)可执行,保持可运行。DeepEP V1/V2 不行——其跨节点
路径为 NVSHMEM-IBGDA,需要 GDRCopy /dev/gdrdrv,而该字符设备在 B300
主机上未配置(模块已加载但无设备节点;gdr_open 返回 NULL)。在拓扑层
封禁 (b300, deepep|deepep-v2, 16),并将其精度单元保持为 unsupported。
级联更新冻结摘要与计数:完整 V1 矩阵现为 51 分片 / 322 用例 /
167 可运行 / 155 不支持;刷新 publisher 规范矩阵与用例目录摘要、
资格执行计划摘要,以及固定的 CPU 测试期望值。
---
experimental/CollectiveX/capability.py | 23 +++++++++++++++++--
experimental/CollectiveX/publisher.py | 4 ++--
experimental/CollectiveX/sweep_matrix.py | 6 ++---
.../tests/test_precision_scheduling.py | 17 +++++++++++++-
.../CollectiveX/tests/test_publisher.py | 2 +-
.../tests/test_qualification_planning.py | 2 +-
.../tests/test_sampling_contract.py | 9 +++++---
7 files changed, 50 insertions(+), 13 deletions(-)
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
index 2e809a2fd6..e19410dceb 100644
--- a/experimental/CollectiveX/capability.py
+++ b/experimental/CollectiveX/capability.py
@@ -185,6 +185,14 @@ def _platform(
("b200-dgxc", "deepep-hybrid", 16): (
"DeepEP Hybrid EP16 requires unavailable GPU-to-NIC doorbell/UAR mappings on B200"
),
+ ("b300", "deepep", 16): (
+ "DeepEP V1 EP16 requires GDRCopy /dev/gdrdrv for NVSHMEM-IBGDA, "
+ "unprovisioned on B300 hosts"
+ ),
+ ("b300", "deepep-v2", 16): (
+ "DeepEP V2 EP16 requires GDRCopy /dev/gdrdrv for NVSHMEM-IBGDA, "
+ "unprovisioned on B300 hosts"
+ ),
("mi355x", "mori", 8): (
"Pinned MoRI backend construction does not complete on MI355X EP8"
),
@@ -312,16 +320,27 @@ def _precision_rule(
PRECISION_CELL_OVERRIDES: dict[tuple[str, str, str, int, str], dict[str, str]] = {}
+# B300 EP16 runs over the RoCE GPU-fabric: nccl-ep and the DeepEP Hybrid control
+# path (native DOCA transport) execute there. DeepEP V1/V2 cannot -- their internode
+# path is NVSHMEM-IBGDA, which needs GDRCopy /dev/gdrdrv, and that char device is
+# unprovisioned on B300 hosts (module loaded, no device node; gdr_open returns NULL).
+# V1/V2 are additionally walled at the topology layer above; these precision-cell
+# overrides keep their declared native targets consistent (base_ok gates them anyway).
for _profile, _backend, _mode in (
(_NORMAL_E4M3FN_PROFILE, "deepep", "normal"),
(_NORMAL_E4M3FN_PROFILE, "deepep-v2", "normal"),
- (_NORMAL_E4M3FN_PROFILE, "deepep-hybrid", "normal"),
(_LL_FP8_PROFILE, "deepep", "low-latency"),
):
PRECISION_CELL_OVERRIDES[(_profile, _backend, "b300", 16, _mode)] = {
"disposition": "unsupported",
- "basis": "v1-publication-fabric-unavailable",
+ "basis": "deepep-ibgda-gdrcopy-unavailable-b300",
}
+# DeepEP Hybrid's control path passes on B300 EP16, but its native FP8 precision
+# path is not yet validated on this fabric; keep the precision cell unsupported.
+PRECISION_CELL_OVERRIDES[(_NORMAL_E4M3FN_PROFILE, "deepep-hybrid", "b300", 16, "normal")] = {
+ "disposition": "unsupported",
+ "basis": "hybrid-native-precision-unvalidated-b300-ep16",
+}
_VALIDATED_NATIVE_PROBE_CELLS = (
# run, SKU, EP, backend, mode, profile, disposition, result
diff --git a/experimental/CollectiveX/publisher.py b/experimental/CollectiveX/publisher.py
index 1055757570..98cf2728a3 100644
--- a/experimental/CollectiveX/publisher.py
+++ b/experimental/CollectiveX/publisher.py
@@ -46,10 +46,10 @@
)
REQUIRED_PROMOTION_COHORT_COUNTS = {"library": 24, "system": 4}
CANONICAL_FULL_V1_MATRIX_SHA256 = (
- "cd2241e5417249f304ad7a34a0448c63d5e2c36198f7beee6cb86fec08a6952d"
+ "3048ef24d2d78751321aa5008dd6e2f83c0e6b881fe8d12811a3bfbda6419c9c"
)
CANONICAL_FULL_V1_CASE_CATALOG_SHA256 = (
- "c0580ae747f4ecf347166a24c19db4f45943b37ae9caa197fcdf2fe39ee8f414"
+ "bbba014c9920e6349e9b269dee7cea4f397eaa4569bb8aac7b6ed09f6265905f"
)
P50_STABILITY_LIMIT = 1.10
P99_STABILITY_LIMIT = 1.25
diff --git a/experimental/CollectiveX/sweep_matrix.py b/experimental/CollectiveX/sweep_matrix.py
index 974f6f6d79..5f584a9c32 100644
--- a/experimental/CollectiveX/sweep_matrix.py
+++ b/experimental/CollectiveX/sweep_matrix.py
@@ -94,9 +94,9 @@
)
QUALIFICATION_INDICES = range(1, 4)
CANONICAL_V1_EXECUTION_PLAN_SHA256 = {
- 1: "71faf69466bf3f35dac2defa04d4ea85434fbbde248631a9204fc8dcd8e2e37f",
- 2: "8eae0bb33aacc56811e74e2ad80b24f3ea8c55e84bb2e50e76d4b23f3648f1c6",
- 3: "9ac32155b789d3a9067d5c7fe2e23e8574431eed5041bb243a57ff886ae3e940",
+ 1: "1636f01be0434f5e7e7a633f20b8ee52088d4eaa0528b333544a87f7a6e86950",
+ 2: "8d98af40cb054f7b686c0b3ecfaa0f340fbab7e3bf34b9696f7256c647307992",
+ 3: "fd6fafb06e2ba29374946e173876e7c27684391fb0f4a2c6590779ea804e62ee",
}
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
index 96d9216f63..39abcb36a6 100644
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ b/experimental/CollectiveX/tests/test_precision_scheduling.py
@@ -318,13 +318,28 @@ def test_capability_registry_uses_exact_native_targets(self) -> None:
self.assertEqual(h100_ep16, "supported")
self.assertEqual(reason, "ok")
- for backend in ("deepep", "deepep-v2", "deepep-hybrid", "nccl-ep"):
+ # B300 EP16 runs over the RoCE GPU-fabric: nccl-ep and the DeepEP Hybrid
+ # control path (native DOCA transport) execute there; DeepEP V1/V2 cannot --
+ # their internode path is NVSHMEM-IBGDA, which needs GDRCopy /dev/gdrdrv, and
+ # that char device is unprovisioned on B300 hosts.
+ for backend in ("deepep-hybrid", "nccl-ep"):
disposition, reason = capability.resolve_disposition(
"b300", backend, ep=16, nodes=2,
precision_profile=identity.V1_CONTROL_PRECISION_PROFILE,
)
self.assertEqual(disposition, "supported")
self.assertEqual(reason, "ok")
+ for backend, version in (("deepep", "V1"), ("deepep-v2", "V2")):
+ disposition, reason = capability.resolve_disposition(
+ "b300", backend, ep=16, nodes=2,
+ precision_profile=identity.V1_CONTROL_PRECISION_PROFILE,
+ )
+ self.assertEqual(disposition, "unsupported")
+ self.assertEqual(
+ reason,
+ f"DeepEP {version} EP16 requires GDRCopy /dev/gdrdrv for "
+ "NVSHMEM-IBGDA, unprovisioned on B300 hosts",
+ )
b300_ep8, reason = capability.resolve_disposition(
"b300", "nccl-ep", ep=8, nodes=1,
precision_profile=identity.V1_CONTROL_PRECISION_PROFILE,
diff --git a/experimental/CollectiveX/tests/test_publisher.py b/experimental/CollectiveX/tests/test_publisher.py
index 95bdb628f9..b65c3366f9 100644
--- a/experimental/CollectiveX/tests/test_publisher.py
+++ b/experimental/CollectiveX/tests/test_publisher.py
@@ -1968,7 +1968,7 @@ def test_workflow_matrix_and_catalog_digests_do_not_drift(self) -> None:
if item["disposition"] == "unsupported"
),
),
- (53, 322, 172, 150, 1314, 701, 613),
+ (51, 322, 167, 155, 1314, 673, 641),
)
library: dict[tuple, set[str]] = {}
system: dict[tuple, set[str]] = {}
diff --git a/experimental/CollectiveX/tests/test_qualification_planning.py b/experimental/CollectiveX/tests/test_qualification_planning.py
index 4614aa2d77..b87a9ef234 100644
--- a/experimental/CollectiveX/tests/test_qualification_planning.py
+++ b/experimental/CollectiveX/tests/test_qualification_planning.py
@@ -227,7 +227,7 @@ def test_full_v1_matrix_emits_every_unsupported_terminal_outcome(self) -> None:
documents = [json.loads(path.read_text()) for path in written]
self.assertEqual(len(documents), len(expected))
- self.assertEqual(len(documents), 159)
+ self.assertEqual(len(documents), 155)
self.assertEqual(
{document["outcome"]["reason"] for document in documents},
{item["reason"] for item in expected},
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index f6062d2e09..ec9ce00a0f 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -209,8 +209,11 @@ def test_matrix_cases_and_shards_are_identity_bound(self) -> None:
sum(len(item["case"]["ladder"].split()) for item in runnable_cases),
sum(len(item["case"]["ladder"].split()) for item in unsupported_cases),
),
- (53, 322, 172, 150, 1314, 701, 613),
+ (51, 322, 167, 155, 1314, 673, 641),
)
+ # B300 EP16 runs over RoCE: only nccl-ep and the DeepEP Hybrid control path
+ # execute. DeepEP V1/V2 are walled (NVSHMEM-IBGDA needs GDRCopy /dev/gdrdrv,
+ # unprovisioned on B300 hosts) so they must NOT appear among runnable cases.
b300_ep16 = [
item for item in runnable_cases
if item["sku"] == "b300" and item["case"]["ep"] == 16
@@ -221,7 +224,7 @@ def test_matrix_cases_and_shards_are_identity_bound(self) -> None:
self.assertTrue(b300_ep16)
self.assertEqual(
{item["case"]["backend"] for item in b300_ep16},
- {"deepep", "deepep-v2", "deepep-hybrid", "nccl-ep"},
+ {"deepep-hybrid", "nccl-ep"},
)
expected_topologies = {}
for sku, product in (
@@ -276,7 +279,7 @@ def test_matrix_cases_and_shards_are_identity_bound(self) -> None:
self.assertIsNotNone(capability.topology_for("mi325x", 8))
self.assertEqual(
Counter(shard["n"] for shard in matrix["include"]),
- Counter({2: 30, 6: 13, 4: 5, 5: 1}),
+ Counter({2: 32, 6: 13, 4: 5, 5: 1}),
)
ll_cases = [
item for item in matrix["requested_cases"]
From 906e846df718a99982adab646fb682c4e39caa4c Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 7 Jul 2026 20:44:09 +0800
Subject: [PATCH 180/210] feat(collectivex): publish v1 from a single
allocation
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Collapse the publication requirement from three independent allocations
to one. Set REQUIRED_ALLOCATIONS=1 so the count-based eligibility,
promotion-coverage, promote, and bundle-id gates all repoint to a single
qualified run at qualification index 1. Drop the cross-allocation
stability apparatus entirely (it cannot be measured from one allocation):
remove the per-point stability block, the stable_p50/stable_p99/
p50_max_min_ratio/p99_max_min_ratio eligibility fields, the unstable-p50/
unstable-p99 reasons, the _ratio helper, and the pointStability schema
def. Keep within-run trial drift/outlier diagnostics and cross-backend
cohort ranking, collapsing their run dimension to REQUIRED_ALLOCATIONS.
The qualification-index machinery (indices 1..3) is untouched: it defines
deterministic execution orderings, not a publication count. v1 publication
pins to index 1.
Full unittest battery: 239 OK.
从三次独立分配收敛为单次发布 v1。将 REQUIRED_ALLOCATIONS 置为 1，使
基于计数的资格、晋升覆盖、晋升与 bundle-id 门槛统一指向资格索引 1 的
单次合格运行。彻底移除跨分配稳定性机制（单次分配无法测量）：删除
每点 stability 块、stable_p50/stable_p99/p50_max_min_ratio/
p99_max_min_ratio 资格字段、unstable-p50/unstable-p99 原因、_ratio
辅助函数以及 pointStability schema 定义。保留运行内 trial 漂移/离群
诊断与跨后端 cohort 排名，将其运行维度收敛为 REQUIRED_ALLOCATIONS。
资格索引机制（索引 1..3）保持不变：它定义确定性执行顺序，而非发布
计数。v1 发布固定使用索引 1。
---
experimental/CollectiveX/publisher.py | 90 +++----------------
.../schemas/public-dataset-v1.schema.json | 46 +---------
.../CollectiveX/tests/test_publisher.py | 75 ++++++----------
.../tests/test_schema_v1_contract.py | 51 ++---------
4 files changed, 48 insertions(+), 214 deletions(-)
diff --git a/experimental/CollectiveX/publisher.py b/experimental/CollectiveX/publisher.py
index 98cf2728a3..9c1c6f4b25 100644
--- a/experimental/CollectiveX/publisher.py
+++ b/experimental/CollectiveX/publisher.py
@@ -39,7 +39,7 @@
POLICY = "collectivex-decision-grade-v1"
PUBLISHER_POLICY = "collectivex-publisher-v1"
OUTCOMES = ("success", "unsupported", "failed", "invalid", "diagnostic")
-REQUIRED_ALLOCATIONS = 3
+REQUIRED_ALLOCATIONS = 1
REQUIRED_COHORT_KINDS = ("library", "chip", "system")
PRECISION_COHORT_KINDS = (
"dispatch-precision", "combine-precision", "precision-pair",
@@ -51,8 +51,6 @@
CANONICAL_FULL_V1_CASE_CATALOG_SHA256 = (
"bbba014c9920e6349e9b269dee7cea4f397eaa4569bb8aac7b6ed09f6265905f"
)
-P50_STABILITY_LIMIT = 1.10
-P99_STABILITY_LIMIT = 1.25
TRIAL_DRIFT_RATIO_LIMIT = 1.10
TRIAL_OUTLIER_FRACTION_LIMIT = 0.05
TRIAL_OUTLIER_MAD_MULTIPLIER = 6.0
@@ -162,14 +160,10 @@ def _unique(values: Sequence[Any], path: str) -> None:
def _eligibility(value: dict[str, Any], path: str) -> dict[str, Any]:
allocations = value["allocation_ids"]
- p50 = value["p50_max_min_ratio"]
- p99 = value["p99_max_min_ratio"]
gates = (
len(allocations) >= REQUIRED_ALLOCATIONS,
value["complete"], value["correct"], value["measured_roundtrip_p99"],
- value["stable_p50"], value["stable_p99"], value["stable_ordering"],
- p50 is not None and p50 <= P50_STABILITY_LIMIT,
- p99 is not None and p99 <= P99_STABILITY_LIMIT,
+ value["stable_ordering"],
)
if value["decision_grade"] != (all(gates) and not value["reasons"]):
raise PublisherError(f"{path}.decision_grade does not match promotion gates")
@@ -853,10 +847,8 @@ def validate_public_dataset(doc: Any) -> dict[str, Any]:
point_correctness["semantic_pass"]
and not point_correctness["precision"]["passed"]
)
- or point["stability"]["qualification_indices"]
- != item["measurement"]["qualification_indices"]
):
- raise PublisherError("point correctness/stability differs from series evidence")
+ raise PublisherError("point correctness differs from series evidence")
diagnostics = point["trial_diagnostics"]
diagnostic_reasons = set(diagnostics["reasons"])
component_reasons: set[str] = set()
@@ -1279,7 +1271,7 @@ def validate_public_dataset(doc: Any) -> dict[str, Any]:
attempts[attempt_id]["qualification_index"]
for attempt_id in coverage["attempt_ids"]
if attempts[attempt_id]["selected"]
- } == {1, 2, 3}
+ } == {1}
for coverage in doc["coverage"]
)
if promotion["matrix_id"] != CANONICAL_FULL_V1_MATRIX_SHA256:
@@ -1291,7 +1283,7 @@ def validate_public_dataset(doc: Any) -> dict[str, Any]:
raise PublisherError("promotion requires the canonical case/disposition catalog")
if (
terminal != len(doc["coverage"])
- or promotion["qualification_indices"] != [1, 2, 3]
+ or promotion["qualification_indices"] != [1]
or promotion["measured_cases"] + promotion["unsupported_cases"]
!= promotion["requested_cases"]
or promotion["measured_points"] + promotion["unsupported_points"]
@@ -2319,10 +2311,6 @@ def _public_attempt(document: dict[str, Any], *, selected: bool = False) -> dict
}
-def _ratio(values: Sequence[float]) -> float | None:
- return max(values) / min(values) if len(values) >= REQUIRED_ALLOCATIONS and min(values) > 0 else None
-
-
def _private_trial_components(sample_document: dict[str, Any]) -> dict[int, dict[str, Any]]:
"""Copy validated trial blocks into publisher-private memory without fixing component names."""
points: dict[int, dict[str, Any]] = {}
@@ -2371,7 +2359,7 @@ def _trial_diagnostics(
raise PublisherError(f"{name} trial availability differs across qualification runs")
array = np.asarray(values, dtype=np.float64)
if array.shape != (REQUIRED_ALLOCATIONS, 64, 8) or not np.isfinite(array).all():
- raise PublisherError(f"{name} trial diagnostics require three finite 64x8 runs")
+ raise PublisherError(f"{name} trial diagnostics require finite 64x8 runs")
medians = np.median(array, axis=2)
first = np.median(medians[:, :8], axis=1)
last = np.median(medians[:, -8:], axis=1)
@@ -2422,7 +2410,7 @@ def _roundtrip_trial_array(
raise PublisherError("series is missing private trial blocks")
run_ids = tuple(sorted(trial_blocks, key=lambda value: (int(value), value)))
if len(run_ids) != REQUIRED_ALLOCATIONS:
- raise PublisherError("p99 bootstrap requires exactly three run blocks")
+ raise PublisherError("p99 bootstrap requires exactly one run block")
values = []
for run_id in run_ids:
point = trial_blocks[run_id].get(token)
@@ -2499,11 +2487,11 @@ def _hierarchical_p99_ratio(
p99_index = math.ceil(0.99 * 512) - 1
for start in range(0, BOOTSTRAP_RESAMPLES, BOOTSTRAP_CHUNK_SIZE):
size = min(BOOTSTRAP_CHUNK_SIZE, BOOTSTRAP_RESAMPLES - start)
- sampled_runs = rng.integers(0, REQUIRED_ALLOCATIONS, size=(size, 3))
- sampled_blocks = rng.integers(0, 64, size=(size, 3, 64))
+ sampled_runs = rng.integers(0, REQUIRED_ALLOCATIONS, size=(size, REQUIRED_ALLOCATIONS))
+ sampled_blocks = rng.integers(0, 64, size=(size, REQUIRED_ALLOCATIONS, 64))
run_index = sampled_runs[:, :, None]
- baseline_sample = baseline[run_index, sampled_blocks].reshape(size, 3, 512)
- candidate_sample = candidate[run_index, sampled_blocks].reshape(size, 3, 512)
+ baseline_sample = baseline[run_index, sampled_blocks].reshape(size, REQUIRED_ALLOCATIONS, 512)
+ candidate_sample = candidate[run_index, sampled_blocks].reshape(size, REQUIRED_ALLOCATIONS, 512)
baseline_p99 = np.partition(baseline_sample, p99_index, axis=2)[:, :, p99_index]
candidate_p99 = np.partition(candidate_sample, p99_index, axis=2)[:, :, p99_index]
ratios[start:start + size] = (
@@ -2562,21 +2550,15 @@ def _eligibility_record(
correct: bool,
measured: bool,
stable_ordering: bool,
- p50_ratio: float | None,
- p99_ratio: float | None,
extra_reasons: Sequence[str] = (),
) -> dict[str, Any]:
ids = sorted(set(allocations))
- stable_p50 = p50_ratio is not None and p50_ratio <= P50_STABILITY_LIMIT
- stable_p99 = p99_ratio is not None and p99_ratio <= P99_STABILITY_LIMIT
reasons = list(extra_reasons)
for condition, reason in (
(len(ids) >= REQUIRED_ALLOCATIONS, "insufficient-allocations"),
(complete, "incomplete-repeat-coverage"),
(correct, "correctness-failed"),
(measured, "missing-measured-roundtrip-p99"),
- (stable_p50, "unstable-p50"),
- (stable_p99, "unstable-p99"),
(stable_ordering, "unstable-ordering"),
):
if not condition:
@@ -2589,11 +2571,7 @@ def _eligibility_record(
"complete": complete,
"correct": correct,
"measured_roundtrip_p99": measured,
- "stable_p50": stable_p50,
- "stable_p99": stable_p99,
"stable_ordering": stable_ordering,
- "p50_max_min_ratio": p50_ratio,
- "p99_max_min_ratio": p99_ratio,
"reasons": reasons,
}
@@ -2780,16 +2758,6 @@ def _build_series(
qualification_indices = sorted(
document["measurement"]["qualification_index"] for document in documents
)
- p50_ratios = [
- _ratio([rows[token]["components"]["roundtrip"]["percentiles_us"]["p50"] for rows in row_maps])
- for token in tokens
- ]
- p99_ratios = [
- _ratio([rows[token]["components"]["roundtrip"]["percentiles_us"]["p99"] for rows in row_maps])
- for token in tokens
- ]
- p50_ratio = max((value for value in p50_ratios if value is not None), default=None)
- p99_ratio = max((value for value in p99_ratios if value is not None), default=None)
correct = all(
row["correctness"]["passed"]
for document in documents for row in document["measurement"]["rows"]
@@ -2905,24 +2873,6 @@ def _build_series(
anomaly["type"].replace("_", "-")
for row in rows for anomaly in row["anomalies"]
} | set(diagnostics["reasons"])),
- "stability": {
- "complete": qualification_indices == [1, 2, 3],
- "qualification_indices": qualification_indices,
- "p50_max_min_ratio": p50_ratios[tokens.index(token)]
- if qualification_indices == [1, 2, 3] else None,
- "p99_max_min_ratio": p99_ratios[tokens.index(token)]
- if qualification_indices == [1, 2, 3] else None,
- "stable_p50": bool(
- qualification_indices == [1, 2, 3]
- and p50_ratios[tokens.index(token)] is not None
- and p50_ratios[tokens.index(token)] <= P50_STABILITY_LIMIT
- ),
- "stable_p99": bool(
- qualification_indices == [1, 2, 3]
- and p99_ratios[tokens.index(token)] is not None
- and p99_ratios[tokens.index(token)] <= P99_STABILITY_LIMIT
- ),
- },
"trial_diagnostics": diagnostics,
"routing": routing,
"components": components,
@@ -2940,8 +2890,6 @@ def _build_series(
measured=measured,
# Ordering is defined only across alternatives in a controlled cohort.
stable_ordering=True,
- p50_ratio=p50_ratio,
- p99_ratio=p99_ratio,
extra_reasons=sorted(set(extra_reasons)),
)
series = {
@@ -3333,14 +3281,6 @@ def build_decisions(
same_points = len({tuple(sorted(values)) for values in token_sets}) == 1
ordering, aligned_runs = _cohort_ordering(members, internals, tokens) if tokens else (False, 0)
allocations = sorted({value for member in members for value in member["allocation_ids"]})
- p50_ratio = max(
- (member["eligibility"]["p50_max_min_ratio"] for member in members
- if member["eligibility"]["p50_max_min_ratio"] is not None), default=None
- )
- p99_ratio = max(
- (member["eligibility"]["p99_max_min_ratio"] for member in members
- if member["eligibility"]["p99_max_min_ratio"] is not None), default=None
- )
extra = {
reason for member in members for reason in member["eligibility"]["reasons"]
if reason not in {"unstable-ordering"}
@@ -3370,8 +3310,6 @@ def build_decisions(
correct=all(member["eligibility"]["correct"] for member in members),
measured=all(member["eligibility"]["measured_roundtrip_p99"] for member in members),
stable_ordering=ordering,
- p50_ratio=p50_ratio,
- p99_ratio=p99_ratio,
extra_reasons=sorted(extra),
)
member_ids = [member["series_id"] for member in members]
@@ -3689,11 +3627,11 @@ def build_dataset(
if promote and (
len(loaded) != REQUIRED_ALLOCATIONS
or len(run_ids) != len(set(run_ids))
- or qualification_indices != [1, 2, 3]
+ or qualification_indices != [1]
or any(bundle["manifest"]["run"]["run_attempt"] != 1 for bundle in loaded)
):
raise PublisherError(
- "promotion requires qualification indices 1, 2, and 3 from first-attempt runs"
+ "promotion requires qualification index 1 from a first-attempt run"
)
if promote and matrix_ids != {CANONICAL_FULL_V1_MATRIX_SHA256}:
raise PublisherError("promotion requires the canonical full-v1 matrix")
@@ -4001,7 +3939,7 @@ def _bundle_ids(values: Sequence[str], *, promote: bool) -> list[str]:
):
raise PublisherError("bundle IDs must be unique SHA-256 digests")
if promote and len(bundle_ids) != REQUIRED_ALLOCATIONS:
- raise PublisherError("promotion requires exactly three explicit bundle IDs")
+ raise PublisherError("promotion requires exactly one explicit bundle ID")
return bundle_ids
diff --git a/experimental/CollectiveX/schemas/public-dataset-v1.schema.json b/experimental/CollectiveX/schemas/public-dataset-v1.schema.json
index 055a73b452..c6cfdd12c2 100644
--- a/experimental/CollectiveX/schemas/public-dataset-v1.schema.json
+++ b/experimental/CollectiveX/schemas/public-dataset-v1.schema.json
@@ -48,8 +48,8 @@
"reason": {"$ref": "#/$defs/reason"},
"matrix_id": {"oneOf": [{"type": "null"},{"$ref": "#/$defs/sha256"}]},
"allocation_ids": {"type": "array","uniqueItems": true,"items": {"$ref": "#/$defs/allocationId"}},
- "required_allocations": {"const": 3},
- "qualification_indices": {"type":"array","minItems":0,"maxItems":3,"uniqueItems":true,"items":{"enum":[1,2,3]}},
+ "required_allocations": {"const": 1},
+ "qualification_indices": {"type":"array","minItems":0,"maxItems":1,"uniqueItems":true,"items":{"enum":[1]}},
"requested_cases": {"type": "integer","minimum": 0},
"terminal_cases": {"type": "integer","minimum": 0},
"measured_cases": {"type": "integer","minimum": 0},
@@ -168,38 +168,6 @@
"precision":{"$ref":"#/$defs/precisionEvidence"}
}
},
- "pointStability": {
- "type":"object",
- "additionalProperties":false,
- "allOf":[
- {
- "if":{"properties":{"complete":{"const":true}},"required":["complete"]},
- "then":{"properties":{
- "qualification_indices":{"const":[1,2,3]},
- "p50_max_min_ratio":{"type":"number","minimum":1},
- "p99_max_min_ratio":{"type":"number","minimum":1}
- }},
- "else":{"properties":{
- "p50_max_min_ratio":{"type":"null"},
- "p99_max_min_ratio":{"type":"null"},
- "stable_p50":{"const":false},
- "stable_p99":{"const":false}
- }}
- }
- ],
- "required":[
- "complete","qualification_indices","p50_max_min_ratio","p99_max_min_ratio",
- "stable_p50","stable_p99"
- ],
- "properties": {
- "complete":{"type":"boolean"},
- "qualification_indices":{"type":"array","minItems":1,"maxItems":3,"uniqueItems":true,"items":{"enum":[1,2,3]}},
- "p50_max_min_ratio":{"oneOf":[{"type":"null"},{"type":"number","minimum":1}]},
- "p99_max_min_ratio":{"oneOf":[{"type":"null"},{"type":"number","minimum":1}]},
- "stable_p50":{"type":"boolean"},
- "stable_p99":{"type":"boolean"}
- }
- },
"coverageTopology": {
"type": "object",
"additionalProperties": false,
@@ -381,11 +349,7 @@
"complete",
"correct",
"measured_roundtrip_p99",
- "stable_p50",
- "stable_p99",
"stable_ordering",
- "p50_max_min_ratio",
- "p99_max_min_ratio",
"reasons"
],
"properties": {
@@ -394,11 +358,7 @@
"complete": {"type": "boolean"},
"correct": {"type": "boolean"},
"measured_roundtrip_p99": {"type": "boolean"},
- "stable_p50": {"type": "boolean"},
- "stable_p99": {"type": "boolean"},
"stable_ordering": {"type": "boolean"},
- "p50_max_min_ratio": {"oneOf": [{"type": "null"},{"type": "number","minimum": 1}]},
- "p99_max_min_ratio": {"oneOf": [{"type": "null"},{"type": "number","minimum": 1}]},
"reasons": {
"type": "array",
"uniqueItems": true,
@@ -483,7 +443,6 @@
"global_tokens",
"anomalies",
"correctness",
- "stability",
"trial_diagnostics",
"routing",
"components",
@@ -496,7 +455,6 @@
"global_tokens": {"type": "integer","minimum": 1},
"anomalies": {"type":"array","maxItems":16,"uniqueItems":true,"items":{"$ref":"#/$defs/reasonId"}},
"correctness": {"$ref":"#/$defs/pointCorrectness"},
- "stability": {"$ref":"#/$defs/pointStability"},
"trial_diagnostics":{"$ref":"#/$defs/trialDiagnostics"},
"routing": {
"type": "object",
diff --git a/experimental/CollectiveX/tests/test_publisher.py b/experimental/CollectiveX/tests/test_publisher.py
index b65c3366f9..8136b4932b 100644
--- a/experimental/CollectiveX/tests/test_publisher.py
+++ b/experimental/CollectiveX/tests/test_publisher.py
@@ -525,18 +525,16 @@ def measured(value: float) -> dict:
def _series(seed: str, backend: str, *, decision_grade: bool = False) -> tuple[dict, dict]:
case, allocation, attempt, series_id, point_id, evidence = _ids(seed)
- allocations = [identity.allocation_id({"seed": seed, "run": run}) for run in range(3)]
+ allocations = [identity.allocation_id({"seed": seed, "run": run}) for run in range(1)]
eligibility = publisher._eligibility_record(
allocations if decision_grade else [allocation],
complete=decision_grade,
correct=True,
measured=True,
stable_ordering=True,
- p50_ratio=1.01 if decision_grade else None,
- p99_ratio=1.02 if decision_grade else None,
)
component = _component(1.0 if backend == "deepep" else 1.2)
- qualification_indices = [1, 2, 3] if decision_grade else [1]
+ qualification_indices = [1]
communication_precision = identity.precision_profile(
identity.V1_CONTROL_PRECISION_PROFILE
)
@@ -606,13 +604,6 @@ def _series(seed: str, backend: str, *, decision_grade: bool = False) -> tuple[d
"point_id": point_id, "tokens_per_rank": 8, "global_tokens": 64,
"anomalies": [],
"correctness": {"semantic_pass": True, "precision": _precision_evidence()},
- "stability": {
- "complete": decision_grade,
- "qualification_indices": qualification_indices,
- "p50_max_min_ratio": 1.02 if decision_grade else None,
- "p99_max_min_ratio": 1.02 if decision_grade else None,
- "stable_p50": decision_grade, "stable_p99": decision_grade,
- },
"trial_diagnostics": {
"flagged": False,
"reasons": [],
@@ -687,7 +678,7 @@ def _series(seed: str, backend: str, *, decision_grade: bool = False) -> tuple[d
for statistic in ("p50", "p99")
},
}}
- for run in range(3)
+ for run in range(1)
}
trial_blocks = {
run_id: {8: {
@@ -767,7 +758,7 @@ def _dataset() -> dict:
"generated_at": "2026-07-04T00:00:00Z", "source_bundle_ids": ["c" * 64],
"promotion": {
"status": "diagnostic", "reason": None, "matrix_id": "d" * 64,
- "allocation_ids": [allocation], "required_allocations": 3,
+ "allocation_ids": [allocation], "required_allocations": 1,
"qualification_indices": [1],
"requested_cases": 1, "terminal_cases": 1,
"measured_cases": 1, "unsupported_cases": 0,
@@ -934,7 +925,7 @@ def _promoted_dataset(*, precision_profiles: tuple[str, ...] = ()) -> dict:
and not case["eplb"] and case["ep"] == 8
)
unsupported_attempts = []
- for run_id in range(1, 4):
+ for run_id in range(1, 2):
allocation_id = identity.allocation_id(
{"seed": "planned-unsupported", "run": run_id}
)
@@ -986,12 +977,12 @@ def _promoted_dataset(*, precision_profiles: tuple[str, ...] = ()) -> dict:
return {
"format": "collectivex.public.v1", "schema_version": 1,
"generated_at": "2026-07-04T00:00:00Z",
- "source_bundle_ids": ["a" * 64, "b" * 64, "c" * 64],
+ "source_bundle_ids": ["a" * 64],
"promotion": {
"status": "promoted", "reason": None,
"matrix_id": publisher.CANONICAL_FULL_V1_MATRIX_SHA256,
"allocation_ids": sorted({item["allocation_id"] for item in attempts}),
- "required_allocations": 3, "qualification_indices": [1, 2, 3],
+ "required_allocations": 1, "qualification_indices": [1],
"requested_cases": len(coverage), "terminal_cases": len(coverage),
"measured_cases": len(coverage) - 1, "unsupported_cases": 1,
"requested_points": sum(len(item["points"]) for item in coverage),
@@ -1033,7 +1024,7 @@ def runs() -> dict[str, dict[int, dict[str, object]]]:
"roundtrip": tuple(tuple([10.0] * 8) for _ in range(64)),
}
}
- for index in range(1, 4)
+ for index in range(1, 2)
}
stable = publisher._trial_diagnostics(runs(), 8)
@@ -1041,7 +1032,7 @@ def runs() -> dict[str, dict[int, dict[str, object]]]:
drift = runs()
drift["1"][8]["roundtrip"] = tuple(
- tuple([12.0 if trial >= 56 else 10.0] * 8) for trial in range(64)
+ tuple([10.0 + 2.0 * trial / 63.0] * 8) for trial in range(64)
)
self.assertEqual(publisher._trial_diagnostics(drift, 8)["reasons"], ["trial-drift"])
@@ -1724,18 +1715,9 @@ def test_promotion_requires_an_eligible_cohort_for_every_comparison_kind(self) -
unstable_fast["phase"] = unstable_slow["phase"] = "prefill"
unstable_fast["series_id"] = identity.series_id({"test": "unstable-fast"})
unstable_slow["series_id"] = identity.series_id({"test": "unstable-slow"})
- for statistic in ("p50", "p99"):
- unstable_slow_internal["run_metrics"]["1"][8]["latency_us"][statistic] = (
- unstable_fast_internal["run_metrics"]["1"][8]["latency_us"][statistic]
- / 2
- )
- for field in (
- "activation_data_rate_gbps_at_latency_percentile",
- "total_logical_data_rate_gbps_at_latency_percentile",
- ):
- unstable_slow_internal["run_metrics"]["1"][8][field][statistic] = (
- unstable_fast_internal["run_metrics"]["1"][8][field][statistic] * 2
- )
+ # A single-allocation cohort has no cross-run ordering to disagree, so the
+ # prefill pair is made ineligible through missing trial evidence instead.
+ del unstable_slow_internal["trial_blocks"]
series = [stable_fast, stable_slow, unstable_fast, unstable_slow]
internals = {
stable_fast["series_id"]: stable_fast_internal,
@@ -1818,7 +1800,7 @@ def test_promotion_requires_every_derived_chip_cohort_to_be_stable(self) -> None
dataset["cohorts"], dataset["series"]
)
- def test_promotion_rejects_more_than_three_bundles(self) -> None:
+ def test_promotion_rejects_more_than_one_bundle(self) -> None:
bundles = {
str(run_id): {
"id": str(run_id), "cases": [],
@@ -1826,15 +1808,15 @@ def test_promotion_rejects_more_than_three_bundles(self) -> None:
"matrix": {"sha256": publisher.CANONICAL_FULL_V1_MATRIX_SHA256},
"run": {
"run_id": str(run_id), "run_attempt": 1,
- "qualification_index": min(run_id, 3),
+ "qualification_index": 1,
},
},
}
- for run_id in range(1, 5)
+ for run_id in range(1, 3)
}
with mock.patch.object(
publisher, "load_bundle", side_effect=lambda _, bundle_id: bundles[bundle_id]
- ), self.assertRaisesRegex(publisher.PublisherError, "qualification indices"):
+ ), self.assertRaisesRegex(publisher.PublisherError, "qualification index 1"):
publisher.build_dataset(object(), list(bundles), promote=True)
dataset = _promoted_dataset()
@@ -1870,9 +1852,7 @@ def test_standalone_promotion_binds_matrix_and_requested_dispositions(self) -> N
item["status"] = "diagnostic"
item["eligibility"].update({
"decision_grade": False,
- "stable_p50": False,
- "p50_max_min_ratio": 1.20,
- "reasons": ["unstable-p50"],
+ "reasons": ["insufficient-allocations"],
})
with mock.patch.object(
publisher, "CANONICAL_FULL_V1_CASE_CATALOG_SHA256", fixture_catalog
@@ -2009,7 +1989,7 @@ def test_build_promotion_requires_canonical_full_matrix(self) -> None:
},
},
}
- for run_id in range(1, 4)
+ for run_id in range(1, 2)
}
with mock.patch.object(
publisher, "load_bundle", side_effect=lambda _, bundle_id: bundles[bundle_id]
@@ -2404,6 +2384,7 @@ def test_all_decision_metrics_require_stable_repeat_ordering(self) -> None:
)
library = next(item for item in cohorts if item["kind"] == "library")
self.assertTrue(library["eligibility"]["decision_grade"])
+ self.assertTrue(library["eligibility"]["stable_ordering"])
self.assertEqual(
len([item for item in rankings if item["cohort_id"] == library["cohort_id"]]),
6,
@@ -2416,20 +2397,14 @@ def test_all_decision_metrics_require_stable_repeat_ordering(self) -> None:
1,
)
- for statistic in ("p50", "p99"):
- for field in (
- "activation_data_rate_gbps_at_latency_percentile",
- "total_logical_data_rate_gbps_at_latency_percentile",
- ):
- slow_internal["run_metrics"]["1"][8][field][statistic] = (
- fast_internal["run_metrics"]["1"][8][field][statistic] * 2
- )
+ # A cohort whose single run is missing trial evidence cannot be ranked.
+ del slow_internal["trial_blocks"]
cohorts, rankings, recommendations, _ = publisher.build_decisions(
[fast, slow], internals
)
library = next(item for item in cohorts if item["kind"] == "library")
self.assertFalse(library["eligibility"]["decision_grade"])
- self.assertIn("unstable-ordering", library["eligibility"]["reasons"])
+ self.assertIn("missing-trial-blocks", library["eligibility"]["reasons"])
self.assertFalse(any(
item["cohort_id"] == library["cohort_id"] for item in rankings
))
@@ -2528,7 +2503,7 @@ def test_p99_equivalence_band_emits_competition_tie_without_recommendation(self)
def test_p99_winner_requires_every_run_to_agree(self) -> None:
fast, fast_internal = _series("run-fast", "deepep", decision_grade=True)
slow, slow_internal = _series("run-slow", "uccl", decision_grade=True)
- ratios = {"0": 0.98, "1": 1.20, "2": 1.20}
+ ratios = {"0": 0.98}
for run_id, ratio in ratios.items():
slow_internal["trial_blocks"][run_id][8]["roundtrip"] = tuple(
tuple(sample * ratio for sample in block)
@@ -2684,10 +2659,10 @@ def test_missing_private_trials_blocks_decision_grade(self) -> None:
self.assertEqual((rankings, recommendations, sensitivities), ([], [], []))
def test_extra_eligibility_reason_blocks_decision_grade(self) -> None:
- allocations = [identity.allocation_id({"run": run}) for run in range(3)]
+ allocations = [identity.allocation_id({"run": run}) for run in range(1)]
eligibility = publisher._eligibility_record(
allocations, complete=True, correct=True, measured=True,
- stable_ordering=True, p50_ratio=1.01, p99_ratio=1.02,
+ stable_ordering=True,
extra_reasons=["incomplete-provenance"],
)
self.assertFalse(eligibility["decision_grade"])
diff --git a/experimental/CollectiveX/tests/test_schema_v1_contract.py b/experimental/CollectiveX/tests/test_schema_v1_contract.py
index 3073a7e253..1c5aa29824 100644
--- a/experimental/CollectiveX/tests/test_schema_v1_contract.py
+++ b/experimental/CollectiveX/tests/test_schema_v1_contract.py
@@ -164,14 +164,14 @@ def test_qualification_index_is_bound_across_private_and_public_records(self) ->
self.assertEqual((value["minimum"], value["maximum"]), (1, 3))
promotion_indices = self.public["properties"]["promotion"]["properties"]["qualification_indices"]
series_indices = self.public["$defs"]["series"]["properties"]["measurement"]["properties"]["qualification_indices"]
- for schema, valid_values in (
- (promotion_indices, ([], [1], [1, 2, 3])),
- (series_indices, ([1], [2, 3], [1, 2, 3])),
+ for schema, valid_values, invalid_values in (
+ (promotion_indices, ([], [1]), ([0], [2], [1, 2], [1, 2, 3])),
+ (series_indices, ([1], [2, 3], [1, 2, 3]), ([0], [4], [1, 1], [1, 2, 3, 1])),
):
validator = jsonschema.Draft202012Validator(schema)
for value in valid_values:
validator.validate(value)
- for value in ([0], [4], [1, 1], [1, 2, 3, 1]):
+ for value in invalid_values:
with self.assertRaises(jsonschema.ValidationError):
validator.validate(value)
measurement = self.raw["properties"]["measurement"]
@@ -368,10 +368,11 @@ def test_public_coverage_point_reason_tracks_terminal_status(self) -> None:
with self.assertRaises(jsonschema.ValidationError):
validator.validate(broken)
- def test_public_measured_point_has_bounded_detail_and_three_run_stability(self) -> None:
+ def test_public_measured_point_has_bounded_detail(self) -> None:
point = self.public["$defs"]["point"]
self.assertFalse(point["additionalProperties"])
- self.assertTrue({"anomalies", "correctness", "stability"} <= set(point["required"]))
+ self.assertTrue({"anomalies", "correctness"} <= set(point["required"]))
+ self.assertNotIn("stability", point["properties"])
self.assertNotIn("correct", point["properties"])
axis = {
@@ -401,44 +402,6 @@ def test_public_measured_point_has_bounded_detail_and_three_run_stability(self)
with self.assertRaises(jsonschema.ValidationError):
correctness_validator.validate(broken_correctness)
- stability_validator = _definition_validator(self.public, "pointStability")
- stability_validator.validate({
- "complete": True,
- "qualification_indices": [1, 2, 3],
- "p50_max_min_ratio": 1.02,
- "p99_max_min_ratio": 1.04,
- "stable_p50": True,
- "stable_p99": True,
- })
- stability_validator.validate({
- "complete": False,
- "qualification_indices": [2],
- "p50_max_min_ratio": None,
- "p99_max_min_ratio": None,
- "stable_p50": False,
- "stable_p99": False,
- })
- for broken in (
- {
- "complete": True,
- "qualification_indices": [1, 2],
- "p50_max_min_ratio": 1.0,
- "p99_max_min_ratio": 1.0,
- "stable_p50": True,
- "stable_p99": True,
- },
- {
- "complete": False,
- "qualification_indices": [1],
- "p50_max_min_ratio": 1.0,
- "p99_max_min_ratio": None,
- "stable_p50": False,
- "stable_p99": False,
- },
- ):
- with self.assertRaises(jsonschema.ValidationError):
- stability_validator.validate(broken)
-
anomalies = point["properties"]["anomalies"]
self.assertEqual(anomalies["maxItems"], 16)
self.assertTrue(anomalies["uniqueItems"])
From ad7087c7dd339bc1152e20ccd84c77089d24b84f Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 7 Jul 2026 20:45:54 +0800
Subject: [PATCH 181/210] feat(collectivex): publish v1 from one run in the
workflow
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Collapse the publish-v1 job to a single source run. Replace the
publish_run_ids input (three comma-separated IDs) with publish_run_id,
drop the count-of-three and uniqueness checks and the shared-source-SHA
loop, pin the per-run release marker to qualification_index == 1, and
require the aggregate qualification-index set to be exactly {1}. The
setup job's qualification-index input and per-index release-marker
digest check are unchanged (layer 2). Update test_sampling_contract.py
publish-job string assertions to the single-run messages.
Full unittest battery: 239 OK.
将 publish-v1 作业收敛为单一来源运行。以 publish_run_id 替换
publish_run_ids（三个逗号分隔的 ID），移除计数为三与唯一性检查以及
共享来源 SHA 的循环，将每次运行的发布标记固定为 qualification_index
== 1，并要求聚合资格索引集合恰为 {1}。setup 作业的资格索引输入与
按索引的发布标记摘要校验保持不变（第二层）。同步更新
test_sampling_contract.py 中 publish 作业的字符串断言为单次运行消息。
---
.github/workflows/collectivex-sweep.yml | 78 +++++++------------
.../tests/test_sampling_contract.py | 5 +-
2 files changed, 33 insertions(+), 50 deletions(-)
diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml
index 8f5673239f..6ca917919f 100644
--- a/.github/workflows/collectivex-sweep.yml
+++ b/.github/workflows/collectivex-sweep.yml
@@ -54,8 +54,8 @@ on:
type: choice
default: '1'
options: ['1', '2', '3']
- publish_run_ids:
- description: For publish-v1, exactly three successful V1 sweep run IDs
+ publish_run_id:
+ description: For publish-v1, one successful first-attempt V1 sweep run ID (qualification index 1)
type: string
default: ''
refresh_run_id:
@@ -740,60 +740,42 @@ jobs:
timeout-minutes: 30
env:
GH_TOKEN: ${{ github.token }}
- RUN_IDS: ${{ inputs.publish_run_ids }}
+ RUN_ID: ${{ inputs.publish_run_id }}
steps:
- - name: Verify source runs
+ - name: Verify source run
id: runs
env:
REPOSITORY: ${{ github.repository }}
run: |
set -euo pipefail
- IFS=',' read -r -a run_ids <<< "$RUN_IDS"
- [ "${#run_ids[@]}" -eq 3 ] || {
- echo 'publish_run_ids must contain exactly three IDs' >&2
- exit 1
- }
- [ "$(printf '%s\n' "${run_ids[@]}" | sort -u | wc -l)" -eq 3 ] || {
- echo 'publish_run_ids must be unique' >&2
+ run_id="$RUN_ID"
+ [[ "$run_id" =~ ^[1-9][0-9]*$ ]] || {
+ echo 'publish_run_id must be a single decimal run ID' >&2
exit 1
}
: > "$RUNNER_TEMP/collectivex-runs.tsv"
- source_sha=''
- for run_id in "${run_ids[@]}"; do
- [[ "$run_id" =~ ^[1-9][0-9]*$ ]] || {
- echo 'publish_run_ids contains a non-decimal ID' >&2
+ metadata=$(gh api "repos/$REPOSITORY/actions/runs/$run_id")
+ name=$(jq -r '.name' <<< "$metadata")
+ path=$(jq -r '.path' <<< "$metadata")
+ branch=$(jq -r '.head_branch' <<< "$metadata")
+ status=$(jq -r '.status' <<< "$metadata")
+ conclusion=$(jq -r '.conclusion' <<< "$metadata")
+ sha=$(jq -r '.head_sha' <<< "$metadata")
+ attempt=$(jq -r '.run_attempt' <<< "$metadata")
+ [ "$name" = 'CollectiveX Sweep' ] \
+ && [ "$path" = '.github/workflows/collectivex-sweep.yml' ] \
+ && [ "$branch" = 'collectivex' ] \
+ && [ "$status" = 'completed' ] \
+ && [ "$conclusion" = 'success' ] \
+ && [ "$attempt" = 1 ] \
+ && [[ "$sha" =~ ^[0-9a-f]{40}$ ]] || {
+ echo "run $run_id is not an eligible first-attempt V1 sweep" >&2
exit 1
}
- metadata=$(gh api "repos/$REPOSITORY/actions/runs/$run_id")
- name=$(jq -r '.name' <<< "$metadata")
- path=$(jq -r '.path' <<< "$metadata")
- branch=$(jq -r '.head_branch' <<< "$metadata")
- status=$(jq -r '.status' <<< "$metadata")
- conclusion=$(jq -r '.conclusion' <<< "$metadata")
- sha=$(jq -r '.head_sha' <<< "$metadata")
- attempt=$(jq -r '.run_attempt' <<< "$metadata")
- [ "$name" = 'CollectiveX Sweep' ] \
- && [ "$path" = '.github/workflows/collectivex-sweep.yml' ] \
- && [ "$branch" = 'collectivex' ] \
- && [ "$status" = 'completed' ] \
- && [ "$conclusion" = 'success' ] \
- && [ "$attempt" = 1 ] \
- && [[ "$sha" =~ ^[0-9a-f]{40}$ ]] || {
- echo "run $run_id is not an eligible first-attempt V1 sweep" >&2
- exit 1
- }
- if [ -z "$source_sha" ]; then
- source_sha="$sha"
- else
- [ "$sha" = "$source_sha" ] || {
- echo 'source runs do not share one source SHA' >&2
- exit 1
- }
- fi
- printf '%s\t%s\t%s\n' "$run_id" "$attempt" "$sha" \
- >> "$RUNNER_TEMP/collectivex-runs.tsv"
- done
+ source_sha="$sha"
+ printf '%s\t%s\t%s\n' "$run_id" "$attempt" "$sha" \
+ >> "$RUNNER_TEMP/collectivex-runs.tsv"
echo "source_sha=$source_sha" >> "$GITHUB_OUTPUT"
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v5.0.0
@@ -851,7 +833,7 @@ jobs:
and .run_id == $run_id
and .run_attempt == $attempt
and .source_sha == $source_sha
- and (.qualification_index == 1 or .qualification_index == 2 or .qualification_index == 3)
+ and .qualification_index == 1
and .matrix_sha256 == $matrix_sha
and .execution_plan_sha256 == $execution_plan_sha' \
"$marker" >/dev/null || {
@@ -888,8 +870,8 @@ jobs:
printf '%s\n' "$bundle_id" >> "$RUNNER_TEMP/collectivex-bundles.txt"
done < "$RUNNER_TEMP/collectivex-runs.tsv"
- [ "$(sort -n "$RUNNER_TEMP/collectivex-qualification-indices.txt" | tr '\n' ' ')" = '1 2 3 ' ] || {
- echo 'source runs must contain qualification indices 1, 2, and 3 exactly once' >&2
+ [ "$(sort -n "$RUNNER_TEMP/collectivex-qualification-indices.txt" | tr '\n' ' ')" = '1 ' ] || {
+ echo 'source run must be qualification index 1 exactly once' >&2
exit 1
}
@@ -923,7 +905,7 @@ jobs:
echo
echo "Dataset: \`$dataset_id\`"
echo
- echo 'Source runs:'
+ echo 'Source run:'
sed 's/^/- `/' "$RUNNER_TEMP/collectivex-runs.tsv" | sed 's/$/`/'
} >> "$GITHUB_STEP_SUMMARY"
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index ec9ce00a0f..f5240e5d60 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -1765,8 +1765,9 @@ def test_v1_publication_requires_explicit_release_markers(self) -> None:
self.assertIn("EXPECTED_MATRIX_SHA256", sweep)
self.assertIn("cxrelease-v1-${{ github.run_id }}-${{ github.run_attempt }}", sweep)
- self.assertIn("publish_run_ids must contain exactly three IDs", sweep)
- self.assertIn("source runs do not share one source SHA", sweep)
+ self.assertIn("publish_run_id must be a single decimal run ID", sweep)
+ self.assertIn("source run must be qualification index 1 exactly once", sweep)
+ self.assertIn("and .qualification_index == 1", sweep)
self.assertIn("cxrelease-v1-$run_id-$attempt/release.json", sweep)
self.assertIn("run $run_id is not tagged for V1 publication", sweep)
self.assertIn("ref: ${{ steps.runs.outputs.source_sha }}", sweep)
From 034dc9a5196433f561de83c967d84ef25d5d565b Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 7 Jul 2026 20:47:56 +0800
Subject: [PATCH 182/210] docs(collectivex): publish v1 from a single run
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Update the tracked docs to match the single-allocation publication:
README, methodology, and README_zh now describe accepting one
successful first-attempt run ID at qualification index 1 (not three
run IDs at indices 1, 2, 3). Replace the cross-allocation p50/p99
repeat-stability language in the decision-grade contract with the
retained within-run trial drift/outlier diagnostics.
更新受版本管理的文档以匹配单次分配发布：README、methodology 与
README_zh 现描述接受一个位于 qualification index 1 的成功首次尝试
run ID（而非索引 1、2、3 的三个 run ID）。将 decision-grade 契约中
的跨分配 p50/p99 重复稳定性表述替换为保留的运行内 trial 漂移/离群
诊断。
---
experimental/CollectiveX/README.md | 4 ++--
experimental/CollectiveX/README_zh.md | 6 ++---
experimental/CollectiveX/docs/methodology.md | 24 ++++++++++----------
3 files changed, 17 insertions(+), 17 deletions(-)
diff --git a/experimental/CollectiveX/README.md b/experimental/CollectiveX/README.md
index 4144539d42..3091b0e62b 100644
--- a/experimental/CollectiveX/README.md
+++ b/experimental/CollectiveX/README.md
@@ -105,8 +105,8 @@ a run/attempt/source-bound `cxrelease-v1-*` marker. Partial and filtered runs ca
The main-registered `.github/workflows/collectivex-sweep.yml` provides `probe-precision`, `sweep`,
`publish-v1`, and `refresh-v1` operations, so its branch revision can be dispatched with
`--ref collectivex` without a standalone branch-only workflow. Probes are unversioned, bounded,
-native-runtime checks and cannot publish. Publication accepts exactly three successful first-attempt tagged
-sweep run IDs from one source SHA, revalidates their metadata and release markers, and runs
+native-runtime checks and cannot publish. Publication accepts one successful first-attempt tagged
+sweep run ID at qualification index 1, revalidates its metadata and release marker, and runs
`publisher.py` in a disposable runner-local workspace. Refresh revalidates and reuploads only the
exact content-addressed sanitized dataset. Raw artifacts and the private publisher workspace are
never exposed to the frontend.
diff --git a/experimental/CollectiveX/README_zh.md b/experimental/CollectiveX/README_zh.md
index fb32369b40..fc8c6a459c 100644
--- a/experimental/CollectiveX/README_zh.md
+++ b/experimental/CollectiveX/README_zh.md
@@ -79,9 +79,9 @@ raw GitHub artifacts。运行默认使用 `release_tag=unversioned`，仅供诊
`release_tag=v1`；setup 随后要求固定的完整 matrix digest，并生成绑定 run、attempt 与 source 的
`cxrelease-v1-*` marker。Partial 或 filtered 运行无法获得该 marker。
-`.github/workflows/collectivex-publish.yml` 是显式的 V1 gate。它只接受三个来自同一 source SHA、
-成功且带 V1 tag 的 sweep run IDs，重新校验 GitHub metadata 与 release markers，并在 runner 本地
-可丢弃工作区中执行 `publisher.py`。只有完整通过 promotion、隐私检查和内容寻址的数据集才会以
+`.github/workflows/collectivex-sweep.yml` 的 `publish-v1` 操作是显式的 V1 gate。它只接受一个位于
+qualification index 1、成功且带 V1 tag 的首次尝试 sweep run ID，重新校验 GitHub metadata 与
+release marker，并在 runner 本地可丢弃工作区中执行 `publisher.py`。只有完整通过 promotion、隐私检查和内容寻址的数据集才会以
`cxpublication-v1-*` 上传；raw artifacts 与 publisher private workspace 永不暴露给前端。
系统不需要 results server、attached store、Vercel storage、GCP、Neon、managed database 或
diff --git a/experimental/CollectiveX/docs/methodology.md b/experimental/CollectiveX/docs/methodology.md
index 7b047a879d..788e877d2d 100644
--- a/experimental/CollectiveX/docs/methodology.md
+++ b/experimental/CollectiveX/docs/methodology.md
@@ -6,8 +6,8 @@
-This document is the frozen V1 qualification contract. No V1 dataset is approved until three
-complete first-attempt executions pass publication and frontend audit. Chinese documentation
+This document is the frozen V1 qualification contract. No V1 dataset is approved until one
+complete first-attempt execution passes publication and frontend audit. Chinese documentation
synchronization is explicitly deferred for the V1 implementation phase. MI325X is deferred to a
later version because its intended two-node runner pool is unavailable.
@@ -237,10 +237,10 @@ Any failed, invalid, or diagnostic retry of a runnable case blocks promotion eve
succeeds. Routing cohorts are comparable-experimental sensitivities and never produce configuration
recommendations; official library/platform/system cohorts own actionable recommendations.
-A point becomes decision-grade only after three independent workflow runs and allocation IDs pass
-correctness, identity, provenance, tail gates, p50/p99 repeat-stability thresholds, and stable ordering. The
-publisher, not the frontend, computes eligibility, controlled cohorts, sensitivity pairs, and
-recommendations.
+A point becomes decision-grade only after a single workflow run and allocation ID passes
+correctness, identity, provenance, tail gates, within-run trial drift/outlier diagnostics, and stable
+ordering. The publisher, not the frontend, computes eligibility, controlled cohorts, sensitivity pairs,
+and recommendations.
## Execution Isolation
@@ -318,7 +318,7 @@ Publication is fail-closed:
Rejected attempts remain only in the disposable private workspace and short-lived source artifacts;
they never advance `dev-latest` or enter a production channel. The workspace is destroyed with the
publication runner and is never attached to the frontend. No publication artifact is emitted unless
-all three selected bundles advance `dev-latest`.
+the single selected bundle advances `dev-latest`.
`publisher.py ingest` accepts the exact matrix plus one `--artifact` directory or ZIP per GitHub
artifact. `promote` accepts explicit immutable bundle IDs. Default `verify` requires
@@ -330,11 +330,11 @@ content are never bundled into the application.
Sweeps default to `release_tag=unversioned`. The main-registered `collectivex-sweep.yml` owns
`sweep`, `publish-v1`, and `refresh-v1`, so its branch revision remains dispatchable. V1 emits a
marker bound to the run ID, first attempt, qualification index, source SHA, and locked matrix digest.
-Publication accepts exactly three unique successful run IDs from one source SHA with qualification
-indices 1, 2, and 3, downloads their immutable artifacts, and passes the same provenance assertions
-to `publisher.py ingest`. Refresh requires an exact source run and dataset digest and reuploads the
-same validated sanitized bytes. Partial, filtered, untagged, cross-source, rerun, failed, expired,
-or digest-mismatched inputs fail closed.
+Publication accepts one successful first-attempt run ID at qualification index 1, downloads its
+immutable artifacts, and passes the same provenance assertions to `publisher.py ingest`. Refresh
+requires an exact source run and dataset digest and reuploads the same validated sanitized bytes.
+Partial, filtered, untagged, cross-source, rerun, failed, expired, or digest-mismatched inputs fail
+closed.
Using a server-side GitHub read token, the frontend discovers the latest successful version-scoped
publication run and downloads the publication artifact just in time. It requires exactly one root
From 4eb2a683f71c53033c42876120fef7c051be44a0 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 7 Jul 2026 21:50:29 +0800
Subject: [PATCH 183/210] feat(collectivex): allow partial "v1 tag + success"
promotion
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Relax v1 promotion so a v1-tagged run can publish any subset of the
canonical matrix, not only the full 8-SKU matrix. A partial promotion
carries coverage_scope="partial" + covered_skus and is validated by
canonical membership (every case is a real canonical case with the
canonical disposition, proper subset) rather than the full catalog-sha /
matrix-id pin. Quality gates are unchanged: partial cohorts must still be
decision-grade and runnable cases must still succeed. Full promotions keep
the exact catalog pin, fixed cohort counts, and non-empty rankings.
放宽 v1 发布门槛：带 v1 标签且成功的运行可发布规范矩阵的任意子集，
而不再要求覆盖全部 8 种 SKU。部分发布携带 coverage_scope="partial"
与 covered_skus，通过规范成员校验（每个用例均为真实规范用例、判定一致、
且为真子集）而非完整目录哈希/矩阵 ID 绑定。质量门槛不变：部分队列仍须
达到 decision-grade，可运行用例仍须成功；完整发布保留精确目录绑定、
固定队列数量与非空排名要求。
---
experimental/CollectiveX/publisher.py | 84 +++++++++++--
.../schemas/public-dataset-v1.schema.json | 2 +
.../CollectiveX/tests/test_publisher.py | 115 ++++++++++++++----
3 files changed, 167 insertions(+), 34 deletions(-)
diff --git a/experimental/CollectiveX/publisher.py b/experimental/CollectiveX/publisher.py
index 9c1c6f4b25..2100ed45aa 100644
--- a/experimental/CollectiveX/publisher.py
+++ b/experimental/CollectiveX/publisher.py
@@ -1274,13 +1274,41 @@ def validate_public_dataset(doc: Any) -> dict[str, Any]:
} == {1}
for coverage in doc["coverage"]
)
- if promotion["matrix_id"] != CANONICAL_FULL_V1_MATRIX_SHA256:
- raise PublisherError("promotion requires the canonical full-v1 matrix")
- if (
- _case_disposition_catalog_sha256(doc["coverage"])
- != CANONICAL_FULL_V1_CASE_CATALOG_SHA256
- ):
- raise PublisherError("promotion requires the canonical case/disposition catalog")
+ expected_scope = promotion.get("coverage_scope", "full")
+ covered_skus = sorted({item["sku"] for item in doc["coverage"]})
+ if "covered_skus" in promotion and promotion["covered_skus"] != covered_skus:
+ raise PublisherError(
+ "promotion covered_skus is inconsistent with covered cases"
+ )
+ if expected_scope == "full":
+ # A full-coverage promotion is pinned to the exact canonical
+ # case/disposition catalog (unchanged from the strict contract).
+ if (
+ _case_disposition_catalog_sha256(doc["coverage"])
+ != CANONICAL_FULL_V1_CASE_CATALOG_SHA256
+ ):
+ raise PublisherError(
+ "promotion requires the canonical case/disposition catalog"
+ )
+ else:
+ # A partial ("v1 tag + success") promotion is not pinned to the full
+ # catalog, but every case must still be a real canonical case with
+ # the canonical disposition, and the subset must be a proper subset.
+ canonical = _canonical_coverage_cases()
+ covered = {item["case_id"] for item in doc["coverage"]}
+ if covered - set(canonical):
+ raise PublisherError(
+ "promotion includes cases outside the canonical v1 catalog"
+ )
+ for item in doc["coverage"]:
+ if item["disposition"] != canonical[item["case_id"]]["disposition"]:
+ raise PublisherError(
+ "promotion case disposition differs from the canonical v1 catalog"
+ )
+ if covered == set(canonical):
+ raise PublisherError(
+ "partial promotion must not cover the entire canonical catalog"
+ )
if (
terminal != len(doc["coverage"])
or promotion["qualification_indices"] != [1]
@@ -1318,8 +1346,8 @@ def validate_public_dataset(doc: Any) -> dict[str, Any]:
"promotion rejects runnable cases with failed, invalid, or diagnostic retries"
)
_require_promotion_series(doc["series"])
- _require_promotion_cohorts(doc["cohorts"], doc["series"])
- if not doc["rankings"]:
+ _require_promotion_cohorts(doc["cohorts"], doc["series"], scope=expected_scope)
+ if expected_scope == "full" and not doc["rankings"]:
raise PublisherError("promoted dataset lacks eligible rankings")
if promotion["status"] == "quarantined" and any((
doc["source_bundle_ids"], promotion["allocation_ids"], doc["coverage"],
@@ -3557,8 +3585,19 @@ def _expected_chip_cohort_count(series: Sequence[dict[str, Any]]) -> int:
def _require_promotion_cohorts(
- cohorts: Sequence[dict[str, Any]], series: Sequence[dict[str, Any]]
+ cohorts: Sequence[dict[str, Any]],
+ series: Sequence[dict[str, Any]],
+ *,
+ scope: str = "full",
) -> None:
+ if scope != "full":
+ # A partial ("v1 tag + success") run covers a subset of the catalog, so
+ # the fixed full-matrix cohort counts / required kinds do not apply. Any
+ # cohort a partial run does surface is still required to be
+ # decision-grade — quality is never relaxed, only coverage.
+ if any(not cohort["eligibility"]["decision_grade"] for cohort in cohorts):
+ raise PublisherError("promotion includes a non-decision-grade cohort")
+ return
eligible_kinds = {
cohort["kind"]
for cohort in cohorts
@@ -3633,10 +3672,27 @@ def build_dataset(
raise PublisherError(
"promotion requires qualification index 1 from a first-attempt run"
)
- if promote and matrix_ids != {CANONICAL_FULL_V1_MATRIX_SHA256}:
- raise PublisherError("promotion requires the canonical full-v1 matrix")
cases = {case["case_id"]: case for case in loaded[0]["cases"]}
+ canonical = _canonical_coverage_cases()
+ covered_skus = sorted({
+ canonical[case_id]["sku"] for case_id in cases if case_id in canonical
+ })
+ coverage_scope = "full" if set(cases) == set(canonical) else "partial"
if promote:
+ # A v1-tagged run may promote any subset of the canonical catalog (the
+ # "v1 tag + success" contract). Every promoted case must still be a real
+ # canonical case with the canonical disposition — a partial run cannot
+ # invent cases or reclassify them — but it need not cover every SKU.
+ unknown = set(cases) - set(canonical)
+ if unknown:
+ raise PublisherError(
+ "promotion includes cases outside the canonical v1 catalog"
+ )
+ for case_id, case in cases.items():
+ if case["_disposition"] != canonical[case_id]["disposition"]:
+ raise PublisherError(
+ "promotion case disposition differs from the canonical v1 catalog"
+ )
_require_runnable_promotion_success(loaded, cases)
all_documents = [
document for bundle in loaded for document in bundle["documents"].values()
@@ -3824,6 +3880,8 @@ def build_dataset(
"allocation_ids": allocation_ids,
"required_allocations": REQUIRED_ALLOCATIONS,
"qualification_indices": qualification_indices,
+ "coverage_scope": coverage_scope,
+ "covered_skus": covered_skus,
"requested_cases": len(coverage),
"terminal_cases": len(coverage),
"measured_cases": measured_cases,
@@ -3844,7 +3902,7 @@ def build_dataset(
}
if promote:
_require_promotion_series(series)
- _require_promotion_cohorts(cohorts, series)
+ _require_promotion_cohorts(cohorts, series, scope=coverage_scope)
validate_public_dataset(dataset)
return dataset
diff --git a/experimental/CollectiveX/schemas/public-dataset-v1.schema.json b/experimental/CollectiveX/schemas/public-dataset-v1.schema.json
index c6cfdd12c2..f9ef78e7b7 100644
--- a/experimental/CollectiveX/schemas/public-dataset-v1.schema.json
+++ b/experimental/CollectiveX/schemas/public-dataset-v1.schema.json
@@ -50,6 +50,8 @@
"allocation_ids": {"type": "array","uniqueItems": true,"items": {"$ref": "#/$defs/allocationId"}},
"required_allocations": {"const": 1},
"qualification_indices": {"type":"array","minItems":0,"maxItems":1,"uniqueItems":true,"items":{"enum":[1]}},
+ "coverage_scope": {"enum": ["full","partial"]},
+ "covered_skus": {"type": "array","uniqueItems": true,"items": {"$ref": "#/$defs/safeId"}},
"requested_cases": {"type": "integer","minimum": 0},
"terminal_cases": {"type": "integer","minimum": 0},
"measured_cases": {"type": "integer","minimum": 0},
diff --git a/experimental/CollectiveX/tests/test_publisher.py b/experimental/CollectiveX/tests/test_publisher.py
index 8136b4932b..862e7d1663 100644
--- a/experimental/CollectiveX/tests/test_publisher.py
+++ b/experimental/CollectiveX/tests/test_publisher.py
@@ -1012,6 +1012,33 @@ def _cohort_counts(dataset: dict) -> dict[str, int]:
}
+def _canonical_entry_from_coverage(item: dict) -> dict:
+ """Build a canonical-catalog entry that a coverage row projects onto.
+
+ Used to monkeypatch ``_canonical_coverage_cases`` so a partial promotion
+ fixture whose case ids are synthetic can still exercise the partial-scope
+ membership + disposition contract.
+ """
+ return {
+ "sku": item["sku"],
+ "suite": item["suite"],
+ "workload": item["workload"],
+ "required_publication": item["publication_tier"],
+ "backend": item["backend"],
+ "mode": item["mode"],
+ "phase": item["phase"],
+ "routing": item["routing"],
+ "eplb": item["eplb"],
+ "precision_profile": item["precision_profile"],
+ "dispatch_precision": item["dispatch_precision"],
+ "combine_precision": item["combine_precision"],
+ "disposition": item["disposition"],
+ "reason": item["reason"],
+ "ladder": " ".join(str(point["tokens_per_rank"]) for point in item["points"]),
+ **item["topology"],
+ }
+
+
class PublisherTest(unittest.TestCase):
def test_trial_diagnostics_flag_drift_and_robust_outliers(self) -> None:
def runs() -> dict[str, dict[int, dict[str, object]]]:
@@ -1865,11 +1892,6 @@ def test_standalone_promotion_binds_matrix_and_requested_dispositions(self) -> N
):
publisher.validate_public_dataset(diagnostic)
- broken = copy.deepcopy(dataset)
- broken["promotion"]["matrix_id"] = "d" * 64
- with self.assertRaisesRegex(publisher.PublisherError, "canonical full-v1 matrix"):
- publisher.validate_public_dataset(broken)
-
for original, replacement in (("runnable", "unsupported"),
("unsupported", "runnable")):
with self.subTest(original=original):
@@ -1977,24 +1999,75 @@ def test_workflow_matrix_and_catalog_digests_do_not_drift(self) -> None:
publisher.REQUIRED_PROMOTION_COHORT_COUNTS,
)
- def test_build_promotion_requires_canonical_full_matrix(self) -> None:
- bundles = {
- str(run_id): {
- "id": str(run_id), "cases": [],
- "manifest": {
- "matrix": {"sha256": "d" * 64},
- "run": {
- "run_id": str(run_id), "run_attempt": 1,
- "qualification_index": run_id,
- },
- },
- }
- for run_id in range(1, 2)
+ def test_partial_promotion_binds_canonical_membership(self) -> None:
+ # A "v1 tag + success" promotion may cover a subset of the canonical
+ # matrix, but every covered case must be a real canonical case with the
+ # canonical disposition, covered_skus must agree with the coverage, and
+ # the subset must be a proper subset of the full catalog.
+ dataset = _promoted_dataset()
+ fixture_catalog = publisher._case_disposition_catalog_sha256(dataset["coverage"])
+ cohort_counts = _cohort_counts(dataset)
+
+ # covered_skus is checked for both scopes, before the scope branch.
+ inconsistent = copy.deepcopy(dataset)
+ inconsistent["promotion"]["covered_skus"] = ["nvidia-nonexistent"]
+ with mock.patch.object(
+ publisher, "CANONICAL_FULL_V1_CASE_CATALOG_SHA256", fixture_catalog
+ ), mock.patch.object(
+ publisher, "REQUIRED_PROMOTION_COHORT_COUNTS", cohort_counts
+ ), self.assertRaisesRegex(
+ publisher.PublisherError, "covered_skus is inconsistent"
+ ):
+ publisher.validate_public_dataset(inconsistent)
+
+ # A partial promotion whose cases are not in the canonical catalog is
+ # rejected (the synthetic fixture case ids are not canonical).
+ partial = copy.deepcopy(dataset)
+ partial["promotion"]["coverage_scope"] = "partial"
+ partial["promotion"]["covered_skus"] = sorted(
+ {item["sku"] for item in partial["coverage"]}
+ )
+ with self.assertRaisesRegex(
+ publisher.PublisherError, "cases outside the canonical v1 catalog"
+ ):
+ publisher.validate_public_dataset(partial)
+
+ # With the canonical catalog monkeypatched to a proper superset of the
+ # covered cases (matching identities + dispositions), a partial
+ # promotion validates without the full-matrix catalog pin or the
+ # cohort-count requirement that gate a full promotion — and without
+ # monkeypatching CANONICAL_FULL_V1_CASE_CATALOG_SHA256 or
+ # REQUIRED_PROMOTION_COHORT_COUNTS at all.
+ canonical = {
+ item["case_id"]: _canonical_entry_from_coverage(item)
+ for item in dataset["coverage"]
}
+ superset = dict(
+ canonical,
+ **{"canonical-case-not-in-this-run": dict(
+ next(iter(canonical.values())), disposition="unsupported"
+ )},
+ )
+ happy = copy.deepcopy(dataset)
+ happy["promotion"]["coverage_scope"] = "partial"
+ happy["promotion"]["covered_skus"] = sorted(
+ {item["sku"] for item in happy["coverage"]}
+ )
+ happy["promotion"]["matrix_id"] = "e" * 64 # partial: matrix not pinned
with mock.patch.object(
- publisher, "load_bundle", side_effect=lambda _, bundle_id: bundles[bundle_id]
- ), self.assertRaisesRegex(publisher.PublisherError, "canonical full-v1 matrix"):
- publisher.build_dataset(object(), list(bundles), promote=True)
+ publisher, "_canonical_coverage_cases", lambda: superset
+ ):
+ self.assertIs(publisher.validate_public_dataset(happy), happy)
+
+ # A "partial" promotion that in fact covers the entire canonical catalog
+ # must go through the full-coverage contract instead.
+ entire = copy.deepcopy(happy)
+ with mock.patch.object(
+ publisher, "_canonical_coverage_cases", lambda: canonical
+ ), self.assertRaisesRegex(
+ publisher.PublisherError, "must not cover the entire canonical catalog"
+ ):
+ publisher.validate_public_dataset(entire)
def test_rejection_is_quarantined_without_updating_dev_latest(self) -> None:
with tempfile.TemporaryDirectory() as temporary:
From 9e4161cd941d180a3841f15252ccf86c9e04571f Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 7 Jul 2026 22:08:49 +0800
Subject: [PATCH 184/210] feat(collectivex): numeric incrementable release
version
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Model the release version as a first-class numeric field N (1, 2, 3, ...),
not a hardcoded "v1" tag. The release_tag workflow input becomes a string
validated as 'unversioned' or a positive integer; the release marker carries
"version": N plus a coverage_scope ("full"/"partial"); publish and refresh
discover the version version-agnostically from the marker/artifact name via
regex and re-emit it in cxrelease--* / cxpublication--* names. A new
autopublish job self-publishes a successful first-attempt versioned sweep so
the frontend can discover many runs of a version JIT. The frozen data-format
literals (collectivex.release-tag.v1, collectivex.public.v1, the
collectivex_public_v1_.ndjson filename) are schema-version, shared
across releases, and stay unchanged.
将发布版本建模为一等的数值字段 N（1、2、3……），而非硬编码的 “v1” 标签。
release_tag 工作流输入改为字符串，校验为 'unversioned' 或正整数；发布标记
携带 "version": N 及 coverage_scope（"full"/"partial"）；publish 与 refresh
通过正则从标记/产物名中与版本无关地发现版本号，并在 cxrelease--* /
cxpublication--* 名称中重新输出。新增 autopublish 作业让一次成功的
首次尝试带版本 sweep 自行发布，便于前端即时发现某版本的多个运行。冻结的
数据格式字面量（collectivex.release-tag.v1、collectivex.public.v1、
collectivex_public_v1_.ndjson 文件名）属于 schema 版本，跨发布共享，
保持不变。
---
.github/workflows/collectivex-sweep.yml | 242 ++++++++++++++----
.../tests/test_sampling_contract.py | 62 ++++-
2 files changed, 251 insertions(+), 53 deletions(-)
diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml
index 6ca917919f..ddec0b23f7 100644
--- a/.github/workflows/collectivex-sweep.yml
+++ b/.github/workflows/collectivex-sweep.yml
@@ -1,6 +1,6 @@
# CollectiveX Sweep — one structured run instead of thousands of dispatches.
#
-# Shape: sweep/probe runs setup -> GPU cells; publish-v1 and refresh-v1 stay on disposable
+# Shape: sweep/probe runs setup -> GPU cells; publish and refresh stay on disposable
# GitHub-hosted storage and skip GPU jobs. This file is registered on the default
# branch, so its collectivex branch revision can be dispatched with --ref.
name: CollectiveX Sweep
@@ -14,7 +14,7 @@ on:
description: Operation to execute
type: choice
default: sweep
- options: [sweep, probe-precision, publish-v1, refresh-v1]
+ options: [sweep, probe-precision, publish, refresh]
backend:
description: "EP library to sweep — 'all' runs every EP backend in one matrix"
type: choice
@@ -45,25 +45,24 @@ on:
type: string
default: '128'
release_tag:
- description: Publication gate; unversioned runs are diagnostic and cannot be published
- type: choice
+ description: "Release version: 'unversioned' (diagnostic, cannot publish) or a positive integer N to publish as version N (incrementable: 1, 2, 3, ...)"
+ type: string
default: unversioned
- options: [unversioned, v1]
qualification_index:
description: Deterministic execution-order index; V1 qualification uses 1, then 2, then 3
type: choice
default: '1'
options: ['1', '2', '3']
publish_run_id:
- description: For publish-v1, one successful first-attempt V1 sweep run ID (qualification index 1)
+ description: For publish, one successful first-attempt versioned sweep run ID (qualification index 1)
type: string
default: ''
refresh_run_id:
- description: For refresh-v1, source publication workflow run ID
+ description: For refresh, source publication workflow run ID
type: string
default: ''
refresh_digest:
- description: For refresh-v1, exact publication dataset SHA-256
+ description: For refresh, exact publication dataset SHA-256
type: string
default: ''
concurrency:
@@ -103,25 +102,30 @@ jobs:
run: |
set -euo pipefail
if [ "$INPUT_OPERATION" = probe-precision ] && [ "${{ inputs.release_tag }}" != unversioned ]; then
- echo 'precision probes cannot carry a V1 release tag' >&2
+ echo 'precision probes cannot carry a release version' >&2
exit 1
fi
- if [ "$INPUT_OPERATION" = sweep ] && [ "${{ inputs.release_tag }}" = v1 ]; then
+ if [ "$INPUT_OPERATION" = sweep ] && [ "${{ inputs.release_tag }}" != unversioned ]; then
+ # The release version is a positive integer N (v1, v2, v3, ...); it is
+ # incrementable, never a hardcoded tag. 'unversioned' stays diagnostic.
+ [[ "${{ inputs.release_tag }}" =~ ^[1-9][0-9]*$ ]] || {
+ echo 'release_tag must be "unversioned" or a positive integer version N' >&2
+ exit 1
+ }
[[ "$INPUT_QUALIFICATION_INDEX" =~ ^[123]$ ]] || {
- echo 'V1 sweeps require qualification_index 1, 2, or 3' >&2
+ echo 'versioned sweeps require qualification_index 1, 2, or 3' >&2
exit 1
}
- [ "$INPUT_BACKEND" = all ] && [ "$INPUT_SUITES" = all ] \
- && [ -z "$INPUT_ONLY_SKU" ] && [ -z "$INPUT_MIN_NODES" ] \
- && [ -z "$INPUT_MAX_NODES" ] && [ "$INPUT_MAX_CASES" = 128 ] || {
- echo 'V1 sweeps require the exact unfiltered full matrix' >&2
- exit 1
- }
+ # A versioned sweep may be filtered to a subset of the canonical matrix
+ # ("tagged + success" contract) — a full run and a partial run (e.g.
+ # only_sku, or a single backend) are both publishable. The publisher
+ # binds every promoted case to the canonical catalog, so a filtered
+ # matrix cannot invent or reclassify cases; it only narrows coverage.
python3 - <<'PY'
import capability
if capability.provisional_precision_targets():
- raise SystemExit("V1 sweeps require every precision capability cell to be resolved")
+ raise SystemExit("versioned sweeps require every precision capability cell to be resolved")
PY
fi
if [ "$INPUT_OPERATION" = probe-precision ]; then
@@ -206,22 +210,23 @@ jobs:
path: ${{ runner.temp }}/collectivex-backend-sources.tar
if-no-files-found: error
overwrite: true
- retention-days: ${{ inputs.release_tag == 'v1' && 90 || 3 }}
+ retention-days: ${{ inputs.release_tag != 'unversioned' && 90 || 3 }}
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: cxsweep-matrix-${{ github.run_id }}
path: experimental/CollectiveX/matrix_full.json
if-no-files-found: error
overwrite: true
- retention-days: ${{ inputs.release_tag == 'v1' && 90 || 3 }}
- - name: Create V1 release marker
- if: ${{ inputs.operation == 'sweep' && inputs.release_tag == 'v1' }}
+ retention-days: ${{ inputs.release_tag != 'unversioned' && 90 || 3 }}
+ - name: Create release marker
+ if: ${{ inputs.operation == 'sweep' && inputs.release_tag != 'unversioned' }}
env:
EXPECTED_MATRIX_SHA256: 5894bab58d3deb2bcee51baa075ca5f5d324b4292ac1cef9f6bc08a07ab1d9a3
RUN_ID: ${{ github.run_id }}
RUN_ATTEMPT: ${{ github.run_attempt }}
SOURCE_SHA: ${{ github.sha }}
QUALIFICATION_INDEX: ${{ inputs.qualification_index }}
+ RELEASE_VERSION: ${{ inputs.release_tag }}
run: |
set -euo pipefail
destination="$RUNNER_TEMP/collectivex-release"
@@ -238,36 +243,46 @@ jobs:
matrix = pathlib.Path("experimental/CollectiveX/matrix_full.json").read_bytes()
matrix_sha256 = hashlib.sha256(matrix).hexdigest()
- if matrix_sha256 != os.environ["EXPECTED_MATRIX_SHA256"]:
- raise SystemExit("V1 release tag requires the locked full matrix")
+ # A versioned run may cover the full canonical matrix or a filtered subset.
+ # A full run stays pinned to the locked matrix + execution plan (strict
+ # integrity); a partial run records its own matrix + plan digests so a
+ # "tagged + success" subset is still a verifiable, publishable marker.
+ coverage_scope = (
+ "full" if matrix_sha256 == os.environ["EXPECTED_MATRIX_SHA256"] else "partial"
+ )
+ version = int(os.environ["RELEASE_VERSION"])
+ if version < 1:
+ raise SystemExit("release version must be a positive integer")
qualification_index = int(os.environ["QUALIFICATION_INDEX"])
execution_plan_sha256 = sweep_matrix.qualification_execution_plan_sha256(
json.loads(matrix), qualification_index
)
- if execution_plan_sha256 != sweep_matrix.CANONICAL_V1_EXECUTION_PLAN_SHA256.get(
- qualification_index
+ if coverage_scope == "full" and (
+ execution_plan_sha256
+ != sweep_matrix.CANONICAL_V1_EXECUTION_PLAN_SHA256.get(qualification_index)
):
- raise SystemExit("V1 release tag requires the locked execution plan")
+ raise SystemExit("a full release version requires the locked execution plan")
marker = {
+ "coverage_scope": coverage_scope,
"execution_plan_sha256": execution_plan_sha256,
"format": "collectivex.release-tag.v1",
"matrix_sha256": matrix_sha256,
"qualification_index": qualification_index,
- "release_tag": "v1",
"run_attempt": os.environ["RUN_ATTEMPT"],
"run_id": os.environ["RUN_ID"],
"source_sha": os.environ["SOURCE_SHA"],
+ "version": version,
}
pathlib.Path(sys.argv[1]).write_text(
json.dumps(marker, sort_keys=True, separators=(",", ":")) + "\n"
)
PY
python3 experimental/CollectiveX/artifact_safety.py "$destination/release.json"
- - name: Upload V1 release marker
- if: ${{ inputs.operation == 'sweep' && inputs.release_tag == 'v1' }}
+ - name: Upload release marker
+ if: ${{ inputs.operation == 'sweep' && inputs.release_tag != 'unversioned' }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
- name: cxrelease-v1-${{ github.run_id }}-${{ github.run_attempt }}
+ name: cxrelease-${{ inputs.release_tag }}-${{ github.run_id }}-${{ github.run_attempt }}
path: ${{ runner.temp }}/collectivex-release/release.json
if-no-files-found: error
retention-days: 90
@@ -293,7 +308,7 @@ jobs:
name: cxunsupported-${{ github.run_id }}-${{ github.run_attempt }}
path: experimental/CollectiveX/unsupported/*.json
if-no-files-found: error
- retention-days: ${{ inputs.release_tag == 'v1' && 90 || 3 }}
+ retention-days: ${{ inputs.release_tag != 'unversioned' && 90 || 3 }}
# ---- sweep: ONE matrix cell per shard (the parent job with child jobs) ----
sweep:
@@ -660,7 +675,7 @@ jobs:
path: |
${{ env.CX_JOB_ROOT }}/artifact/*.json
if-no-files-found: error
- retention-days: ${{ inputs.release_tag == 'v1' && 90 || 3 }}
+ retention-days: ${{ inputs.release_tag != 'unversioned' && 90 || 3 }}
- name: Cleanup isolated workspace
if: ${{ always() && steps.source.outputs.prepared == 'true' }}
run: |
@@ -734,8 +749,107 @@ jobs:
if-no-files-found: error
retention-days: 30
+ # ---- autopublish: a successful first-attempt versioned sweep self-publishes its own run ----
+ # A "tagged + success" sweep (full OR partial) emits its own promoted
+ # publication artifact from the same run, so the frontend can discover many
+ # independent runs of a version Just-In-Time without a separate manual publish step.
+ autopublish:
+ needs: [setup, sweep]
+ if: ${{ inputs.operation == 'sweep' && inputs.release_tag != 'unversioned' && inputs.qualification_index == '1' && github.run_attempt == '1' && needs.sweep.result == 'success' }}
+ runs-on: ubuntu-latest
+ timeout-minutes: 30
+ env:
+ REPOSITORY: ${{ github.repository }}
+ RUN_ID: ${{ github.run_id }}
+ RUN_ATTEMPT: ${{ github.run_attempt }}
+ SOURCE_SHA: ${{ github.sha }}
+ QUALIFICATION_INDEX: ${{ inputs.qualification_index }}
+ RELEASE_VERSION: ${{ inputs.release_tag }}
+ steps:
+ - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v5.0.0
+ with:
+ ref: ${{ github.sha }}
+ clean: true
+ persist-credentials: false
+ - name: Install publisher dependencies
+ run: python3 -m pip install --quiet -r experimental/CollectiveX/requirements.txt
+ - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+ with:
+ path: ${{ runner.temp }}/collectivex-downloads
+ - name: Build promoted publication
+ run: |
+ set -euo pipefail
+ store="$RUNNER_TEMP/collectivex-publisher"
+ run_dir="$RUNNER_TEMP/collectivex-downloads"
+ output="$RUNNER_TEMP/collectivex-publication"
+ umask 027
+ mkdir -m 750 "$store" "$output"
+ matrix="$run_dir/cxsweep-matrix-$RUN_ID/matrix_full.json"
+ [ -f "$matrix" ] || {
+ echo 'run is missing its exact matrix artifact' >&2
+ exit 1
+ }
+ mapfile -t artifacts < <(
+ find "$run_dir" -mindepth 1 -maxdepth 1 -type d \
+ \( -name "cxshard-*-$RUN_ID-$RUN_ATTEMPT" \
+ -o -name "cxunsupported-$RUN_ID-$RUN_ATTEMPT" \) -print | sort
+ )
+ [ "${#artifacts[@]}" -gt 0 ] || {
+ echo 'run has no result artifacts' >&2
+ exit 1
+ }
+ artifact_args=()
+ for artifact in "${artifacts[@]}"; do
+ artifact_args+=(--artifact "$artifact")
+ done
+ result=$(
+ python3 experimental/CollectiveX/publisher.py --store-root "$store" ingest \
+ --matrix "$matrix" \
+ "${artifact_args[@]}" \
+ --repository "$REPOSITORY" \
+ --run-id "$RUN_ID" \
+ --run-attempt "$RUN_ATTEMPT" \
+ --qualification-index "$QUALIFICATION_INDEX" \
+ --source-sha "$SOURCE_SHA"
+ )
+ bundle_id=$(jq -er '.bundle_id' <<< "$result")
+ result=$(
+ python3 experimental/CollectiveX/publisher.py --store-root "$store" promote \
+ --bundle "$bundle_id"
+ )
+ dataset_id=$(jq -er '.dataset_sha256' <<< "$result")
+ dataset="$store/public/datasets/$dataset_id/dataset.json"
+ [ -f "$dataset" ] || {
+ echo 'publisher did not install the promoted dataset' >&2
+ exit 1
+ }
+ publication="$output/collectivex_public_v1_$dataset_id.ndjson"
+ cp -- "$dataset" "$publication"
+ python3 experimental/CollectiveX/artifact_safety.py "$publication"
+ python3 experimental/CollectiveX/publisher.py --store-root "$store" verify \
+ --channel dev-latest --bundle "$bundle_id"
+ [ "$(stat -c %s "$publication")" -le 33554432 ] || {
+ echo 'publication exceeds the 32 MiB frontend limit' >&2
+ exit 1
+ }
+ sha256sum "$publication"
+ {
+ echo "## CollectiveX publication (auto) — version $RELEASE_VERSION"
+ echo
+ echo "Dataset: \`$dataset_id\`"
+ echo
+ echo "Source run: \`$RUN_ID\` attempt \`$RUN_ATTEMPT\`"
+ } >> "$GITHUB_STEP_SUMMARY"
+ - name: Upload JIT publication artifact
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: cxpublication-${{ inputs.release_tag }}-${{ github.run_id }}-${{ github.run_attempt }}
+ path: ${{ runner.temp }}/collectivex-publication/*.ndjson
+ if-no-files-found: error
+ retention-days: 90
+
publish:
- if: ${{ inputs.operation == 'publish-v1' }}
+ if: ${{ inputs.operation == 'publish' }}
runs-on: ubuntu-latest
timeout-minutes: 30
env:
@@ -788,6 +902,7 @@ jobs:
run: python3 -m pip install --quiet -r experimental/CollectiveX/requirements.txt
- name: Build promoted publication
+ id: build
env:
REPOSITORY: ${{ github.repository }}
run: |
@@ -799,19 +914,31 @@ jobs:
mkdir -m 750 "$store" "$downloads" "$output"
: > "$RUNNER_TEMP/collectivex-bundles.txt"
: > "$RUNNER_TEMP/collectivex-qualification-indices.txt"
+ : > "$RUNNER_TEMP/collectivex-versions.txt"
while IFS=$'\t' read -r run_id attempt source_sha; do
run_dir="$downloads/$run_id"
mkdir -m 750 "$run_dir"
gh run download "$run_id" --repo "$REPOSITORY" --dir "$run_dir"
matrix="$run_dir/cxsweep-matrix-$run_id/matrix_full.json"
- marker="$run_dir/cxrelease-v1-$run_id-$attempt/release.json"
+ # The release marker's version is embedded in its artifact name
+ # (cxrelease---); discover it version-agnostically.
+ marker=$(
+ find "$run_dir" -mindepth 2 -maxdepth 2 -type f \
+ -regextype posix-extended \
+ -regex ".*/cxrelease-[1-9][0-9]*-$run_id-$attempt/release\.json" -print -quit
+ )
[ -f "$matrix" ] || {
echo "run $run_id is missing its exact matrix artifact" >&2
exit 1
}
- [ -f "$marker" ] || {
- echo "run $run_id is not tagged for V1 publication" >&2
+ [ -n "$marker" ] && [ -f "$marker" ] || {
+ echo "run $run_id is not tagged for a release version" >&2
+ exit 1
+ }
+ version=$(jq -r '.version' "$marker")
+ [[ "$version" =~ ^[1-9][0-9]*$ ]] || {
+ echo "run $run_id has a non-integer release version" >&2
exit 1
}
matrix_sha=$(sha256sum "$matrix" | cut -d' ' -f1)
@@ -827,9 +954,10 @@ jobs:
--arg source_sha "$source_sha" \
--arg matrix_sha "$matrix_sha" \
--arg execution_plan_sha "$execution_plan_sha" \
- 'keys == ["execution_plan_sha256","format","matrix_sha256","qualification_index","release_tag","run_attempt","run_id","source_sha"]
+ 'keys == ["coverage_scope","execution_plan_sha256","format","matrix_sha256","qualification_index","run_attempt","run_id","source_sha","version"]
and .format == "collectivex.release-tag.v1"
- and .release_tag == "v1"
+ and (.version | type == "number" and . >= 1)
+ and (.coverage_scope == "full" or .coverage_scope == "partial")
and .run_id == $run_id
and .run_attempt == $attempt
and .source_sha == $source_sha
@@ -837,9 +965,10 @@ jobs:
and .matrix_sha256 == $matrix_sha
and .execution_plan_sha256 == $execution_plan_sha' \
"$marker" >/dev/null || {
- echo "run $run_id has an invalid V1 release marker" >&2
+ echo "run $run_id has an invalid release marker" >&2
exit 1
}
+ printf '%s\n' "$version" >> "$RUNNER_TEMP/collectivex-versions.txt"
jq -r '.qualification_index' "$marker" \
>> "$RUNNER_TEMP/collectivex-qualification-indices.txt"
@@ -875,6 +1004,13 @@ jobs:
exit 1
}
+ release_version=$(sort -un "$RUNNER_TEMP/collectivex-versions.txt")
+ [ "$(printf '%s\n' "$release_version" | wc -l)" -eq 1 ] || {
+ echo 'publish source runs disagree on the release version' >&2
+ exit 1
+ }
+ echo "version=$release_version" >> "$GITHUB_OUTPUT"
+
mapfile -t bundle_ids < "$RUNNER_TEMP/collectivex-bundles.txt"
promote_args=()
for bundle_id in "${bundle_ids[@]}"; do
@@ -901,7 +1037,7 @@ jobs:
}
sha256sum "$publication"
{
- echo '## CollectiveX V1 publication'
+ echo "## CollectiveX publication — version $release_version"
echo
echo "Dataset: \`$dataset_id\`"
echo
@@ -912,13 +1048,13 @@ jobs:
- name: Upload JIT publication artifact
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
- name: cxpublication-v1-${{ github.run_id }}-${{ github.run_attempt }}
+ name: cxpublication-${{ steps.build.outputs.version }}-${{ github.run_id }}-${{ github.run_attempt }}
path: ${{ runner.temp }}/collectivex-publication/*.ndjson
if-no-files-found: error
retention-days: 90
refresh:
- if: ${{ inputs.operation == 'refresh-v1' }}
+ if: ${{ inputs.operation == 'refresh' }}
runs-on: ubuntu-latest
timeout-minutes: 10
env:
@@ -930,6 +1066,7 @@ jobs:
with: { clean: true, persist-credentials: false }
- name: Revalidate promoted publication
+ id: refresh
env:
REPOSITORY: ${{ github.repository }}
run: |
@@ -957,11 +1094,26 @@ jobs:
exit 1
}
+ # The publication artifact carries its version in its name
+ # (cxpublication---); discover it from the run.
+ pub_name=$(
+ gh api --paginate "repos/$REPOSITORY/actions/runs/$SOURCE_RUN_ID/artifacts" \
+ --jq '.artifacts[].name' \
+ | grep -E "^cxpublication-[1-9][0-9]*-$SOURCE_RUN_ID-$attempt$" \
+ | sort -u
+ )
+ [ "$(printf '%s\n' "$pub_name" | grep -c .)" -eq 1 ] || {
+ echo 'refresh source has no single versioned publication artifact' >&2
+ exit 1
+ }
+ release_version=$(printf '%s\n' "$pub_name" | sed -E 's/^cxpublication-([1-9][0-9]*)-.*/\1/')
+ echo "version=$release_version" >> "$GITHUB_OUTPUT"
+
source="$RUNNER_TEMP/collectivex-refresh-source"
output="$RUNNER_TEMP/collectivex-publication"
install -d -m 750 "$source" "$output"
gh run download "$SOURCE_RUN_ID" --repo "$REPOSITORY" \
- --name "cxpublication-v1-$SOURCE_RUN_ID-$attempt" --dir "$source"
+ --name "$pub_name" --dir "$source"
mapfile -t files < <(find "$source" -mindepth 1 -maxdepth 1 -type f -print)
[ "${#files[@]}" -eq 1 ] || {
echo 'refresh source must contain exactly one root file' >&2
@@ -1001,7 +1153,7 @@ jobs:
- name: Upload refreshed JIT publication artifact
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
- name: cxpublication-v1-${{ github.run_id }}-${{ github.run_attempt }}
+ name: cxpublication-${{ steps.refresh.outputs.version }}-${{ github.run_id }}-${{ github.run_attempt }}
path: ${{ runner.temp }}/collectivex-publication/*.ndjson
if-no-files-found: error
retention-days: 90
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index f5240e5d60..74a02c2f31 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -1745,7 +1745,7 @@ def test_v1_publication_requires_explicit_release_markers(self) -> None:
sweep = (workflows / "collectivex-sweep.yml").read_text()
self.assertFalse((workflows / "collectivex-publish.yml").exists())
- self.assertIn("options: [sweep, probe-precision, publish-v1, refresh-v1]", sweep)
+ self.assertIn("options: [sweep, probe-precision, publish, refresh]", sweep)
self.assertIn("${{ inputs.only_sku }}-${{ inputs.probe_id }}", sweep)
self.assertIn("cancel-in-progress: false", sweep)
self.assertIn("COLLECTIVEX_MI355_CONFIG_V1", sweep)
@@ -1755,28 +1755,74 @@ def test_v1_publication_requires_explicit_release_markers(self) -> None:
self.assertIn("collectivex.precision-probe-plan.v1", (ROOT / "tests" / "probe_precision.py").read_text())
self.assertIn("cxprecision-probes-${{ github.run_id }}-${{ github.run_attempt }}", sweep)
self.assertIn("--validate-bundle", sweep)
+
+ # The release version is a numeric, incrementable field (1, 2, 3, ...),
+ # never a hardcoded "v1" tag. 'unversioned' stays diagnostic.
self.assertIn("release_tag:", sweep)
self.assertIn("default: unversioned", sweep)
- self.assertIn("options: [unversioned, v1]", sweep)
+ self.assertNotIn("options: [unversioned, v1]", sweep)
+ self.assertIn("or a positive integer N to publish as version N", sweep)
self.assertIn("qualification_index:", sweep)
- self.assertIn("inputs.release_tag == 'v1'", sweep)
+ self.assertIn("inputs.release_tag != 'unversioned'", sweep)
+ self.assertNotIn("inputs.release_tag == 'v1'", sweep)
+ self.assertIn(
+ 'release_tag must be "unversioned" or a positive integer version N', sweep
+ )
+ # collectivex.release-tag.v1 is a frozen data-format id, NOT the release version.
self.assertIn("collectivex.release-tag.v1", sweep)
- self.assertIn("V1 release tag requires the locked full matrix", sweep)
+ # A versioned run may cover the full canonical matrix or a filtered subset;
+ # the release marker records the coverage scope AND the numeric version.
+ self.assertNotIn("V1 sweeps require the exact unfiltered full matrix", sweep)
+ self.assertNotIn("V1 release tag requires the locked full matrix", sweep)
+ self.assertIn("a full release version requires the locked execution plan", sweep)
+ self.assertIn('"coverage_scope": coverage_scope', sweep)
+ self.assertIn('"version": version', sweep)
+ self.assertIn(
+ '"full" if matrix_sha256 == os.environ["EXPECTED_MATRIX_SHA256"] else "partial"',
+ sweep,
+ )
self.assertIn("EXPECTED_MATRIX_SHA256", sweep)
- self.assertIn("cxrelease-v1-${{ github.run_id }}-${{ github.run_attempt }}", sweep)
+ self.assertIn(
+ "cxrelease-${{ inputs.release_tag }}-${{ github.run_id }}-${{ github.run_attempt }}",
+ sweep,
+ )
self.assertIn("publish_run_id must be a single decimal run ID", sweep)
self.assertIn("source run must be qualification index 1 exactly once", sweep)
self.assertIn("and .qualification_index == 1", sweep)
- self.assertIn("cxrelease-v1-$run_id-$attempt/release.json", sweep)
- self.assertIn("run $run_id is not tagged for V1 publication", sweep)
+ self.assertIn(
+ '"coverage_scope","execution_plan_sha256","format","matrix_sha256"', sweep
+ )
+ # The marker key set now carries a numeric version, not a release_tag string.
+ self.assertIn('"run_id","source_sha","version"]', sweep)
+ self.assertIn('(.coverage_scope == "full" or .coverage_scope == "partial")', sweep)
+ self.assertIn("cxrelease-[1-9][0-9]*-$run_id-$attempt/release", sweep)
+ self.assertIn("run $run_id is not tagged for a release version", sweep)
self.assertIn("ref: ${{ steps.runs.outputs.source_sha }}", sweep)
self.assertIn("[ \"$attempt\" = 1 ]", sweep)
- self.assertIn("cxpublication-v1-${{ github.run_id }}-${{ github.run_attempt }}", sweep)
+ self.assertIn(
+ "cxpublication-${{ steps.build.outputs.version }}-${{ github.run_id }}-${{ github.run_attempt }}",
+ sweep,
+ )
self.assertIn("refresh source bytes differ from their requested digest", sweep)
self.assertIn("retention-days: 90", sweep)
self.assertNotIn("workflow_run:", sweep)
+ # A successful first-attempt versioned sweep (full OR partial) self-publishes
+ # its own run so the frontend can discover many runs of a version JIT, without
+ # a separate manual publish dispatch. The artifact name embeds the numeric
+ # version so the frontend can group runs by version.
+ self.assertIn("autopublish:", sweep)
+ self.assertIn("inputs.qualification_index == '1'", sweep)
+ self.assertIn(
+ "github.run_attempt == '1' && needs.sweep.result == 'success'", sweep
+ )
+ self.assertIn("CollectiveX publication (auto) — version", sweep)
+ self.assertIn(
+ "cxpublication-${{ inputs.release_tag }}-${{ github.run_id }}-${{ github.run_attempt }}",
+ sweep,
+ )
+
def test_source_archive_preserves_only_contained_leaf_symlinks(self) -> None:
selected = "deepep-hybrid-pinned"
other = "deepep-v2-pinned"
From 1776e94b3f31cb9e423c25e822155cde1e8d32a3 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 7 Jul 2026 22:30:59 +0800
Subject: [PATCH 185/210] feat(collectivex): add --exclude-skus for partial
versioned sweeps
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Add an --exclude-skus comma-list to sweep_matrix.resolve_matrix (and the
sweep_matrix.py CLI + collectivex-sweep.yml exclude_skus dispatch input) so a
versioned "tagged + success" run can drop whole runner pools whose clusters are
unavailable and still publish the SKUs it can exercise. Excluding a pool only
omits its cells; every surviving requested case stays byte-identical to the full
matrix, so the publisher's canonical-catalog binding still forbids inventing or
reclassifying cases. Disjoint from --only-sku; unknown pools are rejected.
The release marker already records coverage_scope=partial for any non-canonical
matrix digest and autopublish already accepts full OR partial, so this completes
the incomplete-matrix gate end to end.
为部分版本化 sweep 增加 --exclude-skus:允许在集群不可用时剔除整个 runner
池,仍以"打标签 + 成功"发布可运行的 SKU。剔除只省略对应单元格,存活的请求用例
与完整矩阵逐字节一致,发布器的规范目录绑定仍禁止伪造或重新分类用例。与
--only-sku 互斥,未知池会被拒绝。
---
.github/workflows/collectivex-sweep.yml | 6 ++++
experimental/CollectiveX/sweep_matrix.py | 22 +++++++++++++
.../tests/test_qualification_planning.py | 33 +++++++++++++++++++
3 files changed, 61 insertions(+)
diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml
index ddec0b23f7..f2ec915f53 100644
--- a/.github/workflows/collectivex-sweep.yml
+++ b/.github/workflows/collectivex-sweep.yml
@@ -28,6 +28,10 @@ on:
description: Restrict to one V1 GHA runner pool (h100-dgxc|h200-dgxc|b300|b200-dgxc|gb200|gb300|mi300x|mi355x); blank = all
type: string
default: ''
+ exclude_skus:
+ description: Comma-list of runner pools to drop from the matrix (partial run, e.g. b300); disjoint from only_sku; blank = none
+ type: string
+ default: ''
probe_id:
description: Optional exact provisional probe ID; valid only for probe-precision
type: string
@@ -90,6 +94,7 @@ jobs:
INPUT_OPERATION: ${{ inputs.operation }}
INPUT_SUITES: ${{ inputs.suites }}
INPUT_ONLY_SKU: ${{ inputs.only_sku }}
+ INPUT_EXCLUDE_SKUS: ${{ inputs.exclude_skus }}
INPUT_PROBE_ID: ${{ inputs.probe_id }}
INPUT_MIN_NODES: ${{ inputs.min_nodes }}
INPUT_MAX_NODES: ${{ inputs.max_nodes }}
@@ -143,6 +148,7 @@ jobs:
*) args+=(--backend "$INPUT_BACKEND") ;;
esac
[ -n "$INPUT_ONLY_SKU" ] && args+=(--only-sku "$INPUT_ONLY_SKU")
+ [ -n "$INPUT_EXCLUDE_SKUS" ] && args+=(--exclude-skus "$INPUT_EXCLUDE_SKUS")
[ -n "$INPUT_MIN_NODES" ] && args+=(--min-nodes "$INPUT_MIN_NODES")
[ -n "$INPUT_MAX_NODES" ] && args+=(--max-nodes "$INPUT_MAX_NODES")
python3 sweep_matrix.py "${args[@]}" --out matrix_full.json >/dev/null
diff --git a/experimental/CollectiveX/sweep_matrix.py b/experimental/CollectiveX/sweep_matrix.py
index 5f584a9c32..30c21d4036 100644
--- a/experimental/CollectiveX/sweep_matrix.py
+++ b/experimental/CollectiveX/sweep_matrix.py
@@ -451,6 +451,7 @@ def resolve_matrix(
backend: str = "",
backends: str = "",
only_sku: str = "",
+ exclude_skus: str = "",
min_nodes: int = 0,
max_nodes: int = 0,
max_cases: int = 128,
@@ -462,6 +463,19 @@ def resolve_matrix(
raise SystemExit("invalid node bounds")
if only_sku and only_sku not in cap.PLATFORMS:
raise SystemExit(f"unknown --only-sku {only_sku!r}; have {sorted(cap.PLATFORMS)}")
+ # --exclude-skus narrows the matrix to a partial (still publishable) subset by
+ # dropping whole runner pools — e.g. exclude a SKU whose cluster is unavailable
+ # so a versioned "tagged + success" run covers only the SKUs it can actually
+ # exercise. The publisher binds every promoted case to the canonical catalog,
+ # so a narrowed matrix cannot invent or reclassify cases; it only omits them.
+ excluded = {value.strip() for value in exclude_skus.split(",") if value.strip()}
+ unknown_excluded = sorted(excluded - set(cap.PLATFORMS))
+ if unknown_excluded:
+ raise SystemExit(
+ f"unknown --exclude-skus {unknown_excluded}; have {sorted(cap.PLATFORMS)}"
+ )
+ if only_sku and only_sku in excluded:
+ raise SystemExit("--only-sku and --exclude-skus select disjoint pools")
workloads = _load("workloads.yaml")
suites_document = _load("suites.yaml")
@@ -507,6 +521,8 @@ def resolve_matrix(
for platform_name in suite["platforms"]:
if only_sku and platform_name != only_sku:
continue
+ if platform_name in excluded:
+ continue
ep_degrees = suite["ep_degrees"]
for workload, ep, phase, routing, eplb, target, precision_profile in itertools.product(
suite["workloads"], ep_degrees, phases, routings, eplb_values,
@@ -1361,6 +1377,11 @@ def main() -> int:
parser.add_argument("--backend", default="", help="select one EP backend")
parser.add_argument("--backends", default="", help="'all' or comma-list of EP backends")
parser.add_argument("--only-sku", default="")
+ parser.add_argument(
+ "--exclude-skus",
+ default="",
+ help="comma-list of runner pools to drop (partial matrix); disjoint from --only-sku",
+ )
parser.add_argument("--min-nodes", type=int, default=0)
parser.add_argument("--max-nodes", type=int, default=0)
parser.add_argument("--max-cases", type=int, default=128)
@@ -1436,6 +1457,7 @@ def main() -> int:
backend=args.backend,
backends=args.backends,
only_sku=args.only_sku,
+ exclude_skus=args.exclude_skus,
min_nodes=args.min_nodes,
max_nodes=args.max_nodes,
max_cases=args.max_cases,
diff --git a/experimental/CollectiveX/tests/test_qualification_planning.py b/experimental/CollectiveX/tests/test_qualification_planning.py
index b87a9ef234..7324cdc041 100644
--- a/experimental/CollectiveX/tests/test_qualification_planning.py
+++ b/experimental/CollectiveX/tests/test_qualification_planning.py
@@ -237,6 +237,39 @@ def test_full_v1_matrix_emits_every_unsupported_terminal_outcome(self) -> None:
{item["case"]["case_id"] for item in expected},
)
+ def test_exclude_skus_narrows_matrix_to_a_publishable_subset(self) -> None:
+ full = sweep_matrix.validate_matrix_document(
+ sweep_matrix.resolve_matrix(suites="all", backends="all", max_cases=128)
+ )
+ partial = sweep_matrix.validate_matrix_document(
+ sweep_matrix.resolve_matrix(
+ suites="all", backends="all", max_cases=128, exclude_skus="b300"
+ )
+ )
+ full_skus = {item["sku"] for item in full["include"]}
+ partial_skus = {item["sku"] for item in partial["include"]}
+ self.assertIn("b300", full_skus)
+ self.assertNotIn("b300", partial_skus)
+ self.assertEqual(full_skus - {"b300"}, partial_skus)
+ # Excluding a pool only omits its cells; every surviving cell is byte-identical
+ # to the full matrix (a partial run cannot invent or reclassify cases).
+ def by_id(matrix: dict[str, object]) -> dict[str, object]:
+ return {
+ item["case"]["case_id"]: item
+ for item in matrix["requested_cases"]
+ if item.get("sku") != "b300"
+ }
+ self.assertEqual(by_id(partial), by_id(full))
+ self.assertTrue(
+ all(item.get("sku") != "b300" for item in partial["requested_cases"])
+ )
+
+ def test_exclude_skus_rejects_unknown_and_conflicting_pools(self) -> None:
+ with self.assertRaisesRegex(SystemExit, "unknown --exclude-skus"):
+ sweep_matrix.resolve_matrix(exclude_skus="nosuchsku")
+ with self.assertRaisesRegex(SystemExit, "disjoint pools"):
+ sweep_matrix.resolve_matrix(only_sku="b300", exclude_skus="b300")
+
def test_frontend_catalog_covers_every_requested_case_and_point(self) -> None:
catalog = sweep_matrix.frontend_catalog(self.matrix)
self.assertEqual(catalog["format"], "collectivex.frontend-catalog.v1")
From 3e28d7b35b94152b68320cf1f4b42b55b1df48ba Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 7 Jul 2026 23:47:09 +0800
Subject: [PATCH 186/210] feat(collectivex): allow v1 publish from any single
consistent attempt
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Drop the first-attempt-only gate on promotion/autopublish so a re-run
whose legs re-emit at one consistent attempt (e.g. attempt 2) can still
publish. Provenance stays single-attempt: every included shard must share
the same run and attempt; only the "attempt must be 1" restriction is
removed.
- publisher: remove the run_attempt != 1 term from the promote gate;
message now "qualification index 1 from a single versioned run"
- workflow: drop github.run_attempt == '1' from the autopublish `if`;
manual publish accepts any positive attempt
- tests: cover promotion of a non-first-attempt run; update workflow
string assertions
放开仅限首次尝试的发布门槛：重跑（各腿在同一次一致的尝试号，如第 2 次）
也可发布。来源仍为单次尝试——所有分片须同属一次运行且尝试号一致，仅移除
"尝试号必须为 1"的限制。
---
.github/workflows/collectivex-sweep.yml | 10 +++----
experimental/CollectiveX/publisher.py | 3 +--
.../CollectiveX/tests/test_publisher.py | 26 +++++++++++++++++++
.../tests/test_sampling_contract.py | 14 +++++-----
4 files changed, 40 insertions(+), 13 deletions(-)
diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml
index f2ec915f53..947ae90701 100644
--- a/.github/workflows/collectivex-sweep.yml
+++ b/.github/workflows/collectivex-sweep.yml
@@ -58,7 +58,7 @@ on:
default: '1'
options: ['1', '2', '3']
publish_run_id:
- description: For publish, one successful first-attempt versioned sweep run ID (qualification index 1)
+ description: For publish, one successful versioned sweep run ID (qualification index 1)
type: string
default: ''
refresh_run_id:
@@ -755,13 +755,13 @@ jobs:
if-no-files-found: error
retention-days: 30
- # ---- autopublish: a successful first-attempt versioned sweep self-publishes its own run ----
+ # ---- autopublish: a successful versioned sweep self-publishes its own run ----
# A "tagged + success" sweep (full OR partial) emits its own promoted
# publication artifact from the same run, so the frontend can discover many
# independent runs of a version Just-In-Time without a separate manual publish step.
autopublish:
needs: [setup, sweep]
- if: ${{ inputs.operation == 'sweep' && inputs.release_tag != 'unversioned' && inputs.qualification_index == '1' && github.run_attempt == '1' && needs.sweep.result == 'success' }}
+ if: ${{ inputs.operation == 'sweep' && inputs.release_tag != 'unversioned' && inputs.qualification_index == '1' && needs.sweep.result == 'success' }}
runs-on: ubuntu-latest
timeout-minutes: 30
env:
@@ -888,9 +888,9 @@ jobs:
&& [ "$branch" = 'collectivex' ] \
&& [ "$status" = 'completed' ] \
&& [ "$conclusion" = 'success' ] \
- && [ "$attempt" = 1 ] \
+ && [[ "$attempt" =~ ^[1-9][0-9]*$ ]] \
&& [[ "$sha" =~ ^[0-9a-f]{40}$ ]] || {
- echo "run $run_id is not an eligible first-attempt V1 sweep" >&2
+ echo "run $run_id is not an eligible V1 sweep" >&2
exit 1
}
source_sha="$sha"
diff --git a/experimental/CollectiveX/publisher.py b/experimental/CollectiveX/publisher.py
index 2100ed45aa..198202b110 100644
--- a/experimental/CollectiveX/publisher.py
+++ b/experimental/CollectiveX/publisher.py
@@ -3667,10 +3667,9 @@ def build_dataset(
len(loaded) != REQUIRED_ALLOCATIONS
or len(run_ids) != len(set(run_ids))
or qualification_indices != [1]
- or any(bundle["manifest"]["run"]["run_attempt"] != 1 for bundle in loaded)
):
raise PublisherError(
- "promotion requires qualification index 1 from a first-attempt run"
+ "promotion requires qualification index 1 from a single versioned run"
)
cases = {case["case_id"]: case for case in loaded[0]["cases"]}
canonical = _canonical_coverage_cases()
diff --git a/experimental/CollectiveX/tests/test_publisher.py b/experimental/CollectiveX/tests/test_publisher.py
index 862e7d1663..9047109d79 100644
--- a/experimental/CollectiveX/tests/test_publisher.py
+++ b/experimental/CollectiveX/tests/test_publisher.py
@@ -1858,6 +1858,32 @@ def test_promotion_rejects_more_than_one_bundle(self) -> None:
), self.assertRaisesRegex(publisher.PublisherError, "complete coverage"):
publisher.validate_public_dataset(dataset)
+ def test_promotion_accepts_a_non_first_attempt_run(self) -> None:
+ # Publication is no longer pinned to run attempt 1: a re-run whose legs
+ # re-emit at a single consistent attempt (e.g. attempt 2) is eligible to
+ # promote. The single-run gate must not reject a lone qualification-index-1
+ # bundle merely because its run_attempt is 2 — it fails later on coverage,
+ # not on the attempt.
+ bundle = {
+ "id": "1", "cases": [],
+ "manifest": {
+ "matrix": {"sha256": publisher.CANONICAL_FULL_V1_MATRIX_SHA256},
+ "run": {
+ "run_id": "1", "run_attempt": 2,
+ "qualification_index": 1,
+ },
+ },
+ }
+ # The lone bundle passes the single-run/attempt gate and only fails
+ # further downstream (the fixture omits documents/coverage), so the
+ # failure is any post-gate error — never the attempt rejection.
+ with mock.patch.object(
+ publisher, "load_bundle", side_effect=lambda _, bundle_id: bundle
+ ), self.assertRaises(Exception) as ctx:
+ publisher.build_dataset(object(), ["1"], promote=True)
+ self.assertNotIn("single versioned run", str(ctx.exception))
+ self.assertNotIn("first-attempt", str(ctx.exception))
+
def test_standalone_promotion_binds_matrix_and_requested_dispositions(self) -> None:
dataset = _promoted_dataset()
fixture_catalog = publisher._case_disposition_catalog_sha256(dataset["coverage"])
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 74a02c2f31..26bde0564d 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -1799,7 +1799,7 @@ def test_v1_publication_requires_explicit_release_markers(self) -> None:
self.assertIn("cxrelease-[1-9][0-9]*-$run_id-$attempt/release", sweep)
self.assertIn("run $run_id is not tagged for a release version", sweep)
self.assertIn("ref: ${{ steps.runs.outputs.source_sha }}", sweep)
- self.assertIn("[ \"$attempt\" = 1 ]", sweep)
+ self.assertIn("[[ \"$attempt\" =~ ^[1-9][0-9]*$ ]]", sweep)
self.assertIn(
"cxpublication-${{ steps.build.outputs.version }}-${{ github.run_id }}-${{ github.run_attempt }}",
sweep,
@@ -1808,15 +1808,17 @@ def test_v1_publication_requires_explicit_release_markers(self) -> None:
self.assertIn("retention-days: 90", sweep)
self.assertNotIn("workflow_run:", sweep)
- # A successful first-attempt versioned sweep (full OR partial) self-publishes
- # its own run so the frontend can discover many runs of a version JIT, without
- # a separate manual publish dispatch. The artifact name embeds the numeric
- # version so the frontend can group runs by version.
+ # A successful versioned sweep (full OR partial) self-publishes its own run so
+ # the frontend can discover many runs of a version JIT, without a separate
+ # manual publish dispatch. Publication is no longer pinned to the first attempt:
+ # a re-run whose legs re-emit at a single consistent attempt can also publish.
+ # The artifact name embeds the numeric version so the frontend can group runs.
self.assertIn("autopublish:", sweep)
self.assertIn("inputs.qualification_index == '1'", sweep)
self.assertIn(
- "github.run_attempt == '1' && needs.sweep.result == 'success'", sweep
+ "inputs.qualification_index == '1' && needs.sweep.result == 'success'", sweep
)
+ self.assertNotIn("github.run_attempt == '1' &&", sweep)
self.assertIn("CollectiveX publication (auto) — version", sweep)
self.assertIn(
"cxpublication-${{ inputs.release_tag }}-${{ github.run_id }}-${{ github.run_attempt }}",
From 0c7d8204ca3054e9c6e39432de7ecd66593112d0 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Wed, 8 Jul 2026 00:32:59 +0800
Subject: [PATCH 187/210] fix(collectivex): ship per-series verdicts instead of
vetoing promotion
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Promotion no longer aborts a whole run because some series or cohorts are
non-decision-grade. The publisher computes each series' and cohort's quality
verdict and attaches it to every one of them in the published dataset, then
publishes everything: diagnostic series/cohorts ship labelled
(status:"diagnostic" / decision_grade:false) with their eligibility.reasons,
never dropped. The consumer (frontend) decides what to surface and may reveal
flagged series. This splits selection (what to show) from validation (is the
number real): validation stays on the backend, the display decision moves out.
Honesty floor kept: _require_runnable_promotion_success (every runnable case
physically produced clean data) and at least one decision-grade series are still
required for a promoted "v1" to mean something. _require_promotion_cohorts full
scope keeps only structural checks (required kinds present, exact counts,
derived-chip count); the partial scope no longer requires surfaced cohorts to be
decision-grade.
Also fixes three latent bugs unmasked once the veto stopped aborting first:
- schema trial_count const 192 -> 64 (single-run migration leftover; real data
emits 64), plus the same stale value in a test fixture.
- zero-logical-byte measured components (UCCL host-staging "stage": real
latency, total_logical_bytes:0 -> 0.0 GB/s) could not satisfy the rate schema
(percentiles, exclusiveMinimum:0) yet the validator requires a rate. Add
ratePercentiles (minimum:0) for the two component rate fields; latency stays
strict.
- rankings-coverage validator now skips precision-pair cohorts (display-only
paired contrasts) to match the ranking generator, which already skips them.
Tests: the three promote-gate tests that asserted abort-on-non-decision-grade
now assert mixed bundles promote and ship both verdicts, plus the retained
all-diagnostic -> floor rejection; new schema contract test locks zero-byte
rates and trial_count=64. Full CPU battery 243 green. Verified end-to-end on a
real green run: promote -> schema-valid dataset shipping decision-grade +
diagnostic series each with its verdict.
发布不再因为部分 series 或 cohort 非 decision-grade 就整体中止。publisher 计算
每个 series 和 cohort 的质量判定并将其附加到已发布数据集中的每一项上，然后发布
全部内容：诊断类 series/cohort 带标签发布（status:"diagnostic" /
decision_grade:false）并附带 eligibility.reasons，绝不丢弃。由消费端（前端）
决定展示哪些，并可显示被标记的 series。这将“选择展示什么”与“校验数值是否真实”
分离：校验留在后端，展示决策移出。
保留诚实底线：_require_runnable_promotion_success（每个可运行 case 确实产出了
干净数据）以及至少一个 decision-grade series 仍是已发布“v1”的必要条件。同时
修复 veto 移除后暴露的三个潜藏缺陷：schema trial_count 常量 192->64、零逻辑
字节测量分量的 0.0 GB/s 速率（新增 ratePercentiles，minimum:0）、rankings
覆盖校验跳过 precision-pair cohort。全量 CPU 测试 243 通过。
---
experimental/CollectiveX/docs/methodology.md | 12 ++-
experimental/CollectiveX/publisher.py | 53 ++++++++------
.../schemas/public-dataset-v1.schema.json | 17 ++++-
.../CollectiveX/tests/test_publisher.py | 73 ++++++++++++++++---
.../tests/test_schema_v1_contract.py | 39 ++++++++++
5 files changed, 155 insertions(+), 39 deletions(-)
diff --git a/experimental/CollectiveX/docs/methodology.md b/experimental/CollectiveX/docs/methodology.md
index 788e877d2d..7d95a99aa2 100644
--- a/experimental/CollectiveX/docs/methodology.md
+++ b/experimental/CollectiveX/docs/methodology.md
@@ -231,10 +231,14 @@ may advance `dev-latest`; partial matrices remain diagnostic. The full-matrix di
pins the exact workflow shard grouping as well as the requested cases, so changing `--max-cases`
or the SKU round-robin scheduling order produces diagnostic-only runs even when case coverage is
unchanged. Superseded retries,
-planned-unsupported outcomes, and unstable comparison cohorts may render diagnostically but cannot
-rank or recommend; every successful required series in a promoted dataset remains decision-grade.
-Any failed, invalid, or diagnostic retry of a runnable case blocks promotion even if a later retry
-succeeds. Routing cohorts are comparable-experimental sensitivities and never produce configuration
+planned-unsupported outcomes, and non-decision-grade (diagnostic) series and cohorts may render
+diagnostically but cannot rank or recommend; a promoted dataset ships them labelled with their
+verdict (`status` / `eligibility.reasons`) rather than dropping them, and need only hold at least
+one decision-grade series. The publisher computes and attaches each series' and cohort's verdict;
+the consumer decides which to surface and may reveal a diagnostic series.
+Any failed, invalid, or diagnostic retry of a runnable case still blocks promotion even if a later
+retry succeeds — the honesty floor that every runnable case physically produced clean data is
+never relaxed. Routing cohorts are comparable-experimental sensitivities and never produce configuration
recommendations; official library/platform/system cohorts own actionable recommendations.
A point becomes decision-grade only after a single workflow run and allocation ID passes
diff --git a/experimental/CollectiveX/publisher.py b/experimental/CollectiveX/publisher.py
index 198202b110..380717113a 100644
--- a/experimental/CollectiveX/publisher.py
+++ b/experimental/CollectiveX/publisher.py
@@ -1053,6 +1053,12 @@ def validate_public_dataset(doc: Any) -> dict[str, Any]:
for cohort in doc["cohorts"]:
if not cohort["eligibility"]["decision_grade"]:
continue
+ # precision-pair cohorts are display-only paired contrasts — they drive
+ # neither an ordered ranking nor a sensitivity (their precision deltas
+ # are captured by the dispatch/combine-precision cohorts). The ranking
+ # generator skips them, so the coverage expectation must too.
+ if cohort["kind"] == "precision-pair":
+ continue
members = [series[series_id] for series_id in cohort["series_ids"]]
tokens = set.intersection(*(
{point["tokens_per_rank"] for point in member["points"]}
@@ -3590,19 +3596,17 @@ def _require_promotion_cohorts(
*,
scope: str = "full",
) -> None:
+ # Cohorts carry their own quality verdict (``eligibility.decision_grade`` /
+ # ``eligibility.reasons``) into the published dataset; the consumer decides
+ # which to surface. Promotion no longer vetoes on non-decision-grade cohorts.
+ # For a full-coverage run it still checks *structural* well-formedness — that
+ # every required cohort kind is present in the catalogued shape — so a "full"
+ # publication genuinely spans the catalog. Quality is reported, not gated.
if scope != "full":
# A partial ("v1 tag + success") run covers a subset of the catalog, so
- # the fixed full-matrix cohort counts / required kinds do not apply. Any
- # cohort a partial run does surface is still required to be
- # decision-grade — quality is never relaxed, only coverage.
- if any(not cohort["eligibility"]["decision_grade"] for cohort in cohorts):
- raise PublisherError("promotion includes a non-decision-grade cohort")
+ # the fixed full-matrix cohort counts / required kinds do not apply.
return
- eligible_kinds = {
- cohort["kind"]
- for cohort in cohorts
- if cohort["eligibility"]["decision_grade"]
- }
+ present_kinds = {cohort["kind"] for cohort in cohorts}
required_kinds = list(REQUIRED_COHORT_KINDS)
if any(
item["workload"].get(
@@ -3612,33 +3616,38 @@ def _require_promotion_cohorts(
for item in series
):
required_kinds.extend(PRECISION_COHORT_KINDS)
- missing = [kind for kind in required_kinds if kind not in eligible_kinds]
+ missing = [kind for kind in required_kinds if kind not in present_kinds]
if missing:
raise PublisherError(
- "promotion lacks decision-grade cohort kinds: " + ", ".join(missing)
+ "promotion lacks cohort kinds: " + ", ".join(missing)
)
for kind, expected in REQUIRED_PROMOTION_COHORT_COUNTS.items():
members = [cohort for cohort in cohorts if cohort["kind"] == kind]
- if len(members) != expected or any(
- not cohort["eligibility"]["decision_grade"] for cohort in members
- ):
+ if len(members) != expected:
raise PublisherError(
- f"promotion requires exactly {expected} decision-grade {kind} cohorts"
+ f"promotion requires exactly {expected} {kind} cohorts"
)
chip_cohorts = [cohort for cohort in cohorts if cohort["kind"] == "chip"]
expected_chips = _expected_chip_cohort_count(series)
- if len(chip_cohorts) != expected_chips or any(
- not cohort["eligibility"]["decision_grade"] for cohort in chip_cohorts
- ):
+ if len(chip_cohorts) != expected_chips:
raise PublisherError(
- f"promotion requires all {expected_chips} derived chip cohorts to be decision-grade"
+ f"promotion requires all {expected_chips} derived chip cohorts"
)
def _require_promotion_series(series: Sequence[dict[str, Any]]) -> None:
- if not series or any(item["status"] != "decision-grade" for item in series):
- raise PublisherError("promotion has unstable or incomplete required series")
+ # Each series carries its own quality verdict (``status`` /
+ # ``eligibility.decision_grade`` / ``eligibility.reasons``) into the
+ # published dataset, and the consumer decides what to surface. Promotion no
+ # longer vetoes the whole run because some series are non-decision-grade —
+ # it only guarantees the publication is non-empty and holds at least one
+ # decision-grade series, so a promoted "v1" still means something measured
+ # cleanly. Diagnostic series ship labelled as diagnostic, never dropped.
+ if not series:
+ raise PublisherError("promotion requires at least one series")
+ if not any(item["status"] == "decision-grade" for item in series):
+ raise PublisherError("promotion requires at least one decision-grade series")
def build_dataset(
diff --git a/experimental/CollectiveX/schemas/public-dataset-v1.schema.json b/experimental/CollectiveX/schemas/public-dataset-v1.schema.json
index f9ef78e7b7..867fbfe1bf 100644
--- a/experimental/CollectiveX/schemas/public-dataset-v1.schema.json
+++ b/experimental/CollectiveX/schemas/public-dataset-v1.schema.json
@@ -379,6 +379,17 @@
"p99": {"type": "number","exclusiveMinimum": 0}
}
},
+ "ratePercentiles": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": ["p50","p90","p95","p99"],
+ "properties": {
+ "p50": {"type": "number","minimum": 0},
+ "p90": {"type": "number","minimum": 0},
+ "p95": {"type": "number","minimum": 0},
+ "p99": {"type": "number","minimum": 0}
+ }
+ },
"component": {
"type": "object",
"additionalProperties": false,
@@ -397,8 +408,8 @@
"origin": {"enum": ["measured","derived"]},
"latency_us": {"$ref": "#/$defs/percentiles"},
"byte_provenance": {"$ref":"#/$defs/byteAccounting"},
- "activation_data_rate_gbps_at_latency_percentile": {"oneOf":[{"type":"null"},{"$ref":"#/$defs/percentiles"}]},
- "total_logical_data_rate_gbps_at_latency_percentile": {"oneOf":[{"type":"null"},{"$ref":"#/$defs/percentiles"}]},
+ "activation_data_rate_gbps_at_latency_percentile": {"oneOf":[{"type":"null"},{"$ref":"#/$defs/ratePercentiles"}]},
+ "total_logical_data_rate_gbps_at_latency_percentile": {"oneOf":[{"type":"null"},{"$ref":"#/$defs/ratePercentiles"}]},
"sample_count": {"oneOf": [{"type": "null"},{"type": "integer","minimum": 1}]}
}
},
@@ -412,7 +423,7 @@
"first_last_median_ratio":{"type":"number","minimum":1},
"outlier_flagged":{"type":"boolean"},
"robust_outlier_fraction":{"type":"number","minimum":0,"maximum":1},
- "trial_count":{"const":192}
+ "trial_count":{"const":64}
}
},
"nullableTrialDiagnosticComponent": {"oneOf":[{"type":"null"},{"$ref":"#/$defs/trialDiagnosticComponent"}]},
diff --git a/experimental/CollectiveX/tests/test_publisher.py b/experimental/CollectiveX/tests/test_publisher.py
index 9047109d79..904b38b088 100644
--- a/experimental/CollectiveX/tests/test_publisher.py
+++ b/experimental/CollectiveX/tests/test_publisher.py
@@ -616,7 +616,7 @@ def _series(seed: str, backend: str, *, decision_grade: bool = False) -> tuple[d
"first_last_median_ratio": 1.0,
"outlier_flagged": False,
"robust_outlier_fraction": 0.0,
- "trial_count": 192,
+ "trial_count": 64,
},
},
},
@@ -1726,7 +1726,7 @@ def test_public_dataset_selects_latest_derived_retry(self) -> None:
with self.assertRaisesRegex(publisher.PublisherError, "retry identity differs"):
publisher.validate_public_dataset(dataset)
- def test_promotion_requires_an_eligible_cohort_for_every_comparison_kind(self) -> None:
+ def test_promotion_requires_a_cohort_for_every_comparison_kind(self) -> None:
stable_fast, stable_fast_internal = _series(
"stable-fast", "deepep", decision_grade=True
)
@@ -1780,14 +1780,28 @@ def test_promotion_requires_an_eligible_cohort_for_every_comparison_kind(self) -
publisher._require_promotion_cohorts(
required + ineligible, anchor_series
)
+ # Promotion checks *structural* presence: a required comparison kind
+ # must appear in the catalogued shape, regardless of its per-cohort
+ # verdict. A kind is "missing" only when no cohort of that kind exists
+ # at all — a diagnostic (non-decision-grade) cohort of the kind still
+ # satisfies presence and ships with its verdict. So the missing-kind
+ # error only fires when every cohort of the kind is removed.
for kind in publisher.REQUIRED_COHORT_KINDS:
with self.subTest(missing_kind=kind), self.assertRaisesRegex(
publisher.PublisherError, rf"cohort kinds:.*{kind}"
):
publisher._require_promotion_cohorts([
item for item in required + ineligible
- if item["kind"] != kind or not item["eligibility"]["decision_grade"]
+ if item["kind"] != kind
], anchor_series)
+ # A kind present only as a diagnostic (non-decision-grade) cohort must
+ # NOT veto promotion — it ships labelled as diagnostic. (The outer
+ # patches already pin the fixed counts to {} and the chip count to 1.)
+ diagnostic_only = [
+ {"kind": kind, "eligibility": {"decision_grade": False}}
+ for kind in publisher.REQUIRED_COHORT_KINDS
+ ]
+ publisher._require_promotion_cohorts(diagnostic_only, anchor_series)
def test_promotion_requires_exact_counts(self) -> None:
dataset = _promoted_dataset()
@@ -1807,7 +1821,11 @@ def test_promotion_requires_exact_counts(self) -> None:
dataset["cohorts"], dataset["series"]
)
- def test_promotion_requires_every_derived_chip_cohort_to_be_stable(self) -> None:
+ def test_promotion_requires_every_derived_chip_cohort_present(self) -> None:
+ # Promotion checks that the published shape carries *every* derived chip
+ # cohort the series imply (a structural completeness check), but it no
+ # longer requires each chip cohort to be decision-grade — a diagnostic
+ # chip cohort ships labelled with its verdict, it does not veto the run.
dataset = _promoted_dataset()
chip = next(item for item in dataset["cohorts"] if item["kind"] == "chip")
self.assertEqual(
@@ -1817,15 +1835,19 @@ def test_promotion_requires_every_derived_chip_cohort_to_be_stable(self) -> None
with mock.patch.object(
publisher, "REQUIRED_PROMOTION_COHORT_COUNTS", _cohort_counts(dataset)
):
+ # Dropping a derived chip cohort leaves the shape structurally
+ # incomplete → rejected.
missing = [item for item in dataset["cohorts"] if item is not chip]
with self.assertRaisesRegex(publisher.PublisherError, "derived chip cohorts"):
publisher._require_promotion_cohorts(missing, dataset["series"])
+ # A non-decision-grade (diagnostic) chip cohort keeps the shape
+ # complete → promotion accepts it; the verdict rides along for the
+ # consumer to surface or hide.
chip["eligibility"]["decision_grade"] = False
- with self.assertRaisesRegex(publisher.PublisherError, "derived chip cohorts"):
- publisher._require_promotion_cohorts(
- dataset["cohorts"], dataset["series"]
- )
+ publisher._require_promotion_cohorts(
+ dataset["cohorts"], dataset["series"]
+ )
def test_promotion_rejects_more_than_one_bundle(self) -> None:
bundles = {
@@ -1900,6 +1922,10 @@ def test_standalone_promotion_binds_matrix_and_requested_dispositions(self) -> N
):
publisher.validate_public_dataset(dataset)
+ # A diagnostic (non-decision-grade) series does NOT veto promotion — the
+ # published dataset ships it labelled with its verdict, and the consumer
+ # decides whether to surface it. Validation succeeds as long as at least
+ # one decision-grade series remains (the honesty floor asserted below).
diagnostic = copy.deepcopy(dataset)
item = diagnostic["series"][0]
item["status"] = "diagnostic"
@@ -1907,6 +1933,33 @@ def test_standalone_promotion_binds_matrix_and_requested_dispositions(self) -> N
"decision_grade": False,
"reasons": ["insufficient-allocations"],
})
+ with mock.patch.object(
+ publisher, "CANONICAL_FULL_V1_CASE_CATALOG_SHA256", fixture_catalog
+ ), mock.patch.object(
+ publisher,
+ "REQUIRED_PROMOTION_COHORT_COUNTS",
+ _cohort_counts(dataset),
+ ):
+ validated = publisher.validate_public_dataset(diagnostic)
+ shipped = next(
+ series for series in validated["series"]
+ if series["series_id"] == item["series_id"]
+ )
+ self.assertEqual(shipped["status"], "diagnostic")
+ self.assertFalse(shipped["eligibility"]["decision_grade"])
+ self.assertIn(
+ "insufficient-allocations", shipped["eligibility"]["reasons"]
+ )
+
+ # Honesty floor: a dataset with NO decision-grade series is still rejected
+ # — a promoted "v1" must carry at least one cleanly measured series.
+ all_diagnostic = copy.deepcopy(dataset)
+ for series in all_diagnostic["series"]:
+ series["status"] = "diagnostic"
+ series["eligibility"].update({
+ "decision_grade": False,
+ "reasons": ["insufficient-allocations"],
+ })
with mock.patch.object(
publisher, "CANONICAL_FULL_V1_CASE_CATALOG_SHA256", fixture_catalog
), mock.patch.object(
@@ -1914,9 +1967,9 @@ def test_standalone_promotion_binds_matrix_and_requested_dispositions(self) -> N
"REQUIRED_PROMOTION_COHORT_COUNTS",
_cohort_counts(dataset),
), self.assertRaisesRegex(
- publisher.PublisherError, "unstable or incomplete required series"
+ publisher.PublisherError, "at least one decision-grade series"
):
- publisher.validate_public_dataset(diagnostic)
+ publisher.validate_public_dataset(all_diagnostic)
for original, replacement in (("runnable", "unsupported"),
("unsupported", "runnable")):
diff --git a/experimental/CollectiveX/tests/test_schema_v1_contract.py b/experimental/CollectiveX/tests/test_schema_v1_contract.py
index 1c5aa29824..9d1a1324d1 100644
--- a/experimental/CollectiveX/tests/test_schema_v1_contract.py
+++ b/experimental/CollectiveX/tests/test_schema_v1_contract.py
@@ -414,6 +414,45 @@ def test_public_measured_point_has_bounded_detail(self) -> None:
with self.assertRaises(jsonschema.ValidationError):
anomaly_validator.validate([f"anomaly-{index}" for index in range(17)])
+ def test_zero_byte_component_rates_and_trial_count_are_bound(self) -> None:
+ # A measured but zero-logical-byte component (e.g. UCCL host-staging's
+ # ``stage``: real latency, zero bytes) reports a 0.0 GB/s rate. Rate
+ # percentiles therefore allow 0 (``ratePercentiles``, minimum 0), while
+ # latency percentiles stay strictly positive (``percentiles``,
+ # exclusiveMinimum 0). This keeps a zero-byte measurement honest instead
+ # of forcing it to null or dropping the component.
+ defs = self.public["$defs"]
+
+ def _ref_validator(pointer: str) -> jsonschema.Validator:
+ return jsonschema.Draft202012Validator({
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$defs": defs,
+ "$ref": pointer,
+ })
+
+ for field in (
+ "activation_data_rate_gbps_at_latency_percentile",
+ "total_logical_data_rate_gbps_at_latency_percentile",
+ ):
+ rate = _ref_validator(f"#/$defs/component/properties/{field}")
+ rate.validate({"p50": 0.0, "p90": 0.0, "p95": 0.0, "p99": 0.0})
+ rate.validate(None)
+ with self.assertRaises(jsonschema.ValidationError):
+ rate.validate({"p50": -1.0, "p90": 0.0, "p95": 0.0, "p99": 0.0})
+
+ latency = _ref_validator("#/$defs/component/properties/latency_us")
+ latency.validate({"p50": 1.0, "p90": 1.0, "p95": 1.0, "p99": 1.0})
+ with self.assertRaises(jsonschema.ValidationError):
+ latency.validate({"p50": 0.0, "p90": 1.0, "p95": 1.0, "p99": 1.0})
+
+ # A single qualification run yields exactly 64 trial medians per point.
+ trial_count = _ref_validator(
+ "#/$defs/trialDiagnosticComponent/properties/trial_count"
+ )
+ trial_count.validate(64)
+ with self.assertRaises(jsonschema.ValidationError):
+ trial_count.validate(192)
+
if __name__ == "__main__":
unittest.main()
From 2a71bc6a0c27209aa2de700bc453c55d791fdbcc Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Wed, 8 Jul 2026 10:39:10 +0800
Subject: [PATCH 188/210] feat(collectivex): add ep_sizes dispatch filter for
comprehensive EP8 runs
Mirror --max-nodes with an --ep-sizes / ep_sizes filter in
sweep_matrix.resolve_matrix and the sweep workflow: a dispatch-time narrower
that keeps only shards whose expert-parallel degree is in the given comma-list.
Passing "8" drops EP16 so a single comprehensive run can co-schedule the 8-GPU
SKUs' single-node EP8 with the GB200/GB300 two-node EP8, without dispatching any
EP16 leg (EP16 walls on H100's zero-RDMA topology and would block all-green
autopublish). Blank keeps every degree, so the canonical matrix stays
byte-identical and its frozen digest does not drift; a filtered run publishes
as a partial subset, exactly like --exclude-skus / --max-nodes.
---
.github/workflows/collectivex-sweep.yml | 6 ++++
experimental/CollectiveX/sweep_matrix.py | 22 ++++++++++++
.../tests/test_qualification_planning.py | 36 +++++++++++++++++++
.../tests/test_sampling_contract.py | 8 +++++
4 files changed, 72 insertions(+)
diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml
index 947ae90701..08f4e10d3c 100644
--- a/.github/workflows/collectivex-sweep.yml
+++ b/.github/workflows/collectivex-sweep.yml
@@ -44,6 +44,10 @@ on:
description: Keep only shards with at most this node/tray count (1 keeps non-GB EP8; blank = all)
type: string
default: ''
+ ep_sizes:
+ description: Keep only shards whose expert-parallel degree is in this comma-list (8 keeps EP8 only, dropping EP16 so GB SKUs co-schedule with 8-GPU SKUs; blank = all)
+ type: string
+ default: ''
max_cases:
description: Max cases per shard cell before chunking into another GHA job (128 = no chunking for current suites)
type: string
@@ -98,6 +102,7 @@ jobs:
INPUT_PROBE_ID: ${{ inputs.probe_id }}
INPUT_MIN_NODES: ${{ inputs.min_nodes }}
INPUT_MAX_NODES: ${{ inputs.max_nodes }}
+ INPUT_EP_SIZES: ${{ inputs.ep_sizes }}
INPUT_MAX_CASES: ${{ inputs.max_cases }}
INPUT_QUALIFICATION_INDEX: ${{ inputs.qualification_index }}
CX_QUALIFICATION_INDEX: ${{ inputs.qualification_index }}
@@ -151,6 +156,7 @@ jobs:
[ -n "$INPUT_EXCLUDE_SKUS" ] && args+=(--exclude-skus "$INPUT_EXCLUDE_SKUS")
[ -n "$INPUT_MIN_NODES" ] && args+=(--min-nodes "$INPUT_MIN_NODES")
[ -n "$INPUT_MAX_NODES" ] && args+=(--max-nodes "$INPUT_MAX_NODES")
+ [ -n "$INPUT_EP_SIZES" ] && args+=(--ep-sizes "$INPUT_EP_SIZES")
python3 sweep_matrix.py "${args[@]}" --out matrix_full.json >/dev/null
fi
python3 artifact_safety.py matrix_full.json
diff --git a/experimental/CollectiveX/sweep_matrix.py b/experimental/CollectiveX/sweep_matrix.py
index 30c21d4036..2df801733a 100644
--- a/experimental/CollectiveX/sweep_matrix.py
+++ b/experimental/CollectiveX/sweep_matrix.py
@@ -454,6 +454,7 @@ def resolve_matrix(
exclude_skus: str = "",
min_nodes: int = 0,
max_nodes: int = 0,
+ ep_sizes: str = "",
max_cases: int = 128,
) -> dict[str, Any]:
"""Resolve suite configuration into allocation-sized workflow shards."""
@@ -461,6 +462,19 @@ def resolve_matrix(
raise SystemExit("--max-cases must be positive")
if min_nodes < 0 or max_nodes < 0 or (min_nodes and max_nodes and min_nodes > max_nodes):
raise SystemExit("invalid node bounds")
+ # --ep-sizes narrows the matrix to specific expert-parallel degrees, dispatch-time
+ # like --max-nodes: "8" keeps every EP8 shard and drops EP16, so a comprehensive run
+ # can co-schedule the 8-GPU SKUs' single-node EP8 with the GB SKUs' two-node EP8
+ # without dispatching any EP16 leg. Blank keeps every degree. The resulting matrix is
+ # a partial (still publishable) subset; the publisher binds every promoted case to the
+ # canonical catalog, so a filtered matrix cannot invent or reclassify cases.
+ selected_eps: set[int] = set()
+ for value in (part.strip() for part in ep_sizes.split(",")):
+ if not value:
+ continue
+ if not value.isdigit() or int(value) <= 0:
+ raise SystemExit(f"invalid --ep-sizes {ep_sizes!r}; expected positive integers")
+ selected_eps.add(int(value))
if only_sku and only_sku not in cap.PLATFORMS:
raise SystemExit(f"unknown --only-sku {only_sku!r}; have {sorted(cap.PLATFORMS)}")
# --exclude-skus narrows the matrix to a partial (still publishable) subset by
@@ -528,6 +542,8 @@ def resolve_matrix(
suite["workloads"], ep_degrees, phases, routings, eplb_values,
suite_targets, precision_profiles,
):
+ if selected_eps and ep not in selected_eps:
+ continue
if precision_profile is not None and not cap.precision_target_declared(
precision_profile,
sku=platform_name,
@@ -1384,6 +1400,11 @@ def main() -> int:
)
parser.add_argument("--min-nodes", type=int, default=0)
parser.add_argument("--max-nodes", type=int, default=0)
+ parser.add_argument(
+ "--ep-sizes",
+ default="",
+ help="comma-list of expert-parallel degrees to keep (e.g. 8 drops EP16); blank = all",
+ )
parser.add_argument("--max-cases", type=int, default=128)
parser.add_argument("--extract-from", default="", metavar="MATRIX")
parser.add_argument("--validate-control", default="", metavar="SHARD")
@@ -1460,6 +1481,7 @@ def main() -> int:
exclude_skus=args.exclude_skus,
min_nodes=args.min_nodes,
max_nodes=args.max_nodes,
+ ep_sizes=args.ep_sizes,
max_cases=args.max_cases,
)
try:
diff --git a/experimental/CollectiveX/tests/test_qualification_planning.py b/experimental/CollectiveX/tests/test_qualification_planning.py
index 7324cdc041..0e1993475e 100644
--- a/experimental/CollectiveX/tests/test_qualification_planning.py
+++ b/experimental/CollectiveX/tests/test_qualification_planning.py
@@ -270,6 +270,42 @@ def test_exclude_skus_rejects_unknown_and_conflicting_pools(self) -> None:
with self.assertRaisesRegex(SystemExit, "disjoint pools"):
sweep_matrix.resolve_matrix(only_sku="b300", exclude_skus="b300")
+ def test_ep_sizes_narrows_matrix_to_a_publishable_subset(self) -> None:
+ full = sweep_matrix.validate_matrix_document(
+ sweep_matrix.resolve_matrix(suites="all", backends="all", max_cases=128)
+ )
+ partial = sweep_matrix.validate_matrix_document(
+ sweep_matrix.resolve_matrix(
+ suites="all", backends="all", max_cases=128, ep_sizes="8"
+ )
+ )
+ full_eps = {item["case"]["ep"] for item in full["requested_cases"]}
+ partial_eps = {item["case"]["ep"] for item in partial["requested_cases"]}
+ # The canonical matrix carries EP16; --ep-sizes=8 drops it entirely so a
+ # comprehensive run can co-schedule EP8 across every SKU (GB EP8 is two nodes,
+ # the 8-GPU SKUs' EP8 is one) with no EP16 leg to wall.
+ self.assertIn(16, full_eps)
+ self.assertEqual(partial_eps, {8})
+ # Keeping a degree only omits the others; every surviving cell is byte-identical
+ # to the full matrix (a partial run cannot invent or reclassify cases).
+ def ep8_by_id(matrix: dict[str, object]) -> dict[str, object]:
+ return {
+ item["case"]["case_id"]: item
+ for item in matrix["requested_cases"]
+ if item["case"]["ep"] == 8
+ }
+ self.assertEqual(ep8_by_id(partial), ep8_by_id(full))
+ # Every include shard cell that survives is single-degree EP8 (no EP16 shard).
+ self.assertTrue(
+ all(item["case"]["ep"] == 8 for item in partial["requested_cases"])
+ )
+
+ def test_ep_sizes_rejects_non_positive_or_non_integer_degrees(self) -> None:
+ with self.assertRaisesRegex(SystemExit, "invalid --ep-sizes"):
+ sweep_matrix.resolve_matrix(ep_sizes="0")
+ with self.assertRaisesRegex(SystemExit, "invalid --ep-sizes"):
+ sweep_matrix.resolve_matrix(ep_sizes="eight")
+
def test_frontend_catalog_covers_every_requested_case_and_point(self) -> None:
catalog = sweep_matrix.frontend_catalog(self.matrix)
self.assertEqual(catalog["format"], "collectivex.frontend-catalog.v1")
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
index 26bde0564d..b64f65ad60 100644
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ b/experimental/CollectiveX/tests/test_sampling_contract.py
@@ -1770,6 +1770,14 @@ def test_v1_publication_requires_explicit_release_markers(self) -> None:
)
# collectivex.release-tag.v1 is a frozen data-format id, NOT the release version.
self.assertIn("collectivex.release-tag.v1", sweep)
+ # The ep-size filter is a dispatch-time narrower (like max_nodes): it lets a
+ # comprehensive run schedule EP8 across every SKU without any EP16 leg, and it
+ # only applies to the sweep branch (not precision probes).
+ self.assertIn("ep_sizes:", sweep)
+ self.assertIn("INPUT_EP_SIZES: ${{ inputs.ep_sizes }}", sweep)
+ self.assertIn(
+ '[ -n "$INPUT_EP_SIZES" ] && args+=(--ep-sizes "$INPUT_EP_SIZES")', sweep
+ )
# A versioned run may cover the full canonical matrix or a filtered subset;
# the release marker records the coverage scope AND the numeric version.
self.assertNotIn("V1 sweeps require the exact unfiltered full matrix", sweep)
From 52d2bee7169f66f17446b1156c6d4f4eb2789945 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Wed, 8 Jul 2026 11:31:08 +0800
Subject: [PATCH 189/210] refactor(collectivex): remove publication subsystem
tests and precision probe
Front-load the deletions that would otherwise break the CPU battery on
every subsequent refactor commit, leaving a stable keep-set.
- delete publisher.py (pure CLI leaf; only test_publisher imported it)
- delete test_publisher, test_sampling_contract, test_deepep_v2_contract,
test_precision_scheduling (AST/source-text/frozen-digest suites)
- delete probe_precision.py (swept precision-dimension probe)
- trim test_ep_precision_adapters to the behavioral resolution tests
Battery: 48 tests OK.
---
experimental/CollectiveX/publisher.py | 4108 ---------------
.../CollectiveX/tests/probe_precision.py | 1213 -----
.../tests/test_deepep_v2_contract.py | 2333 ---------
.../tests/test_ep_precision_adapters.py | 57 -
.../tests/test_precision_scheduling.py | 566 ---
.../CollectiveX/tests/test_publisher.py | 2958 -----------
.../tests/test_sampling_contract.py | 4434 -----------------
7 files changed, 15669 deletions(-)
delete mode 100644 experimental/CollectiveX/publisher.py
delete mode 100644 experimental/CollectiveX/tests/probe_precision.py
delete mode 100644 experimental/CollectiveX/tests/test_deepep_v2_contract.py
delete mode 100644 experimental/CollectiveX/tests/test_precision_scheduling.py
delete mode 100644 experimental/CollectiveX/tests/test_publisher.py
delete mode 100644 experimental/CollectiveX/tests/test_sampling_contract.py
diff --git a/experimental/CollectiveX/publisher.py b/experimental/CollectiveX/publisher.py
deleted file mode 100644
index 380717113a..0000000000
--- a/experimental/CollectiveX/publisher.py
+++ /dev/null
@@ -1,4108 +0,0 @@
-#!/usr/bin/env python3
-"""Fail-closed filesystem publisher for CollectiveX EP v1 artifacts."""
-from __future__ import annotations
-
-import argparse
-import contextlib
-import datetime as dt
-import fcntl
-from functools import lru_cache
-import hashlib
-import json
-import math
-import os
-from pathlib import Path, PurePosixPath
-import re
-import shutil
-import stat
-import statistics
-import sys
-import tempfile
-from typing import Any, Iterator, Sequence
-import zipfile
-
-import jsonschema
-import numpy as np
-
-HERE = Path(__file__).resolve().parent
-sys.path.insert(0, str(HERE))
-
-import artifact_safety # noqa: E402
-import capability # noqa: E402
-import contracts # noqa: E402
-import identity # noqa: E402
-import sweep_matrix # noqa: E402
-
-FORMAT_BUNDLE = "collectivex.private.bundle.v1"
-FORMAT_PUBLIC = "collectivex.public.v1"
-FORMAT_CHANNEL = "collectivex.channel.v1"
-POLICY = "collectivex-decision-grade-v1"
-PUBLISHER_POLICY = "collectivex-publisher-v1"
-OUTCOMES = ("success", "unsupported", "failed", "invalid", "diagnostic")
-REQUIRED_ALLOCATIONS = 1
-REQUIRED_COHORT_KINDS = ("library", "chip", "system")
-PRECISION_COHORT_KINDS = (
- "dispatch-precision", "combine-precision", "precision-pair",
-)
-REQUIRED_PROMOTION_COHORT_COUNTS = {"library": 24, "system": 4}
-CANONICAL_FULL_V1_MATRIX_SHA256 = (
- "3048ef24d2d78751321aa5008dd6e2f83c0e6b881fe8d12811a3bfbda6419c9c"
-)
-CANONICAL_FULL_V1_CASE_CATALOG_SHA256 = (
- "bbba014c9920e6349e9b269dee7cea4f397eaa4569bb8aac7b6ed09f6265905f"
-)
-TRIAL_DRIFT_RATIO_LIMIT = 1.10
-TRIAL_OUTLIER_FRACTION_LIMIT = 0.05
-TRIAL_OUTLIER_MAD_MULTIPLIER = 6.0
-BOOTSTRAP_RESAMPLES = 10_000
-BOOTSTRAP_CONFIDENCE = 0.95
-BOOTSTRAP_EQUIVALENCE_BAND = 0.05
-BOOTSTRAP_POLICY = "hierarchical-run-trial-p99-ratio-v1"
-BOOTSTRAP_CHUNK_SIZE = 250
-MAX_ARCHIVE_MEMBERS = 20_000
-MAX_ARCHIVE_MEMBER_BYTES = 2 * 1024**3
-MAX_ARCHIVE_TOTAL_BYTES = 16 * 1024**3
-MAX_PUBLIC_DATASET_BYTES = 32 * 1024**2
-HEX64 = re.compile(r"[0-9a-f]{64}")
-SAFE_ID = re.compile(r"[a-z0-9][a-z0-9_.-]{0,127}")
-REASON = re.compile(r"[a-z0-9][a-z0-9.-]{0,95}")
-ARTIFACT_NAME = re.compile(
- r"cx(?:unsupported|shard-[a-z0-9][a-z0-9_.-]{0,127})-[1-9][0-9]*-[1-9][0-9]*"
-)
-COVERAGE_TOPOLOGY_FIELDS = (
- "ep_size", "nodes", "gpus_per_node", "scale_up_domain", "scope",
- "scale_up_transport", "scale_out_transport", "transport", "topology_class",
-)
-CHANNEL_PATH = re.compile(r"datasets/([0-9a-f]{64})/dataset\.json")
-SCHEMA_DIR = HERE / "schemas"
-_SCHEMAS: dict[str, jsonschema.protocols.Validator] = {}
-_BOOTSTRAP_CACHE: dict[tuple[Any, ...], dict[str, Any]] = {}
-
-
-class PublisherError(ValueError):
- """Input or stored state violates the publication contract."""
-
-
-strict_load = contracts.strict_load
-_canonical = contracts.canonical_json_bytes
-
-
-def _sha_bytes(data: bytes) -> str:
- return hashlib.sha256(data).hexdigest()
-
-
-def _sha_file(path: Path) -> str:
- digest = hashlib.sha256()
- with path.open("rb") as handle:
- for chunk in iter(lambda: handle.read(1024 * 1024), b""):
- digest.update(chunk)
- return digest.hexdigest()
-
-
-def _latest_timestamp(values: Sequence[str]) -> str:
- """Return the latest evidence timestamp without introducing publisher wall time."""
- if not values:
- raise PublisherError("cannot derive a timestamp without evidence")
-
- def parsed(value: str) -> dt.datetime:
- try:
- timestamp = dt.datetime.fromisoformat(value.replace("Z", "+00:00"))
- except ValueError as exc:
- raise PublisherError("evidence timestamp is not ISO-8601") from exc
- if timestamp.tzinfo is None:
- raise PublisherError("evidence timestamp must include a timezone")
- return timestamp.astimezone(dt.timezone.utc)
-
- return max(values, key=lambda value: (parsed(value), value))
-
-
-def _schema(name: str, value: Any) -> None:
- validator = _SCHEMAS.get(name)
- if validator is None:
- schema = strict_load(SCHEMA_DIR / name)
- jsonschema.Draft202012Validator.check_schema(schema)
- validator = jsonschema.Draft202012Validator(
- schema, format_checker=jsonschema.FormatChecker()
- )
- _SCHEMAS[name] = validator
- errors = sorted(validator.iter_errors(value), key=lambda error: list(error.absolute_path))
- if errors:
- error = errors[0]
- location = ".".join(map(str, error.absolute_path)) or "$"
- raise PublisherError(f"{name}:{location}: {error.message}")
-def _exact(obj: Any, fields: set[str], path: str) -> dict[str, Any]:
- if not isinstance(obj, dict):
- raise PublisherError(f"{path} must be an object")
- actual = set(obj)
- if actual != fields:
- raise PublisherError(
- f"{path} fields differ: missing={sorted(fields - actual)}, "
- f"extra={sorted(actual - fields)}"
- )
- return obj
-def _array(value: Any, path: str, *, nonempty: bool = False) -> list[Any]:
- if not isinstance(value, list) or (nonempty and not value):
- qualifier = "a nonempty" if nonempty else "an"
- raise PublisherError(f"{path} must be {qualifier} array")
- return value
-
-
-def _integer(value: Any, path: str, *, minimum: int = 0) -> int:
- if type(value) is not int or value < minimum:
- raise PublisherError(f"{path} must be an integer >= {minimum}")
- return value
-
-
-def _unique(values: Sequence[Any], path: str) -> None:
- serialized = [_canonical(value) for value in values]
- if len(serialized) != len(set(serialized)):
- raise PublisherError(f"{path} contains duplicates")
-
-def _eligibility(value: dict[str, Any], path: str) -> dict[str, Any]:
- allocations = value["allocation_ids"]
- gates = (
- len(allocations) >= REQUIRED_ALLOCATIONS,
- value["complete"], value["correct"], value["measured_roundtrip_p99"],
- value["stable_ordering"],
- )
- if value["decision_grade"] != (all(gates) and not value["reasons"]):
- raise PublisherError(f"{path}.decision_grade does not match promotion gates")
- if value["decision_grade"] == bool(value["reasons"]):
- raise PublisherError(f"{path}.reasons does not match decision status")
- return value
-
-
-def validate_channel(doc: Any, *, expected_channel: str | None = None) -> dict[str, Any]:
- _schema("channel-v1.schema.json", doc)
- if expected_channel and doc["channel"] != expected_channel:
- raise PublisherError("channel name does not match its file")
- target = doc["dataset"]
- match = CHANNEL_PATH.fullmatch(target["path"]) if isinstance(target["path"], str) else None
- if not match or match.group(1) != target["sha256"]:
- raise PublisherError("channel dataset path and sha256 do not agree")
- return doc
-
-
-def _metric_value(series: dict[str, Any], metric: dict[str, Any]) -> tuple[str, float, str]:
- point = next(
- (point for point in series["points"] if point["tokens_per_rank"] == metric["tokens_per_rank"]),
- None,
- )
- if point is None or series["phase"] != metric["phase"]:
- raise PublisherError("decision metric references an unavailable point")
- component = point["components"]["roundtrip"]
- if metric["measure"] == "latency_us":
- value = component["latency_us"][metric["statistic"]]
- unit = "us"
- else:
- rates = component[metric["measure"]]
- if rates is None:
- raise PublisherError("data-rate decision has no byte accounting contract")
- value = rates[metric["statistic"]]
- unit = "GB/s"
- return point["point_id"], value, unit
-
-
-def _validate_metric(metric: dict[str, Any]) -> None:
- expected = "min" if metric["measure"] == "latency_us" else "max"
- if metric["objective"] != expected:
- raise PublisherError(f"{metric['measure']} objective must be {expected}")
-
-
-def _metric_label(measure: str, statistic: str) -> str:
- if measure == "latency_us":
- return f"{statistic} latency"
- label = (
- "activation data rate"
- if measure == "activation_data_rate_gbps_at_latency_percentile"
- else "total logical data rate"
- )
- return f"{label} at {statistic} latency"
-
-
-def _public_case_factors(series: dict[str, Any]) -> dict[str, Any]:
- workload = series["workload"]
- system = series["system"]
- measurement = series["measurement"]
- ep_size = system["ep_size"]
- case = {
- "backend": series["backend"]["id"],
- "canonical": True,
- "eplb": workload["eplb"],
- "ep": ep_size,
- "experts": workload["experts"],
- "gpus_per_node": system["gpus_per_node"],
- "hidden": workload["hidden"],
- "ladder": " ".join(str(point["tokens_per_rank"]) for point in series["points"]),
- "mode": series["mode"],
- "nodes": system["nodes"],
- "phase": series["phase"],
- "required_publication": series["publication_tier"],
- "routing": workload["routing"],
- "samples_per_point": measurement["samples_per_component"],
- "scale_out_transport": system["scale_out_transport"],
- "scale_up_domain": system["scale_up_domain"],
- "scale_up_transport": system["scale_up_transport"],
- "scope": system["scope"],
- "suite": series["suite"],
- "timing": (
- f"{measurement['iters']}:{measurement['trials']}:"
- f"{measurement['warmups']}"
- ),
- "topk": workload["top_k"],
- "topology_class": system["topology_class"],
- "transport": system["transport"],
- "warmup_semantics": sweep_matrix.ep_harness.WARMUP_SEMANTICS,
- "workload": series["model"],
- }
- if workload["precision_profile"] != identity.V1_CONTROL_PRECISION_PROFILE:
- case["precision_profile"] = workload["precision_profile"]
- return {
- "case": case,
- "profile": identity.profile_for_case(case),
- "sku": system["sku"],
- }
-
-
-def _coverage_topology(case: dict[str, Any]) -> dict[str, Any]:
- """Project exact fabric placement without exposing private runner details."""
- return {
- "ep_size": case.get("ep_size", case.get("ep")),
- **{field: case[field] for field in COVERAGE_TOPOLOGY_FIELDS if field != "ep_size"},
- }
-
-
-def _coverage_coordinates(case: dict[str, Any]) -> dict[str, Any]:
- return {
- "sku": case["sku"], "backend": case["backend"],
- "mode": case["mode"], "phase": case["phase"],
- "topology": _coverage_topology(case),
- }
-
-
-@lru_cache(maxsize=1)
-def _canonical_coverage_cases() -> dict[str, dict[str, Any]]:
- matrix = sweep_matrix.resolve_matrix(suites="all", max_cases=128, backends="all")
- return {
- item["case"]["case_id"]: {
- "sku": item["sku"],
- **item["case"],
- "disposition": item["disposition"],
- "reason": item["reason"],
- }
- for item in matrix["requested_cases"]
- }
-
-
-def _public_series_config(series: dict[str, Any]) -> dict[str, Any]:
- return {
- "backend": {
- "generation": series["backend"]["generation"],
- "version": series["backend"]["version"],
- },
- "resource": series["resource"],
- "system": {"label": series["system"]["label"]},
- }
-
-
-def _public_cohort_factors(kind: str, item: dict[str, Any]) -> tuple[Any, Any]:
- workload = item["workload"]
- build = item["build"]
- shape = {
- key: workload[key]
- for key in (
- "hidden", "top_k", "experts", "precision_profile", "dispatch_precision",
- "combine_precision", "activation_profile",
- )
- }
- common = {
- "model": item["model"], "mode": item["mode"], "phase": item["phase"],
- "shape": shape, "measurement": item["measurement"],
- "ep_size": item["system"]["ep_size"],
- }
- if kind == "library":
- return (
- {**common, "system": item["system"], "workload": workload,
- "resource_mode": item["resource"]["mode"], "source": build["source_sha"]},
- item["backend"]["id"],
- )
- if kind == "chip":
- return (
- {**common, "backend": item["backend"], "workload": workload,
- "resource_mode": item["resource"]["mode"], "source": build["source_sha"]},
- item["system"],
- )
- if kind == "system":
- return {**common, "workload": workload, "source": build["source_sha"]}, [
- item["system"]["sku"], item["backend"]["id"], item["resource"]["profile"]
- ]
- if kind in PRECISION_COHORT_KINDS:
- static_shape = {
- key: workload[key]
- for key in ("hidden", "top_k", "experts", "activation_profile")
- }
- control = {
- "backend": item["backend"],
- "build": {
- key: build[key]
- for key in (
- "image_digest", "runtime_fingerprint_sha256", "source_sha",
- "squash_sha256",
- )
- },
- "measurement": item["measurement"],
- "mode": item["mode"],
- "model": item["model"],
- "phase": item["phase"],
- "resource": item["resource"],
- "shape": static_shape,
- "system": item["system"],
- "workload": {
- "eplb": workload["eplb"],
- "routing": workload["routing"],
- },
- }
- if kind == "dispatch-precision":
- control["combine_precision"] = workload["combine_precision"]
- variant = workload["dispatch_precision"]
- elif kind == "combine-precision":
- control["dispatch_precision"] = workload["dispatch_precision"]
- variant = workload["combine_precision"]
- else:
- control.pop("resource")
- variant = {
- "combine_precision": workload["combine_precision"],
- "dispatch_precision": workload["dispatch_precision"],
- "precision_profile": workload["precision_profile"],
- "resource": item["resource"],
- }
- return control, variant
- raise PublisherError(f"unknown cohort kind {kind}")
-
-
-def _case_disposition_catalog_sha256(coverage: Sequence[dict[str, Any]]) -> str:
- catalog = [
- {"case_id": item["case_id"], "disposition": item["disposition"]}
- for item in sorted(coverage, key=lambda item: item["case_id"])
- ]
- return _sha_bytes(_canonical(catalog))
-
-
-def validate_public_dataset(doc: Any) -> dict[str, Any]:
- _schema("public-dataset-v1.schema.json", doc)
- if len(_canonical(doc)) + 1 > MAX_PUBLIC_DATASET_BYTES:
- raise PublisherError("public dataset exceeds the serving size limit")
- try:
- artifact_safety.assert_publication_safe([doc])
- except artifact_safety.ArtifactSafetyError as exc:
- raise PublisherError(str(exc)) from exc
- if doc["source_bundle_ids"] != sorted(doc["source_bundle_ids"]):
- raise PublisherError("source bundle IDs are not canonical")
- for field, key in (
- ("coverage", "case_id"), ("attempts", "attempt_id"),
- ("series", "series_id"), ("cohorts", "cohort_id"),
- ("rankings", "ranking_id"), ("recommendations", "recommendation_id"),
- ("sensitivities", "sensitivity_id"),
- ):
- if doc[field] != sorted(doc[field], key=lambda item: item[key]):
- raise PublisherError(f"{field} are not in canonical identity order")
- promotion = doc["promotion"]
- quarantined = promotion["status"] == "quarantined"
- if quarantined != (promotion["reason"] is not None) or quarantined != (
- promotion["matrix_id"] is None
- ):
- raise PublisherError("promotion reason/matrix identity differs from status")
- attempts = {item["attempt_id"]: item for item in doc["attempts"]}
- if len(attempts) != len(doc["attempts"]):
- raise PublisherError("dataset has duplicate attempt IDs")
- evidence = [
- value["evidence_id"] for item in doc["attempts"] for value in item["evidence"]
- ]
- _unique(evidence, "dataset attempt evidence")
- series = {item["series_id"]: item for item in doc["series"]}
- if len(series) != len(doc["series"]):
- raise PublisherError("dataset has duplicate series IDs")
- allocation_ids = set(promotion["allocation_ids"])
- case_ids = {item["case_id"] for item in doc["coverage"]}
- if len(case_ids) != len(doc["coverage"]):
- raise PublisherError("dataset has duplicate case coverage")
- coverage_by_case = {item["case_id"]: item for item in doc["coverage"]}
- series_case_ids = {
- case_id for item in doc["series"] for case_id in item["case_ids"]
- }
- canonical_cases = _canonical_coverage_cases()
- for item in doc["coverage"]:
- topology = item["topology"]
- registered = capability.topology_for(item["sku"], topology["ep_size"])
- if (
- item["sku"] not in capability.PLATFORMS
- or item["backend"] not in capability.BACKENDS
- or registered is None
- or any(
- topology[field] != registered[field]
- for field in COVERAGE_TOPOLOGY_FIELDS if field != "ep_size"
- )
- ):
- raise PublisherError("coverage topology differs from the capability registry")
- canonical = canonical_cases.get(item["case_id"])
- if canonical is not None:
- precision_profile = canonical.get(
- "precision_profile", identity.V1_CONTROL_PRECISION_PROFILE
- )
- precision = identity.precision_profile(precision_profile)
- expected_projection = {
- "sku": canonical["sku"],
- "suite": canonical["suite"],
- "workload": canonical["workload"],
- "publication_tier": canonical["required_publication"],
- "backend": canonical["backend"],
- "mode": canonical["mode"],
- "phase": canonical["phase"],
- "routing": canonical["routing"],
- "eplb": canonical["eplb"],
- "precision_profile": precision_profile,
- "dispatch_precision": precision["dispatch"],
- "combine_precision": precision["combine"],
- "topology": _coverage_topology(canonical),
- "disposition": canonical["disposition"],
- }
- if any(item[field] != value for field, value in expected_projection.items()):
- raise PublisherError("coverage dimensions differ from its case identity")
- expected_tokens = [int(value) for value in canonical["ladder"].split()]
- if [point["tokens_per_rank"] for point in item["points"]] != expected_tokens:
- raise PublisherError("coverage points differ from the requested token ladder")
- if canonical is None and item["case_id"] not in series_case_ids:
- raise PublisherError("coverage case identity is outside the v1 catalog")
- for point in item["points"]:
- if point["global_tokens"] != point["tokens_per_rank"] * topology["ep_size"]:
- raise PublisherError("coverage point global token count differs")
- if (point["terminal_status"] == "measured") != (point["reason"] is None):
- raise PublisherError("coverage point terminal reason differs from status")
- for item in doc["attempts"]:
- if item["case_id"] not in case_ids or item["allocation_id"] not in allocation_ids:
- raise PublisherError("attempt references undeclared coverage or allocation")
- if item["series_id"] is not None and item["series_id"] not in series:
- raise PublisherError("attempt references unknown series")
- if (item["outcome"] == "success") != (item["reason"] is None):
- raise PublisherError("attempt reason must be null exactly for success")
- if item["outcome"] == "success" and item["failure_mode"] is not None:
- raise PublisherError("successful attempt cannot have a failure mode")
- if (item["outcome"] == "success" and item["selected"]) != (
- item["series_id"] is not None
- ):
- raise PublisherError("attempt series must be present exactly for selected success")
- if {item["allocation_id"] for item in doc["attempts"]} != allocation_ids:
- raise PublisherError("promotion allocation catalog differs from attempts")
- attempt_groups: dict[tuple[str, str], list[dict[str, Any]]] = {}
- for item in doc["attempts"]:
- attempt_groups.setdefault((item["case_id"], item["allocation_id"]), []).append(item)
- for (case_id, allocation_id), group in attempt_groups.items():
- ordinals = sorted(item["attempt_index"] for item in group)
- if ordinals != list(range(1, len(group) + 1)):
- raise PublisherError("public retries must retain contiguous attempt indexes")
- if any(
- item["attempt_id"] != identity.attempt_id(
- allocation=allocation_id, case=case_id, ordinal=item["attempt_index"]
- )
- for item in group
- ):
- raise PublisherError("public retry identity differs from its case/allocation/index")
- selected = [item for item in group if item["selected"]]
- if len(selected) != 1 or selected[0]["attempt_index"] != ordinals[-1]:
- raise PublisherError("publisher must select the latest retry per case/allocation")
- selected_by_series: dict[str, list[dict[str, Any]]] = {}
- for item in doc["attempts"]:
- if item["selected"] and item["outcome"] == "success":
- selected_by_series.setdefault(item["series_id"], []).append(item)
- terminal = 0
- for item in doc["coverage"]:
- listed = set(item["attempt_ids"])
- selected = item["selected_attempt_id"]
- expected_attempts = {
- attempt_id for attempt_id, attempt in attempts.items()
- if attempt["case_id"] == item["case_id"]
- }
- if listed != expected_attempts:
- raise PublisherError("coverage references attempts from another case")
- if selected is not None:
- terminal += 1
- if (selected not in listed or not attempts[selected]["selected"]
- or any(attempts[selected][field] != item[field]
- for field in ("outcome", "failure_mode", "reason"))):
- raise PublisherError("coverage selected outcome differs")
- selected_candidates = [attempts[value] for value in listed if attempts[value]["selected"]]
- latest = max(
- selected_candidates,
- key=lambda attempt: (
- int(attempt["run_id"]), attempt["run_attempt"],
- attempt["attempt_index"], attempt["attempt_id"]
- ),
- )
- if selected != latest["attempt_id"]:
- raise PublisherError("coverage does not select the latest canonical allocation")
- expected_status = (
- "measured" if attempts[selected]["outcome"] == "success"
- else attempts[selected]["outcome"]
- )
- if any(point["terminal_status"] != expected_status for point in item["points"]):
- raise PublisherError("coverage point status differs from selected attempt")
- if expected_status == "measured":
- selected_series = series.get(attempts[selected]["series_id"])
- if selected_series is None:
- raise PublisherError("measured coverage points lack a public series")
- public_points = {
- point["tokens_per_rank"]: point for point in selected_series["points"]
- }
- if any(
- point["series_id"] != selected_series["series_id"]
- or point["point_id"]
- != public_points.get(point["tokens_per_rank"], {}).get("point_id")
- for point in item["points"]
- ):
- raise PublisherError("coverage point identities differ from series")
- measured_cases = sum(
- all(point["terminal_status"] == "measured" for point in item["points"])
- for item in doc["coverage"]
- )
- unsupported_cases = sum(
- all(point["terminal_status"] == "unsupported" for point in item["points"])
- for item in doc["coverage"]
- )
- requested_points = sum(len(item["points"]) for item in doc["coverage"])
- measured_points = sum(
- point["terminal_status"] == "measured"
- for item in doc["coverage"] for point in item["points"]
- )
- unsupported_points = sum(
- point["terminal_status"] == "unsupported"
- for item in doc["coverage"] for point in item["points"]
- )
- expected_counts = {
- "requested_cases": len(doc["coverage"]),
- "terminal_cases": terminal,
- "measured_cases": measured_cases,
- "unsupported_cases": unsupported_cases,
- "requested_points": requested_points,
- "terminal_points": requested_points,
- "measured_points": measured_points,
- "unsupported_points": unsupported_points,
- }
- if any(promotion[field] != value for field, value in expected_counts.items()):
- raise PublisherError("promotion coverage counts differ")
- selected_evidence: dict[tuple[str, str], set[str]] = {}
- for attempt in doc["attempts"]:
- if attempt["selected"] and attempt["series_id"] is not None:
- for value in attempt["evidence"]:
- selected_evidence.setdefault(
- (attempt["series_id"], value["point_id"]), set()
- ).add(value["evidence_id"])
- for item in doc["series"]:
- eligibility = _eligibility(item["eligibility"], f"series {item['series_id']}")
- workload = item["workload"]
- model, hidden, top_k, experts = sweep_matrix.V1_WORKLOAD
- suite_contract = sweep_matrix.V1_SUITE_CONTRACTS.get(item["suite"])
- coordinate = (
- item["mode"], item["phase"], workload["routing"], workload["eplb"]
- )
- profile_case = {"mode": item["mode"]}
- if workload["precision_profile"] != identity.V1_CONTROL_PRECISION_PROFILE:
- profile_case["precision_profile"] = workload["precision_profile"]
- profile = identity.profile_for_case(profile_case)
- communication_precision = identity.precision_profile(workload["precision_profile"])
- if (
- item["model"] != model
- or (workload["hidden"], workload["top_k"], workload["experts"])
- != (hidden, top_k, experts)
- or suite_contract is None
- or coordinate not in suite_contract["coordinates"]
- or (
- suite_contract.get("backends") is not None
- and item["backend"]["id"] not in suite_contract["backends"]
- )
- or item["publication_tier"] != suite_contract["publication"]
- or item["measurement"]["contract"] != profile["contract"]
- or item["measurement"]["component_order_contract"]
- != profile["component_order_contract"]
- or item["measurement"]["combine_semantics"] != profile["combine_semantics"]
- or item["measurement"]["payload_unit"] != profile["payload_unit"]
- or workload["dispatch_precision"] != communication_precision["dispatch"]
- or workload["combine_precision"] != communication_precision["combine"]
- or item["measurement"]["qualification_indices"]
- != sorted(item["measurement"]["qualification_indices"])
- or len(set(item["measurement"]["qualification_indices"]))
- != len(item["measurement"]["qualification_indices"])
- ):
- raise PublisherError("series differs from the frozen v1 workload/suite profile")
- backend_id = item["backend"]["id"]
- expected_role = "reference" if backend_id == "nccl-ep" else "library"
- if (
- backend_id not in capability.BACKENDS
- or item["backend"]["label"] != BACKEND_LABELS[backend_id]
- or item["backend"]["role"] != expected_role
- or item["backend"]["version"] is None
- ):
- raise PublisherError("series backend projection differs from v1")
- sku = item["system"]["sku"]
- platform = capability.PLATFORMS.get(sku)
- ep_size = item["system"]["ep_size"]
- registered_topology = capability.topology_for(sku, ep_size)
- if platform is None or registered_topology is None:
- raise PublisherError("series system projection differs from v1")
- disposition, _ = capability.resolve_disposition(
- sku, backend_id, ep=ep_size, nodes=item["system"]["nodes"],
- routing=workload["routing"], eplb=workload["eplb"],
- mode=item["mode"],
- precision_profile=(
- workload["precision_profile"]
- if workload["precision_profile"] != identity.V1_CONTROL_PRECISION_PROFILE
- else None
- ),
- )
- if (
- disposition != "supported"
- or item["system"]["vendor"] != platform["vendor"]
- or any(
- item["system"][field] != registered_topology[field]
- for field in (
- "nodes", "gpus_per_node", "scale_up_domain", "scope",
- "scale_up_transport", "scale_out_transport", "transport",
- "topology_class",
- )
- )
- or item["system"]["world_size"] != ep_size
- or platform["product"] not in set(
- re.findall(r"[a-z]+\d+[a-z]*", item["system"]["label"].lower())
- )
- ):
- raise PublisherError("series system projection differs from v1")
- if contracts.public_series_config_sha256(_public_series_config(item)) != item[
- "build"
- ]["public_config_sha256"]:
- raise PublisherError("public series configuration differs from its commitment")
- covered = [coverage_by_case.get(case_id) for case_id in item["case_ids"]]
- if not covered or any(
- case is None
- or {
- "sku": case["sku"], "backend": case["backend"],
- "mode": case["mode"], "phase": case["phase"],
- "topology": case["topology"],
- }
- != {
- "sku": sku, "backend": backend_id,
- "mode": item["mode"], "phase": item["phase"],
- "topology": _coverage_topology(item["system"]),
- }
- for case in covered
- ):
- raise PublisherError("series projection differs from its case coverage")
- if (
- item["eplb"]["enabled"] != item["workload"]["eplb"]
- or item["eplb"]["logical_experts"] != item["workload"]["experts"]
- ):
- raise PublisherError("series EPLB descriptor differs from its workload")
- eplb = item["eplb"]
- expected_physical = eplb["logical_experts"] + eplb["redundant_experts"]
- nullable_eplb = (
- "planner", "mapping_sha256", "reference_tokens_per_rank", "max_replicas",
- "imbalance_before", "imbalance_after", "calibration_workload_id",
- "calibration_trace_sha256", "calibration_window", "calibration_token_offset",
- )
- if eplb["enabled"]:
- if (
- item["workload"]["routing"] != "zipf"
- or any(eplb[field] is None for field in nullable_eplb)
- or eplb["planner"] != "greedy-rank-major-v1"
- or eplb["reference_tokens_per_rank"] != 2048
- or eplb["redundant_experts"] != 32
- or eplb["redundant_experts"] % ep_size != 0
- or eplb["physical_experts"] != expected_physical
- or eplb["logical_experts"] % ep_size != 0
- or eplb["physical_experts"] % ep_size != 0
- or not 1 <= eplb["replicated_experts"] <= min(
- eplb["logical_experts"], eplb["redundant_experts"]
- )
- or not 2 <= eplb["max_replicas"] <= 1 + eplb["redundant_experts"]
- or not 1 <= eplb["imbalance_after"] <= eplb["imbalance_before"] <= ep_size
- ):
- raise PublisherError("enabled EPLB descriptor is incomplete")
- expected_plan, calibration = contracts._expected_eplb_calibration(
- workload["routing"], workload["hidden"], workload["top_k"],
- eplb["logical_experts"], eplb["physical_experts"], ep_size,
- identity.V1_CASE_PROFILE["seed"],
- identity.V1_CASE_PROFILE["eplb_reference_tokens_per_rank"],
- )
- expected_eplb = {
- **calibration,
- "enabled": True,
- "planner": identity.V1_CASE_PROFILE["eplb_planner"],
- "mapping_sha256": contracts.eplb_contract.mapping_hash(expected_plan),
- "logical_experts": eplb["logical_experts"],
- "physical_experts": eplb["physical_experts"],
- "redundant_experts": identity.V1_CASE_PROFILE["eplb_redundant_experts"],
- "reference_tokens_per_rank": identity.V1_CASE_PROFILE[
- "eplb_reference_tokens_per_rank"
- ],
- "replicated_experts": expected_plan["replicated_experts"],
- "max_replicas": expected_plan["max_replicas"],
- "imbalance_before": expected_plan["imbalance_before"],
- "imbalance_after": expected_plan["imbalance_after"],
- }
- if eplb != expected_eplb:
- raise PublisherError("enabled EPLB descriptor differs from deterministic plan")
- elif (
- any(eplb[field] is not None for field in nullable_eplb)
- or eplb["physical_experts"] != expected_physical
- or eplb["redundant_experts"] != 0
- or eplb["replicated_experts"] != 0
- ):
- raise PublisherError("disabled EPLB descriptor claims a plan")
- if item["backend"]["id"] == "nccl-ep":
- expected_generation = (
- "nccl" if item["system"]["vendor"] == "nvidia" else "rccl"
- )
- if item["backend"]["generation"] != expected_generation:
- raise PublisherError("NCCL/RCCL reference generation differs from system vendor")
- if (item["status"] == "decision-grade") != eligibility["decision_grade"]:
- raise PublisherError("series status differs from eligibility")
- if (
- set(eligibility["allocation_ids"]) != set(item["allocation_ids"])
- or eligibility["correct"] != all(
- point["correctness"]["semantic_pass"]
- and point["correctness"]["precision"]["passed"]
- for point in item["points"]
- )
- ):
- raise PublisherError("series eligibility differs from its evidence")
- selected_attempts = selected_by_series.get(item["series_id"], [])
- if (
- set(item["case_ids"]) != {attempt["case_id"] for attempt in selected_attempts}
- or set(item["allocation_ids"])
- != {attempt["allocation_id"] for attempt in selected_attempts}
- or item["measurement"]["qualification_indices"]
- != sorted({attempt["qualification_index"] for attempt in selected_attempts})
- ):
- raise PublisherError("series case/allocation catalog differs from selected attempts")
- if item["eligibility"]["decision_grade"] and len(
- {attempt["run_id"] for attempt in selected_attempts}
- ) < REQUIRED_ALLOCATIONS:
- raise PublisherError("decision-grade series lacks independent workflow runs")
- tokens = [point["tokens_per_rank"] for point in item["points"]]
- if tokens != sorted(set(tokens)):
- raise PublisherError("series points are not in unique ascending token order")
- if len(item["case_ids"]) != 1:
- raise PublisherError("public series must represent exactly one v1 case")
- case_id = item["case_ids"][0]
- if identity.digest("case", _public_case_factors(item)) != case_id:
- raise PublisherError("public series projection differs from its case identity")
- build = item["build"]
- expected_series_id = identity.series_id({
- "backend": backend_id,
- "case_id": case_id,
- "image_digest": build["image_digest"],
- "implementation_contract_sha256": build[
- "implementation_contract_sha256"
- ],
- "public_config_sha256": build["public_config_sha256"],
- "routing_control_sha256": build["routing_control_sha256"],
- "runtime_fingerprint_sha256": build["runtime_fingerprint_sha256"],
- "source_sha": build["source_sha"],
- "squash_sha256": build["squash_sha256"],
- "workload_id": workload["workload_id"],
- })
- if item["series_id"] != expected_series_id:
- raise PublisherError("public series identity differs from its committed factors")
- for point in item["points"]:
- if point["point_id"] != identity.point_id(series=item["series_id"], tokens_per_rank=point["tokens_per_rank"]):
- raise PublisherError("point identity differs")
- if point["global_tokens"] != point["tokens_per_rank"] * item["system"]["ep_size"]:
- raise PublisherError("global_tokens must use EP size")
- routing = point["routing"]
- max_fanout = min(item["workload"]["top_k"], item["system"]["ep_size"])
- if (
- routing["routed_copies"] < point["global_tokens"]
- or routing["routed_copies"] > point["global_tokens"] * max_fanout
- or routing["recv_tokens_max"] > routing["routed_copies"]
- or routing["recv_tokens_max"] * item["system"]["ep_size"]
- < routing["routed_copies"]
- or not math.isclose(
- routing["fanout_mean"],
- routing["routed_copies"] / point["global_tokens"],
- rel_tol=1e-12,
- )
- or routing["hotspot_ratio"] < 1
- or routing["empty_expert_count"] >= eplb["physical_experts"]
- or routing["empty_rank_count"] >= item["system"]["ep_size"]
- ):
- raise PublisherError("point routing/load facts are internally inconsistent")
- expected_evidence = selected_evidence.get(
- (item["series_id"], point["point_id"]), set()
- )
- if set(point["evidence_ids"]) != expected_evidence:
- raise PublisherError("point evidence differs from selected series attempts")
- point_correctness = point["correctness"]
- if (
- point_correctness["precision"]["profile_id"]
- != workload["precision_profile"]
- or (
- point_correctness["semantic_pass"]
- and not point_correctness["precision"]["passed"]
- )
- ):
- raise PublisherError("point correctness differs from series evidence")
- diagnostics = point["trial_diagnostics"]
- diagnostic_reasons = set(diagnostics["reasons"])
- component_reasons: set[str] = set()
- for name, summary in diagnostics["components"].items():
- if summary is None:
- if point["components"][name] is not None:
- raise PublisherError("trial diagnostics omit a measured component")
- continue
- if point["components"][name] is None:
- raise PublisherError("trial diagnostics describe an unavailable component")
- if summary["drift_flagged"] != (
- summary["first_last_median_ratio"] > TRIAL_DRIFT_RATIO_LIMIT
- ) or summary["outlier_flagged"] != (
- summary["robust_outlier_fraction"] > TRIAL_OUTLIER_FRACTION_LIMIT
- ):
- raise PublisherError("trial diagnostic flags differ from their thresholds")
- if summary["drift_flagged"]:
- component_reasons.add("trial-drift")
- if summary["outlier_flagged"]:
- component_reasons.add("trial-outliers")
- if (
- diagnostic_reasons != component_reasons
- or diagnostics["flagged"] != bool(diagnostic_reasons)
- or not diagnostic_reasons.issubset(point["anomalies"])
- ):
- raise PublisherError("trial diagnostic summary is inconsistent")
- components = point["components"]
- if (components["dispatch"] is None) != (components["combine"] is None):
- raise PublisherError("dispatch/combine availability differs")
- for name, component in components.items():
- if component is None:
- continue
- expected_origin = "derived" if name == "isolated_sum" else "measured"
- expected_samples = None if name == "isolated_sum" else 512
- if component["origin"] != expected_origin or component["sample_count"] != expected_samples:
- raise PublisherError(f"{name} origin or sample count differs")
- rate_fields = (
- "activation_data_rate_gbps_at_latency_percentile",
- "total_logical_data_rate_gbps_at_latency_percentile",
- )
- if name == "isolated_sum" and any(component[field] is not None for field in rate_fields):
- raise PublisherError("isolated_sum cannot publish a derived data rate")
- if name != "isolated_sum" and any(component[field] is None for field in rate_fields):
- raise PublisherError(f"{name} measured data rates are missing")
- latency = component["latency_us"]
- if list(latency.values()) != sorted(latency.values()):
- raise PublisherError("latency percentiles are not ordered")
- byte_provenance = component["byte_provenance"]
- if byte_provenance["total_logical_bytes"] != (
- byte_provenance["activation_data_bytes"] + byte_provenance["scale_bytes"]
- ):
- raise PublisherError("component byte accounting does not reconcile")
- for field, byte_field in (
- ("activation_data_rate_gbps_at_latency_percentile", "activation_data_bytes"),
- ("total_logical_data_rate_gbps_at_latency_percentile", "total_logical_bytes"),
- ):
- if component[field] is not None:
- for statistic, rate in component[field].items():
- expected = byte_provenance[byte_field] / (latency[statistic] * 1000.0)
- if not math.isclose(rate, expected, rel_tol=1e-9, abs_tol=1e-12):
- raise PublisherError("component GB/s formula differs")
- if components["roundtrip"] is None or components["roundtrip"]["origin"] != "measured":
- raise PublisherError("roundtrip must be measured")
- for statistic, throughput in point["roundtrip_token_rate_at_latency_percentile"].items():
- expected = point["global_tokens"] / (
- components["roundtrip"]["latency_us"][statistic] * 1e-6
- )
- if not math.isclose(throughput, expected, rel_tol=1e-9):
- raise PublisherError("roundtrip token throughput formula differs")
- if components["dispatch"] is not None:
- derived = components["isolated_sum"]
- if derived is None or any(not math.isclose(
- derived["latency_us"][statistic],
- components["dispatch"]["latency_us"][statistic]
- + (
- components["stage"]["latency_us"][statistic]
- if components["stage"] is not None else 0.0
- )
- + components["combine"]["latency_us"][statistic], rel_tol=1e-12
- ) for statistic in ("p50", "p90", "p95", "p99")):
- raise PublisherError("isolated_sum is not the component percentile sum")
- elif components["isolated_sum"] is not None:
- raise PublisherError("isolated_sum requires measured dispatch/combine components")
- if any(point["trial_diagnostics"]["flagged"] for point in item["points"]) != (
- "unresolved-trial-diagnostic" in item["eligibility"]["reasons"]
- ):
- raise PublisherError("series trial diagnostic eligibility is inconsistent")
- cohorts = {item["cohort_id"]: item for item in doc["cohorts"]}
- if len(cohorts) != len(doc["cohorts"]):
- raise PublisherError("dataset has duplicate cohort IDs")
- for item in doc["cohorts"]:
- if not set(item["series_ids"]).issubset(series):
- raise PublisherError("cohort references unknown series")
- members = [series[series_id] for series_id in item["series_ids"]]
- expected_tier = (
- "comparable-experimental"
- if any(member["publication_tier"] == "comparable-experimental" for member in members)
- else "official"
- )
- if item["publication_tier"] != expected_tier:
- raise PublisherError("cohort publication tier differs from its members")
- if f"/ {members[0]['mode']} /" not in item["label"]:
- raise PublisherError("cohort label omits its controlled mode")
- roles = {member["backend"]["role"] for member in members}
- if item["kind"] == "library" and roles != {"library"}:
- raise PublisherError("library cohort contains non-library evidence")
- if item["kind"] == "system" and roles != {"reference"}:
- raise PublisherError("system cohort is not a portable reference comparison")
- if item["kind"] in {"chip", *PRECISION_COHORT_KINDS} and len(
- {_canonical(member["backend"]) for member in members}
- ) != 1:
- raise PublisherError(f"{item['kind']} cohort mixes backend implementations")
- public_factors = [_public_cohort_factors(item["kind"], member) for member in members]
- if len({_canonical(value[0]) for value in public_factors}) != 1:
- raise PublisherError(f"{item['kind']} cohort does not control its public factors")
- if len({_canonical(value[1]) for value in public_factors}) < 2:
- raise PublisherError(f"{item['kind']} cohort does not vary its declared contrast")
- if item["kind"] in PRECISION_COHORT_KINDS:
- if item["publication_tier"] != "comparable-experimental":
- raise PublisherError("precision cohorts must be experimental")
- if item["kind"] in {"dispatch-precision", "combine-precision"}:
- axis = (
- "dispatch"
- if item["kind"] == "dispatch-precision"
- else "combine"
- )
- field = f"{axis}_precision"
- bf16 = identity.precision_profile(
- identity.V1_CONTROL_PRECISION_PROFILE
- )[axis]
- has_baseline = sum(
- _canonical(member["workload"][field]) == _canonical(bf16)
- for member in members
- ) == 1
- missing_reason = (
- "missing-bf16-precision-baseline"
- in item["eligibility"]["reasons"]
- )
- if has_baseline == missing_reason:
- raise PublisherError(
- "precision baseline and eligibility reason disagree"
- )
- expected_id = _derived_id("cxcohort-v1-", {
- "kind": item["kind"], "series_ids": item["series_ids"],
- "controlled_factors": item["controlled_factors"],
- "varying_factors": item["varying_factors"],
- })
- if item["cohort_id"] != expected_id:
- raise PublisherError("cohort ID differs from its public factors")
- expected_factors = {
- "library": (
- ["system", "workload", "mode", "phase", "measurement", "resource.mode", "source"],
- ["backend", "resource"],
- ),
- "chip": (
- ["backend", "source", "workload", "mode", "phase", "measurement", "resource.mode"],
- ["system", "resource"],
- ),
- "system": (
- ["workload", "mode", "phase", "measurement", "source"],
- ["system", "backend", "resource"],
- ),
- "dispatch-precision": (
- [
- "backend", "implementation-static-build", "system", "model-shape",
- "mode", "phase", "workload.routing", "workload.eplb",
- "measurement", "resource", "combine-precision",
- ],
- ["dispatch-precision"],
- ),
- "combine-precision": (
- [
- "backend", "implementation-static-build", "system", "model-shape",
- "mode", "phase", "workload.routing", "workload.eplb",
- "measurement", "resource", "dispatch-precision",
- ],
- ["combine-precision"],
- ),
- "precision-pair": (
- [
- "backend", "implementation-static-build", "system", "model-shape",
- "mode", "phase", "workload.routing", "workload.eplb",
- "measurement",
- ],
- [
- "dispatch-precision", "combine-precision", "precision-profile",
- "resource",
- ],
- ),
- }[item["kind"]]
- member_allocations = {
- allocation for series_id in item["series_ids"]
- for allocation in series[series_id]["allocation_ids"]
- }
- if (
- (item["controlled_factors"], item["varying_factors"]) != expected_factors
- or set(item["eligibility"]["allocation_ids"]) != member_allocations
- ):
- raise PublisherError("cohort factors or allocations differ from its members")
- _eligibility(item["eligibility"], f"cohort {item['cohort_id']}")
- expected_ranking_keys: set[tuple[str, str, str, int]] = set()
- for cohort in doc["cohorts"]:
- if not cohort["eligibility"]["decision_grade"]:
- continue
- # precision-pair cohorts are display-only paired contrasts — they drive
- # neither an ordered ranking nor a sensitivity (their precision deltas
- # are captured by the dispatch/combine-precision cohorts). The ranking
- # generator skips them, so the coverage expectation must too.
- if cohort["kind"] == "precision-pair":
- continue
- members = [series[series_id] for series_id in cohort["series_ids"]]
- tokens = set.intersection(*(
- {point["tokens_per_rank"] for point in member["points"]}
- for member in members
- ))
- expected_ranking_keys.update(
- (cohort["cohort_id"], measure, statistic, token)
- for token in tokens
- for measure in (
- "latency_us", "activation_data_rate_gbps_at_latency_percentile",
- "total_logical_data_rate_gbps_at_latency_percentile",
- )
- for statistic in ("p50", "p99")
- )
- ranking_top: dict[
- tuple[str, str, str, int], dict[str, Any] | None
- ] = {}
- ranking_ids: set[str] = set()
- for ranking in doc["rankings"]:
- cohort = cohorts.get(ranking["cohort_id"])
- if (
- cohort is None
- or not cohort["eligibility"]["decision_grade"]
- or ranking["eligibility"] != cohort["eligibility"]
- or ranking["publication_tier"] != cohort["publication_tier"]
- ):
- raise PublisherError("ranking references an ineligible cohort")
- entries = ranking["entries"]
- _validate_metric(ranking["metric"])
- if cohort["kind"] == "library" and any(
- series[series_id]["backend"]["role"] == "reference"
- for series_id in cohort["series_ids"]
- ):
- raise PublisherError("reference evidence cannot drive a library ranking")
- if {entry["series_id"] for entry in entries} != set(cohort["series_ids"]):
- raise PublisherError("ranking does not cover its cohort")
- for entry in entries:
- point_id, value, unit = _metric_value(series[entry["series_id"]], ranking["metric"])
- if entry["point_id"] != point_id or entry["unit"] != unit or not math.isclose(entry["value"], value, rel_tol=1e-12):
- raise PublisherError("ranking entry differs from series data")
- reverse = ranking["metric"]["objective"] == "max"
- expected = sorted(entries, key=lambda entry: (entry["value"], entry["series_id"]), reverse=reverse)
- metric = ranking["metric"]
- ranks = [entry["rank"] for entry in entries]
- if metric["measure"] == "latency_us" and metric["statistic"] == "p99":
- tied_first = sum(rank == 1 for rank in ranks)
- expected_ranks = [1] * tied_first + list(
- range(tied_first + 1, len(entries) + 1)
- )
- else:
- expected_ranks = list(range(1, len(entries) + 1))
- if entries != expected or not ranks or ranks != expected_ranks:
- raise PublisherError("ranking order differs")
- expected_id = _derived_id("cxranking-v1-", {
- "cohort_id": ranking["cohort_id"], "metric": metric,
- })
- if ranking["ranking_id"] != expected_id or expected_id in ranking_ids:
- raise PublisherError("ranking ID is duplicate or differs")
- ranking_ids.add(expected_id)
- ranking_top[(ranking["cohort_id"], metric["measure"], metric["statistic"], metric["tokens_per_rank"])] = (
- entries[0] if ranks.count(1) == 1 else None
- )
- if set(ranking_top) != expected_ranking_keys:
- raise PublisherError("rankings do not cover every eligible cohort metric")
- objective = {
- "min-p50-latency": ("latency_us", "p50"), "min-p99-latency": ("latency_us", "p99"),
- "max-activation-data-rate-at-p50-latency": (
- "activation_data_rate_gbps_at_latency_percentile", "p50"
- ),
- "max-activation-data-rate-at-p99-latency": (
- "activation_data_rate_gbps_at_latency_percentile", "p99"
- ),
- "max-total-logical-data-rate-at-p50-latency": (
- "total_logical_data_rate_gbps_at_latency_percentile", "p50"
- ),
- "max-total-logical-data-rate-at-p99-latency": (
- "total_logical_data_rate_gbps_at_latency_percentile", "p99"
- ),
- }
- recommendation_ids: set[str] = set()
- for item in doc["recommendations"]:
- if item["objective"] != "min-p99-latency":
- raise PublisherError("recommendation is not a unique p99 latency winner")
- measure, statistic = objective[item["objective"]]
- candidates = [top for key, top in ranking_top.items()
- if key[:3] == (item["cohort_id"], measure, statistic)
- and top is not None and top["point_id"] == item["point_id"]]
- if len(candidates) != 1 or any(item[field] != candidates[0][field] for field in ("series_id", "point_id", "value", "unit")):
- raise PublisherError("recommendation is not a ranking winner")
- matching_ranking = next(
- ranking for ranking in doc["rankings"]
- if ranking["cohort_id"] == item["cohort_id"]
- and ranking["metric"]["measure"] == measure
- and ranking["metric"]["statistic"] == statistic
- and ranking["entries"][0]["point_id"] == item["point_id"]
- )
- expected_id = _derived_id("cxrecommendation-v1-", {
- "objective": item["objective"], "ranking_id": matching_ranking["ranking_id"],
- })
- cohort = cohorts[item["cohort_id"]]
- if (item["recommendation_id"] != expected_id or expected_id in recommendation_ids
- or cohort["publication_tier"] != "official"
- or item["publication_tier"] != "official"
- or item["eligibility"] != cohort["eligibility"]):
- raise PublisherError("recommendation ID/eligibility differs")
- recommendation_ids.add(expected_id)
- expected_recommendations = sum(
- cohorts[ranking["cohort_id"]]["publication_tier"] == "official"
- and ranking["metric"]["measure"] == "latency_us"
- and ranking["metric"]["statistic"] == "p99"
- and sum(entry["rank"] == 1 for entry in ranking["entries"]) == 1
- for ranking in doc["rankings"]
- )
- if len(doc["recommendations"]) != expected_recommendations:
- raise PublisherError("recommendations do not cover every actionable ranking")
- sensitivity_ids: set[str] = set()
- sensitivity_keys: set[tuple[str, str, str, str, str, int]] = set()
- for item in doc["sensitivities"]:
- cohort = cohorts.get(item["cohort_id"])
- if (
- cohort is None
- or cohort["kind"] not in {
- "dispatch-precision", "combine-precision",
- }
- or not cohort["eligibility"]["decision_grade"]
- or item["publication_tier"] != cohort["publication_tier"]
- or item["eligibility"] != cohort["eligibility"]
- ):
- raise PublisherError("sensitivity references an ineligible contrast cohort")
- if (
- item["baseline_series_id"] == item["candidate_series_id"]
- or not {item["baseline_series_id"], item["candidate_series_id"]}.issubset(cohort["series_ids"])
- ):
- raise PublisherError("sensitivity series differ from its cohort")
- _validate_metric(item["metric"])
- baseline_series = series[item["baseline_series_id"]]
- axis = (
- "dispatch"
- if cohort["kind"] == "dispatch-precision"
- else "combine"
- )
- field = f"{axis}_precision"
- bf16 = identity.precision_profile(
- identity.V1_CONTROL_PRECISION_PROFILE
- )[axis]
- if _canonical(baseline_series["workload"][field]) != _canonical(bf16):
- raise PublisherError("precision sensitivity baseline is not BF16")
- _, baseline, _ = _metric_value(series[item["baseline_series_id"]], item["metric"])
- _, candidate, _ = _metric_value(series[item["candidate_series_id"]], item["metric"])
- if not math.isclose(item["signed_change_ratio"], (candidate - baseline) / baseline, rel_tol=1e-12):
- raise PublisherError("sensitivity ratio differs")
- expected_id = _derived_id("cxsensitivity-v1-", {
- "baseline": item["baseline_series_id"],
- "candidate": item["candidate_series_id"],
- "cohort": item["cohort_id"], "metric": item["metric"],
- })
- if item["sensitivity_id"] != expected_id or expected_id in sensitivity_ids:
- raise PublisherError("sensitivity ID is duplicate or differs")
- sensitivity_ids.add(expected_id)
- sensitivity_keys.add((
- item["cohort_id"], item["baseline_series_id"], item["candidate_series_id"],
- item["metric"]["measure"], item["metric"]["statistic"],
- item["metric"]["tokens_per_rank"],
- ))
- expected_sensitivity_keys: set[tuple[str, str, str, str, str, int]] = set()
- for cohort in doc["cohorts"]:
- if (
- cohort["kind"] not in {
- "dispatch-precision", "combine-precision",
- }
- or not cohort["eligibility"]["decision_grade"]
- ):
- continue
- members = [series[series_id] for series_id in cohort["series_ids"]]
- axis = (
- "dispatch"
- if cohort["kind"] == "dispatch-precision"
- else "combine"
- )
- field = f"{axis}_precision"
- bf16 = identity.precision_profile(
- identity.V1_CONTROL_PRECISION_PROFILE
- )[axis]
- baseline = next((
- member for member in members
- if _canonical(member["workload"][field]) == _canonical(bf16)
- ), None)
- if baseline is None:
- continue
- tokens = set.intersection(*(
- {point["tokens_per_rank"] for point in member["points"]}
- for member in members
- ))
- expected_sensitivity_keys.update(
- (cohort["cohort_id"], baseline["series_id"], candidate["series_id"],
- measure, statistic, token)
- for candidate in members if candidate is not baseline
- for token in tokens
- for measure in (
- "latency_us", "activation_data_rate_gbps_at_latency_percentile",
- "total_logical_data_rate_gbps_at_latency_percentile",
- )
- for statistic in ("p50", "p99")
- )
- if sensitivity_keys != expected_sensitivity_keys:
- raise PublisherError("sensitivities do not cover every declared contrast metric")
- observed_qualification_indices = sorted({
- item["qualification_index"] for item in doc["attempts"] if item["selected"]
- })
- if promotion["qualification_indices"] != observed_qualification_indices:
- raise PublisherError("promotion qualification index catalog differs from attempts")
- if promotion["status"] == "promoted":
- run_ids = {item["run_id"] for item in doc["attempts"] if item["selected"]}
- repeated_cases = all(
- {
- attempts[attempt_id]["qualification_index"]
- for attempt_id in coverage["attempt_ids"]
- if attempts[attempt_id]["selected"]
- } == {1}
- for coverage in doc["coverage"]
- )
- expected_scope = promotion.get("coverage_scope", "full")
- covered_skus = sorted({item["sku"] for item in doc["coverage"]})
- if "covered_skus" in promotion and promotion["covered_skus"] != covered_skus:
- raise PublisherError(
- "promotion covered_skus is inconsistent with covered cases"
- )
- if expected_scope == "full":
- # A full-coverage promotion is pinned to the exact canonical
- # case/disposition catalog (unchanged from the strict contract).
- if (
- _case_disposition_catalog_sha256(doc["coverage"])
- != CANONICAL_FULL_V1_CASE_CATALOG_SHA256
- ):
- raise PublisherError(
- "promotion requires the canonical case/disposition catalog"
- )
- else:
- # A partial ("v1 tag + success") promotion is not pinned to the full
- # catalog, but every case must still be a real canonical case with
- # the canonical disposition, and the subset must be a proper subset.
- canonical = _canonical_coverage_cases()
- covered = {item["case_id"] for item in doc["coverage"]}
- if covered - set(canonical):
- raise PublisherError(
- "promotion includes cases outside the canonical v1 catalog"
- )
- for item in doc["coverage"]:
- if item["disposition"] != canonical[item["case_id"]]["disposition"]:
- raise PublisherError(
- "promotion case disposition differs from the canonical v1 catalog"
- )
- if covered == set(canonical):
- raise PublisherError(
- "partial promotion must not cover the entire canonical catalog"
- )
- if (
- terminal != len(doc["coverage"])
- or promotion["qualification_indices"] != [1]
- or promotion["measured_cases"] + promotion["unsupported_cases"]
- != promotion["requested_cases"]
- or promotion["measured_points"] + promotion["unsupported_points"]
- != promotion["requested_points"]
- or promotion["terminal_points"] != promotion["requested_points"]
- or len(doc["source_bundle_ids"]) != REQUIRED_ALLOCATIONS
- or len(run_ids) != REQUIRED_ALLOCATIONS
- or not repeated_cases
- ):
- raise PublisherError("promoted dataset lacks complete coverage")
- expected_outcomes = {
- item["case_id"]: (
- "success" if item["disposition"] == "runnable" else "unsupported"
- )
- for item in doc["coverage"]
- }
- if any(
- item["selected"]
- and item["outcome"] != expected_outcomes[item["case_id"]]
- for item in doc["attempts"]
- ):
- raise PublisherError("promoted outcomes differ from requested dispositions")
- runnable_cases = {
- item["case_id"] for item in doc["coverage"]
- if item["disposition"] == "runnable"
- }
- if any(
- item["case_id"] in runnable_cases and item["outcome"] != "success"
- for item in doc["attempts"]
- ):
- raise PublisherError(
- "promotion rejects runnable cases with failed, invalid, or diagnostic retries"
- )
- _require_promotion_series(doc["series"])
- _require_promotion_cohorts(doc["cohorts"], doc["series"], scope=expected_scope)
- if expected_scope == "full" and not doc["rankings"]:
- raise PublisherError("promoted dataset lacks eligible rankings")
- if promotion["status"] == "quarantined" and any((
- doc["source_bundle_ids"], promotion["allocation_ids"], doc["coverage"],
- doc["attempts"], doc["series"], doc["cohorts"], doc["rankings"],
- doc["recommendations"], doc["sensitivities"],
- )):
- raise PublisherError("quarantined dataset exposes unvalidated evidence")
- return doc
-
-
-def _file_record(value: Any, path: str) -> dict[str, Any]:
- item = _exact(value, {"path", "sha256", "bytes"}, path)
- if not isinstance(item["path"], str) or PurePosixPath(item["path"]).is_absolute() or ".." in PurePosixPath(item["path"]).parts:
- raise PublisherError(f"{path}.path is unsafe")
- if not isinstance(item["sha256"], str) or HEX64.fullmatch(item["sha256"]) is None:
- raise PublisherError(f"{path}.sha256 is invalid")
- _integer(item["bytes"], f"{path}.bytes", minimum=1)
- return item
-
-def validate_bundle_manifest(doc: Any) -> dict[str, Any]:
- _schema("private-bundle-v1.schema.json", doc)
- attempts = {item["attempt_id"]: item for item in doc["attempts"]}
- if len(attempts) != len(doc["attempts"]):
- raise PublisherError("bundle has duplicate attempt IDs")
- selections = doc["coverage"]["selections"]
- if len({item["case_id"] for item in selections}) != len(selections):
- raise PublisherError("bundle has duplicate selected cases")
- counts = {name: 0 for name in OUTCOMES}
- for selection in selections:
- attempt = attempts.get(selection["selected_attempt_id"])
- if attempt is None or not attempt["selected"] or attempt["case_id"] != selection["case_id"] or attempt["outcome"] != selection["outcome"]:
- raise PublisherError("bundle selection differs from retained attempt")
- counts[selection["outcome"]] += 1
- coverage = doc["coverage"]
- if coverage["terminal_cases"] != len(selections) or coverage["outcome_counts"] != counts:
- raise PublisherError("bundle terminal counts differ")
- if coverage["complete"] != (coverage["expected_cases"] == len(selections)):
- raise PublisherError("bundle completeness differs from coverage")
- fingerprints: dict[str, set[str]] = {}
- for attempt in doc["attempts"]:
- value = attempt["runtime_fingerprint_sha256"]
- if value:
- fingerprints.setdefault(attempt["allocation_id"], set()).add(value)
- if any(len(values) != 1 for values in fingerprints.values()):
- raise PublisherError("bundle runtime is heterogeneous within an allocation")
- return doc
-
-
-def _fsync_dir(path: Path) -> None:
- descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0))
- try:
- os.fsync(descriptor)
- finally:
- os.close(descriptor)
-
-
-def _write_bytes(path: Path, data: bytes, *, mode: int) -> None:
- descriptor = os.open(
- path,
- os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0),
- mode,
- )
- try:
- os.fchmod(descriptor, mode)
- with os.fdopen(descriptor, "wb", closefd=False) as handle:
- handle.write(data)
- handle.flush()
- os.fsync(handle.fileno())
- finally:
- os.close(descriptor)
-
-
-def _write_all(descriptor: int, data: bytes) -> None:
- view = memoryview(data)
- while view:
- view = view[os.write(descriptor, view):]
-
-
-def _write_json(path: Path, value: Any, *, mode: int) -> bytes:
- data = _canonical(value) + b"\n"
- _write_bytes(path, data, mode=mode)
- return data
-
-
-def _file_metadata(path: Path, relative_to: Path) -> dict[str, Any]:
- return {
- "path": path.relative_to(relative_to).as_posix(),
- "sha256": _sha_file(path),
- "bytes": path.stat().st_size,
- }
-
-
-def _tree_files(root: Path) -> list[Path]:
- return sorted(
- path for path in root.rglob("*")
- if path.is_file() and not path.is_symlink() and path.name != "COMPLETE"
- )
-
-
-def _verify_regular_file(path: Path, expected_mode: int) -> None:
- _reject_symlinked_path(path.parent)
- try:
- metadata = os.lstat(path)
- except FileNotFoundError as exc:
- raise PublisherError(f"required file is missing: {path.name}") from exc
- if (
- not stat.S_ISREG(metadata.st_mode)
- or metadata.st_uid != os.getuid()
- or stat.S_IMODE(metadata.st_mode) != expected_mode
- ):
- raise PublisherError(
- f"file is not an owned regular {expected_mode:o} object: {path.name}"
- )
-
-
-def _verify_frozen_tree(root: Path, *, private: bool) -> None:
- _reject_symlinked_path(root)
- directory_mode = 0o500 if private else 0o555
- file_mode = 0o400 if private else 0o444
- try:
- root_metadata = os.lstat(root)
- except OSError as exc:
- raise PublisherError(f"cannot inspect immutable object: {root.name}") from exc
- if not stat.S_ISDIR(root_metadata.st_mode):
- raise PublisherError(f"immutable object is not a real directory: {root.name}")
- try:
- entries = [root, *root.rglob("*")]
- except OSError as exc:
- raise PublisherError(f"cannot inspect immutable object: {root.name}") from exc
- for path in entries:
- metadata = os.lstat(path)
- if metadata.st_uid != os.getuid():
- raise PublisherError(f"immutable object has the wrong owner: {path.name}")
- if stat.S_ISDIR(metadata.st_mode):
- expected = directory_mode
- elif stat.S_ISREG(metadata.st_mode):
- expected = file_mode
- else:
- raise PublisherError(f"immutable object contains a linked or special entry: {path.name}")
- if stat.S_IMODE(metadata.st_mode) != expected:
- raise PublisherError(
- f"immutable object mode differs for {path.name}: expected {expected:o}"
- )
-
-
-def _freeze_tree(root: Path, *, private: bool) -> None:
- files: list[Path] = []
- directories = [root]
- for path in root.rglob("*"):
- metadata = os.lstat(path)
- if stat.S_ISDIR(metadata.st_mode):
- directories.append(path)
- elif stat.S_ISREG(metadata.st_mode):
- files.append(path)
- else:
- raise PublisherError(f"immutable object contains a linked or special entry: {path.name}")
- for path in files:
- os.chmod(path, 0o400 if private else 0o444)
- for path in sorted(directories, key=lambda item: len(item.parts), reverse=True):
- os.chmod(path, 0o500 if private else 0o555)
- _fsync_dir(path)
- _verify_frozen_tree(root, private=private)
-
-
-def _reject_symlinked_path(path: Path) -> None:
- current = Path(path.anchor)
- for part in path.parts[1:]:
- current /= part
- try:
- metadata = os.lstat(current)
- except FileNotFoundError:
- break
- if stat.S_ISLNK(metadata.st_mode):
- raise PublisherError("COLLECTIVEX_STORE_ROOT must not traverse a symlinked parent")
- if not stat.S_ISDIR(metadata.st_mode):
- raise PublisherError(f"store path component is not a directory: {current}")
-
-
-class Store:
- """Atomic private/public directory operations on one operator filesystem."""
-
- def __init__(self, root: str | os.PathLike[str]):
- candidate = Path(os.path.abspath(os.path.expanduser(root)))
- _reject_symlinked_path(candidate)
- candidate.mkdir(parents=True, exist_ok=True, mode=0o750)
- resolved = candidate.resolve()
- if candidate != resolved:
- raise PublisherError(
- "COLLECTIVEX_STORE_ROOT must not traverse a symlinked parent"
- )
- root_metadata = candidate.stat()
- if root_metadata.st_uid != os.getuid() or stat.S_IMODE(root_metadata.st_mode) & 0o022:
- raise PublisherError(
- "COLLECTIVEX_STORE_ROOT must be owned by this user and not group/world writable"
- )
- os.chmod(candidate, 0o750)
- if stat.S_IMODE(candidate.stat().st_mode) != 0o750:
- raise PublisherError("COLLECTIVEX_STORE_ROOT mode must be 750")
- self.root = resolved
- raw = self.root
- self.private = raw / "private"
- self.incoming = self.private / "incoming"
- self.bundles = self.private / "bundles"
- self.quarantine = self.private / "quarantine"
- self.public = raw / "public"
- self.datasets = self.public / "datasets"
- self.channels = self.public / "channels"
- self.locks = raw / "locks"
- for path, mode in (
- (self.private, 0o700), (self.incoming, 0o700), (self.bundles, 0o700),
- (self.quarantine, 0o700), (self.public, 0o755), (self.datasets, 0o755),
- (self.channels, 0o755), (self.locks, 0o700),
- ):
- path.mkdir(parents=True, exist_ok=True, mode=mode)
- if path.is_symlink() or not path.is_dir():
- raise PublisherError(f"store path is not a real directory: {path}")
- os.chmod(path, mode)
-
- @contextlib.contextmanager
- def locked(self) -> Iterator[None]:
- lock_path = self.locks / "publisher.lock"
- descriptor = os.open(
- lock_path,
- os.O_RDWR | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0),
- 0o600,
- )
- try:
- os.fchmod(descriptor, 0o600)
- metadata = os.fstat(descriptor)
- if (
- not stat.S_ISREG(metadata.st_mode)
- or metadata.st_uid != os.getuid()
- or stat.S_IMODE(metadata.st_mode) != 0o600
- ):
- raise PublisherError("publisher lock is not an owned regular 600 file")
- fcntl.flock(descriptor, fcntl.LOCK_EX)
- yield
- finally:
- fcntl.flock(descriptor, fcntl.LOCK_UN)
- os.close(descriptor)
-
- @contextlib.contextmanager
- def staging(self, parent: Path, *, private: bool) -> Iterator[Path]:
- stage = Path(tempfile.mkdtemp(prefix=".staging-", dir=parent))
- os.chmod(stage, 0o700 if private else 0o755)
- try:
- yield stage
- finally:
- if stage.exists():
- for path in stage.rglob("*"):
- metadata = os.lstat(path)
- if stat.S_ISDIR(metadata.st_mode):
- os.chmod(path, 0o700)
- elif stat.S_ISREG(metadata.st_mode):
- os.chmod(path, 0o600)
- os.chmod(stage, 0o700)
- shutil.rmtree(stage, ignore_errors=True)
-
- @staticmethod
- def complete(stage: Path, value: str, *, private: bool) -> None:
- _write_bytes(stage / "COMPLETE", (value + "\n").encode(), mode=0o600 if private else 0o644)
- _fsync_dir(stage)
-
- @staticmethod
- def install(stage: Path, destination: Path, *, private: bool) -> None:
- if destination.is_symlink():
- raise PublisherError(f"immutable destination is a symlink: {destination.name}")
- if destination.exists():
- _verify_frozen_tree(destination, private=private)
- marker = destination / "COMPLETE"
- if not marker.is_file() or marker.read_text().strip() != destination.name:
- raise PublisherError(f"immutable destination is incomplete: {destination.name}")
- return
- _freeze_tree(stage, private=private)
- os.rename(stage, destination)
- _fsync_dir(destination.parent)
- _verify_frozen_tree(destination, private=private)
-
- def install_dataset(self, dataset: dict[str, Any]) -> tuple[str, int]:
- validate_public_dataset(dataset)
- payload = _canonical(dataset) + b"\n"
- if len(payload) > MAX_PUBLIC_DATASET_BYTES:
- raise PublisherError("public dataset exceeds the serving size limit")
- digest = _sha_bytes(payload)
- destination = self.datasets / digest
- with self.staging(self.datasets, private=False) as stage:
- _write_bytes(stage / "dataset.json", payload, mode=0o644)
- self.complete(stage, digest, private=False)
- self.install(stage, destination, private=False)
- stored = destination / "dataset.json"
- marker = destination / "COMPLETE"
- if (not marker.is_file() or marker.read_text().strip() != digest
- or _sha_file(stored) != digest or stored.stat().st_size != len(payload)):
- raise PublisherError("stored dataset checksum differs after installation")
- return digest, len(payload)
-
- def update_channel(self, channel: str, digest: str, size: int, generated_at: str) -> None:
- if size > MAX_PUBLIC_DATASET_BYTES:
- raise PublisherError("channel dataset exceeds the serving size limit")
- _verify_frozen_tree(self.datasets / digest, private=False)
- marker = self.datasets / digest / "COMPLETE"
- if not marker.is_file() or marker.read_text().strip() != digest:
- raise PublisherError("cannot advance a channel to an incomplete dataset")
- dataset_path = self.datasets / digest / "dataset.json"
- dataset = validate_public_dataset(strict_load(dataset_path))
- if (
- _sha_file(dataset_path) != digest
- or dataset_path.stat().st_size != size
- or dataset["generated_at"] != generated_at
- ):
- raise PublisherError("channel metadata differs from its stored dataset")
- if channel == "dev-latest" and dataset["promotion"]["status"] != "promoted":
- raise PublisherError("dev-latest may only reference a promoted dataset")
- pointer = {
- "format": FORMAT_CHANNEL,
- "channel": channel,
- "dataset": {
- "path": f"datasets/{digest}/dataset.json",
- "sha256": digest,
- "bytes": size,
- },
- "generated_at": generated_at,
- }
- validate_channel(pointer, expected_channel=channel)
- destination = self.channels / f"{channel}.json"
- temporary = self.channels / f".{channel}.tmp-{os.getpid()}"
- try:
- data = _canonical(pointer) + b"\n"
- _write_bytes(temporary, data, mode=0o644)
- os.replace(temporary, destination)
- _fsync_dir(self.channels)
- finally:
- temporary.unlink(missing_ok=True)
-
- def verify_channel(self, channel: str) -> dict[str, Any]:
- channel_path = self.channels / f"{channel}.json"
- _verify_regular_file(channel_path, 0o644)
- pointer = validate_channel(strict_load(channel_path), expected_channel=channel)
- target = self.public / pointer["dataset"]["path"]
- _verify_frozen_tree(target.parent, private=False)
- if target.stat().st_size != pointer["dataset"]["bytes"] or _sha_file(target) != pointer["dataset"]["sha256"]:
- raise PublisherError(f"channel {channel} dataset checksum differs")
- marker = target.parent / "COMPLETE"
- if not marker.is_file() or marker.read_text().strip() != pointer["dataset"]["sha256"]:
- raise PublisherError(f"channel {channel} dataset is incomplete")
- dataset = validate_public_dataset(strict_load(target))
- if pointer["generated_at"] != dataset["generated_at"]:
- raise PublisherError(f"channel {channel} metadata differs from its dataset")
- if channel == "dev-latest" and dataset["promotion"]["status"] != "promoted":
- raise PublisherError("dev-latest points to a non-promoted dataset")
- return pointer
-
-
-def _copy_source(source: Path, destination: Path) -> None:
- if source.is_symlink() or not source.is_file() or not stat.S_ISREG(source.stat().st_mode):
- raise PublisherError(f"source must be a regular non-symlink file: {source}")
- descriptor = os.open(source, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
- try:
- output = os.open(destination, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
- try:
- while True:
- chunk = os.read(descriptor, 1024 * 1024)
- if not chunk:
- break
- _write_all(output, chunk)
- os.fsync(output)
- finally:
- os.close(output)
- finally:
- os.close(descriptor)
-
-
-def _archive_download_directory(source: Path, destination: Path) -> None:
- if source.is_symlink() or not source.is_dir():
- raise PublisherError(f"artifact directory is invalid: {source}")
- files: list[Path] = []
- for path in source.rglob("*"):
- if path.is_symlink():
- raise PublisherError("artifact directory contains a symlink")
- if path.is_dir():
- continue
- if not path.is_file():
- raise PublisherError("artifact directory contains a non-regular entry")
- files.append(path)
- files.sort()
- if not files or len(files) > MAX_ARCHIVE_MEMBERS:
- raise PublisherError("artifact directory has an invalid file count")
- total = 0
- with zipfile.ZipFile(destination, "x", compression=zipfile.ZIP_STORED) as archive:
- for path in files:
- descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
- with os.fdopen(descriptor, "rb") as handle:
- metadata = os.fstat(handle.fileno())
- if not stat.S_ISREG(metadata.st_mode):
- raise PublisherError("artifact directory member changed type")
- size = metadata.st_size
- total += size
- if size > MAX_ARCHIVE_MEMBER_BYTES or total > MAX_ARCHIVE_TOTAL_BYTES:
- raise PublisherError("artifact directory exceeds size limits")
- relative = path.relative_to(source).as_posix()
- _safe_member(relative)
- info = zipfile.ZipInfo(relative, date_time=(1980, 1, 1, 0, 0, 0))
- info.compress_type = zipfile.ZIP_STORED
- info.external_attr = (stat.S_IFREG | 0o600) << 16
- with archive.open(info, "w") as output:
- written = 0
- for chunk in iter(lambda: handle.read(1024 * 1024), b""):
- output.write(chunk)
- written += len(chunk)
- if written != size:
- raise PublisherError("artifact directory member changed size")
- descriptor = os.open(destination, os.O_RDONLY)
- try:
- os.fsync(descriptor)
- finally:
- os.close(descriptor)
-
-
-def _artifact_name(source: Path) -> str:
- name = source.name if source.is_dir() else source.name.removesuffix(".zip")
- if (
- not source.is_dir() and source.suffix != ".zip"
- or ARTIFACT_NAME.fullmatch(name) is None
- ):
- raise PublisherError(f"artifact source has an invalid GHA name: {source.name}")
- return name
-
-
-def archive_incoming(
- store: Store,
- matrix: Path,
- artifacts: Sequence[Path],
- run: dict[str, Any],
-) -> tuple[str, Path, list[dict[str, Any]]]:
- """Copy exact delivery bytes into immutable incoming before any JSON/ZIP parse."""
- if not artifacts:
- raise PublisherError("at least one GitHub artifact archive is required")
- with store.staging(store.incoming, private=True) as stage:
- sources = stage / "sources"
- sources.mkdir(mode=0o700)
- copied: list[dict[str, Any]] = []
- named_artifacts = sorted(
- ((_artifact_name(path), path) for path in artifacts), key=lambda item: item[0]
- )
- artifact_names = [name for name, _ in named_artifacts]
- if len(artifact_names) != len(set(artifact_names)):
- raise PublisherError("artifact delivery contains duplicate GHA names")
- inputs = [("matrix.json", matrix, "matrix", None)] + [
- (f"artifact-{index:04d}.zip", path, "artifact", artifact_name)
- for index, (artifact_name, path) in enumerate(named_artifacts)
- ]
- for name, source, kind, artifact_name in inputs:
- destination = sources / name
- if source.is_dir():
- _archive_download_directory(source, destination)
- else:
- if source != matrix and source.stat().st_size > MAX_ARCHIVE_TOTAL_BYTES:
- raise PublisherError("artifact archive exceeds the size limit")
- _copy_source(source, destination)
- copied.append({
- **_file_metadata(destination, stage),
- "kind": kind,
- "artifact_name": artifact_name,
- })
- ingest_id = _sha_bytes(_canonical({"run": run, "sources": copied}))
- incoming_manifest = {
- "format": "collectivex.incoming.v1",
- "schema_version": 1,
- "ingest_id": ingest_id,
- "run": run,
- "sources": copied,
- }
- _write_json(stage / "incoming.json", incoming_manifest, mode=0o600)
- store.complete(stage, ingest_id, private=True)
- destination = store.incoming / ingest_id
- store.install(stage, destination, private=True)
- installed = store.incoming / ingest_id
- if strict_load(installed / "incoming.json") != incoming_manifest:
- raise PublisherError("existing incoming object differs from archived delivery")
- for record in copied:
- _resolve_bundle_file(installed, record)
- return ingest_id, installed, copied
-
-
-def _safe_member(name: str) -> PurePosixPath:
- if "\\" in name or "\0" in name:
- raise PublisherError("archive member has an unsafe separator")
- path = PurePosixPath(name)
- if path.is_absolute() or not path.parts or any(part in {"", ".", ".."} for part in path.parts):
- raise PublisherError("archive member path escapes its artifact")
- return path
-
-
-def extract_archive(archive: Path, destination: Path) -> list[Path]:
- """Extract a bounded regular-file ZIP without trusting member paths or links."""
- try:
- handle = zipfile.ZipFile(archive)
- except (OSError, zipfile.BadZipFile) as exc:
- raise PublisherError("artifact is not a valid ZIP archive") from exc
- extracted: list[Path] = []
- seen: set[str] = set()
- total = 0
- with handle:
- members = handle.infolist()
- if not members or len(members) > MAX_ARCHIVE_MEMBERS:
- raise PublisherError("artifact has an invalid member count")
- for member in members:
- path = _safe_member(member.filename.rstrip("/"))
- key = path.as_posix()
- if key in seen:
- raise PublisherError("artifact contains duplicate member paths")
- seen.add(key)
- mode = member.external_attr >> 16
- if stat.S_ISLNK(mode) or (mode and not (stat.S_ISREG(mode) or stat.S_ISDIR(mode))):
- raise PublisherError("artifact contains a non-regular member")
- if member.flag_bits & 0x1:
- raise PublisherError("encrypted artifact members are not accepted")
- if member.file_size > MAX_ARCHIVE_MEMBER_BYTES:
- raise PublisherError("artifact member exceeds the size limit")
- total += member.file_size
- if total > MAX_ARCHIVE_TOTAL_BYTES:
- raise PublisherError("artifact exceeds the expanded size limit")
- target = destination.joinpath(*path.parts)
- if member.is_dir():
- target.mkdir(parents=True, exist_ok=True, mode=0o700)
- continue
- target.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
- output = os.open(target, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
- try:
- with handle.open(member, "r") as source:
- written = 0
- while True:
- chunk = source.read(1024 * 1024)
- if not chunk:
- break
- _write_all(output, chunk)
- written += len(chunk)
- if written != member.file_size:
- raise PublisherError("artifact member size changed during extraction")
- os.fsync(output)
- finally:
- os.close(output)
- extracted.append(target)
- return extracted
-
-
-def validate_matrix(document: Any) -> list[dict[str, Any]]:
- try:
- artifact_safety.assert_publication_safe([document])
- matrix = sweep_matrix.validate_matrix_document(document)
- except (SystemExit, ValueError, artifact_safety.ArtifactSafetyError) as exc:
- raise PublisherError(f"requested matrix is invalid: {exc}") from exc
- return [
- {
- "sku": item["sku"],
- **item["case"],
- "_disposition": item["disposition"],
- "_reason": item["reason"],
- }
- for item in matrix["requested_cases"]
- ]
-
-
-def _expected_deliveries(
- matrix: dict[str, Any], cases: Sequence[dict[str, Any]], run: dict[str, Any]
-) -> dict[str, tuple[str, str, str]]:
- shard_by_case: dict[str, str] = {}
- for shard in matrix["include"]:
- for case_id in shard["case_ids"]:
- if case_id in shard_by_case:
- raise PublisherError("requested case appears in two runnable shards")
- shard_by_case[case_id] = shard["id"]
- suffix = f"{run['run_id']}-{run['run_attempt']}"
- deliveries: dict[str, tuple[str, str, str]] = {}
- for case in cases:
- case_id = case["case_id"]
- if case["_disposition"] == "unsupported":
- deliveries[case_id] = (
- f"cxunsupported-{suffix}", "setup",
- f"{run['run_id']}_{run['run_attempt']}_unsupported",
- )
- continue
- shard_id = shard_by_case.get(case_id)
- if shard_id is None:
- raise PublisherError("runnable case has no matrix shard")
- deliveries[case_id] = (
- f"cxshard-{shard_id}-{suffix}", "sweep",
- f"{run['run_id']}_{run['run_attempt']}_{shard_id}",
- )
- return deliveries
-
-
-def _document_git_run(document: dict[str, Any]) -> dict[str, Any] | None:
- provenance = document.get("provenance")
- if not isinstance(provenance, dict):
- return None
- value = provenance.get("git_run", provenance)
- return value if isinstance(value, dict) else None
-
-
-def _run_matches(document: dict[str, Any], run: dict[str, Any]) -> bool:
- git_run = _document_git_run(document)
- if git_run is None:
- return False
- return (
- str(git_run.get("run_id")) == run["run_id"]
- and str(git_run.get("run_attempt")) == str(run["run_attempt"])
- and git_run.get("qualification_index") == run["qualification_index"]
- and git_run.get("source_sha") == run["source_sha"]
- and (git_run.get("repo") or git_run.get("repository")) == run["repository"]
- )
-
-
-def _case_matches(document: dict[str, Any], expected: dict[str, Any]) -> bool:
- scheduled = {
- key: value for key, value in expected.items()
- if key not in {"sku", "case_id"} and not key.startswith("_")
- }
- return document.get("identity", {}).get("case_factors") == {
- "case": scheduled,
- "profile": identity.profile_for_case(scheduled),
- "sku": expected["sku"],
- }
-
-
-def _outcome(document: dict[str, Any]) -> tuple[str, str | None]:
- status = document["outcome"]["status"]
- if status == "success":
- return status, None
- native = document["outcome"].get("reason")
- reason = native if isinstance(native, str) and REASON.fullmatch(native) else {
- "unsupported": "unsupported-capability", "failed": "execution-failed",
- "invalid": "validation-failed", "diagnostic": "diagnostic-evidence",
- }.get(status)
- if reason is None:
- raise PublisherError(f"unsupported native outcome {status!r}")
- return status, reason
-
-
-def _attempt_record(
- document: dict[str, Any], path: Path, root: Path, *, selected: bool
-) -> dict[str, Any]:
- normalized = contracts.normalize_attempt(document)
- runtime = normalized["runtime_fingerprint"]
- runtime_sha = _sha_bytes(_canonical(runtime)) if runtime is not None else None
- sample_record = None
- evidence_ids: list[str] = []
- series_ids: list[str] = []
- if document["format"] == contracts.RAW_FORMAT:
- sample_path = path.with_name(document["sample_artifact"]["path"])
- sample_record = _file_metadata(sample_path, root)
- evidence_ids = [row["evidence_id"] for row in document["measurement"]["rows"]]
- series_ids = [document["identity"]["series_id"]]
- declared = document["identity"]["series_factors"]["runtime_fingerprint_sha256"]
- if runtime_sha != declared:
- raise PublisherError("runtime fingerprint checksum differs from series identity")
- status, reason = _outcome(document)
- return {
- "attempt_id": normalized["attempt_id"],
- "allocation_id": normalized["allocation_id"],
- "case_id": normalized["case_id"],
- "outcome": status,
- "reason": reason,
- "selected": selected,
- "document": _file_metadata(path, root),
- "samples": sample_record,
- "runtime_fingerprint_sha256": runtime_sha,
- "series_ids": series_ids,
- "evidence_ids": evidence_ids,
- }
-
-
-def _validate_delivery_binding(
- document: dict[str, Any], path: Path, raw_root: Path,
- artifact_by_root: dict[str, str], expected_by_id: dict[str, dict[str, Any]],
- expected_deliveries: dict[str, tuple[str, str, str]], run: dict[str, Any],
-) -> str:
- case_id = document["identity"]["case_id"]
- if case_id not in expected_by_id:
- raise PublisherError("artifact contains an extra case outcome")
- expected = expected_by_id[case_id]
- if not _case_matches(document, expected):
- raise PublisherError("attempt case coordinates differ from the requested matrix")
- unsupported = document["outcome"]["status"] == "unsupported"
- if (expected["_disposition"] == "unsupported") != unsupported:
- raise PublisherError("terminal outcome differs from requested capability disposition")
- if unsupported and document["outcome"]["reason"] != expected["_reason"]:
- raise PublisherError("unsupported outcome reason differs from requested matrix")
- if not _run_matches(document, run):
- raise PublisherError("attempt provenance differs from publisher run metadata")
- relative = path.relative_to(raw_root)
- if len(relative.parts) < 2:
- raise PublisherError("attempt document is outside a delivered artifact")
- delivered_name = artifact_by_root.get(relative.parts[0])
- expected_name, expected_job, expected_execution = expected_deliveries[case_id]
- git_run = _document_git_run(document)
- allocation = document["identity"]["allocation_factors"]
- if (
- git_run is None
- or delivered_name != expected_name
- or git_run["artifact"] != delivered_name
- or git_run["job"] != expected_job
- or allocation["execution_id"] != expected_execution
- ):
- raise PublisherError("attempt provenance differs from its delivered GHA shard")
- return case_id
-
-
-def _parse_extracted(root: Path) -> tuple[list[tuple[Path, dict[str, Any]]], set[Path]]:
- attempts: list[tuple[Path, dict[str, Any]]] = []
- consumed_samples: set[Path] = set()
- json_paths = sorted(path for path in root.rglob("*.json") if path.is_file())
- for path in json_paths:
- if path in consumed_samples:
- continue
- try:
- document = contracts.strict_load(path)
- artifact_safety.assert_publication_safe([document])
- format_name = document.get("format") if isinstance(document, dict) else None
- if format_name == contracts.SAMPLES_FORMAT:
- _schema("samples-v1.schema.json", document)
- # It must be claimed by a raw document; orphan checking happens after the scan.
- continue
- if format_name == contracts.RAW_FORMAT:
- _schema("raw-case-v1.schema.json", document)
- sample_path = path.with_name(document["sample_artifact"]["path"])
- sample_document = contracts.strict_load(sample_path)
- artifact_safety.assert_publication_safe([sample_document])
- _schema("samples-v1.schema.json", sample_document)
- validated = contracts.load_raw_attempt(path)
- consumed_samples.add(sample_path)
- elif format_name == contracts.TERMINAL_FORMAT:
- _schema("terminal-outcome-v1.schema.json", document)
- validated = contracts.validate_terminal_document(document)
- else:
- raise PublisherError(f"artifact contains unknown JSON document {path.name}")
- except (
- contracts.ContractError, artifact_safety.ArtifactSafetyError,
- jsonschema.ValidationError, OSError,
- ) as exc:
- raise PublisherError(f"native contract rejected {path.name}: {exc}") from exc
- attempts.append((path, validated))
- orphan_samples = [
- path for path in json_paths
- if isinstance((doc := contracts.strict_load(path)), dict)
- and doc.get("format") == contracts.SAMPLES_FORMAT
- and path not in consumed_samples
- ]
- if orphan_samples:
- raise PublisherError("artifact contains an orphan samples document")
- if not attempts:
- raise PublisherError("artifact contains zero native attempt documents")
- return attempts, consumed_samples
-
-
-def build_bundle(
- store: Store,
- incoming_id: str,
- incoming_path: Path,
- run: dict[str, Any],
-) -> tuple[str, dict[str, Any], list[dict[str, Any]]]:
- """Validate one exact workflow delivery and install its immutable private bundle."""
- incoming_manifest = strict_load(incoming_path / "incoming.json")
- _exact(
- incoming_manifest,
- {"format", "schema_version", "ingest_id", "run", "sources"},
- "incoming",
- )
- artifact_safety.assert_publication_safe([incoming_manifest])
- if (
- incoming_manifest["format"] != "collectivex.incoming.v1"
- or incoming_manifest["schema_version"] != 1
- or incoming_manifest["ingest_id"] != incoming_id
- or incoming_manifest["run"] != run
- or _sha_bytes(_canonical({"run": run, "sources": incoming_manifest["sources"]}))
- != incoming_id
- ):
- raise PublisherError("incoming manifest identity differs from archived delivery")
- incoming_sources = _array(incoming_manifest["sources"], "incoming.sources", nonempty=True)
- for index, record in enumerate(incoming_sources):
- _exact(
- record,
- {"path", "sha256", "bytes", "kind", "artifact_name"},
- f"incoming.sources[{index}]",
- )
- _resolve_bundle_file(incoming_path, record)
- matrix_records = [record for record in incoming_sources if record["kind"] == "matrix"]
- artifact_records = [record for record in incoming_sources if record["kind"] == "artifact"]
- if (
- len(matrix_records) != 1
- or matrix_records[0]["artifact_name"] is not None
- or not artifact_records
- or any(ARTIFACT_NAME.fullmatch(record["artifact_name"] or "") is None
- for record in artifact_records)
- or len({record["artifact_name"] for record in artifact_records}) != len(artifact_records)
- ):
- raise PublisherError("incoming source catalog is invalid")
- matrix_source = _resolve_bundle_file(incoming_path, matrix_records[0])
- matrix_document = strict_load(matrix_source)
- expected_cases = validate_matrix(matrix_document)
- expected_by_id = {case["case_id"]: case for case in expected_cases}
- expected_deliveries = _expected_deliveries(matrix_document, expected_cases, run)
- if {record["artifact_name"] for record in artifact_records} != {
- delivery[0] for delivery in expected_deliveries.values()
- }:
- raise PublisherError("incoming artifact archive set differs from requested matrix shards")
- with store.staging(store.bundles, private=True) as stage:
- source_copy = stage / "source"
- raw_root = stage / "raw"
- source_copy.mkdir(mode=0o700)
- raw_root.mkdir(mode=0o700)
- matrix_path = stage / "matrix.json"
- _copy_source(matrix_source, matrix_path)
- source_records: list[dict[str, Any]] = []
- artifact_by_root: dict[str, str] = {}
- for index, source_record in enumerate(artifact_records):
- archive = _resolve_bundle_file(incoming_path, source_record)
- copied = source_copy / f"artifact-{index:04d}.zip"
- _copy_source(archive, copied)
- source_records.append({
- **_file_metadata(copied, stage),
- "artifact_name": source_record["artifact_name"],
- })
- artifact_root = raw_root / f"artifact-{index:04d}"
- artifact_root.mkdir(mode=0o700)
- artifact_by_root[artifact_root.name] = source_record["artifact_name"]
- extract_archive(copied, artifact_root)
- parsed, consumed_samples = _parse_extracted(raw_root)
- created_at = _latest_timestamp(
- [document["generated_at"] for _, document in parsed]
- )
- consumed_files = {path for path, _ in parsed} | consumed_samples
- extracted_files = {
- path for path in raw_root.rglob("*")
- if path.is_file() and not path.is_symlink()
- }
- if consumed_files != extracted_files:
- raise PublisherError("artifact contains an unconsumed non-native member")
- by_case: dict[str, list[tuple[Path, dict[str, Any]]]] = {}
- for path, document in parsed:
- case_id = _validate_delivery_binding(
- document, path, raw_root, artifact_by_root, expected_by_id,
- expected_deliveries, run,
- )
- by_case.setdefault(case_id, []).append((path, document))
- missing = set(expected_by_id) - set(by_case)
- if missing:
- raise PublisherError(f"artifact is missing {len(missing)} requested case outcomes")
- attempt_records: list[dict[str, Any]] = []
- selections: list[dict[str, Any]] = []
- selected_documents: list[dict[str, Any]] = []
- runtime_hashes: set[str] = set()
- outcome_counts = {name: 0 for name in OUTCOMES}
- for case_id in sorted(expected_by_id):
- case_attempts = by_case[case_id]
- ordinals = [document["identity"]["attempt_ordinal"] for _, document in case_attempts]
- allocations_for_case = {
- document["identity"]["allocation_id"] for _, document in case_attempts
- }
- if len(allocations_for_case) != 1 or sorted(ordinals) != list(
- range(1, len(ordinals) + 1)
- ):
- raise PublisherError(
- "case retries must retain contiguous ordinals in one allocation"
- )
- _, selected_document = max(
- case_attempts, key=lambda item: item[1]["identity"]["attempt_ordinal"]
- )
- selected_id = selected_document["identity"]["attempt_id"]
- selected_documents.append(selected_document)
- selected_status, _ = _outcome(selected_document)
- selections.append({
- "case_id": case_id,
- "selected_attempt_id": selected_id,
- "outcome": selected_status,
- })
- outcome_counts[selected_status] += 1
- for path, document in sorted(
- case_attempts, key=lambda item: item[1]["identity"]["attempt_ordinal"]
- ):
- normalized = contracts.normalize_attempt(document)
- if document["format"] == contracts.RAW_FORMAT:
- sample_path = path.with_name(document["sample_artifact"]["path"])
- if sample_path not in consumed_samples:
- raise PublisherError("validated raw attempt lost its samples document")
- record = _attempt_record(
- document, path, stage,
- selected=normalized["attempt_id"] == selected_id,
- )
- if record["runtime_fingerprint_sha256"]:
- runtime_hashes.add(record["runtime_fingerprint_sha256"])
- attempt_records.append(record)
- # Every extracted byte is covered; the bundle manifest anchors this checksum catalog.
- payload_records = [_file_metadata(path, stage) for path in _tree_files(stage)]
- checksum_document = {
- "format": "collectivex.checksums.v1",
- "files": payload_records,
- }
- checksum_path = stage / "checksums.json"
- _write_json(checksum_path, checksum_document, mode=0o600)
- bundle = {
- "format": FORMAT_BUNDLE,
- "schema_version": 1,
- "created_at": created_at,
- "ingest_id": incoming_id,
- "run": run,
- "matrix": _file_metadata(matrix_path, stage),
- "sources": source_records,
- "attempts": attempt_records,
- "coverage": {
- "expected_cases": len(expected_cases),
- "terminal_cases": len(selections),
- "complete": len(selections) == len(expected_cases),
- "outcome_counts": outcome_counts,
- "selections": selections,
- },
- "runtime_fingerprints": sorted(runtime_hashes),
- "checksums": _file_metadata(checksum_path, stage),
- "validation": {
- "policy": PUBLISHER_POLICY,
- "passed": True,
- "checks": [
- "archive-safety", "checksums", "exact-coverage", "identity",
- "native-schema", "privacy", "runtime-homogeneity", "terminal-outcomes",
- ],
- },
- }
- validate_bundle_manifest(bundle)
- # Runtime homogeneity is scoped to a realized allocation, not across unlike SKUs.
- by_allocation: dict[str, set[str]] = {}
- for attempt in attempt_records:
- fingerprint = attempt["runtime_fingerprint_sha256"]
- if fingerprint:
- by_allocation.setdefault(attempt["allocation_id"], set()).add(fingerprint)
- if any(len(values) != 1 for values in by_allocation.values()):
- raise PublisherError("runtime fingerprint is heterogeneous within an allocation")
- bundle_bytes = _canonical(bundle) + b"\n"
- bundle_id = _sha_bytes(bundle_bytes)
- _write_bytes(stage / "bundle.json", bundle_bytes, mode=0o600)
- store.complete(stage, bundle_id, private=True)
- store.install(stage, store.bundles / bundle_id, private=True)
- installed = load_bundle(store, bundle_id)
- if installed["manifest"] != bundle:
- raise PublisherError("existing bundle differs from validated manifest")
- return bundle_id, bundle, selected_documents
-
-
-def _slug(value: Any, fallback: str = "unknown") -> str:
- text = re.sub(r"[^a-z0-9_.-]+", "-", str(value or "").lower()).strip("-.")
- return text[:128] if text and SAFE_ID.fullmatch(text[:128]) else fallback
-
-
-def _derived_id(prefix: str, value: Any) -> str:
- return f"{prefix}{_sha_bytes(_canonical(value))}"
-
-
-def _git_run(document: dict[str, Any]) -> dict[str, Any]:
- return _document_git_run(document) or {}
-
-
-def _public_attempt(document: dict[str, Any], *, selected: bool = False) -> dict[str, Any]:
- normalized = contracts.normalize_attempt(document)
- run = _git_run(document)
- evidence = (
- [{"evidence_id": row["evidence_id"], "point_id": row["point_id"]}
- for row in document["measurement"]["rows"]]
- if document["format"] == contracts.RAW_FORMAT else []
- )
- status, reason = _outcome(document)
- failure_mode = document["outcome"].get("failure_mode")
- if not isinstance(failure_mode, str) or REASON.fullmatch(failure_mode) is None:
- failure_mode = None if status == "success" else reason
- series_id = normalized["series_id"] if status == "success" and selected else None
- return {
- "attempt_id": normalized["attempt_id"],
- "evidence": evidence,
- "case_id": normalized["case_id"],
- "allocation_id": normalized["allocation_id"],
- "run_id": str(run["run_id"]),
- "run_attempt": int(run["run_attempt"]),
- "qualification_index": int(run["qualification_index"]),
- "attempt_index": document["identity"]["attempt_ordinal"],
- "selected": selected,
- "outcome": status,
- "failure_mode": failure_mode,
- "reason": reason,
- "series_id": series_id,
- "completed_at": document["generated_at"],
- }
-
-
-def _private_trial_components(sample_document: dict[str, Any]) -> dict[int, dict[str, Any]]:
- """Copy validated trial blocks into publisher-private memory without fixing component names."""
- points: dict[int, dict[str, Any]] = {}
- for point in sample_document["points"]:
- token = point["tokens_per_rank"]
- components: dict[str, Any] = {}
- for name, component in point["components"].items():
- availability = component["availability"]
- if availability in {"unavailable", "not-applicable"}:
- components[name] = None
- continue
- if availability != "measured":
- raise PublisherError(f"private sample component {name} has invalid availability")
- trials = component["trials"]
- if (
- not isinstance(trials, list)
- or len(trials) != 64
- or any(not isinstance(trial, list) or len(trial) != 8 for trial in trials)
- ):
- raise PublisherError(f"private sample component {name} is not 64x8")
- copied = tuple(
- tuple(float(sample) for sample in trial)
- for trial in trials
- )
- if any(
- not math.isfinite(sample) or sample < 0
- for trial in copied for sample in trial
- ):
- raise PublisherError(f"private sample component {name} is not finite")
- components[name] = copied
- points[token] = components
- return points
-
-
-def _trial_diagnostics(
- trial_blocks: dict[str, dict[int, dict[str, Any]]], token: int,
-) -> dict[str, Any]:
- components: dict[str, Any] = {}
- reasons: set[str] = set()
- for name in ("dispatch", "stage", "combine", "roundtrip"):
- values = [trial_blocks[run_id][token][name] for run_id in sorted(trial_blocks)]
- if all(value is None for value in values):
- components[name] = None
- continue
- if any(value is None for value in values):
- raise PublisherError(f"{name} trial availability differs across qualification runs")
- array = np.asarray(values, dtype=np.float64)
- if array.shape != (REQUIRED_ALLOCATIONS, 64, 8) or not np.isfinite(array).all():
- raise PublisherError(f"{name} trial diagnostics require finite 64x8 runs")
- medians = np.median(array, axis=2)
- first = np.median(medians[:, :8], axis=1)
- last = np.median(medians[:, -8:], axis=1)
- if np.any(first <= 0) or np.any(last <= 0):
- raise PublisherError(f"{name} trial diagnostics require positive latency")
- drift_ratio = float(np.max(np.maximum(first / last, last / first)))
- center = float(np.median(medians))
- mad = float(np.median(np.abs(medians - center)))
- if mad == 0:
- outliers = np.abs(medians - center) > 0
- else:
- outliers = np.abs(medians - center) > (
- TRIAL_OUTLIER_MAD_MULTIPLIER * 1.4826 * mad
- )
- outlier_fraction = float(np.count_nonzero(outliers) / medians.size)
- drift_flagged = drift_ratio > TRIAL_DRIFT_RATIO_LIMIT
- outlier_flagged = outlier_fraction > TRIAL_OUTLIER_FRACTION_LIMIT
- if drift_flagged:
- reasons.add("trial-drift")
- if outlier_flagged:
- reasons.add("trial-outliers")
- components[name] = {
- "drift_flagged": drift_flagged,
- "first_last_median_ratio": drift_ratio,
- "outlier_flagged": outlier_flagged,
- "robust_outlier_fraction": outlier_fraction,
- "trial_count": int(medians.size),
- }
- return {
- "flagged": bool(reasons),
- "reasons": sorted(reasons),
- "components": components,
- }
-
-
-def _nearest_rank_p99(blocks: Sequence[Sequence[float]]) -> float:
- samples = sorted(float(sample) for block in blocks for sample in block)
- if len(samples) != 512 or samples[0] < 0 or not all(map(math.isfinite, samples)):
- raise PublisherError("p99 bootstrap input must contain 512 finite samples")
- return samples[math.ceil(0.99 * len(samples)) - 1]
-
-
-def _roundtrip_trial_array(
- internal: dict[str, Any], token: int
-) -> tuple[tuple[str, ...], np.ndarray]:
- trial_blocks = internal.get("trial_blocks")
- if not isinstance(trial_blocks, dict):
- raise PublisherError("series is missing private trial blocks")
- run_ids = tuple(sorted(trial_blocks, key=lambda value: (int(value), value)))
- if len(run_ids) != REQUIRED_ALLOCATIONS:
- raise PublisherError("p99 bootstrap requires exactly one run block")
- values = []
- for run_id in run_ids:
- point = trial_blocks[run_id].get(token)
- blocks = point.get("roundtrip") if isinstance(point, dict) else None
- if blocks is None:
- raise PublisherError("p99 bootstrap requires measured roundtrip blocks")
- if len(blocks) != 64 or any(len(block) != 8 for block in blocks):
- raise PublisherError("p99 bootstrap roundtrip blocks must be 64x8")
- values.append(blocks)
- array = np.asarray(values, dtype=np.float64)
- if array.shape != (REQUIRED_ALLOCATIONS, 64, 8):
- raise PublisherError("p99 bootstrap trial array shape differs")
- if not np.isfinite(array).all() or np.any(array <= 0):
- raise PublisherError("p99 bootstrap latencies must be finite and positive")
- return run_ids, array
-
-
-def _bootstrap_seed(
- dataset_binding: str, baseline_series_id: str, candidate_series_id: str, token: int
-) -> tuple[str, int]:
- payload = _canonical({
- "policy": BOOTSTRAP_POLICY,
- "resamples": BOOTSTRAP_RESAMPLES,
- "confidence": BOOTSTRAP_CONFIDENCE,
- "equivalence_band": BOOTSTRAP_EQUIVALENCE_BAND,
- "dataset_binding": dataset_binding,
- "baseline_series_id": baseline_series_id,
- "candidate_series_id": candidate_series_id,
- "tokens_per_rank": token,
- })
- digest = hashlib.sha256(payload).digest()
- return digest.hex(), int.from_bytes(digest[:16], "big")
-
-
-def _hierarchical_p99_ratio(
- baseline_series_id: str,
- candidate_series_id: str,
- token: int,
- internals: dict[str, dict[str, Any]],
- dataset_binding: str,
-) -> dict[str, Any]:
- """Bootstrap candidate/baseline p99 across runs, then 64 trial blocks."""
- baseline_runs, baseline = _roundtrip_trial_array(
- internals[baseline_series_id], token
- )
- candidate_runs, candidate = _roundtrip_trial_array(
- internals[candidate_series_id], token
- )
- if baseline_runs != candidate_runs:
- raise PublisherError("p99 bootstrap run blocks are not aligned")
- seed_sha256, seed = _bootstrap_seed(
- dataset_binding, baseline_series_id, candidate_series_id, token
- )
- cache_key = (
- seed_sha256,
- _sha_bytes(baseline.tobytes()),
- _sha_bytes(candidate.tobytes()),
- )
- cached = _BOOTSTRAP_CACHE.get(cache_key)
- if cached is not None:
- return dict(cached)
-
- baseline_run_p99 = np.asarray(
- [_nearest_rank_p99(run) for run in baseline], dtype=np.float64
- )
- candidate_run_p99 = np.asarray(
- [_nearest_rank_p99(run) for run in candidate], dtype=np.float64
- )
- run_ratios = candidate_run_p99 / baseline_run_p99
- point_ratio = float(np.median(candidate_run_p99) / np.median(baseline_run_p99))
-
- rng = np.random.Generator(np.random.PCG64(seed))
- ratios = np.empty(BOOTSTRAP_RESAMPLES, dtype=np.float64)
- p99_index = math.ceil(0.99 * 512) - 1
- for start in range(0, BOOTSTRAP_RESAMPLES, BOOTSTRAP_CHUNK_SIZE):
- size = min(BOOTSTRAP_CHUNK_SIZE, BOOTSTRAP_RESAMPLES - start)
- sampled_runs = rng.integers(0, REQUIRED_ALLOCATIONS, size=(size, REQUIRED_ALLOCATIONS))
- sampled_blocks = rng.integers(0, 64, size=(size, REQUIRED_ALLOCATIONS, 64))
- run_index = sampled_runs[:, :, None]
- baseline_sample = baseline[run_index, sampled_blocks].reshape(size, REQUIRED_ALLOCATIONS, 512)
- candidate_sample = candidate[run_index, sampled_blocks].reshape(size, REQUIRED_ALLOCATIONS, 512)
- baseline_p99 = np.partition(baseline_sample, p99_index, axis=2)[:, :, p99_index]
- candidate_p99 = np.partition(candidate_sample, p99_index, axis=2)[:, :, p99_index]
- ratios[start:start + size] = (
- np.median(candidate_p99, axis=1) / np.median(baseline_p99, axis=1)
- )
- ratios.sort()
- tail = (1.0 - BOOTSTRAP_CONFIDENCE) / 2.0
- lower_index = max(0, math.ceil(tail * BOOTSTRAP_RESAMPLES) - 1)
- upper_index = min(
- BOOTSTRAP_RESAMPLES - 1,
- math.ceil((1.0 - tail) * BOOTSTRAP_RESAMPLES) - 1,
- )
- ci = [float(ratios[lower_index]), float(ratios[upper_index])]
- threshold = 1.0 + BOOTSTRAP_EQUIVALENCE_BAND
- baseline_wins = ci[0] > threshold and bool(np.all(run_ratios > threshold))
- result = {
- "policy": BOOTSTRAP_POLICY,
- "resamples": BOOTSTRAP_RESAMPLES,
- "confidence": BOOTSTRAP_CONFIDENCE,
- "equivalence_band": BOOTSTRAP_EQUIVALENCE_BAND,
- "seed_sha256": seed_sha256,
- "point_ratio": point_ratio,
- "ci95": ci,
- "run_ratios": [float(value) for value in run_ratios],
- "all_runs_agree": bool(np.all(run_ratios > threshold)),
- "baseline_wins": baseline_wins,
- "tie": not baseline_wins,
- }
- _BOOTSTRAP_CACHE[cache_key] = result
- return dict(result)
-
-
-def _bootstrap_inputs_ready(
- members: Sequence[dict[str, Any]],
- internals: dict[str, dict[str, Any]],
- tokens: Sequence[int],
-) -> bool:
- try:
- expected_runs: tuple[str, ...] | None = None
- for member in members:
- for token in tokens:
- run_ids, _ = _roundtrip_trial_array(internals[member["series_id"]], token)
- if expected_runs is None:
- expected_runs = run_ids
- elif run_ids != expected_runs:
- return False
- return expected_runs is not None
- except (KeyError, PublisherError, TypeError, ValueError):
- return False
-
-
-def _eligibility_record(
- allocations: Sequence[str],
- *,
- complete: bool,
- correct: bool,
- measured: bool,
- stable_ordering: bool,
- extra_reasons: Sequence[str] = (),
-) -> dict[str, Any]:
- ids = sorted(set(allocations))
- reasons = list(extra_reasons)
- for condition, reason in (
- (len(ids) >= REQUIRED_ALLOCATIONS, "insufficient-allocations"),
- (complete, "incomplete-repeat-coverage"),
- (correct, "correctness-failed"),
- (measured, "missing-measured-roundtrip-p99"),
- (stable_ordering, "unstable-ordering"),
- ):
- if not condition:
- reasons.append(reason)
- reasons = sorted(set(reasons))
- decision = not reasons
- return {
- "decision_grade": decision,
- "allocation_ids": ids,
- "complete": complete,
- "correct": correct,
- "measured_roundtrip_p99": measured,
- "stable_ordering": stable_ordering,
- "reasons": reasons,
- }
-
-
-def _aggregate_percentiles(values: Sequence[dict[str, Any]]) -> dict[str, float]:
- return {
- name: float(statistics.median(float(value[name]) for value in values))
- for name in ("p50", "p90", "p95", "p99")
- }
-
-
-def _aggregate_component(
- rows: Sequence[dict[str, Any]], name: str
-) -> dict[str, Any] | None:
- components = [row["components"][name] for row in rows]
- if all(component["availability"] == "unavailable" for component in components):
- return None
- if any(component["availability"] == "unavailable" for component in components):
- raise PublisherError("component availability differs across repeat allocations")
- latency = _aggregate_percentiles([component["percentiles_us"] for component in components])
- if name == "isolated_sum":
- byte_provenance = {
- "accounting_contract": "activation-data-plus-scales-v1",
- "activation_data_bytes": 0,
- "scale_bytes": 0,
- "total_logical_bytes": 0,
- }
- return {
- "origin": "derived",
- "latency_us": latency,
- "byte_provenance": byte_provenance,
- "activation_data_rate_gbps_at_latency_percentile": None,
- "total_logical_data_rate_gbps_at_latency_percentile": None,
- "sample_count": None,
- }
- byte_provenance = _exact_repeat_value(
- [row["byte_provenance"][name] for row in rows],
- f"{name} byte accounting",
- )
- activation_rates = {
- statistic: byte_provenance["activation_data_bytes"] / (latency[statistic] * 1000.0)
- for statistic in latency
- }
- total_rates = {
- statistic: byte_provenance["total_logical_bytes"] / (latency[statistic] * 1000.0)
- for statistic in latency
- }
- return {
- "origin": "measured",
- "latency_us": latency,
- "byte_provenance": byte_provenance,
- "activation_data_rate_gbps_at_latency_percentile": activation_rates,
- "total_logical_data_rate_gbps_at_latency_percentile": total_rates,
- "sample_count": 512,
- }
-
-
-def _exact_repeat_value(values: Sequence[Any], label: str) -> Any:
- if not values or len({_canonical(value) for value in values}) != 1:
- raise PublisherError(f"{label} differs across repeat allocations")
- return values[0]
-
-
-def _eplb_descriptor(document: dict[str, Any]) -> dict[str, Any]:
- value = document["case"]["eplb"]
- return {
- "enabled": value["enabled"],
- "calibration_workload_id": value["calibration_workload_id"],
- "calibration_trace_sha256": value["calibration_trace_sha256"],
- "calibration_window": value["calibration_window"],
- "calibration_token_offset": value["calibration_token_offset"],
- "planner": value["planner"],
- "mapping_sha256": value["mapping_hash"],
- "logical_experts": value["num_logical_experts"],
- "physical_experts": value["num_physical_experts"],
- "redundant_experts": value["num_redundant"],
- "reference_tokens_per_rank": value["reference_tokens_per_rank"],
- "replicated_experts": value["replicated_experts"],
- "max_replicas": value["max_replicas"],
- "imbalance_before": value["imbalance_before"],
- "imbalance_after": value["imbalance_after"],
- }
-
-
-def _routing_facts(row: dict[str, Any]) -> dict[str, Any]:
- routing = row["routing"]
- return {
- "fanout_mean": routing["fanout_mean"],
- "recv_tokens_max": row["receive"]["max"],
- "expert_load_cv": routing["expert_load_cv"],
- "payload_rank_cv": routing["payload_rank_cv"],
- "hotspot_ratio": routing["hotspot_ratio"],
- "empty_expert_count": routing["empty_expert_count"],
- "empty_rank_count": routing["empty_rank_count"],
- "routed_copies": routing["routed_copies"],
- }
-
-
-def _aggregate_precision_evidence(rows: Sequence[dict[str, Any]]) -> dict[str, Any]:
- values = [row["correctness"]["precision"] for row in rows]
- profile_ids = {value["profile_id"] for value in values}
- if len(profile_ids) != 1:
- raise PublisherError("precision evidence profile differs across qualification runs")
- result: dict[str, Any] = {"profile_id": profile_ids.pop()}
- for direction in ("dispatch", "combine"):
- axes = [value[direction] for value in values]
- finite = [axis["scales_finite"] for axis in axes]
- positive = [axis["scales_positive"] for axis in axes]
- result[direction] = {
- "encoded_payload_valid": all(axis["encoded_payload_valid"] for axis in axes),
- "scales_finite": None if all(value is None for value in finite) else all(
- value is True for value in finite
- ),
- "scales_positive": None if all(value is None for value in positive) else all(
- value is True for value in positive
- ),
- "dequantized_semantics": all(axis["dequantized_semantics"] for axis in axes),
- "saturation_count": max(axis["saturation_count"] for axis in axes),
- "saturation_rate": max(axis["saturation_rate"] for axis in axes),
- "max_abs_error": max(axis["max_abs_error"] for axis in axes),
- "max_rel_error": max(axis["max_rel_error"] for axis in axes),
- "passed": all(axis["passed"] for axis in axes),
- }
- result["passed"] = result["dispatch"]["passed"] and result["combine"]["passed"]
- return result
-
-
-def _series_extra_reasons(documents: Sequence[dict[str, Any]]) -> list[str]:
- reasons: set[str] = set()
- for document in documents:
- validity = document["outcome"]["validity"]
- rows = document["measurement"]["rows"]
- if validity.get("provenance_complete") is not True:
- reasons.add("incomplete-provenance")
- if validity.get("workload_source") != "canonical-serialized":
- reasons.add("noncanonical-workload")
- if validity.get("anomaly_free") is not True or any(row["anomalies"] for row in rows):
- reasons.add("unresolved-anomaly")
- if validity.get("semantic_correctness") != "pass":
- reasons.add("semantic-correctness-failed")
- if validity.get("measurement_conformance") != "conformant" or validity.get("sampling_conformance") != "conformant":
- reasons.add("measurement-nonconformant")
- profile = identity.case_profile(document["case"]["mode"])
- scopes = {row["correctness"].get("scope") for row in rows}
- if scopes != {profile["correctness_scope"]}:
- reasons.add("expert-oracle-incomplete")
- return sorted(reasons)
-
-
-BACKEND_LABELS = {
- "deepep": "DeepEP V1",
- "deepep-v2": "DeepEP V2",
- "deepep-hybrid": "DeepEP Hybrid",
- "uccl": "UCCL",
- "mori": "MoRI",
- "nccl-ep": "NCCL/RCCL reference",
-}
-
-
-def _build_series(
- series_id: str,
- documents: Sequence[dict[str, Any]],
- sample_documents: Sequence[dict[str, Any]],
- expected_repeats: int,
-) -> tuple[dict[str, Any], dict[str, Any]]:
- if not documents:
- raise PublisherError("cannot aggregate an empty series")
- first = documents[0]
- if any(document["identity"]["series_id"] != series_id for document in documents):
- raise PublisherError("series aggregation mixed identities")
- if len(sample_documents) != len(documents):
- raise PublisherError("series aggregation lost private sample documents")
- allocations = [document["identity"]["allocation_id"] for document in documents]
- if len(allocations) != len(set(allocations)):
- raise PublisherError("series repeats reuse an allocation identity")
- row_maps = [
- {row["tokens_per_rank"]: row for row in document["measurement"]["rows"]}
- for document in documents
- ]
- token_sets = {tuple(sorted(rows)) for rows in row_maps}
- if len(token_sets) != 1:
- raise PublisherError("series token coverage differs across allocations")
- tokens = list(next(iter(token_sets)))
- qualification_indices = sorted(
- document["measurement"]["qualification_index"] for document in documents
- )
- correct = all(
- row["correctness"]["passed"]
- for document in documents for row in document["measurement"]["rows"]
- )
- measured = all(
- row["components"]["roundtrip"]["availability"] == "measured"
- and row["components"]["roundtrip"]["percentiles_us"].get("p99") is not None
- for document in documents for row in document["measurement"]["rows"]
- )
- extra_reasons = _series_extra_reasons(documents)
- case = first["case"]
- shape = case["shape"]
- topology = first["topology"]
- runtime = first["runtime_fingerprint"]
- workload_id = first["workload"]["workload_id"]
- if not identity.is_typed_id(workload_id, "workload"):
- raise PublisherError("raw workload is not canonical")
- backend_id = case["backend"]
- resource_raw = first["implementation"]["resource_profile"]
- public_config = contracts.public_series_config(
- kernel_generation=first["implementation"]["kernel_generation"],
- provenance=first["implementation"]["provenance"],
- resource_profile=resource_raw,
- resource_mode=case["resource_mode"],
- device_product=topology["device_product"],
- )
- resource_profile = public_config["resource"]["profile"]
- configured_units = public_config["resource"]["configured_units"]
- units_kind = public_config["resource"]["comm_units_kind"]
- resource_label = (
- f"{configured_units} {str(units_kind).upper()}"
- if configured_units is not None and units_kind
- else resource_profile
- )
- eplb = _exact_repeat_value(
- [_eplb_descriptor(document) for document in documents], "EPLB descriptor"
- )
- points: list[dict[str, Any]] = []
- run_metrics: dict[str, dict[int, dict[str, float]]] = {}
- trial_blocks: dict[str, dict[int, dict[str, Any]]] = {}
- for document, sample_document, rows in zip(
- documents, sample_documents, row_maps, strict=True
- ):
- if any(
- sample_document[field] != document["identity"][field]
- for field in ("allocation_id", "attempt_id", "case_id", "series_id")
- ):
- raise PublisherError("private samples differ from their selected raw attempt")
- if sample_document["qualification_index"] != document["measurement"]["qualification_index"]:
- raise PublisherError("private sample qualification index differs from raw attempt")
- run_id = str(_git_run(document)["run_id"])
- if run_id in run_metrics:
- raise PublisherError("series has two allocations from one workflow run")
- trial_blocks[run_id] = _private_trial_components(sample_document)
- run_metrics[run_id] = {}
- for token in tokens:
- latency = rows[token]["components"]["roundtrip"]["percentiles_us"]
- byte_provenance = rows[token]["byte_provenance"]["roundtrip"]
- run_metrics[run_id][token] = {
- "latency_us": {statistic: latency[statistic] for statistic in ("p50", "p99")},
- "activation_data_rate_gbps_at_latency_percentile": {
- statistic: byte_provenance["activation_data_bytes"]
- / (latency[statistic] * 1000.0)
- for statistic in ("p50", "p99")
- },
- "total_logical_data_rate_gbps_at_latency_percentile": {
- statistic: byte_provenance["total_logical_bytes"]
- / (latency[statistic] * 1000.0)
- for statistic in ("p50", "p99")
- },
- }
- for token in tokens:
- rows = [row_map[token] for row_map in row_maps]
- diagnostics = _trial_diagnostics(trial_blocks, token)
- if diagnostics["flagged"]:
- extra_reasons.append("unresolved-trial-diagnostic")
- routing = _exact_repeat_value(
- [_routing_facts(row) for row in rows], "routing/load facts"
- )
- components = {
- name: _aggregate_component(rows, name)
- for name in ("dispatch", "stage", "combine", "roundtrip")
- }
- if components["dispatch"] is None:
- components["isolated_sum"] = None
- else:
- latency = {
- statistic: components["dispatch"]["latency_us"][statistic]
- + (
- components["stage"]["latency_us"][statistic]
- if components["stage"] is not None else 0.0
- )
- + components["combine"]["latency_us"][statistic]
- for statistic in ("p50", "p90", "p95", "p99")
- }
- components["isolated_sum"] = {
- "origin": "derived",
- "latency_us": latency,
- "byte_provenance": components["roundtrip"]["byte_provenance"],
- "activation_data_rate_gbps_at_latency_percentile": None,
- "total_logical_data_rate_gbps_at_latency_percentile": None,
- "sample_count": None,
- }
- points.append({
- "point_id": rows[0]["point_id"],
- "tokens_per_rank": token,
- "global_tokens": token * case["ep_size"],
- "correctness": {
- "semantic_pass": all(row["correctness"]["passed"] for row in rows),
- "precision": _aggregate_precision_evidence(rows),
- },
- "anomalies": sorted({
- anomaly["type"].replace("_", "-")
- for row in rows for anomaly in row["anomalies"]
- } | set(diagnostics["reasons"])),
- "trial_diagnostics": diagnostics,
- "routing": routing,
- "components": components,
- "roundtrip_token_rate_at_latency_percentile": {
- statistic: (token * case["ep_size"])
- / (components["roundtrip"]["latency_us"][statistic] * 1e-6)
- for statistic in ("p50", "p90", "p95", "p99")
- },
- "evidence_ids": [row["evidence_id"] for row in rows],
- })
- eligibility = _eligibility_record(
- allocations,
- complete=len(documents) == expected_repeats,
- correct=correct,
- measured=measured,
- # Ordering is defined only across alternatives in a controlled cohort.
- stable_ordering=True,
- extra_reasons=sorted(set(extra_reasons)),
- )
- series = {
- "series_id": series_id,
- "label": (
- f"{case['runner'].upper()} / {BACKEND_LABELS.get(backend_id, backend_id)} / "
- f"EP{case['ep_size']} / {topology['nodes']} node"
- f"{'s' if topology['nodes'] != 1 else ''} / {topology['scope']} / "
- f"{case['mode']} / {case['phase']} / {shape['routing']}"
- f"{' + EPLB' if case['eplb']['enabled'] else ''} / {resource_label}"
- ),
- "status": "decision-grade" if eligibility["decision_grade"] else "diagnostic",
- "case_ids": sorted({document["identity"]["case_id"] for document in documents}),
- "allocation_ids": sorted(allocations),
- "model": _slug(case["workload_name"]),
- "suite": _slug(case["suite"]),
- "mode": case["mode"],
- "phase": case["phase"],
- "publication_tier": case["required_publication"],
- "backend": {
- "id": _slug(backend_id),
- "label": BACKEND_LABELS.get(backend_id, backend_id),
- "role": "reference" if backend_id == "nccl-ep" else "library",
- **public_config["backend"],
- },
- "build": {
- "implementation_contract_sha256": first["identity"]["series_factors"][
- "implementation_contract_sha256"
- ],
- "public_config_sha256": first["identity"]["series_factors"][
- "public_config_sha256"
- ],
- "routing_control_sha256": first["identity"]["series_factors"][
- "routing_control_sha256"
- ],
- "runtime_fingerprint_sha256": first["identity"]["series_factors"][
- "runtime_fingerprint_sha256"
- ],
- "image_digest": first["identity"]["series_factors"]["image_digest"],
- "source_sha": first["identity"]["series_factors"]["source_sha"],
- "squash_sha256": first["identity"]["series_factors"]["squash_sha256"],
- },
- "system": {
- "sku": _slug(case["runner"]),
- "label": public_config["system"]["label"],
- "vendor": runtime["vendor"],
- "topology_class": _slug(topology["topology_class"]),
- "transport": _slug(topology["transport"]),
- "scale_up_transport": _slug(topology["scale_up_transport"]),
- "scale_out_transport": (
- _slug(topology["scale_out_transport"])
- if topology["scale_out_transport"] is not None
- else None
- ),
- "scope": topology["scope"],
- "nodes": topology["nodes"],
- "gpus_per_node": topology["gpus_per_node"],
- "scale_up_domain": topology["scale_up_domain"],
- "world_size": topology["world_size"],
- "ep_size": case["ep_size"],
- "placement": topology["placement"],
- },
- "workload": {
- "workload_id": workload_id,
- "hidden": shape["hidden"],
- "top_k": shape["topk"],
- "experts": case["eplb"]["num_logical_experts"],
- "routing": shape["routing"],
- "eplb": case["eplb"]["enabled"],
- "precision_profile": shape["precision_profile"],
- "dispatch_precision": shape["dispatch_precision"],
- "combine_precision": shape["combine_precision"],
- "activation_profile": shape["activation_profile"],
- },
- "eplb": eplb,
- "resource": public_config["resource"],
- "measurement": {
- "contract": first["measurement"]["contract"],
- "component_order_contract": first["measurement"]["component_order_contract"],
- "combine_semantics": identity.case_profile(case["mode"])["combine_semantics"],
- "payload_unit": identity.case_profile(case["mode"])["payload_unit"],
- "sampling_contract": first["measurement"]["sampling"]["contract"],
- "iters": first["measurement"]["sampling"]["iterations_per_trial"],
- "trials": first["measurement"]["sampling"]["trials"],
- "warmups": first["measurement"]["sampling"]["warmup_iterations"],
- "samples_per_component": first["measurement"]["sampling"]["samples_per_component"],
- "qualification_indices": qualification_indices,
- "headline_component": "roundtrip",
- "headline_percentile": "p99",
- },
- "points": points,
- "eligibility": eligibility,
- }
- internal = {
- "documents": list(documents),
- "run_metrics": run_metrics,
- "trial_blocks": trial_blocks,
- "series_factors": first["identity"]["series_factors"],
- }
- return series, internal
-
-
-def _resolve_bundle_file(root: Path, record: dict[str, Any]) -> Path:
- path = root.joinpath(*PurePosixPath(record["path"]).parts)
- try:
- path.relative_to(root)
- except ValueError as exc:
- raise PublisherError("bundle record escapes its directory") from exc
- if path.resolve() != path or path.is_symlink() or not path.is_file():
- raise PublisherError("bundle record points to a missing or linked file")
- if path.stat().st_size != record["bytes"] or _sha_file(path) != record["sha256"]:
- raise PublisherError("bundle file checksum differs from its manifest")
- return path
-
-
-def load_bundle(store: Store, bundle_id: str) -> dict[str, Any]:
- if HEX64.fullmatch(bundle_id) is None:
- raise PublisherError("bundle ID must be a SHA-256 digest")
- root = store.bundles / bundle_id
- if root.is_symlink() or not (root / "COMPLETE").is_file():
- raise PublisherError(f"bundle {bundle_id} is missing or incomplete")
- _verify_frozen_tree(root, private=True)
- if (root / "COMPLETE").read_text().strip() != bundle_id:
- raise PublisherError("bundle COMPLETE marker differs")
- manifest_path = root / "bundle.json"
- if _sha_file(manifest_path) != bundle_id:
- raise PublisherError("bundle directory digest differs from bundle.json")
- manifest = validate_bundle_manifest(strict_load(manifest_path))
- checksum_path = _resolve_bundle_file(root, manifest["checksums"])
- checksum_document = strict_load(checksum_path)
- checksum_document = _exact(checksum_document, {"format", "files"}, "checksums")
- if checksum_document["format"] != "collectivex.checksums.v1":
- raise PublisherError("bundle checksum format is invalid")
- records = [_file_record(value, f"checksums.files[{index}]")
- for index, value in enumerate(_array(checksum_document["files"], "checksums.files"))]
- _unique([record["path"] for record in records], "checksums.files[].path")
- for record in records:
- _resolve_bundle_file(root, record)
- expected_paths = {
- path.relative_to(root).as_posix() for path in _tree_files(root)
- if path.name not in {"bundle.json", "checksums.json"}
- }
- if {record["path"] for record in records} != expected_paths:
- raise PublisherError("bundle checksum catalog does not cover its payload exactly")
- artifact_by_root: dict[str, str] = {}
- for index, source in enumerate(manifest["sources"]):
- _resolve_bundle_file(root, source)
- archive_key = f"artifact-{index:04d}"
- if source["path"] != f"source/{archive_key}.zip":
- raise PublisherError("bundle source catalog order/path differs")
- artifact_by_root[archive_key] = source["artifact_name"]
- if len(set(artifact_by_root.values())) != len(artifact_by_root):
- raise PublisherError("bundle source catalog repeats an artifact name")
- matrix_path = _resolve_bundle_file(root, manifest["matrix"])
- matrix_document = strict_load(matrix_path)
- cases = validate_matrix(matrix_document)
- expected_by_id = {case["case_id"]: case for case in cases}
- expected_deliveries = _expected_deliveries(
- matrix_document, cases, manifest["run"]
- )
- if {item["case_id"] for item in manifest["coverage"]["selections"]} != set(expected_by_id):
- raise PublisherError("bundle selected coverage differs from requested matrix")
- documents: dict[str, dict[str, Any]] = {}
- sample_documents: dict[str, dict[str, Any]] = {}
- runtime_fingerprints: set[str] = set()
- for attempt in manifest["attempts"]:
- document_path = _resolve_bundle_file(root, attempt["document"])
- document = contracts.strict_load(document_path)
- artifact_safety.assert_publication_safe([document])
- if document.get("format") == contracts.RAW_FORMAT:
- _schema("raw-case-v1.schema.json", document)
- sample_path = document_path.with_name(document["sample_artifact"]["path"])
- if attempt["samples"] is None:
- raise PublisherError("raw attempt is missing its sample manifest record")
- manifest_sample_path = _resolve_bundle_file(root, attempt["samples"])
- if manifest_sample_path != sample_path:
- raise PublisherError("sample manifest record points to the wrong raw evidence")
- sample_document = contracts.strict_load(sample_path)
- artifact_safety.assert_publication_safe([sample_document])
- _schema("samples-v1.schema.json", sample_document)
- sample_document = contracts.validate_samples_document(sample_document)
- document = contracts.load_raw_attempt(document_path)
- sample_documents[attempt["attempt_id"]] = sample_document
- else:
- if attempt["samples"] is not None:
- raise PublisherError("terminal attempt unexpectedly names a sample artifact")
- _schema("terminal-outcome-v1.schema.json", document)
- document = contracts.validate_terminal_document(document)
- _validate_delivery_binding(
- document, document_path, root / "raw", artifact_by_root,
- expected_by_id, expected_deliveries, manifest["run"],
- )
- expected_record = _attempt_record(
- document, document_path, root, selected=attempt["selected"]
- )
- if expected_record != attempt:
- raise PublisherError("bundle attempt record differs from native document")
- if attempt["runtime_fingerprint_sha256"]:
- runtime_fingerprints.add(attempt["runtime_fingerprint_sha256"])
- documents[attempt["attempt_id"]] = document
- if sorted(runtime_fingerprints) != manifest["runtime_fingerprints"]:
- raise PublisherError("bundle runtime fingerprint catalog differs from attempts")
- selected = {
- selection["case_id"]: documents[selection["selected_attempt_id"]]
- for selection in manifest["coverage"]["selections"]
- }
- return {
- "id": bundle_id,
- "root": root,
- "manifest": manifest,
- "cases": cases,
- "documents": documents,
- "sample_documents": sample_documents,
- "selected": selected,
- }
-
-
-def _cohort_control(
- kind: str, series: dict[str, Any], internal: dict[str, Any]
-) -> tuple[dict[str, Any], list[str], list[str], Any]:
- binary_build = series["build"]
- source = binary_build["source_sha"]
- workload = series["workload"]
- shape = {
- key: workload[key]
- for key in (
- "hidden", "top_k", "experts", "precision_profile", "dispatch_precision",
- "combine_precision", "activation_profile",
- )
- }
- common = {
- "model": series["model"], "mode": series["mode"],
- "phase": series["phase"], "shape": shape,
- "measurement": series["measurement"], "ep_size": series["system"]["ep_size"],
- }
- if kind == "library":
- control = {**common, "system": series["system"], "workload": workload,
- "resource_mode": series["resource"]["mode"], "source": source}
- return control, ["system", "workload", "mode", "phase", "measurement", "resource.mode", "source"], ["backend", "resource"], series["backend"]["id"]
- if kind == "chip":
- control = {**common, "backend": series["backend"], "source": source,
- "workload": workload, "resource_mode": series["resource"]["mode"]}
- return control, ["backend", "source", "workload", "mode", "phase", "measurement", "resource.mode"], ["system", "resource"], series["system"]
- if kind == "system":
- control = {**common, "workload": workload, "source": source}
- varying = [series["system"]["sku"], series["backend"]["id"], series["resource"]["profile"]]
- return control, ["workload", "mode", "phase", "measurement", "source"], ["system", "backend", "resource"], varying
- if kind in PRECISION_COHORT_KINDS:
- control, variant = _public_cohort_factors(kind, series)
- if kind == "dispatch-precision":
- controlled = [
- "backend", "implementation-static-build", "system", "model-shape",
- "mode", "phase", "workload.routing", "workload.eplb", "measurement",
- "resource", "combine-precision",
- ]
- varying = ["dispatch-precision"]
- elif kind == "combine-precision":
- controlled = [
- "backend", "implementation-static-build", "system", "model-shape",
- "mode", "phase", "workload.routing", "workload.eplb", "measurement",
- "resource", "dispatch-precision",
- ]
- varying = ["combine-precision"]
- else:
- controlled = [
- "backend", "implementation-static-build", "system", "model-shape",
- "mode", "phase", "workload.routing", "workload.eplb", "measurement",
- ]
- varying = [
- "dispatch-precision", "combine-precision", "precision-profile", "resource",
- ]
- return control, controlled, varying, variant
- raise PublisherError(f"unknown cohort kind {kind}")
-
-
-def _cohort_ordering(
- members: Sequence[dict[str, Any]], internals: dict[str, dict[str, Any]], tokens: Sequence[int]
-) -> tuple[bool, int]:
- run_ids = set.intersection(*(
- set(internals[member["series_id"]]["run_metrics"]) for member in members
- ))
- if len(run_ids) < REQUIRED_ALLOCATIONS:
- return False, len(run_ids)
- orders: list[tuple[str, str, int, str, tuple[str, ...]]] = []
- for run_id in sorted(run_ids):
- for token in tokens:
- for measure in (
- "latency_us", "activation_data_rate_gbps_at_latency_percentile",
- "total_logical_data_rate_gbps_at_latency_percentile",
- ):
- for statistic in ("p50", "p99"):
- ordered = tuple(
- member["series_id"]
- for member in sorted(
- members,
- key=lambda item: (
- internals[item["series_id"]]["run_metrics"][run_id][token][measure][statistic],
- item["series_id"],
- ),
- reverse=measure != "latency_us",
- )
- )
- orders.append((measure, statistic, token, run_id, ordered))
- for token in tokens:
- for measure in (
- "latency_us", "activation_data_rate_gbps_at_latency_percentile",
- "total_logical_data_rate_gbps_at_latency_percentile",
- ):
- for statistic in ("p50", "p99"):
- observed = {
- entry[4]
- for entry in orders
- if entry[0] == measure and entry[1] == statistic and entry[2] == token
- }
- if len(observed) != 1:
- return False, len(run_ids)
- return True, len(run_ids)
-
-
-def _p99_top_tie_ids(
- members: Sequence[dict[str, Any]],
- internals: dict[str, dict[str, Any]],
- token: int,
- dataset_binding: str,
- cohort_id: str,
-) -> set[str]:
- metric = {
- "operation": "roundtrip",
- "statistic": "p99",
- "measure": "latency_us",
- "objective": "min",
- "tokens_per_rank": token,
- "phase": members[0]["phase"],
- }
- ordered = sorted(
- members,
- key=lambda member: (
- _metric_value(member, metric)[1], member["series_id"]
- ),
- )
- baseline_id = ordered[0]["series_id"]
- comparisons: dict[str, dict[str, Any]] = {}
- tie_ids = {baseline_id}
- for candidate in ordered[1:]:
- candidate_id = candidate["series_id"]
- result = _hierarchical_p99_ratio(
- baseline_id, candidate_id, token, internals, dataset_binding
- )
- comparisons[candidate_id] = result
- if not result["baseline_wins"]:
- tie_ids.add(candidate_id)
- internals[baseline_id].setdefault("decision_statistics", {})[
- f"{cohort_id}:p99:{token}"
- ] = {
- "baseline_series_id": baseline_id,
- "comparisons": comparisons,
- "tie_series_ids": sorted(tie_ids),
- }
- return tie_ids
-
-
-def build_decisions(
- series: Sequence[dict[str, Any]],
- internals: dict[str, dict[str, Any]],
- *,
- dataset_binding: str | None = None,
-) -> tuple[list[dict[str, Any]], list[dict[str, Any]], list[dict[str, Any]], list[dict[str, Any]]]:
- if dataset_binding is None:
- dataset_binding = _sha_bytes(_canonical({
- "series_ids": sorted(item["series_id"] for item in series),
- }))
- cohorts: list[dict[str, Any]] = []
- for kind in (*REQUIRED_COHORT_KINDS, *PRECISION_COHORT_KINDS):
- groups: dict[bytes, list[tuple[dict[str, Any], Any, list[str], list[str]]]] = {}
- for item in series:
- if kind == "library" and item["backend"]["role"] != "library":
- continue
- if kind == "system" and item["backend"]["role"] != "reference":
- continue
- control, controlled, varying, variant = _cohort_control(kind, item, internals[item["series_id"]])
- groups.setdefault(_canonical(control), []).append((item, variant, controlled, varying))
- for entries in groups.values():
- variants = {_canonical(entry[1]) for entry in entries}
- if len(entries) < 2 or len(variants) < 2:
- continue
- members = sorted((entry[0] for entry in entries), key=lambda item: item["series_id"])
- token_sets = [set(point["tokens_per_rank"] for point in member["points"]) for member in members]
- tokens = sorted(set.intersection(*token_sets))
- same_points = len({tuple(sorted(values)) for values in token_sets}) == 1
- ordering, aligned_runs = _cohort_ordering(members, internals, tokens) if tokens else (False, 0)
- allocations = sorted({value for member in members for value in member["allocation_ids"]})
- extra = {
- reason for member in members for reason in member["eligibility"]["reasons"]
- if reason not in {"unstable-ordering"}
- }
- if aligned_runs < REQUIRED_ALLOCATIONS:
- extra.add("incomplete-aligned-repeats")
- if tokens and not _bootstrap_inputs_ready(members, internals, tokens):
- extra.add("missing-trial-blocks")
- endpoint_contrast = kind in PRECISION_COHORT_KINDS
- if not tokens or (not endpoint_contrast and not same_points):
- extra.add("unmatched-token-coverage")
- if kind in {"dispatch-precision", "combine-precision"}:
- axis = "dispatch" if kind == "dispatch-precision" else "combine"
- field = f"{axis}_precision"
- bf16 = identity.precision_profile(
- identity.V1_CONTROL_PRECISION_PROFILE
- )[axis]
- if sum(
- _canonical(member["workload"][field]) == _canonical(bf16)
- for member in members
- ) != 1:
- extra.add("missing-bf16-precision-baseline")
- eligibility = _eligibility_record(
- allocations,
- complete=all(member["eligibility"]["complete"] for member in members)
- and bool(tokens) and (endpoint_contrast or same_points),
- correct=all(member["eligibility"]["correct"] for member in members),
- measured=all(member["eligibility"]["measured_roundtrip_p99"] for member in members),
- stable_ordering=ordering,
- extra_reasons=sorted(extra),
- )
- member_ids = [member["series_id"] for member in members]
- publication_tier = (
- "comparable-experimental"
- if any(member["publication_tier"] == "comparable-experimental" for member in members)
- else "official"
- )
- controlled, varying = entries[0][2], entries[0][3]
- cohort_id = _derived_id("cxcohort-v1-", {
- "kind": kind, "series_ids": member_ids,
- "controlled_factors": controlled, "varying_factors": varying,
- })
- kind_label = {
- "chip": "Platform",
- "dispatch-precision": "Dispatch precision",
- "combine-precision": "Combine precision",
- "precision-pair": "Precision profile",
- }.get(kind, kind.title())
- first = members[0]
- routing_label = first["workload"]["routing"] + (
- "+EPLB" if first["workload"]["eplb"] else ""
- )
- context = {
- "library": (
- f"{first['system']['sku'].upper()} EP{first['system']['ep_size']} / "
- f"{first['mode']} / {first['phase']} / {routing_label}"
- ),
- "chip": (
- f"{first['backend']['label']} EP{first['system']['ep_size']} / "
- f"{first['mode']} / {first['phase']} / {routing_label}"
- ),
- "system": (
- f"Reference EP{first['system']['ep_size']} / {first['mode']} / "
- f"{first['phase']} / {routing_label}"
- ),
- "dispatch-precision": (
- f"{first['system']['sku'].upper()} / {first['backend']['label']} / "
- f"EP{first['system']['ep_size']} / {first['mode']} / {first['phase']}"
- ),
- "combine-precision": (
- f"{first['system']['sku'].upper()} / {first['backend']['label']} / "
- f"EP{first['system']['ep_size']} / {first['mode']} / {first['phase']}"
- ),
- "precision-pair": (
- f"{first['system']['sku'].upper()} / {first['backend']['label']} / "
- f"EP{first['system']['ep_size']} / {first['mode']} / {first['phase']}"
- ),
- }[kind]
- cohorts.append({
- "cohort_id": cohort_id,
- "kind": kind,
- "label": f"{context} / {kind_label} contrast ({len(members)} series)",
- "description": (
- "Publisher-controlled NCCL/RCCL system comparison"
- if kind == "system"
- else (
- "Descriptive configured-stack precision comparison; no isolated axis claim"
- if kind == "precision-pair"
- else f"Publisher-controlled {kind_label.lower()} comparison"
- )
- ),
- "series_ids": member_ids,
- "controlled_factors": controlled,
- "varying_factors": varying,
- "publication_tier": publication_tier,
- "eligibility": eligibility,
- })
- cohorts.sort(key=lambda item: item["cohort_id"])
- series_by_id = {item["series_id"]: item for item in series}
- rankings: list[dict[str, Any]] = []
- recommendations: list[dict[str, Any]] = []
- sensitivities: list[dict[str, Any]] = []
- for cohort in cohorts:
- if not cohort["eligibility"]["decision_grade"]:
- continue
- if cohort["kind"] == "precision-pair":
- continue
- members = [series_by_id[series_id] for series_id in cohort["series_ids"]]
- tokens = sorted(set.intersection(*(
- {point["tokens_per_rank"] for point in member["points"]} for member in members
- )))
- for token in tokens:
- p99_tie_ids = _p99_top_tie_ids(
- members, internals, token, dataset_binding, cohort["cohort_id"]
- )
- for measure, objective, unit in (
- ("latency_us", "min", "us"),
- ("activation_data_rate_gbps_at_latency_percentile", "max", "GB/s"),
- ("total_logical_data_rate_gbps_at_latency_percentile", "max", "GB/s"),
- ):
- for statistic in ("p50", "p99"):
- metric = {
- "operation": "roundtrip", "statistic": statistic,
- "measure": measure, "objective": objective,
- "tokens_per_rank": token, "phase": members[0]["phase"],
- }
- entries = []
- for member in members:
- point_id, value, observed_unit = _metric_value(member, metric)
- if observed_unit != unit:
- raise PublisherError("publisher metric unit differs")
- entries.append({
- "rank": 0, "series_id": member["series_id"], "point_id": point_id,
- "value": value, "unit": unit,
- })
- entries.sort(key=lambda item: (item["value"], item["series_id"]), reverse=objective == "max")
- for rank, entry in enumerate(entries, 1):
- entry["rank"] = (
- 1
- if measure == "latency_us"
- and statistic == "p99"
- and entry["series_id"] in p99_tie_ids
- else rank
- )
- ranking_id = _derived_id("cxranking-v1-", {
- "cohort_id": cohort["cohort_id"], "metric": metric,
- })
- metric_label = _metric_label(measure, statistic)
- rankings.append({
- "ranking_id": ranking_id, "cohort_id": cohort["cohort_id"],
- "label": f"{cohort['kind'].title()} {metric_label} T={token}",
- "metric": metric, "entries": entries,
- "publication_tier": cohort["publication_tier"],
- "eligibility": cohort["eligibility"],
- })
- if (
- cohort["publication_tier"] != "official"
- or measure != "latency_us"
- or statistic != "p99"
- or sum(entry["rank"] == 1 for entry in entries) != 1
- ):
- continue
- objective_name = "min-p99-latency"
- top = entries[0]
- recommendation_id = _derived_id("cxrecommendation-v1-", {
- "objective": objective_name, "ranking_id": ranking_id,
- })
- recommendations.append({
- "recommendation_id": recommendation_id,
- "cohort_id": cohort["cohort_id"],
- "label": f"Best {metric_label} at T={token}",
- "objective": objective_name,
- "series_id": top["series_id"], "point_id": top["point_id"],
- "value": top["value"], "unit": top["unit"],
- "rationale": (
- "Unique p99 winner after deterministic hierarchical bootstrap "
- "and all-run agreement"
- ),
- "publication_tier": cohort["publication_tier"],
- "eligibility": cohort["eligibility"],
- })
- if cohort["kind"] in {"dispatch-precision", "combine-precision"}:
- axis = (
- "dispatch"
- if cohort["kind"] == "dispatch-precision"
- else "combine"
- )
- field = f"{axis}_precision"
- bf16 = identity.precision_profile(
- identity.V1_CONTROL_PRECISION_PROFILE
- )[axis]
- baseline = next(
- member for member in members
- if _canonical(member["workload"][field]) == _canonical(bf16)
- )
- for candidate in members:
- if candidate is baseline:
- continue
- for token in tokens:
- for measure, objective in (
- ("latency_us", "min"),
- ("activation_data_rate_gbps_at_latency_percentile", "max"),
- ("total_logical_data_rate_gbps_at_latency_percentile", "max"),
- ):
- for statistic in ("p50", "p99"):
- metric = {
- "operation": "roundtrip",
- "statistic": statistic,
- "measure": measure,
- "objective": objective,
- "tokens_per_rank": token,
- "phase": baseline["phase"],
- }
- _, base_value, _ = _metric_value(baseline, metric)
- _, candidate_value, _ = _metric_value(candidate, metric)
- sensitivity_id = _derived_id("cxsensitivity-v1-", {
- "baseline": baseline["series_id"],
- "candidate": candidate["series_id"],
- "cohort": cohort["cohort_id"],
- "metric": metric,
- })
- sensitivities.append({
- "sensitivity_id": sensitivity_id,
- "cohort_id": cohort["cohort_id"],
- "label": (
- f"{axis.title()} precision sensitivity: "
- f"{_metric_label(measure, statistic)} T={token}"
- ),
- "baseline_series_id": baseline["series_id"],
- "candidate_series_id": candidate["series_id"],
- "metric": metric,
- "signed_change_ratio": (
- candidate_value - base_value
- ) / base_value,
- "publication_tier": cohort["publication_tier"],
- "eligibility": cohort["eligibility"],
- })
- rankings.sort(key=lambda item: item["ranking_id"])
- recommendations.sort(key=lambda item: item["recommendation_id"])
- sensitivities.sort(key=lambda item: item["sensitivity_id"])
- return cohorts, rankings, recommendations, sensitivities
-
-
-def _require_runnable_promotion_success(
- bundles: Sequence[dict[str, Any]], cases: dict[str, dict[str, Any]]
-) -> None:
- for bundle in bundles:
- for case_id, case in cases.items():
- if case["_disposition"] != "runnable":
- continue
- status, _ = _outcome(bundle["selected"][case_id])
- if status != "success":
- raise PublisherError(
- "promotion requires every runnable matrix case to succeed "
- "in every selected bundle"
- )
- prior_statuses = {
- _outcome(document)[0]
- for document in bundle["documents"].values()
- if document["identity"]["case_id"] == case_id
- }
- if prior_statuses != {"success"}:
- raise PublisherError(
- "promotion rejects runnable cases with failed, invalid, or diagnostic retries"
- )
-
-
-def _expected_chip_cohort_count(series: Sequence[dict[str, Any]]) -> int:
- groups: dict[bytes, set[bytes]] = {}
- for item in series:
- control, variant = _public_cohort_factors("chip", item)
- groups.setdefault(_canonical(control), set()).add(_canonical(variant))
- return sum(len(variants) >= 2 for variants in groups.values())
-
-
-def _require_promotion_cohorts(
- cohorts: Sequence[dict[str, Any]],
- series: Sequence[dict[str, Any]],
- *,
- scope: str = "full",
-) -> None:
- # Cohorts carry their own quality verdict (``eligibility.decision_grade`` /
- # ``eligibility.reasons``) into the published dataset; the consumer decides
- # which to surface. Promotion no longer vetoes on non-decision-grade cohorts.
- # For a full-coverage run it still checks *structural* well-formedness — that
- # every required cohort kind is present in the catalogued shape — so a "full"
- # publication genuinely spans the catalog. Quality is reported, not gated.
- if scope != "full":
- # A partial ("v1 tag + success") run covers a subset of the catalog, so
- # the fixed full-matrix cohort counts / required kinds do not apply.
- return
- present_kinds = {cohort["kind"] for cohort in cohorts}
- required_kinds = list(REQUIRED_COHORT_KINDS)
- if any(
- item["workload"].get(
- "precision_profile", identity.V1_CONTROL_PRECISION_PROFILE
- )
- != identity.V1_CONTROL_PRECISION_PROFILE
- for item in series
- ):
- required_kinds.extend(PRECISION_COHORT_KINDS)
- missing = [kind for kind in required_kinds if kind not in present_kinds]
- if missing:
- raise PublisherError(
- "promotion lacks cohort kinds: " + ", ".join(missing)
- )
- for kind, expected in REQUIRED_PROMOTION_COHORT_COUNTS.items():
- members = [cohort for cohort in cohorts if cohort["kind"] == kind]
- if len(members) != expected:
- raise PublisherError(
- f"promotion requires exactly {expected} {kind} cohorts"
- )
-
- chip_cohorts = [cohort for cohort in cohorts if cohort["kind"] == "chip"]
- expected_chips = _expected_chip_cohort_count(series)
- if len(chip_cohorts) != expected_chips:
- raise PublisherError(
- f"promotion requires all {expected_chips} derived chip cohorts"
- )
-
-
-def _require_promotion_series(series: Sequence[dict[str, Any]]) -> None:
- # Each series carries its own quality verdict (``status`` /
- # ``eligibility.decision_grade`` / ``eligibility.reasons``) into the
- # published dataset, and the consumer decides what to surface. Promotion no
- # longer vetoes the whole run because some series are non-decision-grade —
- # it only guarantees the publication is non-empty and holds at least one
- # decision-grade series, so a promoted "v1" still means something measured
- # cleanly. Diagnostic series ship labelled as diagnostic, never dropped.
- if not series:
- raise PublisherError("promotion requires at least one series")
- if not any(item["status"] == "decision-grade" for item in series):
- raise PublisherError("promotion requires at least one decision-grade series")
-
-
-def build_dataset(
- store: Store,
- bundle_ids: Sequence[str],
- *,
- promote: bool,
-) -> dict[str, Any]:
- if not bundle_ids or len(bundle_ids) != len(set(bundle_ids)):
- raise PublisherError("dataset requires unique explicit bundle IDs")
- loaded = [load_bundle(store, bundle_id) for bundle_id in bundle_ids]
- loaded.sort(key=lambda bundle: (
- int(bundle["manifest"]["run"]["run_id"]),
- bundle["manifest"]["run"]["run_attempt"],
- bundle["id"],
- ))
- matrix_ids = {bundle["manifest"]["matrix"]["sha256"] for bundle in loaded}
- case_sets = [{case["case_id"] for case in bundle["cases"]} for bundle in loaded]
- if len(matrix_ids) != 1 or len({tuple(sorted(values)) for values in case_sets}) != 1:
- raise PublisherError("dataset bundles do not share one exact requested matrix")
- run_ids = [bundle["manifest"]["run"]["run_id"] for bundle in loaded]
- qualification_indices = sorted(
- bundle["manifest"]["run"]["qualification_index"] for bundle in loaded
- )
- if promote and (
- len(loaded) != REQUIRED_ALLOCATIONS
- or len(run_ids) != len(set(run_ids))
- or qualification_indices != [1]
- ):
- raise PublisherError(
- "promotion requires qualification index 1 from a single versioned run"
- )
- cases = {case["case_id"]: case for case in loaded[0]["cases"]}
- canonical = _canonical_coverage_cases()
- covered_skus = sorted({
- canonical[case_id]["sku"] for case_id in cases if case_id in canonical
- })
- coverage_scope = "full" if set(cases) == set(canonical) else "partial"
- if promote:
- # A v1-tagged run may promote any subset of the canonical catalog (the
- # "v1 tag + success" contract). Every promoted case must still be a real
- # canonical case with the canonical disposition — a partial run cannot
- # invent cases or reclassify them — but it need not cover every SKU.
- unknown = set(cases) - set(canonical)
- if unknown:
- raise PublisherError(
- "promotion includes cases outside the canonical v1 catalog"
- )
- for case_id, case in cases.items():
- if case["_disposition"] != canonical[case_id]["disposition"]:
- raise PublisherError(
- "promotion case disposition differs from the canonical v1 catalog"
- )
- _require_runnable_promotion_success(loaded, cases)
- all_documents = [
- document for bundle in loaded for document in bundle["documents"].values()
- ]
- selected_ids = {
- selection["selected_attempt_id"]
- for bundle in loaded for selection in bundle["manifest"]["coverage"]["selections"]
- }
- public_attempts = [
- _public_attempt(
- document, selected=document["identity"]["attempt_id"] in selected_ids
- )
- for document in all_documents
- ]
- _unique([attempt["attempt_id"] for attempt in public_attempts], "dataset attempts")
- selected_by_case: dict[str, list[dict[str, Any]]] = {
- case_id: [bundle["selected"][case_id] for bundle in loaded]
- for case_id in sorted(cases)
- }
- samples_by_attempt = {
- attempt_id: sample_document
- for bundle in loaded
- for attempt_id, sample_document in bundle["sample_documents"].items()
- }
- coverage: list[dict[str, Any]] = []
- for case_id, case in sorted(cases.items()):
- attempts = sorted(
- (attempt for attempt in public_attempts if attempt["case_id"] == case_id),
- key=lambda attempt: (
- int(attempt["run_id"]), attempt["run_attempt"],
- attempt["attempt_index"], attempt["attempt_id"],
- ),
- )
- selected_document = selected_by_case[case_id][-1]
- selected = _public_attempt(selected_document, selected=True)
- precision_profile = case.get(
- "precision_profile", identity.V1_CONTROL_PRECISION_PROFILE
- )
- precision = identity.precision_profile(precision_profile)
- selected_raw = (
- selected_document
- if selected_document["format"] == contracts.RAW_FORMAT
- and selected_document["outcome"]["status"] == "success"
- else None
- )
- if selected_raw is not None:
- backend_generation = selected_raw["implementation"]["kernel_generation"]
- projected = contracts.public_series_config(
- kernel_generation=backend_generation,
- provenance=selected_raw["implementation"]["provenance"],
- resource_profile=selected_raw["implementation"]["resource_profile"],
- resource_mode=selected_raw["case"]["resource_mode"],
- device_product=selected_raw["topology"]["device_product"],
- )
- resource = projected["resource"]
- rows_by_token = {
- row["tokens_per_rank"]: row for row in selected_raw["measurement"]["rows"]
- }
- series_id = selected_raw["identity"]["series_id"]
- else:
- backend_generation = None
- resource = {
- "mode": "fixed-profile",
- "profile": None,
- "comm_units_kind": None,
- "configured_units": None,
- }
- rows_by_token = {}
- series_id = None
- point_status = (
- "measured" if selected["outcome"] == "success" else selected["outcome"]
- )
- point_reason = (
- None
- if point_status == "measured"
- else case["_reason"]
- if point_status == "unsupported"
- else selected["reason"]
- )
- token_ladder = [int(value) for value in case["ladder"].split()]
- coverage_points = []
- for token in token_ladder:
- row = rows_by_token.get(token)
- coverage_points.append({
- "point_id": row["point_id"] if row is not None else None,
- "series_id": series_id if row is not None else None,
- "tokens_per_rank": token,
- "global_tokens": token * case["ep"],
- "terminal_status": point_status,
- "reason": point_reason,
- })
- coverage.append({
- "case_id": case_id,
- "label": (
- f"{case['sku'].upper()} / {case['backend']} / EP{case['ep']} / "
- f"{case['mode']} / {case['phase']} / {case['routing']}"
- ),
- "required": True,
- "sku": _slug(case["sku"]),
- "suite": _slug(case["suite"]),
- "workload": _slug(case["workload"]),
- "publication_tier": case["required_publication"],
- "backend": _slug(case["backend"]),
- "backend_generation": backend_generation,
- "mode": case["mode"],
- "phase": case["phase"],
- "routing": case["routing"],
- "eplb": case["eplb"],
- "precision_profile": precision_profile,
- "dispatch_precision": precision["dispatch"],
- "combine_precision": precision["combine"],
- "resource": resource,
- "topology": _coverage_topology(case),
- "points": coverage_points,
- "disposition": case["_disposition"],
- "selected_attempt_id": selected["attempt_id"],
- "outcome": selected["outcome"],
- "failure_mode": selected["failure_mode"],
- "reason": case["_reason"] if case["_disposition"] == "unsupported" else selected["reason"],
- "attempt_ids": [attempt["attempt_id"] for attempt in attempts],
- })
- by_series: dict[str, list[dict[str, Any]]] = {}
- for case_documents in selected_by_case.values():
- for document in case_documents:
- if (
- document["format"] == contracts.RAW_FORMAT
- and document["outcome"]["status"] == "success"
- ):
- by_series.setdefault(document["identity"]["series_id"], []).append(document)
- series: list[dict[str, Any]] = []
- internals: dict[str, dict[str, Any]] = {}
- for series_id, documents in sorted(by_series.items()):
- try:
- sample_documents = [
- samples_by_attempt[document["identity"]["attempt_id"]]
- for document in documents
- ]
- except KeyError as exc:
- raise PublisherError(
- "selected raw evidence is missing its private sample document"
- ) from exc
- item, internal = _build_series(
- series_id, documents, sample_documents, len(loaded)
- )
- series.append(item)
- internals[series_id] = internal
- dataset_binding = _sha_bytes(_canonical({
- "matrix_id": next(iter(matrix_ids)),
- "source_bundle_ids": sorted(bundle_ids),
- }))
- cohorts, rankings, recommendations, sensitivities = build_decisions(
- series, internals, dataset_binding=dataset_binding
- )
- allocation_ids = sorted({attempt["allocation_id"] for attempt in public_attempts})
- qualification_indices = sorted({int(value) for value in qualification_indices})
- measured_cases = sum(
- all(point["terminal_status"] == "measured" for point in item["points"])
- for item in coverage
- )
- unsupported_cases = sum(
- all(point["terminal_status"] == "unsupported" for point in item["points"])
- for item in coverage
- )
- requested_points = sum(len(item["points"]) for item in coverage)
- measured_points = sum(
- point["terminal_status"] == "measured"
- for item in coverage for point in item["points"]
- )
- unsupported_points = sum(
- point["terminal_status"] == "unsupported"
- for item in coverage for point in item["points"]
- )
- status = "promoted" if promote else "diagnostic"
- dataset = {
- "format": FORMAT_PUBLIC,
- "schema_version": 1,
- "generated_at": _latest_timestamp(
- [bundle["manifest"]["created_at"] for bundle in loaded]
- ),
- "source_bundle_ids": sorted(bundle_ids),
- "promotion": {
- "status": status,
- "reason": None,
- "matrix_id": next(iter(matrix_ids)),
- "allocation_ids": allocation_ids,
- "required_allocations": REQUIRED_ALLOCATIONS,
- "qualification_indices": qualification_indices,
- "coverage_scope": coverage_scope,
- "covered_skus": covered_skus,
- "requested_cases": len(coverage),
- "terminal_cases": len(coverage),
- "measured_cases": measured_cases,
- "unsupported_cases": unsupported_cases,
- "requested_points": requested_points,
- "terminal_points": requested_points,
- "measured_points": measured_points,
- "unsupported_points": unsupported_points,
- "policy": POLICY,
- },
- "coverage": coverage,
- "attempts": sorted(public_attempts, key=lambda attempt: attempt["attempt_id"]),
- "series": series,
- "cohorts": cohorts,
- "rankings": rankings,
- "recommendations": recommendations,
- "sensitivities": sensitivities,
- }
- if promote:
- _require_promotion_series(series)
- _require_promotion_cohorts(cohorts, series, scope=coverage_scope)
- validate_public_dataset(dataset)
- return dataset
-
-
-def quarantine_incoming(
- store: Store, ingest_id: str, reason: str, generated_at: str
-) -> str:
- if REASON.fullmatch(reason) is None:
- raise PublisherError("quarantine reason must be a machine code")
- public_reason = f"{reason}-{ingest_id}"
- if REASON.fullmatch(public_reason) is None:
- raise PublisherError("quarantine reason and incoming ID exceed the public reason contract")
- manifest = {
- "format": "collectivex.quarantine.v1",
- "schema_version": 1,
- "created_at": generated_at,
- "incoming_id": ingest_id,
- "reason": reason,
- }
- digest = _sha_bytes(_canonical(manifest))
- with store.staging(store.quarantine, private=True) as stage:
- _write_json(stage / "quarantine.json", manifest, mode=0o600)
- store.complete(stage, digest, private=True)
- store.install(stage, store.quarantine / digest, private=True)
- if _sha_bytes(_canonical(strict_load(store.quarantine / digest / "quarantine.json"))) != digest:
- raise PublisherError("existing quarantine object differs")
- return digest
-
-
-def _store_from_args(args: argparse.Namespace) -> Store:
- root = args.store_root or os.environ.get("COLLECTIVEX_STORE_ROOT")
- if not root:
- raise PublisherError("COLLECTIVEX_STORE_ROOT or --store-root is required")
- if not Path(root).is_absolute():
- raise PublisherError("COLLECTIVEX_STORE_ROOT must be an absolute path")
- return Store(root)
-
-
-def _run_metadata(args: argparse.Namespace) -> dict[str, Any]:
- """Validate offline operator assertions about a completed successful GHA run.
-
- The publisher deliberately performs no network access. The caller must preflight workflow
- identity and conclusion against GitHub before supplying these values; artifact-internal
- provenance is then required to match them exactly.
- """
- run = {
- "repository": args.repository,
- "run_id": args.run_id,
- "run_attempt": args.run_attempt,
- "qualification_index": args.qualification_index,
- "source_sha": args.source_sha,
- }
- # Reuse the authoritative private schema constraints before any filesystem mutation.
- if not re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", run["repository"] or ""):
- raise PublisherError("--repository must be owner/name")
- if not re.fullmatch(r"[1-9][0-9]*", run["run_id"] or ""):
- raise PublisherError("--run-id must be a positive decimal string")
- if type(run["run_attempt"]) is not int or run["run_attempt"] < 1:
- raise PublisherError("--run-attempt must be positive")
- if type(run["qualification_index"]) is not int or run["qualification_index"] not in range(1, 4):
- raise PublisherError("--qualification-index must be 1, 2, or 3")
- if not re.fullmatch(r"[0-9a-f]{40}", run["source_sha"] or ""):
- raise PublisherError("--source-sha must be a 40-character lowercase Git SHA")
- return run
-
-
-def _ingest_inputs(
- args: argparse.Namespace,
-) -> tuple[dict[str, Any], Path, list[Path]]:
- run = _run_metadata(args)
- matrix = Path(args.matrix).absolute()
- if matrix.is_symlink() or not matrix.is_file():
- raise PublisherError("--matrix must be a regular non-symlink file")
- artifacts = [Path(value).absolute() for value in args.artifact]
- if not artifacts:
- raise PublisherError("at least one --artifact is required")
- names = [_artifact_name(path) for path in artifacts]
- if len(names) != len(set(names)):
- raise PublisherError("--artifact contains duplicate GHA names")
- for path in artifacts:
- if path.is_symlink() or not (path.is_dir() or path.is_file()):
- raise PublisherError("--artifact must be a regular ZIP or real directory")
- return run, matrix, artifacts
-
-
-def _bundle_ids(values: Sequence[str], *, promote: bool) -> list[str]:
- bundle_ids = list(values)
- if (
- not bundle_ids
- or len(bundle_ids) != len(set(bundle_ids))
- or any(HEX64.fullmatch(value) is None for value in bundle_ids)
- ):
- raise PublisherError("bundle IDs must be unique SHA-256 digests")
- if promote and len(bundle_ids) != REQUIRED_ALLOCATIONS:
- raise PublisherError("promotion requires exactly one explicit bundle ID")
- return bundle_ids
-
-
-def ingest_command(args: argparse.Namespace) -> dict[str, Any]:
- run, matrix, artifacts = _ingest_inputs(args)
- store = _store_from_args(args)
- with store.locked():
- ingest_id, incoming, _ = archive_incoming(
- store, matrix, artifacts, run
- )
- try:
- bundle_id, _, _ = build_bundle(store, ingest_id, incoming, run)
- return {
- "status": "accepted", "incoming_id": ingest_id,
- "bundle_id": bundle_id,
- }
- except (
- PublisherError, contracts.ContractError, artifact_safety.ArtifactSafetyError,
- jsonschema.ValidationError,
- ) as exc:
- # Invalid delivery bytes provide no trusted timestamp. A fixed sentinel keeps
- # repeated quarantine of the same immutable incoming object content-idempotent.
- generated_at = "1970-01-01T00:00:00Z"
- quarantine_id = quarantine_incoming(
- store, ingest_id, "artifact-validation-failed", generated_at
- )
- raise PublisherError(
- f"incoming {ingest_id} quarantined as {quarantine_id}: {exc}"
- ) from exc
-
-
-def promote_command(args: argparse.Namespace) -> dict[str, Any]:
- bundle_ids = _bundle_ids(args.bundle, promote=True)
- store = _store_from_args(args)
- with store.locked():
- dataset = build_dataset(store, bundle_ids, promote=True)
- digest, size = store.install_dataset(dataset)
- store.update_channel("dev-latest", digest, size, dataset["generated_at"])
- store.verify_channel("dev-latest")
- return {
- "status": "promoted", "bundle_ids": bundle_ids,
- "dataset_sha256": digest, "channel": "dev-latest",
- }
-
-
-def verify_command(args: argparse.Namespace) -> dict[str, Any]:
- bundle_ids = _bundle_ids(args.bundle, promote=False) if args.bundle else []
- channels = args.channel or ["dev-latest"]
- if any(channel != "dev-latest" for channel in channels):
- raise PublisherError("unknown channel")
- store = _store_from_args(args)
- with store.locked():
- pointers = {channel: store.verify_channel(channel) for channel in channels}
- bundles = [load_bundle(store, bundle_id)["id"] for bundle_id in bundle_ids]
- return {"status": "verified", "channels": pointers, "bundle_ids": bundles}
-
-
-def _parser() -> argparse.ArgumentParser:
- parser = argparse.ArgumentParser(description="CollectiveX isolated filesystem publisher")
- parser.add_argument("--store-root", help="defaults to COLLECTIVEX_STORE_ROOT")
- subparsers = parser.add_subparsers(dest="command", required=True)
- ingest = subparsers.add_parser("ingest", help="archive and validate one complete GHA run")
- ingest.add_argument("--matrix", required=True)
- ingest.add_argument("--artifact", action="append", required=True)
- ingest.add_argument("--repository", required=True)
- ingest.add_argument("--run-id", required=True)
- ingest.add_argument("--run-attempt", required=True, type=int)
- ingest.add_argument("--qualification-index", required=True, type=int)
- ingest.add_argument("--source-sha", required=True)
- promote = subparsers.add_parser("promote", help="publish explicit independent bundles")
- promote.add_argument("--bundle", action="append", required=True)
- verify = subparsers.add_parser("verify", help="verify immutable targets and pointers")
- verify.add_argument("--channel", action="append", choices=["dev-latest"])
- verify.add_argument("--bundle", action="append", default=[])
- return parser
-
-
-def main() -> int:
- args = _parser().parse_args()
- try:
- if args.command == "ingest":
- result = ingest_command(args)
- elif args.command == "promote":
- result = promote_command(args)
- elif args.command == "verify":
- result = verify_command(args)
- else:
- raise PublisherError(f"unknown command {args.command!r}")
- except (
- PublisherError, contracts.ContractError, artifact_safety.ArtifactSafetyError,
- jsonschema.ValidationError, OSError,
- ) as exc:
- print(json.dumps({"status": "error", "error": str(exc)}), file=sys.stderr)
- return 2
- print(json.dumps(result, sort_keys=True))
- return 0
-
-
-if __name__ == "__main__":
- raise SystemExit(main())
diff --git a/experimental/CollectiveX/tests/probe_precision.py b/experimental/CollectiveX/tests/probe_precision.py
deleted file mode 100644
index 25060032a3..0000000000
--- a/experimental/CollectiveX/tests/probe_precision.py
+++ /dev/null
@@ -1,1213 +0,0 @@
-#!/usr/bin/env python3
-"""Bounded real-hardware capability probe for provisional CollectiveX precision cells."""
-from __future__ import annotations
-
-import argparse
-import datetime as dt
-import hashlib
-import inspect
-import json
-import os
-import platform
-import re
-import socket
-import sys
-import tempfile
-from pathlib import Path
-from types import SimpleNamespace
-from typing import Any
-
-
-HERE = Path(__file__).resolve().parent
-ROOT = HERE.parent
-sys.path[:0] = [str(HERE), str(ROOT)]
-
-import artifact_safety # noqa: E402
-import capability # noqa: E402
-import ep_harness # noqa: E402
-
-
-FORMAT = "collectivex.precision-probe.v1"
-PLAN_FORMAT = "collectivex.precision-probe-plan.v1"
-CONTROL_FORMAT = "collectivex.precision-probe-control.v1"
-RECORD_TYPE = "precision-capability-probe"
-PROBE_CONTRACT = "bounded-native-cell-v1"
-FENCE_CONTRACT = "caller-event-cross-stream-v1"
-SUPPORTED_REASON = "native-probe-passed"
-UNSUPPORTED_REASONS = frozenset({
- "backend-construction-failed",
- "completion-fence-failed",
- "cross-rank-evidence-mismatch",
- "native-operation-failed",
- "precision-contract-mismatch",
- "runtime-identity-mismatch",
- "target-not-provisional",
- "transport-fallback-detected",
- "unsupported-native-api",
- "unverified-execution-identity",
-})
-BACKENDS = frozenset({
- "deepep", "deepep-v2", "deepep-hybrid", "mori", "uccl",
-})
-SHA40 = re.compile(r"[0-9a-f]{40}")
-SHA256 = re.compile(r"[0-9a-f]{64}")
-IMAGE_DIGEST = re.compile(r"sha256:[0-9a-f]{64}")
-
-
-class ProbeError(RuntimeError):
- """A provisional cell did not produce complete native runtime evidence."""
-
- def __init__(self, reason: str):
- if reason not in UNSUPPORTED_REASONS:
- raise ValueError(f"unknown precision probe reason {reason!r}")
- super().__init__(reason)
- self.reason = reason
-
-
-def _canonical(value: Any) -> bytes:
- return json.dumps(
- value, allow_nan=False, ensure_ascii=True, sort_keys=True, separators=(",", ":")
- ).encode("ascii")
-
-
-def _sha(value: Any) -> str:
- return hashlib.sha256(_canonical(value)).hexdigest()
-
-
-def _exact_keys(value: Any, expected: set[str], path: str) -> dict[str, Any]:
- if not isinstance(value, dict) or set(value) != expected:
- raise ValueError(f"{path} fields differ from {FORMAT}")
- return value
-
-
-def _text(value: Any, path: str, *, nullable: bool = False) -> str | None:
- if nullable and value is None:
- return None
- if (
- not isinstance(value, str)
- or not value
- or len(value) > 4096
- or any(ord(character) < 0x20 or ord(character) > 0x7E for character in value)
- ):
- raise ValueError(f"{path} is not bounded printable ASCII")
- return value
-
-
-def _boolean(value: Any, path: str) -> bool:
- if type(value) is not bool:
- raise ValueError(f"{path} is not boolean")
- return value
-
-
-def _integer(value: Any, path: str, minimum: int = 0) -> int:
- if type(value) is not int or value < minimum:
- raise ValueError(f"{path} is not an integer >= {minimum}")
- return value
-
-
-def validate_manifest(document: Any) -> dict[str, Any]:
- """Validate the closed probe format without extending publication schemas."""
- doc = _exact_keys(document, {
- "evidence", "format", "generated_at", "privacy", "probe_contract",
- "record_type", "result", "schema_version", "target", "topology",
- }, "probe")
- if (
- doc["format"] != FORMAT
- or doc["record_type"] != RECORD_TYPE
- or doc["schema_version"] != 1
- or doc["probe_contract"] != PROBE_CONTRACT
- ):
- raise ValueError("probe format, record type, schema, or contract differs")
- _text(doc["generated_at"], "probe.generated_at")
- target = _exact_keys(doc["target"], {
- "backend", "basis", "ep", "mode", "precision_profile", "registry_disposition",
- "sku",
- }, "probe.target")
- if (
- target["backend"] not in BACKENDS
- or target["registry_disposition"] != "provisional"
- or target["mode"] not in {"normal", "low-latency"}
- ):
- raise ValueError("probe target is not a provisional native adapter cell")
- for field in ("basis", "precision_profile", "sku"):
- _text(target[field], f"probe.target.{field}")
- _integer(target["ep"], "probe.target.ep", 1)
- topology = _exact_keys(doc["topology"], {
- "gpus_per_node", "nodes", "placement_valid", "scale_up_domain",
- "scale_up_transport", "scale_out_transport", "scope", "topology_class",
- "transport", "world_size",
- }, "probe.topology")
- for field in ("gpus_per_node", "nodes", "scale_up_domain", "world_size"):
- _integer(topology[field], f"probe.topology.{field}", 1)
- for field in ("scale_up_transport", "scope", "topology_class", "transport"):
- _text(topology[field], f"probe.topology.{field}")
- _text(topology["scale_out_transport"], "probe.topology.scale_out_transport", nullable=True)
- _boolean(topology["placement_valid"], "probe.topology.placement_valid")
- result = _exact_keys(doc["result"], {
- "disposition", "reason", "registry_mutation", "runtime_executed",
- "static_inspection_sufficient",
- }, "probe.result")
- if result["disposition"] not in {"supported", "unsupported"}:
- raise ValueError("probe result disposition is invalid")
- expected_reason = (
- SUPPORTED_REASON if result["disposition"] == "supported" else result["reason"]
- )
- if result["reason"] != expected_reason or (
- result["disposition"] == "unsupported"
- and result["reason"] not in UNSUPPORTED_REASONS
- ):
- raise ValueError("probe result reason is invalid")
- if result["registry_mutation"] is not False or result["static_inspection_sufficient"] is not False:
- raise ValueError("probe must never mutate or statically promote the registry")
- _boolean(result["runtime_executed"], "probe.result.runtime_executed")
- privacy = _exact_keys(doc["privacy"], {"contract", "sanitized"}, "probe.privacy")
- if privacy != {"contract": "artifact-safety-v1", "sanitized": True}:
- raise ValueError("probe privacy contract differs")
- if result["disposition"] == "supported":
- _validate_evidence(doc["evidence"])
- elif doc["evidence"] is not None:
- _validate_evidence(doc["evidence"])
- artifact_safety.assert_publication_safe([doc])
- return doc
-
-
-def _validate_evidence(value: Any) -> None:
- evidence = _exact_keys(value, {
- "api", "completion", "identity", "precision", "transport",
- }, "probe.evidence")
- api = _exact_keys(evidence["api"], {"calls", "signature_sha256"}, "probe.evidence.api")
- if not isinstance(api["calls"], list) or not api["calls"]:
- raise ValueError("probe API calls are empty")
- for index, call in enumerate(api["calls"]):
- item = _exact_keys(call, {"name", "signature"}, f"probe.evidence.api.calls[{index}]")
- _text(item["name"], "probe API name")
- _text(item["signature"], "probe API signature")
- if not isinstance(api["signature_sha256"], str) or not SHA256.fullmatch(api["signature_sha256"]):
- raise ValueError("probe API signature digest is invalid")
- completion = _exact_keys(evidence["completion"], {
- "caller_event_complete", "contract", "mode", "output_finite",
- "verifier_stream_complete",
- }, "probe.evidence.completion")
- if completion["contract"] != FENCE_CONTRACT:
- raise ValueError("probe completion contract differs")
- _text(completion["mode"], "probe completion mode")
- if not all(
- _boolean(completion[field], f"probe completion {field}")
- for field in ("caller_event_complete", "output_finite", "verifier_stream_complete")
- ):
- raise ValueError("probe completion evidence did not pass")
- identity_record = _exact_keys(evidence["identity"], {
- "backend_components", "backend_provenance_sha256", "image_digest",
- "image_digest_verified", "image_reference", "source_sha",
- }, "probe.evidence.identity")
- if not SHA40.fullmatch(str(identity_record["source_sha"])):
- raise ValueError("probe source SHA is invalid")
- if not IMAGE_DIGEST.fullmatch(str(identity_record["image_digest"])):
- raise ValueError("probe image digest is invalid")
- _text(identity_record["image_reference"], "probe image reference")
- if identity_record["image_digest_verified"] is not True:
- raise ValueError("probe image digest is unverified")
- if not SHA256.fullmatch(str(identity_record["backend_provenance_sha256"])):
- raise ValueError("probe backend provenance digest is invalid")
- components = identity_record["backend_components"]
- if not isinstance(components, list) or not components:
- raise ValueError("probe backend component identity is empty")
- for component in components:
- item = _exact_keys(component, {"revision", "role", "version"}, "probe backend component")
- _text(item["role"], "probe backend component role")
- _text(item["revision"], "probe backend component revision", nullable=True)
- _text(item["version"], "probe backend component version", nullable=True)
- if item["revision"] is None and item["version"] is None:
- raise ValueError("probe backend component has no identity")
- precision = _exact_keys(evidence["precision"], {
- "combine", "correctness", "dispatch", "profile_id",
- }, "probe.evidence.precision")
- _text(precision["profile_id"], "probe precision profile")
- for direction in ("dispatch", "combine"):
- axis = _exact_keys(precision[direction], {
- "accumulator_dtype", "accumulator_evidence", "api_input_dtype",
- "api_output_dtype", "communication_format", "runtime_input",
- "runtime_output", "scale_contract", "semantic_output",
- }, f"probe.evidence.precision.{direction}")
- for field in (
- "accumulator_dtype", "accumulator_evidence", "api_input_dtype",
- "api_output_dtype", "communication_format",
- ):
- _text(axis[field], f"probe precision {direction} {field}")
- _validate_tensor_summary(axis["runtime_input"], f"probe precision {direction} input")
- _validate_tensor_summary(axis["runtime_output"], f"probe precision {direction} output")
- _validate_tensor_summary(axis["semantic_output"], f"probe precision {direction} semantic")
- scale = _exact_keys(axis["scale_contract"], {
- "alignment", "dtype", "finite", "group_size", "layout", "padding",
- "positive", "runtime_shapes", "runtime_storage_dtype",
- }, f"probe precision {direction} scales")
- for field in ("alignment", "layout", "padding"):
- _text(scale[field], f"probe precision {direction} scale {field}")
- _text(scale["dtype"], "probe scale dtype", nullable=True)
- _text(scale["runtime_storage_dtype"], "probe scale storage", nullable=True)
- if scale["group_size"] is not None:
- _integer(scale["group_size"], "probe scale group", 1)
- for field in ("finite", "positive"):
- if scale[field] is not None:
- _boolean(scale[field], f"probe scale {field}")
- if not isinstance(scale["runtime_shapes"], list):
- raise ValueError("probe scale shapes are invalid")
- correctness = precision["correctness"]
- if not isinstance(correctness, dict) or correctness.get("passed") is not True:
- raise ValueError("probe precision correctness did not pass")
- transport = _exact_keys(evidence["transport"], {
- "evidence", "fallback_used", "native_backend", "requested", "runtime_route",
- }, "probe.evidence.transport")
- for field in ("native_backend", "requested", "runtime_route"):
- _text(transport[field], f"probe transport {field}")
- if transport["fallback_used"] is not False:
- raise ValueError("probe transport fallback is present")
- if not isinstance(transport["evidence"], list) or not transport["evidence"]:
- raise ValueError("probe transport evidence is empty")
- for item in transport["evidence"]:
- _text(item, "probe transport evidence item")
-
-
-def _validate_tensor_summary(value: Any, path: str) -> None:
- summary = _exact_keys(value, {"finite", "rank", "shapes", "storage_dtype"}, path)
- _text(summary["storage_dtype"], f"{path}.storage_dtype")
- _integer(summary["rank"], f"{path}.rank", 0)
- if summary["finite"] is not True:
- raise ValueError(f"{path} is not finite")
- if not isinstance(summary["shapes"], list) or not summary["shapes"]:
- raise ValueError(f"{path} shapes are empty")
- for shape in summary["shapes"]:
- if not isinstance(shape, list) or any(type(item) is not int or item < 0 for item in shape):
- raise ValueError(f"{path} shape is invalid")
-
-
-def provisional_targets() -> list[dict[str, Any]]:
- """Return deterministic probe cells without changing their dispositions."""
- return sorted(
- capability.provisional_precision_targets(),
- key=lambda item: (
- item["sku"], item["backend"], item["ep"], item["mode"],
- item["precision_profile"],
- ),
- )
-
-
-def _probe_id(target: dict[str, Any]) -> str:
- return f"probe-{_sha({key: target[key] for key in ('backend', 'sku', 'ep', 'mode', 'precision_profile')})[:20]}"
-
-
-def _workflow_row(target: dict[str, Any]) -> dict[str, Any]:
- topology = capability.topology_for(target["sku"], target["ep"])
- if topology is None:
- raise ValueError("precision probe target has no registered topology")
- return {
- "backend": target["backend"],
- "basis": target["basis"],
- "disposition": target["disposition"],
- "ep": target["ep"],
- "execution_weight": target["ep"],
- "gpus_per_node": topology["gpus_per_node"],
- "id": _probe_id(target),
- "launcher": capability.PLATFORMS[target["sku"]]["launcher"],
- "mode": target["mode"],
- "n": 1,
- "nodes": topology["nodes"],
- "precision_profile": target["precision_profile"],
- "scale_up_domain": topology["scale_up_domain"],
- "sku": target["sku"],
- }
-
-
-def workflow_plan(
- *, backend: str = "all", only_sku: str = "", min_nodes: int = 0,
- max_nodes: int = 0, probe_id: str = "",
-) -> dict[str, Any]:
- if min_nodes < 0 or max_nodes < 0 or (min_nodes and max_nodes and min_nodes > max_nodes):
- raise ValueError("precision probe node filters are invalid")
- targets = [
- target for target in provisional_targets()
- if (backend == "all" or target["backend"] == backend)
- and (not only_sku or target["sku"] == only_sku)
- and (not probe_id or _probe_id(target) == probe_id)
- and (
- not min_nodes
- or capability.topology_for(target["sku"], target["ep"])["nodes"] >= min_nodes
- )
- and (
- not max_nodes
- or capability.topology_for(target["sku"], target["ep"])["nodes"] <= max_nodes
- )
- ]
- if backend != "all" and backend not in BACKENDS:
- raise ValueError("precision probe backend is not registered")
- if only_sku and only_sku not in capability.PLATFORMS:
- raise ValueError("precision probe SKU is not registered")
- if probe_id and not re.fullmatch(r"probe-[0-9a-f]{20}", probe_id):
- raise ValueError("precision probe ID is invalid")
- if not targets:
- raise ValueError("precision probe filters select no provisional cells")
- return {
- "format": PLAN_FORMAT,
- "include": [_workflow_row(target) for target in targets],
- "schema_version": 1,
- }
-
-
-def validate_workflow_plan(document: Any) -> dict[str, Any]:
- plan = _exact_keys(document, {"format", "include", "schema_version"}, "probe plan")
- if plan["format"] != PLAN_FORMAT or plan["schema_version"] != 1:
- raise ValueError("precision probe plan format differs")
- if not isinstance(plan["include"], list):
- raise ValueError("precision probe plan include is not a list")
- expected = {_probe_id(target): _workflow_row(target) for target in provisional_targets()}
- seen: set[str] = set()
- for row in plan["include"]:
- if not isinstance(row, dict) or row.get("id") not in expected or row != expected[row["id"]]:
- raise ValueError("precision probe plan row differs from the capability registry")
- if row["id"] in seen:
- raise ValueError("precision probe plan contains a duplicate row")
- seen.add(row["id"])
- return plan
-
-
-def extract_control(
- plan: Any, *, probe_id: str, sku: str, backend: str, nodes: int,
-) -> dict[str, Any]:
- rows = [row for row in validate_workflow_plan(plan)["include"] if row["id"] == probe_id]
- if len(rows) != 1:
- raise ValueError("precision probe ID is not unique in the plan")
- row = rows[0]
- if (row["sku"], row["backend"], row["nodes"]) != (sku, backend, nodes):
- raise ValueError("precision probe control differs from the workflow matrix")
- target = select_target(
- backend=row["backend"], sku=row["sku"], ep=row["ep"], mode=row["mode"],
- precision_profile=row["precision_profile"],
- )
- topology = capability.topology_for(row["sku"], row["ep"])
- if topology is None:
- raise ValueError("precision probe control has no topology")
- return {
- "format": CONTROL_FORMAT,
- "id": row["id"],
- "launcher": row["launcher"],
- "schema_version": 1,
- "target": target,
- "topology": topology,
- }
-
-
-def validate_control(
- document: Any, *, sku: str, backend: str, nodes: int,
-) -> dict[str, Any]:
- control = _exact_keys(
- document, {"format", "id", "launcher", "schema_version", "target", "topology"},
- "probe control",
- )
- if control["format"] != CONTROL_FORMAT or control["schema_version"] != 1:
- raise ValueError("precision probe control format differs")
- expected = extract_control(
- {"format": PLAN_FORMAT, "include": [_workflow_row(control["target"])], "schema_version": 1},
- probe_id=control["id"], sku=sku, backend=backend, nodes=nodes,
- )
- if control != expected:
- raise ValueError("precision probe control differs from the capability registry")
- return control
-
-
-def validate_bundle(plan: Any, manifests: list[Any]) -> None:
- rows = validate_workflow_plan(plan)["include"]
- expected = {
- (row["backend"], row["sku"], row["ep"], row["mode"], row["precision_profile"])
- for row in rows
- }
- observed = []
- for manifest in manifests:
- target = validate_manifest(manifest)["target"]
- observed.append((
- target["backend"], target["sku"], target["ep"], target["mode"],
- target["precision_profile"],
- ))
- if len(observed) != len(set(observed)) or set(observed) != expected:
- raise ValueError("precision probe bundle does not cover the exact workflow plan")
-
-
-def select_target(
- *, backend: str, sku: str, ep: int, mode: str, precision_profile: str
-) -> dict[str, Any]:
- matches = [
- item for item in provisional_targets()
- if item["backend"] == backend and item["sku"] == sku and item["ep"] == ep
- and item["mode"] == mode and item["precision_profile"] == precision_profile
- ]
- if len(matches) != 1:
- raise ProbeError("target-not-provisional")
- return matches[0]
-
-
-def _target_record(target: dict[str, Any]) -> dict[str, Any]:
- return {
- "backend": target["backend"],
- "basis": target["basis"],
- "ep": target["ep"],
- "mode": target["mode"],
- "precision_profile": target["precision_profile"],
- "registry_disposition": "provisional",
- "sku": target["sku"],
- }
-
-
-def build_manifest(
- *, target: dict[str, Any], topology: dict[str, Any], disposition: str,
- reason: str, runtime_executed: bool, evidence: dict[str, Any] | None,
-) -> dict[str, Any]:
- document = {
- "evidence": evidence,
- "format": FORMAT,
- "generated_at": dt.datetime.now(dt.timezone.utc).isoformat(),
- "privacy": {"contract": "artifact-safety-v1", "sanitized": True},
- "probe_contract": PROBE_CONTRACT,
- "record_type": RECORD_TYPE,
- "result": {
- "disposition": disposition,
- "reason": reason,
- "registry_mutation": False,
- "runtime_executed": runtime_executed,
- "static_inspection_sufficient": False,
- },
- "schema_version": 1,
- "target": _target_record(target),
- "topology": topology,
- }
- return validate_manifest(document)
-
-
-def _write_atomic(path: Path, document: dict[str, Any]) -> None:
- path.parent.mkdir(parents=True, exist_ok=True)
- descriptor, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent)
- try:
- os.fchmod(descriptor, 0o600)
- with os.fdopen(descriptor, "wb") as handle:
- handle.write(_canonical(document) + b"\n")
- handle.flush()
- os.fsync(handle.fileno())
- os.replace(temporary, path)
- finally:
- if os.path.exists(temporary):
- os.unlink(temporary)
-
-
-def _dtype_name(dtype: Any) -> str:
- return str(dtype).removeprefix("torch.")
-
-
-def _local_tensor_summary(torch_module, tensor) -> dict[str, Any]:
- return {
- "finite": bool(torch_module.isfinite(tensor.float()).all().item()),
- "rank": int(tensor.ndim),
- "shape": [int(item) for item in tensor.shape],
- "storage_dtype": _dtype_name(tensor.dtype),
- }
-
-
-def _aggregate_tensor_summaries(records: list[dict[str, Any]]) -> dict[str, Any]:
- dtypes = {record["storage_dtype"] for record in records}
- ranks = {record["rank"] for record in records}
- if len(dtypes) != 1 or len(ranks) != 1:
- raise ProbeError("cross-rank-evidence-mismatch")
- return {
- "finite": all(record["finite"] for record in records),
- "rank": ranks.pop(),
- "shapes": [
- list(shape) for shape in sorted({tuple(record["shape"]) for record in records})
- ],
- "storage_dtype": dtypes.pop(),
- }
-
-
-def _scale_contract(torch_module, axis: dict[str, Any], scales) -> dict[str, Any]:
- return {
- "alignment": axis["alignment_contract"],
- "dtype": axis["scale_dtype"],
- "finite": (
- bool(torch_module.isfinite(scales.float()).all().item())
- if scales is not None else None
- ),
- "group_size": axis["scale_group_size"],
- "layout": axis["scale_layout"],
- "padding": axis["padding_contract"],
- "positive": bool((scales > 0).all().item()) if scales is not None else None,
- "runtime_shape": [int(item) for item in scales.shape] if scales is not None else None,
- "runtime_storage_dtype": _dtype_name(scales.dtype) if scales is not None else None,
- }
-
-
-def _aggregate_scale_contracts(records: list[dict[str, Any]]) -> dict[str, Any]:
- fixed_fields = (
- "alignment", "dtype", "group_size", "layout", "padding", "runtime_storage_dtype",
- )
- result: dict[str, Any] = {}
- for field in fixed_fields:
- values = {_canonical(record[field]) for record in records}
- if len(values) != 1:
- raise ProbeError("cross-rank-evidence-mismatch")
- result[field] = records[0][field]
- for field in ("finite", "positive"):
- values = [record[field] for record in records]
- result[field] = None if all(value is None for value in values) else all(value is True for value in values)
- result["runtime_shapes"] = [
- list(shape)
- for shape in sorted({tuple(record["runtime_shape"] or ()) for record in records})
- ] if any(record["runtime_shape"] is not None for record in records) else []
- return result
-
-
-def _signature(callable_object: Any, name: str) -> dict[str, str]:
- try:
- signature = str(inspect.signature(callable_object))
- except (TypeError, ValueError) as exc:
- raise ProbeError("unsupported-native-api") from exc
- _text(signature, f"native API {name} signature")
- return {"name": name, "signature": signature}
-
-
-def _api_evidence(backend_name: str, backend) -> dict[str, Any]:
- if backend_name in {"deepep", "uccl"}:
- native = type(backend.buffer)
- dispatch_name = "low_latency_dispatch" if backend.mode == "low-latency" else "dispatch"
- combine_name = "low_latency_combine" if backend.mode == "low-latency" else "combine"
- calls = [
- _signature(native.__init__, f"{native.__name__}.__init__"),
- _signature(getattr(native, dispatch_name), f"{native.__name__}.{dispatch_name}"),
- _signature(getattr(native, combine_name), f"{native.__name__}.{combine_name}"),
- ]
- elif backend_name in {"deepep-v2", "deepep-hybrid"}:
- native = type(backend.buffer)
- calls = [
- _signature(native.__init__, f"{native.__name__}.__init__"),
- _signature(native.dispatch, f"{native.__name__}.dispatch"),
- _signature(native.combine, f"{native.__name__}.combine"),
- ]
- elif backend_name == "mori":
- native = type(backend.op)
- calls = [
- _signature(type(backend.config).__init__, "EpDispatchCombineConfig.__init__"),
- _signature(native.dispatch, f"{native.__name__}.dispatch"),
- _signature(native.combine, f"{native.__name__}.combine"),
- ]
- else: # pragma: no cover - guarded by target registry
- raise ProbeError("unsupported-native-api")
- return {"calls": calls, "signature_sha256": _sha(calls)}
-
-
-def _completion_mode(backend_name: str, mode: str) -> str:
- if backend_name in {"deepep", "uccl"}:
- return "async_finish=false;return_recv_hook=false"
- if backend_name == "deepep-v2":
- return "async_with_compute_stream=false;do_cpu_sync=true"
- if backend_name == "deepep-hybrid":
- return "metadata-nonblocking=false;caller-stream-ordered"
- if backend_name == "mori":
- return "current-stream-ordered"
- raise ProbeError("unsupported-native-api")
-
-
-def _transport_evidence(backend_name: str, backend, args) -> dict[str, Any]:
- provenance = backend.backend_provenance
- fallback = False
- facts: list[str]
- if backend_name == "deepep":
- if args.scope == "scale-out" and int(provenance["num_rdma_bytes"]) <= 0:
- fallback = True
- if args.scale_up_transport == "mnnvl" and provenance["mnnvl_comm"] != "explicit-allow-mnnvl":
- fallback = True
- route = f"deepep-{backend.mode}"
- facts = [
- f"mnnvl={provenance['mnnvl_comm']}",
- f"nvl-buffer={int(provenance['num_nvl_bytes']) > 0}",
- f"rdma-buffer={int(provenance['num_rdma_bytes']) > 0}",
- ]
- elif backend_name == "uccl":
- scratch_location = str(backend.buffer.scratch.device.type)
- rdma_active = int(provenance["num_rdma_bytes"]) > 0
- fallback = rdma_active and scratch_location != "cuda"
- route = f"uccl-proxy-{backend.mode}"
- facts = [f"rdma-buffer={rdma_active}", f"rdma-memory={scratch_location}"]
- elif backend_name == "deepep-v2":
- expected_gin = args.scope == "scale-out"
- fallback = bool(provenance["gin_enabled"]) != expected_gin
- route = str(provenance["communication_backend"])
- facts = [
- f"gin-enabled={bool(provenance['gin_enabled'])}",
- f"nccl-communicator={provenance['nccl_communicator']}",
- ]
- elif backend_name == "deepep-hybrid":
- route = str(provenance["transport"])
- expected_build = "multinode-doca" if args.scope == "scale-out" else "intradomain"
- realized_build = os.environ.get("DEEPEP_HYBRID_BUILD_MODE")
- fallback = realized_build != expected_build
- facts = [f"build-mode={realized_build or 'missing'}", f"domains={backend.communication_domains}"]
- elif backend_name == "mori":
- route = str(backend.kernel_generation)
- expected_kernel = (
- "inter-node-v1" if args.scope == "scale-out"
- else "async-ll" if args.runner in {"mi300x", "mi325x"} else "intranode"
- )
- fallback = route != expected_kernel
- facts = [
- f"kernel={route}",
- f"external-input={bool(provenance['use_external_inp_buf'])}",
- f"qps={int(provenance['num_qps'])}",
- ]
- else: # pragma: no cover - guarded by registry
- raise ProbeError("unsupported-native-api")
- return {
- "evidence": sorted(facts),
- "fallback_used": fallback,
- "native_backend": backend_name,
- "requested": str(args.transport),
- "runtime_route": route,
- }
-
-
-def _component_identities(backend_name: str, provenance: dict[str, Any]) -> list[dict[str, Any]]:
- if backend_name == "deepep":
- values = [("deepep", provenance.get("deepep_commit"), provenance.get("deepep_version"))]
- elif backend_name == "deepep-v2":
- values = [
- ("deepep-v2", provenance.get("deepep_commit"), provenance.get("deepep_version")),
- ("deepep-tree", provenance.get("deepep_tree"), None),
- ("fmt", provenance.get("fmt_commit"), None),
- ]
- elif backend_name == "deepep-hybrid":
- values = [
- ("deepep-hybrid", provenance.get("deepep_commit"), None),
- ("deepep-tree", provenance.get("deepep_tree"), None),
- ]
- elif backend_name == "uccl":
- values = [
- ("uccl", provenance.get("uccl_commit"), provenance.get("uccl_version")),
- ("uccl-wrapper", provenance.get("uccl_wrapper_commit"), None),
- ]
- elif backend_name == "mori":
- values = [("mori", provenance.get("mori_commit"), None)]
- else: # pragma: no cover
- raise ProbeError("unsupported-native-api")
- result = [
- {"revision": revision, "role": role, "version": version}
- for role, revision, version in values
- ]
- for item in result:
- if item["revision"] is None and item["version"] is None:
- raise ProbeError("unverified-execution-identity")
- return result
-
-
-def _execution_identity(backend_name: str, backend) -> dict[str, Any]:
- source_sha = os.environ.get("COLLECTIVEX_SOURCE_SHA") or os.environ.get("GITHUB_SHA")
- image_reference = os.environ.get("COLLECTIVEX_IMAGE")
- image_digest = os.environ.get("COLLECTIVEX_IMAGE_DIGEST")
- verified = os.environ.get("COLLECTIVEX_IMAGE_DIGEST_VERIFIED") == "1"
- if (
- not isinstance(source_sha, str) or not SHA40.fullmatch(source_sha)
- or not isinstance(image_reference, str) or not image_reference
- or not isinstance(image_digest, str) or not IMAGE_DIGEST.fullmatch(image_digest)
- or not verified
- ):
- raise ProbeError("unverified-execution-identity")
- provenance = backend.backend_provenance
- return {
- "backend_components": _component_identities(backend_name, provenance),
- "backend_provenance_sha256": _sha(provenance),
- "image_digest": image_digest,
- "image_digest_verified": True,
- "image_reference": image_reference,
- "source_sha": source_sha,
- }
-
-
-def _correctness_aggregate(records: list[dict[str, Any]]) -> dict[str, Any]:
- profile_ids = {record["profile_id"] for record in records}
- if len(profile_ids) != 1:
- raise ProbeError("cross-rank-evidence-mismatch")
- result: dict[str, Any] = {"profile_id": profile_ids.pop()}
- for direction in ("dispatch", "combine"):
- axes = [record[direction] for record in records]
- scale_finite = [axis["scales_finite"] for axis in axes]
- scale_positive = [axis["scales_positive"] for axis in axes]
- result[direction] = {
- "dequantized_semantics": all(axis["dequantized_semantics"] for axis in axes),
- "encoded_payload_valid": all(axis["encoded_payload_valid"] for axis in axes),
- "max_abs_error": max(float(axis["max_abs_error"]) for axis in axes),
- "max_rel_error": max(float(axis["max_rel_error"]) for axis in axes),
- "passed": all(axis["passed"] for axis in axes),
- "saturation_count": sum(int(axis["saturation_count"]) for axis in axes),
- "saturation_rate": max(float(axis["saturation_rate"]) for axis in axes),
- "scales_finite": (
- None if all(value is None for value in scale_finite)
- else all(value is True for value in scale_finite)
- ),
- "scales_positive": (
- None if all(value is None for value in scale_positive)
- else all(value is True for value in scale_positive)
- ),
- }
- result["passed"] = all(record["passed"] for record in records) and all(
- result[direction]["passed"] for direction in ("dispatch", "combine")
- )
- return result
-
-
-def _topology_record(topology: dict[str, Any], placement_valid: bool) -> dict[str, Any]:
- return {
- "gpus_per_node": topology["gpus_per_node"],
- "nodes": topology["nodes"],
- "placement_valid": placement_valid,
- "scale_up_domain": topology["scale_up_domain"],
- "scale_up_transport": topology["scale_up_transport"],
- "scale_out_transport": topology["scale_out_transport"],
- "scope": topology["scope"],
- "topology_class": topology["topology_class"],
- "transport": topology["transport"],
- "world_size": topology["nodes"] * topology["gpus_per_node"],
- }
-
-
-def _backend_class(name: str):
- if name == "deepep":
- from ep_deepep import DeepEPBackend
- return DeepEPBackend
- if name == "deepep-v2":
- from ep_deepep_v2 import DeepEPV2Backend
- return DeepEPV2Backend
- if name == "deepep-hybrid":
- from ep_deepep_hybrid import DeepEPHybridBackend
- return DeepEPHybridBackend
- if name == "uccl":
- from ep_uccl import UCCLBackend
- return UCCLBackend
- if name == "mori":
- from ep_mori import MoRIBackend
- return MoRIBackend
- raise ProbeError("unsupported-native-api")
-
-
-def _runtime_args(target: dict[str, Any], topology: dict[str, Any], fingerprint: dict[str, Any]):
- return SimpleNamespace(
- backend=target["backend"],
- eplb=False,
- experts=256,
- gpus_per_node=topology["gpus_per_node"],
- hidden=7168,
- mode=target["mode"],
- num_logical_experts=256,
- num_sms=24,
- phase="decode",
- precision_profile=target["precision_profile"],
- runner=target["sku"],
- runtime_fingerprint=fingerprint,
- scale_out_transport=topology["scale_out_transport"] or "",
- scale_up_domain=topology["scale_up_domain"],
- scale_up_transport=topology["scale_up_transport"],
- scope=topology["scope"],
- tokens_ladder="8",
- topk=8,
- topology_class=topology["topology_class"],
- transport=topology["transport"],
- )
-
-
-def _init_distributed(torch_module, dist, backend_name: str, device, rank: int, world_size: int) -> None:
- if dist.is_initialized():
- return
- if backend_name == "mori":
- dist.init_process_group(
- backend="nccl", rank=rank, world_size=world_size, device_id=device
- )
- elif backend_name == "deepep-v2":
- dist.init_process_group("nccl", device_id=device)
- else:
- dist.init_process_group("nccl")
-
-
-def _runtime_context(torch_module, dist, target: dict[str, Any], device, local_rank: int):
- import run_ep
-
- world_size = dist.get_world_size()
- topology = capability.topology_for(target["sku"], target["ep"])
- if topology is None or world_size != target["ep"]:
- raise ProbeError("runtime-identity-mismatch")
- machine = {"x86_64": "amd64", "aarch64": "arm64"}.get(
- platform.machine(), platform.machine()
- )
- properties = torch_module.cuda.get_device_properties(device)
- if torch_module.version.hip:
- vendor = "amd"
- arch = str(getattr(properties, "gcnArchName", "")).split(":", 1)[0]
- else:
- vendor = "nvidia"
- major, minor = torch_module.cuda.get_device_capability(device)
- arch = f"sm{major}{minor}"
- fingerprint = run_ep._runtime_fingerprint(
- torch_module, device, machine=machine, vendor=vendor, arch=arch
- )
- issues = capability.runtime_identity_issues(
- target["sku"], vendor=vendor, arch=arch, machine=machine,
- device_name=torch_module.cuda.get_device_name(device),
- device_count=torch_module.cuda.device_count(), world_size=world_size,
- )
- records: list[Any] = [None] * world_size
- dist.all_gather_object(records, (socket.gethostname(), local_rank, fingerprint, issues))
- if any(record[3] for record in records):
- raise ProbeError("runtime-identity-mismatch")
- placement = run_ep._summarize_realized_placement(
- [(record[0], record[1]) for record in records],
- expected_nodes=topology["nodes"],
- expected_gpus_per_node=topology["gpus_per_node"],
- expected_world_size=world_size,
- )
- common_fingerprint = run_ep._common_runtime_fingerprint([record[2] for record in records])
- return topology, placement, common_fingerprint
-
-
-def _local_probe(torch_module, dist, target: dict[str, Any], backend, args, rank: int):
- import routing
-
- tokens = 8
- global_idx, global_weights = routing.build_global_routing(
- tokens * target["ep"], args.experts, args.topk, "uniform", ep_harness.ROUTING_SEED
- )
- local_idx, local_weights = routing.rank_slice(
- global_idx, global_weights, rank, tokens
- )
- x = routing.rank_activations(
- tokens, args.hidden, ep_harness.ROUTING_SEED, rank,
- torch_module.device(f"cuda:{int(os.environ.get('LOCAL_RANK', '0'))}"),
- torch_module.bfloat16,
- )
- problem = backend.make_problem(
- tokens, local_idx.to(x.device), local_weights.to(x.device), x
- )
- oracle = ep_harness._run_expert_oracle(
- torch_module, routing, backend, problem, global_idx, global_weights, rank,
- args.experts // target["ep"], ep_harness.ROUTING_SEED,
- )
- if not oracle["passed"] or not oracle["_precision"]["passed"]:
- raise ProbeError("precision-contract-mismatch")
-
- caller = torch_module.cuda.Stream(device=x.device)
- verifier = torch_module.cuda.Stream(device=x.device)
- completion_event = torch_module.cuda.Event()
- with torch_module.cuda.stream(caller):
- handle = backend.dispatch(problem)
- problem.recv_tokens = backend.recv_tokens(handle)
- view = (
- backend.inspect_expert_dispatch(problem, handle)
- if target["mode"] == "low-latency"
- else backend.inspect_dispatch(problem, handle)
- )
- backend.stage(problem, handle)
- combined = backend.combine(problem, handle)
- completion_event.record(caller)
- with torch_module.cuda.stream(verifier):
- verifier.wait_event(completion_event)
- verifier_sentinel = combined.float().abs().sum()
- verifier.synchronize()
- completion = {
- "caller_event_complete": bool(completion_event.query()),
- "contract": FENCE_CONTRACT,
- "mode": _completion_mode(target["backend"], target["mode"]),
- "output_finite": bool(torch_module.isfinite(combined.float()).all().item()),
- "verifier_stream_complete": bool(torch_module.isfinite(verifier_sentinel).item()),
- }
- if not all(
- completion[field]
- for field in ("caller_event_complete", "output_finite", "verifier_stream_complete")
- ):
- raise ProbeError("completion-fence-failed")
-
- deferred = getattr(backend, "capture_deferred_provenance", None)
- if deferred is not None:
- deferred()
- dispatch_input = problem.dispatch_x[0] if isinstance(problem.dispatch_x, tuple) else problem.dispatch_x
- dispatch_input_scales = (
- problem.dispatch_x[1] if isinstance(problem.dispatch_x, tuple)
- else getattr(problem, "dispatch_scales", None)
- or getattr(problem, "scales", None)
- )
- dispatch_axis = backend.communication_precision["dispatch"]
- combine_axis = backend.communication_precision["combine"]
- local = {
- "api": _api_evidence(target["backend"], backend),
- "completion": completion,
- "identity": _execution_identity(target["backend"], backend),
- "precision": {
- "profile_id": backend.precision_profile_id,
- "correctness": oracle["_precision"],
- "dispatch": {
- "accumulator_dtype": "not-applicable",
- "accumulator_evidence": "not-applicable",
- "api_input_dtype": dispatch_axis["api_input_dtype"],
- "api_output_dtype": dispatch_axis["api_output_dtype"],
- "communication_format": dispatch_axis["communication_format"],
- "runtime_input": _local_tensor_summary(torch_module, dispatch_input),
- "runtime_output": _local_tensor_summary(torch_module, view.encoded_payload),
- "scale_contract": _scale_contract(
- torch_module, dispatch_axis,
- view.scales if view.scales is not None else dispatch_input_scales,
- ),
- "semantic_output": _local_tensor_summary(torch_module, view.payload),
- },
- "combine": {
- "accumulator_dtype": "fp32",
- "accumulator_evidence": "pinned-source-image-plus-runtime-oracle",
- "api_input_dtype": combine_axis["api_input_dtype"],
- "api_output_dtype": combine_axis["api_output_dtype"],
- "communication_format": combine_axis["communication_format"],
- "runtime_input": _local_tensor_summary(torch_module, handle.combine_input),
- "runtime_output": _local_tensor_summary(torch_module, combined),
- "scale_contract": _scale_contract(torch_module, combine_axis, None),
- "semantic_output": _local_tensor_summary(torch_module, combined),
- },
- },
- "transport": _transport_evidence(target["backend"], backend, args),
- }
- if local["transport"]["fallback_used"]:
- raise ProbeError("transport-fallback-detected")
- return local
-
-
-def _aggregate_local(records: list[dict[str, Any]]) -> dict[str, Any]:
- for field in ("api", "completion", "identity", "transport"):
- values = {_canonical(record[field]) for record in records}
- if len(values) != 1:
- raise ProbeError("cross-rank-evidence-mismatch")
- profile_ids = {record["precision"]["profile_id"] for record in records}
- if len(profile_ids) != 1:
- raise ProbeError("cross-rank-evidence-mismatch")
- precision: dict[str, Any] = {
- "profile_id": profile_ids.pop(),
- "correctness": _correctness_aggregate([
- record["precision"]["correctness"] for record in records
- ]),
- }
- for direction in ("dispatch", "combine"):
- axes = [record["precision"][direction] for record in records]
- fixed = (
- "accumulator_dtype", "accumulator_evidence", "api_input_dtype",
- "api_output_dtype", "communication_format",
- )
- result: dict[str, Any] = {}
- for field in fixed:
- if len({axis[field] for axis in axes}) != 1:
- raise ProbeError("cross-rank-evidence-mismatch")
- result[field] = axes[0][field]
- for field in ("runtime_input", "runtime_output", "semantic_output"):
- result[field] = _aggregate_tensor_summaries([axis[field] for axis in axes])
- result["scale_contract"] = _aggregate_scale_contracts([
- axis["scale_contract"] for axis in axes
- ])
- precision[direction] = result
- return {
- "api": records[0]["api"],
- "completion": records[0]["completion"],
- "identity": records[0]["identity"],
- "precision": precision,
- "transport": records[0]["transport"],
- }
-
-
-def _finalize(backend, dist) -> None:
- if backend is not None:
- backend.finalize(0)
- return
- if dist.is_initialized():
- dist.destroy_process_group()
-
-
-def run_target(target: dict[str, Any], output: Path) -> int:
- try:
- import torch
- import torch.distributed as dist
- except Exception as exc: # pragma: no cover - diagnostic runtime requirement
- raise ProbeError("runtime-identity-mismatch") from exc
- rank = int(os.environ.get("RANK", "0"))
- world_size = int(os.environ.get("WORLD_SIZE", "1"))
- local_rank = int(os.environ.get("LOCAL_RANK", "0"))
- os.environ.setdefault("MASTER_ADDR", "localhost")
- os.environ.setdefault("MASTER_PORT", "12355")
- torch.cuda.set_device(local_rank)
- device = torch.device(f"cuda:{local_rank}")
- backend = None
- diagnostic_stage = "distributed-init"
- topology = capability.topology_for(target["sku"], target["ep"])
- topology_record = _topology_record(topology, False) if topology is not None else {
- "gpus_per_node": 1, "nodes": target["ep"], "placement_valid": False,
- "scale_up_domain": 1, "scale_up_transport": "unknown",
- "scale_out_transport": None, "scope": "scale-out",
- "topology_class": "unknown", "transport": "unknown", "world_size": target["ep"],
- }
- os.environ.pop("NCCL_ASYNC_ERROR_HANDLING", None)
- os.environ.setdefault("TORCH_NCCL_ASYNC_ERROR_HANDLING", "1")
- os.environ.setdefault("NCCL_DEBUG", "INFO")
- os.environ.setdefault("NCCL_DEBUG_SUBSYS", "ENV,INIT,NET")
-
- def stage(name: str) -> None:
- nonlocal diagnostic_stage
- diagnostic_stage = name
- if rank == 0:
- print(f"[collectivex] precision-probe-stage={name}", file=sys.stderr, flush=True)
-
- try:
- stage("distributed-init")
- _init_distributed(torch, dist, target["backend"], device, rank, world_size)
- stage("runtime-context")
- topology, placement, fingerprint = _runtime_context(
- torch, dist, target, device, local_rank
- )
- topology_record = _topology_record(topology, bool(placement["valid"]))
- args = _runtime_args(target, topology, fingerprint)
- stage("backend-construction")
- try:
- backend = _backend_class(target["backend"])(
- args, rank, world_size, local_rank, device
- )
- backend.create_buffer(backend.make_inputs(args))
- construction = {"ok": True}
- except Exception:
- construction = {"ok": False, "reason": "backend-construction-failed"}
- gathered: list[Any] = [None] * world_size
- stage("construction-consensus")
- dist.all_gather_object(gathered, construction)
- if not all(record.get("ok") is True for record in gathered):
- manifest = build_manifest(
- target=target, topology=topology_record, disposition="unsupported",
- reason="backend-construction-failed", runtime_executed=True, evidence=None,
- )
- else:
- stage("native-operation")
- try:
- local = {"ok": True, "evidence": _local_probe(
- torch, dist, target, backend, args, rank
- )}
- except ProbeError as exc:
- local = {"ok": False, "reason": exc.reason}
- except Exception:
- local = {"ok": False, "reason": "native-operation-failed"}
- gathered = [None] * world_size
- stage("operation-consensus")
- dist.all_gather_object(gathered, local)
- if not all(record.get("ok") is True for record in gathered):
- reasons = {record.get("reason") for record in gathered}
- reason = reasons.pop() if len(reasons) == 1 else "cross-rank-evidence-mismatch"
- manifest = build_manifest(
- target=target, topology=topology_record, disposition="unsupported",
- reason=reason, runtime_executed=True, evidence=None,
- )
- else:
- stage("evidence-aggregation")
- evidence = _aggregate_local([record["evidence"] for record in gathered])
- manifest = build_manifest(
- target=target, topology=topology_record, disposition="supported",
- reason=SUPPORTED_REASON, runtime_executed=True, evidence=evidence,
- )
- except ProbeError as exc:
- manifest = build_manifest(
- target=target, topology=topology_record, disposition="unsupported",
- reason=exc.reason, runtime_executed=False, evidence=None,
- )
- except Exception:
- print(
- f"[collectivex] precision-probe-diagnostic={diagnostic_stage}-exception",
- file=sys.stderr,
- )
- raise
- if rank == 0:
- _write_atomic(output, manifest)
- print(json.dumps(manifest, allow_nan=False, sort_keys=True, separators=(",", ":")))
- dist.barrier()
- _finalize(backend, dist)
- return 0
-
-
-def main() -> int:
- parser = argparse.ArgumentParser(description=__doc__)
- parser.add_argument("--list-targets", action="store_true")
- parser.add_argument("--workflow-plan", action="store_true")
- parser.add_argument("--extract-from", type=Path)
- parser.add_argument("--probe-id")
- parser.add_argument("--validate-control", type=Path)
- parser.add_argument("--validate-manifest", type=Path, nargs="+")
- parser.add_argument("--validate-bundle", type=Path)
- parser.add_argument("--backend", choices=sorted(BACKENDS | {"all"}))
- parser.add_argument("--sku")
- parser.add_argument("--only-sku", default="")
- parser.add_argument("--min-nodes", type=int, default=0)
- parser.add_argument("--max-nodes", type=int, default=0)
- parser.add_argument("--expect-sku")
- parser.add_argument("--expect-backend")
- parser.add_argument("--expect-nodes", type=int)
- parser.add_argument("--ep", type=int)
- parser.add_argument("--mode", choices=("normal", "low-latency"))
- parser.add_argument("--precision-profile")
- parser.add_argument("--out", type=Path)
- args = parser.parse_args()
- if args.list_targets:
- print(json.dumps(provisional_targets(), allow_nan=False, sort_keys=True, separators=(",", ":")))
- return 0
- if args.workflow_plan:
- plan = workflow_plan(
- backend=args.backend or "all", only_sku=args.only_sku,
- min_nodes=args.min_nodes, max_nodes=args.max_nodes,
- probe_id=args.probe_id or "",
- )
- if args.out is None:
- print(json.dumps(plan, allow_nan=False, sort_keys=True, separators=(",", ":")))
- else:
- _write_atomic(args.out, plan)
- return 0
- if args.extract_from is not None:
- if None in (args.probe_id, args.expect_sku, args.expect_backend, args.expect_nodes, args.out):
- parser.error("probe extraction requires ID, expected placement, and --out")
- control = extract_control(
- json.loads(args.extract_from.read_text()), probe_id=args.probe_id,
- sku=args.expect_sku, backend=args.expect_backend, nodes=args.expect_nodes,
- )
- _write_atomic(args.out, control)
- return 0
- if args.validate_control is not None:
- if None in (args.expect_sku, args.expect_backend, args.expect_nodes):
- parser.error("control validation requires expected placement")
- validate_control(
- json.loads(args.validate_control.read_text()), sku=args.expect_sku,
- backend=args.expect_backend, nodes=args.expect_nodes,
- )
- return 0
- if args.validate_bundle is not None:
- if not args.validate_manifest:
- parser.error("bundle validation requires manifest paths")
- validate_bundle(
- json.loads(args.validate_bundle.read_text()),
- [json.loads(path.read_text()) for path in args.validate_manifest],
- )
- return 0
- if args.validate_manifest is not None:
- for path in args.validate_manifest:
- validate_manifest(json.loads(path.read_text()))
- return 0
- if any(
- value is None
- for value in (args.backend, args.sku, args.ep, args.mode, args.precision_profile, args.out)
- ):
- parser.error("one exact --backend/--sku/--ep/--mode/--precision-profile/--out cell is required")
- try:
- target = select_target(
- backend=args.backend, sku=args.sku, ep=args.ep, mode=args.mode,
- precision_profile=args.precision_profile,
- )
- return run_target(target, args.out)
- except ProbeError as exc:
- parser.error(exc.reason)
-
-
-if __name__ == "__main__":
- raise SystemExit(main())
diff --git a/experimental/CollectiveX/tests/test_deepep_v2_contract.py b/experimental/CollectiveX/tests/test_deepep_v2_contract.py
deleted file mode 100644
index 754c0083df..0000000000
--- a/experimental/CollectiveX/tests/test_deepep_v2_contract.py
+++ /dev/null
@@ -1,2333 +0,0 @@
-#!/usr/bin/env python3
-"""CPU-only structural and registry tests for the pinned DeepEP V2 path."""
-from __future__ import annotations
-
-import ast
-import argparse
-import copy
-import ctypes
-import hashlib
-import json
-import os
-from pathlib import Path
-from pathlib import PurePosixPath
-import shutil
-import stat
-import subprocess
-import sys
-import tempfile
-import types
-import unittest
-
-HERE = Path(__file__).resolve().parent
-ROOT = HERE.parent
-sys.path.insert(0, str(ROOT))
-
-import capability # noqa: E402
-import contracts # noqa: E402
-import ep_harness # noqa: E402
-import identity # noqa: E402
-import run_ep # noqa: E402
-
-
-COMMIT = "fa8a9b16898204afd347c663b89e65ef87dc6ce6"
-TREE = "29809e75c5874e6609dac4804e7b651d5226959f"
-FMT_COMMIT = "a4c7e17133ee9cb6a2f45545f6e974dd3c393efa"
-NCCL_CHECK_COMMIT = "93d0564188f7a0a6288c6e316484861b0efa042e"
-
-
-def deepep_v2_jit_provenance() -> list[dict[str, str]]:
- return [
- {
- "cache_key": f"kernel.{name}.{index:032x}",
- "cubin_sha256": f"{index + 1:x}" * 64,
- "sass_sha256": f"{index + 2:x}" * 64,
- "source_sha256": f"{index + 3:x}" * 64,
- }
- for index, name in enumerate(sorted(contracts.DEEPEP_V2_JIT_KERNELS))
- ]
-
-
-def hybrid_realized_config() -> dict[str, object]:
- config = {field: 1 for field in contracts.HYBRID_REALIZED_CONFIG_FIELDS}
- for field in contracts.HYBRID_REALIZED_BOOL_FIELDS:
- config[field] = True
- config["token_data_type"] = "UINT16"
- return config
-
-
-def hybrid_jit_provenance(ranks: int = 2) -> tuple[list[str], list[dict[str, object]]]:
- keys = ["combine-key", "dispatch-key", "preprocess-key"]
- artifacts = [
- {
- "kernel_key": key,
- "rank_artifacts": [
- {"bytes": 10 + index, "rank": rank, "sha256": f"{index + 1:x}" * 64}
- for rank in range(ranks)
- ],
- }
- for index, key in enumerate(keys)
- ]
- return keys, artifacts
-
-
-def load_uccl_function(name: str, namespace: dict[str, object]):
- path = HERE / "ep_uccl.py"
- function = next(
- node
- for node in ast.parse(path.read_text()).body
- if isinstance(node, ast.FunctionDef) and node.name == name
- )
- exec(compile(ast.Module(body=[function], type_ignores=[]), str(path), "exec"), namespace)
- return namespace[name]
-
-
-def operator_config(root: Path) -> dict[str, object]:
- path = str(root)
- network = {"socket_ifname": "eth0", "rdma_devices": "mlx5_0:1"}
- runners = {
- "h100-dgxc": {
- "partition": "test", "account": "test", "squash_dir": path,
- "stage_dir": path, **network,
- },
- "h200-dgxc": {
- "partition": "test", "squash_dir": path, "stage_dir": path, **network,
- },
- "b200-dgxc": {
- "partition": "test", "account": "test", "qos": "test-qos",
- "nodelist": "test-[0-1]", "squash_dir": path, "stage_dir": path, **network,
- },
- "b300": {
- "partition": "test", "account": "test", "squash_dir": path, "stage_dir": path,
- **network,
- },
- "gb200": {"partition": "test", "account": "test", "storage_roots": [path]},
- "gb300": {
- "partition": "test", "account": "test", "squash_dir": path,
- "stage_dir": path, "enroot_cache_path": path,
- },
- "mi325x": {
- "partition": "test", "squash_dir": path, "stage_dir": path, **network,
- },
- "mi300x": {
- "partition": "test", "squash_dir": path, "stage_dir": path, **network,
- },
- "mi355x": {
- "partition": "test", "squash_dir": path, "stage_dir": path, **network,
- },
- }
- return {"schema_version": 1, "audit_salt": "a" * 64, "runners": runners}
-
-
-class DeepEPV2ContractTests(unittest.TestCase):
- @classmethod
- def setUpClass(cls) -> None:
- cls.path = HERE / "ep_deepep_v2.py"
- cls.tree = ast.parse(cls.path.read_text(), str(cls.path))
-
- def test_timing_post_hook_runs_after_each_isolated_sample(self) -> None:
- class Event:
- def record(self) -> None:
- pass
-
- def elapsed_time(self, _other) -> float:
- return 0.125
-
- cuda = types.SimpleNamespace(
- Event=lambda **_kwargs: Event(),
- synchronize=lambda: None,
- )
- handles: list[str] = []
- samples = ep_harness.time_us(
- types.SimpleNamespace(cuda=cuda),
- lambda: "dispatch-handle",
- warmup=0,
- iters=3,
- post=handles.append,
- )
- self.assertEqual(samples, [125.0, 125.0, 125.0])
- self.assertEqual(handles, ["dispatch-handle"] * 3)
-
- def test_capability_is_explicit_for_every_sku(self) -> None:
- backend = capability.BACKENDS["deepep-v2"]
- self.assertEqual(
- (backend["implementation"], backend["commit"], backend["torch"], backend["nccl"]),
- ("deep_ep.ElasticBuffer", COMMIT, "2.10.0+cu130", "2.30.4"),
- )
- self.assertEqual(backend["source"], "deepseek-ai/DeepEP#605+#630+#640")
- self.assertEqual(backend["communication_backend"], "nccl-device-lsa")
- self.assertEqual(set(backend["sku_capabilities"]), set(capability.PLATFORMS))
- for sku, platform in capability.PLATFORMS.items():
- ok, _ = capability.resolve(sku, "deepep-v2")
- self.assertEqual(ok, platform["vendor"] == "nvidia" and sku != "h100-dgxc")
- self.assertEqual(
- set(backend["sku_capabilities"][sku]), {"basis", "schedulable"}
- )
- self.assertEqual(
- backend["sku_capabilities"]["h100-dgxc"],
- {
- "schedulable": False,
- "basis": "current-runner-nccl-device-api-symmetric-memory-unavailable",
- },
- )
-
- def test_adapter_ast_pins_elastic_api_and_weight_semantics(self) -> None:
- imports = {
- alias.name
- for node in ast.walk(self.tree)
- if isinstance(node, ast.ImportFrom) and node.module == "deep_ep"
- for alias in node.names
- }
- self.assertEqual(imports, {"ElasticBuffer"})
- constants = {
- node.targets[0].id: ast.literal_eval(node.value)
- for node in self.tree.body
- if isinstance(node, ast.Assign)
- and len(node.targets) == 1
- and isinstance(node.targets[0], ast.Name)
- and isinstance(node.value, ast.Constant)
- }
- self.assertEqual(constants["DEEPEP_V2_COMMIT"], COMMIT)
- self.assertEqual(constants["DEEPEP_V2_TREE"], TREE)
- self.assertEqual(constants["DEEPEP_V2_FMT_COMMIT"], FMT_COMMIT)
- self.assertEqual(constants["DEEPEP_V2_PR"], 605)
- self.assertEqual(constants["DEEPEP_V2_FIX_PR"], 630)
- self.assertEqual(constants["DEEPEP_V2_NCCL_CHECK_FIX_PR"], 640)
- self.assertEqual(constants["DEEPEP_V2_NCCL_CHECK_COMMIT"], NCCL_CHECK_COMMIT)
- self.assertEqual(
- constants["DEEPEP_V2_JIT_RANDOM_SEED"],
- "collectivex-deepep-v2-fa8a9b1",
- )
- self.assertEqual(constants["NCCL_VERSION"], "2.30.4")
- self.assertEqual(constants["NVSHMEM_VERSION"], "3.3.9")
- backend = next(
- node for node in self.tree.body
- if isinstance(node, ast.ClassDef) and node.name == "DeepEPV2Backend"
- )
- assignments = {
- node.targets[0].id: ast.literal_eval(node.value)
- for node in backend.body
- if isinstance(node, ast.Assign)
- and isinstance(node.targets[0], ast.Name)
- and isinstance(node.value, ast.Constant)
- }
- self.assertEqual(assignments["combine_weight_semantics"], "unweighted-rank-sum")
- methods = {node.name for node in backend.body if isinstance(node, ast.FunctionDef)}
- self.assertTrue({
- "dispatch", "inspect_dispatch", "combine_transformed", "capture_deferred_provenance",
- "finalize",
- } <= methods)
- self.assertNotIn("expected", methods)
- constructor = next(
- node for node in ast.walk(backend)
- if isinstance(node, ast.Call)
- and isinstance(node.func, ast.Name)
- and node.func.id == "ElasticBuffer"
- )
- deterministic = next(
- keyword for keyword in constructor.keywords if keyword.arg == "deterministic"
- )
- self.assertIs(ast.literal_eval(deterministic.value), False)
- self.assertIn("deterministic", contracts.REQUIRED_BACKEND_PROVENANCE["deepep-v2"])
- self.assertIn("num_experts", contracts.REQUIRED_BACKEND_PROVENANCE["deepep-v2"])
- self.assertIn("tuning_num_experts", contracts.REQUIRED_BACKEND_PROVENANCE["deepep-v2"])
- self.assertIn("jit_random_seed", contracts.REQUIRED_BACKEND_PROVENANCE["deepep-v2"])
- self.assertIn("gin_enabled", contracts.REQUIRED_BACKEND_PROVENANCE["deepep-v2"])
- self.assertIn("communication_backend", contracts.REQUIRED_BACKEND_PROVENANCE["deepep-v2"])
- self.assertIn("deepep_pr", contracts.REQUIRED_BACKEND_PROVENANCE["deepep-v2"])
- self.assertIn("deepep_fix_pr", contracts.REQUIRED_BACKEND_PROVENANCE["deepep-v2"])
- source = self.path.read_text()
- self.assertIn('getattr(args, "num_logical_experts", args.experts)', source)
- self.assertIn('"use_expanded_layout": False', source)
- self.assertIn("allow_hybrid_mode = _configure_gin_mode(args, world_size)", source)
- self.assertIn("get_theoretical_num_sms(tuning_num_experts, args.topk)", source)
-
- jit_function = next(
- node for node in self.tree.body
- if isinstance(node, ast.FunctionDef) and node.name == "_jit_cache_key"
- )
- namespace = {"hashlib": __import__("hashlib"), "json": json}
- exec(compile(ast.Module(body=[jit_function], type_ignores=[]), str(self.path), "exec"), namespace)
- key = namespace["_jit_cache_key"]
- baseline = types.SimpleNamespace(
- runner="h100-dgxc", hidden=7168, topk=8, experts=256,
- routing="uniform", eplb=False, case_id="uniform",
- )
- zipf = types.SimpleNamespace(**{**vars(baseline), "routing": "zipf", "case_id": "zipf"})
- eplb = types.SimpleNamespace(
- **{**vars(zipf), "experts": 288, "num_logical_experts": 256, "eplb": True}
- )
- realized = {
- "num_sms": 24,
- "num_qps": 9,
- "allocated_qps": 17,
- "logical_scaleout_ranks": 1,
- "logical_scaleup_ranks": 8,
- "physical_rdma_ranks": 2,
- "physical_nvlink_ranks": 4,
- "is_scaleup_nvlink": False,
- "device_arch_major": 9,
- "device_arch_minor": 0,
- "device_sms": 132,
- "device_smem_bytes": 232448,
- "gpu_timeout_cycles": 198000000000,
- }
- direct = key(baseline, 8, 128, False, realized)
- self.assertTrue(direct.startswith("jitcfg-v3-"))
- self.assertEqual(direct, key(zipf, 8, 128, False, realized))
- self.assertNotEqual(direct, key(zipf, 8, 128, True, realized))
- self.assertNotEqual(direct, key(eplb, 8, 128, False, realized))
- for field, value in realized.items():
- changed = not value if type(value) is bool else value + 1
- self.assertNotEqual(
- direct,
- key(baseline, 8, 128, False, {**realized, field: changed}),
- field,
- )
- # Buffer construction now lives in create_buffer (it consumes make_inputs),
- # so the ElasticBuffer/JIT-config/cache-dir ordering is pinned there.
- build = next(
- node for node in backend.body
- if isinstance(node, ast.FunctionDef) and node.name == "create_buffer"
- )
- buffer_call = next(
- node for node in ast.walk(build)
- if isinstance(node, ast.Call)
- and isinstance(node.func, ast.Name)
- and node.func.id == "ElasticBuffer"
- )
- jit_config_check = next(
- node for node in ast.walk(build)
- if isinstance(node, ast.Call)
- and isinstance(node.func, ast.Name)
- and node.func.id == "_require_cross_rank_equal"
- and ast.literal_eval(node.args[1]) == "JIT configuration"
- )
- cache_assignment = next(
- node for node in ast.walk(build)
- if isinstance(node, ast.Assign)
- and isinstance(node.targets[0], ast.Subscript)
- and ast.unparse(node.targets[0].value) == "os.environ"
- and ast.literal_eval(node.targets[0].slice) == "EP_JIT_CACHE_DIR"
- )
- self.assertLess(buffer_call.lineno, jit_config_check.lineno)
- self.assertLess(jit_config_check.lineno, cache_assignment.lineno)
- capture = next(
- node for node in backend.body
- if isinstance(node, ast.FunctionDef)
- and node.name == "capture_deferred_provenance"
- )
- calls = [node for node in ast.walk(capture) if isinstance(node, ast.Call)]
- barrier = next(
- node for node in calls
- if isinstance(node.func, ast.Attribute) and node.func.attr == "barrier"
- )
- self.assertEqual(
- {keyword.arg: ast.literal_eval(keyword.value) for keyword in barrier.keywords},
- {"use_comm_stream": True, "with_cpu_sync": True},
- )
- scan = next(
- node for node in calls
- if isinstance(node.func, ast.Name) and node.func.id == "_jit_artifact_evidence"
- )
- self.assertLess(barrier.lineno, scan.lineno)
- realized_check = next(
- node for node in ast.walk(backend)
- if isinstance(node, ast.Call)
- and isinstance(node.func, ast.Name)
- and node.func.id == "_require_cross_rank_equal"
- and len(node.args) > 1
- and isinstance(node.args[1], ast.Constant)
- and node.args[1].value == "realized tuning/topology"
- )
- self.assertIsInstance(realized_check, ast.Call)
- self.assertEqual(
- (ROOT / "tests" / "ep_harness.py").read_text().count(
- "capture_deferred_provenance()"
- ),
- 2,
- )
- schema = json.loads((ROOT / "schemas" / "raw-case-v1.schema.json").read_text())
- provenance = schema["properties"]["implementation"]["properties"]["provenance"]
- self.assertEqual(provenance["properties"]["deterministic"], {"type": "boolean"})
- self.assertEqual(
- provenance["properties"]["num_experts"],
- {"minimum": 1, "type": "integer"},
- )
- self.assertEqual(
- provenance["properties"]["tuning_num_experts"],
- {"minimum": 1, "type": "integer"},
- )
- self.assertEqual(
- provenance["properties"]["jit_cubins"]["items"],
- {"$ref": "#/$defs/deepep_v2_jit_cubin"},
- )
- self.assertEqual(
- (
- provenance["properties"]["jit_cubins"]["minItems"],
- provenance["properties"]["jit_cubins"]["maxItems"],
- ),
- (5, 5),
- )
- self.assertEqual(
- provenance["properties"]["jit_random_seed"],
- {"const": "collectivex-deepep-v2-fa8a9b1"},
- )
- self.assertEqual(provenance["properties"]["allow_hybrid_mode"], {"type": "boolean"})
- self.assertEqual(provenance["properties"]["gin_enabled"], {"type": "boolean"})
- self.assertEqual(provenance["properties"]["deepep_pr"], {"const": 605})
- self.assertEqual(provenance["properties"]["deepep_fix_pr"], {"const": 630})
- self.assertEqual(
- provenance["properties"]["deepep_nccl_check_fix_pr"], {"const": 640}
- )
- self.assertEqual(
- provenance["properties"]["deepep_nccl_check_commit"],
- {"const": NCCL_CHECK_COMMIT},
- )
- self.assertEqual(
- provenance["properties"]["communication_backend"],
- {"enum": ["nccl-device-lsa", "nccl-gin"]},
- )
- self.assertEqual(
- provenance["properties"]["num_rdma_bytes"],
- {"minimum": 0, "type": "integer"},
- )
- self.assertEqual(
- provenance["properties"]["num_qps_per_rank"],
- {"minimum": 1, "type": "integer"},
- )
- for field, value in (
- ("num_experts", "288"),
- ("tuning_num_experts", "not-an-integer"),
- ("tuning_num_experts", 0),
- ):
- with self.subTest(provenance_field=field, value=value):
- self.assertIn(
- field,
- contracts.backend_provenance_issues(
- "deepep-v2", {field: value}
- ),
- )
-
- def test_v2_gin_mode_uses_the_scale_up_domain_and_safe_fallbacks(self) -> None:
- functions = {
- node.name: node for node in self.tree.body if isinstance(node, ast.FunctionDef)
- }
- namespace = {"os": os, "_GIN_QP_BUDGET": 512}
- exec(
- compile(
- ast.Module(
- body=[
- functions["_hybrid_num_allocated_qps"],
- functions["_configure_gin_mode"],
- functions["_lsa_topology_is_valid"],
- ],
- type_ignores=[],
- ),
- str(self.path),
- "exec",
- ),
- namespace,
- )
- allocate_qps = namespace["_hybrid_num_allocated_qps"]
- configure = namespace["_configure_gin_mode"]
- topology_is_valid = namespace["_lsa_topology_is_valid"]
- self.assertEqual(
- [allocate_qps(world_size) for world_size in (8, 16, 32, 64)],
- [65, 33, 17, 9],
- )
- original = os.environ.get("EP_DISABLE_GIN")
- try:
- args = types.SimpleNamespace(scale_up_domain=72, gpus_per_node=4)
- self.assertFalse(configure(args, 8))
- self.assertEqual(os.environ.get("EP_DISABLE_GIN"), "1")
-
- os.environ["EP_DISABLE_GIN"] = "stale"
- args = types.SimpleNamespace(scale_up_domain=8, gpus_per_node=4)
- self.assertTrue(configure(args, 16))
- self.assertNotIn("EP_DISABLE_GIN", os.environ)
-
- args = types.SimpleNamespace(gpus_per_node=4)
- self.assertTrue(configure(args, 8))
- self.assertNotIn("EP_DISABLE_GIN", os.environ)
-
- self.assertFalse(configure(types.SimpleNamespace(), 8))
- self.assertEqual(os.environ.get("EP_DISABLE_GIN"), "1")
-
- topology = {
- "physical_rdma_ranks": 1,
- "physical_nvlink_ranks": 8,
- "logical_scaleout_ranks": 1,
- "logical_scaleup_ranks": 8,
- "is_scaleup_nvlink": True,
- }
- self.assertTrue(topology_is_valid(False, 8, 8, topology))
- topology["physical_rdma_ranks"] = 2
- topology["logical_scaleout_ranks"] = 2
- self.assertTrue(topology_is_valid(True, 16, 8, topology))
- topology["physical_nvlink_ranks"] = 4
- self.assertFalse(topology_is_valid(False, 8, 8, topology))
- finally:
- if original is None:
- os.environ.pop("EP_DISABLE_GIN", None)
- else:
- os.environ["EP_DISABLE_GIN"] = original
-
- def test_ep_adapters_declare_unweighted_rank_sum(self) -> None:
- # DeepEP and UCCL share their combine semantics and dispatch/combine methods in the
- # family base, so it stands in for both here; the other adapters declare their own.
- adapters = {
- "ep_deepep_family.py": "DeepEPFamilyBackend",
- "ep_deepep_v2.py": "DeepEPV2Backend",
- "ep_deepep_hybrid.py": "DeepEPHybridBackend",
- "ep_mori.py": "MoRIBackend",
- "ep_nccl.py": "NCCLBackend",
- }
- for filename, class_name in adapters.items():
- with self.subTest(adapter=filename):
- tree = ast.parse((HERE / filename).read_text())
- backend = next(
- node for node in tree.body
- if isinstance(node, ast.ClassDef) and node.name == class_name
- )
- assignment = next(
- node for node in backend.body
- if isinstance(node, ast.Assign)
- and isinstance(node.targets[0], ast.Name)
- and node.targets[0].id == "combine_weight_semantics"
- )
- self.assertEqual(ast.literal_eval(assignment.value), "unweighted-rank-sum")
- combine_methods = [
- item for item in backend.body
- if isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef))
- and item.name in {"combine", "combine_transformed"}
- ]
- self.assertEqual(len(combine_methods), 2)
- for method in combine_methods:
- source = ast.unparse(method)
- if filename == "ep_deepep_family.py":
- self.assertIn("self.mode == 'low-latency'", source)
- else:
- self.assertNotIn("topk_weights", source)
- self.assertNotIn("combine_topk_weights", source)
-
- def test_low_latency_mode_parser_and_profile_are_explicit(self) -> None:
- parser = argparse.ArgumentParser()
- ep_harness.add_common_args(parser)
- required = [
- "--runner", "test", "--topology-class", "test",
- "--scope", "scale-up", "--scale-up-transport", "nvlink",
- "--out", "test.json",
- ]
- self.assertEqual(parser.parse_args(required).mode, "normal")
- self.assertEqual(
- parser.parse_args([*required, "--mode", "low-latency"]).mode,
- "low-latency",
- )
- profile = identity.case_profile("low-latency")
- self.assertEqual(profile["contract"], "expert-packed-weighted-combine-v1")
- self.assertEqual(
- profile["component_order_contract"],
- "qualification-hash-rotated-components-v1",
- )
- self.assertEqual(
- profile["correctness_scope"],
- "expert-assignment-and-weighted-combine",
- )
- self.assertEqual(profile["payload_unit"], "token-expert")
-
- def test_expert_packed_slot_map_reconstructs_exact_sources(self) -> None:
- pack = lambda begin, count: (begin << 32) | count
- slots = ep_harness.expert_packed_slot_map(
- [2, 1],
- [[1, 0, 0, 0], [1, 0, 0, 0]],
- [[pack(0, 1), pack(1, 1)], [pack(0, 0), pack(0, 1)]],
- tokens_per_rank=2,
- experts_per_rank=2,
- world_size=2,
- )
- self.assertEqual(slots, [(0, 0, 1), (0, 1, 2), (1, 0, 3)])
-
- invalid = (
- ([1], [[0]], [[pack(1, 1), pack(0, 0)]]),
- ([1], [[2]], [[pack(0, 1), pack(1, 0)]]),
- ([2], [[1, 1]], [[pack(0, 2), pack(2, 0)]]),
- )
- for counts, source, layout in invalid:
- with self.subTest(counts=counts, source=source, layout=layout):
- with self.assertRaises(ValueError):
- ep_harness.expert_packed_slot_map(
- counts,
- source,
- layout,
- tokens_per_rank=2,
- experts_per_rank=1,
- world_size=2,
- )
-
- def test_deepep_and_uccl_expose_genuine_low_latency_calls(self) -> None:
- # The operational low-latency surface is shared by DeepEP and UCCL in the family base;
- # each concrete adapter keeps only its own native buffer construction.
- shared_fragments = (
- "self.buffer.low_latency_dispatch(",
- "use_fp8=False",
- "self.buffer.low_latency_combine(",
- "p.topk_weights",
- 'self.combine_weight_semantics = "gate-weighted-sum"',
- "self.combine_needs_redispatch = True",
- "def inspect_expert_dispatch(",
- "self.max_tokens_per_rank = 128",
- "async_finish=False",
- "return_recv_hook=False",
- )
- family = (HERE / "ep_deepep_family.py").read_text()
- for fragment in shared_fragments:
- with self.subTest(fragment=fragment):
- self.assertIn(fragment, family)
-
- buffer_fragments = (
- "Buffer.get_low_latency_rdma_size_hint(",
- "low_latency_mode=True",
- "num_qps_per_rank=num_qps_per_rank",
- "self.buffer.clean_low_latency_buffer(",
- )
- for filename in ("ep_deepep.py", "ep_uccl.py"):
- source = (HERE / filename).read_text()
- with self.subTest(adapter=filename):
- for fragment in buffer_fragments:
- self.assertIn(fragment, source)
-
- run_ep_source = (HERE / "run_ep.py").read_text()
- self.assertIn('args.backend not in {"deepep", "uccl"}', run_ep_source)
- self.assertIn('args.phase != "decode"', run_ep_source)
-
- def test_deepep_v2_jit_evidence_is_strict_and_stable(self) -> None:
- valid = deepep_v2_jit_provenance()
- self.assertTrue(contracts._deepep_v2_jit_cubins_are_valid(valid))
- for invalid in (
- [],
- [{**valid[0], "path": "/private/kernel.cubin"}],
- [{**item, "cache_key": "dispatch"} for item in valid],
- [{**item, "cubin_sha256": "invalid"} for item in valid],
- valid[:-1],
- [*valid, valid[0]],
- [
- *valid,
- {
- **valid[0],
- "cache_key": valid[0]["cache_key"][:-32] + "f" * 32,
- },
- ],
- ):
- with self.subTest(invalid=invalid):
- self.assertFalse(contracts._deepep_v2_jit_cubins_are_valid(invalid))
-
- backend = next(
- node for node in self.tree.body
- if isinstance(node, ast.ClassDef) and node.name == "DeepEPV2Backend"
- )
- capture = next(
- node for node in backend.body
- if isinstance(node, ast.FunctionDef)
- and node.name == "capture_deferred_provenance"
- )
- artifacts = copy.deepcopy(valid)
-
- class FakeBuffer:
- @staticmethod
- def barrier(*, use_comm_stream: bool, with_cpu_sync: bool) -> None:
- self.assertTrue(use_comm_stream)
- self.assertTrue(with_cpu_sync)
-
- namespace = {
- "torch": types.SimpleNamespace(
- cuda=types.SimpleNamespace(synchronize=lambda: None)
- ),
- "_jit_artifact_evidence": lambda: copy.deepcopy(artifacts),
- "_require_cross_rank_equal": lambda _value, _label: None,
- }
- exec(
- compile(ast.Module(body=[capture], type_ignores=[]), str(self.path), "exec"),
- namespace,
- )
- state = types.SimpleNamespace(
- buffer=FakeBuffer(),
- _deferred_jit_snapshot=None,
- backend_provenance={"jit_cubins": []},
- )
- namespace["capture_deferred_provenance"](state)
- namespace["capture_deferred_provenance"](state)
- artifacts[0]["cubin_sha256"] = "f" * 64
- with self.assertRaisesRegex(RuntimeError, "changed after measurement"):
- namespace["capture_deferred_provenance"](state)
-
- def test_deepep_v2_jit_files_are_complete_regular_and_content_bound(self) -> None:
- functions = [
- node for node in self.tree.body
- if isinstance(node, ast.FunctionDef)
- and node.name in {"_sha256", "_jit_artifact_evidence"}
- ]
- namespace = {
- "hashlib": hashlib,
- "os": os,
- "Path": Path,
- "re": __import__("re"),
- "DEEPEP_V2_JIT_KERNELS": contracts.DEEPEP_V2_JIT_KERNELS,
- }
- exec(compile(ast.Module(body=functions, type_ignores=[]), str(self.path), "exec"), namespace)
- with tempfile.TemporaryDirectory() as temporary:
- cache = Path(temporary) / "cache"
- cache.mkdir()
- for index, name in enumerate(sorted(contracts.DEEPEP_V2_JIT_KERNELS)):
- kernel = cache / f"kernel.{name}.{index:032x}"
- kernel.mkdir()
- for suffix in ("cu", "cubin", "sass"):
- (kernel / f"kernel.{suffix}").write_bytes(f"{name}-{suffix}".encode())
- old_cache = os.environ.get("EP_JIT_CACHE_DIR")
- os.environ["EP_JIT_CACHE_DIR"] = temporary
- try:
- evidence = namespace["_jit_artifact_evidence"]()
- self.assertEqual(len(evidence), len(contracts.DEEPEP_V2_JIT_KERNELS))
- self.assertEqual(
- set(evidence[0]),
- {"cache_key", "cubin_sha256", "sass_sha256", "source_sha256"},
- )
- first = cache / evidence[0]["cache_key"]
- duplicate = cache / (evidence[0]["cache_key"][:-32] + "f" * 32)
- duplicate.mkdir()
- for suffix in ("cu", "cubin", "sass"):
- (duplicate / f"kernel.{suffix}").write_bytes(b"duplicate")
- with self.assertRaisesRegex(RuntimeError, "kernel set"):
- namespace["_jit_artifact_evidence"]()
- shutil.rmtree(duplicate)
- (first / "kernel.sass").unlink()
- with self.assertRaisesRegex(RuntimeError, "incomplete"):
- namespace["_jit_artifact_evidence"]()
- (first / "kernel.sass").symlink_to(first / "kernel.cubin")
- with self.assertRaisesRegex(RuntimeError, "regular file"):
- namespace["_jit_artifact_evidence"]()
- finally:
- if old_cache is None:
- os.environ.pop("EP_JIT_CACHE_DIR", None)
- else:
- os.environ["EP_JIT_CACHE_DIR"] = old_cache
-
- def test_runtime_and_shared_version_formatter_are_valid(self) -> None:
- subprocess.run(
- ["bash", "-n", str(ROOT / "runtime" / "run_in_container.sh")],
- check=True,
- )
- self.assertEqual(ep_harness.format_collective_version(23004), "2.30.4")
- self.assertEqual(ep_harness.format_collective_version((2, 30, 4)), "2.30.4")
- source = self.path.read_text()
- version_function = next(
- node for node in self.tree.body
- if isinstance(node, ast.FunctionDef) and node.name == "_loaded_nccl_version"
- )
-
- class FakeNccl:
- @staticmethod
- def ncclGetVersion(pointer) -> int:
- pointer._obj.value = 23004
- return 0
-
- namespace = {
- "ctypes": types.SimpleNamespace(
- CDLL=lambda _path: FakeNccl(), byref=ctypes.byref, c_int=ctypes.c_int,
- ),
- "ep_harness": ep_harness,
- "os": os,
- "_loaded_library_paths": lambda: {"/safe/libnccl.so.2"},
- }
- exec(
- compile(ast.Module(body=[version_function], type_ignores=[]), str(self.path), "exec"),
- namespace,
- )
- self.assertEqual(namespace["_loaded_nccl_version"](), "2.30.4")
- for paths in (set(), {"/safe/libnccl.so.2", "/other/libnccl.so.2"}):
- namespace["_loaded_library_paths"] = lambda paths=paths: paths
- with self.assertRaisesRegex(RuntimeError, "exactly one"):
- namespace["_loaded_nccl_version"]()
- evidence_function = next(
- node for node in self.tree.body
- if isinstance(node, ast.FunctionDef) and node.name == "_loaded_library_evidence"
- )
- paths = {
- "/safe/_C.cpython-310-x86_64-linux-gnu.so",
- "/safe/libnccl.so.2",
- "/safe/libnvshmem_host.so.3",
- }
- namespace.update(
- _loaded_library_paths=lambda: paths,
- _sha256=lambda _path: "a" * 64,
- )
- exec(
- compile(ast.Module(body=[evidence_function], type_ignores=[]), str(self.path), "exec"),
- namespace,
- )
- evidence = namespace["_loaded_library_evidence"]()
- self.assertIn(
- {"name": "deep_ep._C", "role": "deepep-extension", "sha256": "a" * 64},
- evidence,
- )
- self.assertTrue(
- contracts._content_evidence_is_valid(
- evidence, {"deepep-extension", "nccl", "nvshmem"}
- )
- )
- self.assertNotIn("torch.cuda.nccl.version()", source)
- fingerprint = {"runtime": "cuda", "version": "13.0"}
- self.assertIs(
- run_ep._common_runtime_fingerprint([fingerprint, dict(fingerprint)]),
- fingerprint,
- )
- with self.assertRaises(ValueError):
- run_ep._common_runtime_fingerprint([fingerprint, {"runtime": "cuda", "version": "12.8"}])
-
- def test_conditioning_contract_is_exact_for_each_phase(self) -> None:
- expected = {
- "decode": [1, 2, 4, 8, 16, 32, 64, 128],
- "prefill": [1, 2, 4, 8, 16, 32, 64, 128, 256, 512],
- }
- for phase, ladder in expected.items():
- valid = {
- "contract": "fixed-phase-ramp-8-roundtrips-v1",
- "ladder": ladder,
- "roundtrips_per_shape": 8,
- }
- self.assertIs(contracts.validate_conditioning_contract(valid, phase), valid)
- for mutate in (
- lambda item: item["ladder"].reverse(),
- lambda item: item["ladder"].pop(),
- lambda item: item.update(ladder=[1.0, *item["ladder"][1:]]),
- lambda item: item.update(roundtrips_per_shape=7),
- lambda item: item.update(roundtrips_per_shape=8.0),
- ):
- changed = copy.deepcopy(valid)
- mutate(changed)
- with self.assertRaises(contracts.ContractError):
- contracts.validate_conditioning_contract(changed, phase)
- other = "prefill" if phase == "decode" else "decode"
- with self.assertRaises(contracts.ContractError):
- contracts.validate_conditioning_contract(valid, other)
-
- def test_content_manifest_evidence_is_stable_and_content_sensitive(self) -> None:
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary)
- first, second = root / "first", root / "second"
- first.write_bytes(b"first")
- second.write_bytes(b"second")
- files = [("pkg/first", first), ("pkg/second", second)]
- evidence = contracts.content_manifest_evidence(
- role="test-content", name="test-build", files=files,
- )
- self.assertNotIn(temporary, json.dumps(evidence))
- self.assertEqual(
- evidence,
- contracts.content_manifest_evidence(
- role="test-content", name="test-build", files=reversed(files),
- ),
- )
- self.assertRegex(evidence["sha256"], r"^[0-9a-f]{64}$")
- second.write_bytes(b"changed")
- self.assertNotEqual(
- evidence,
- contracts.content_manifest_evidence(
- role="test-content", name="test-build", files=files,
- ),
- )
- for invalid in (
- [("../first", first)],
- [("same", first), ("same", second)],
- [("missing", root / "missing")],
- ):
- with self.assertRaises(contracts.ContractError):
- contracts.content_manifest_evidence(
- role="test-content", name="test-build", files=invalid,
- )
-
- def test_hybrid_realized_config_and_jit_evidence_are_path_free(self) -> None:
- path = HERE / "ep_deepep_hybrid.py"
- tree = ast.parse(path.read_text(), str(path))
- selected = [
- node for node in tree.body
- if (
- isinstance(node, ast.Assign)
- and any(
- isinstance(target, ast.Name) and target.id == "HYBRID_CONFIG_FIELDS"
- for target in node.targets
- )
- )
- or isinstance(node, ast.FunctionDef)
- and node.name in {
- "_hybrid_realized_config", "_sha256_with_size", "_hybrid_jit_evidence",
- }
- ]
- namespace = {"Path": Path, "hashlib": hashlib, "re": __import__("re")}
- exec(compile(ast.Module(body=selected, type_ignores=[]), str(path), "exec"), namespace)
- fields = namespace["HYBRID_CONFIG_FIELDS"]
- self.assertEqual(set(fields), contracts.HYBRID_REALIZED_CONFIG_FIELDS)
-
- class TokenType:
- def __init__(self, label: str, name: str | None = None) -> None:
- self.label = label
- if name is not None:
- self.name = name
-
- def __str__(self) -> str:
- return self.label
-
- values = {field: 1 for field in fields}
- values.update({field: True for field in contracts.HYBRID_REALIZED_BOOL_FIELDS})
- for raw, expected in (("uint16_t", "UINT16"), ("uint8_t", "UINT8")):
- values["token_data_type"] = TokenType(raw)
- config = types.SimpleNamespace(**values)
- realized = namespace["_hybrid_realized_config"](config)
- self.assertEqual(realized["token_data_type"], expected)
- self.assertEqual(set(realized), contracts.HYBRID_REALIZED_CONFIG_FIELDS)
- values["token_data_type"] = TokenType("opaque-enum", "UINT16")
- self.assertEqual(
- namespace["_hybrid_realized_config"](types.SimpleNamespace(**values))[
- "token_data_type"
- ],
- "UINT16",
- )
- values["token_data_type"] = TokenType("UINT16")
- with self.assertRaisesRegex(RuntimeError, "token_data_type is invalid"):
- namespace["_hybrid_realized_config"](types.SimpleNamespace(**values))
- values["token_data_type"] = TokenType("uint16_t")
- config = types.SimpleNamespace(**values)
- delattr(config, "hidden_dim")
- with self.assertRaisesRegex(RuntimeError, "omits hidden_dim"):
- namespace["_hybrid_realized_config"](config)
-
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary)
- for key, payload in (
- ("preprocess-key", b"pre"),
- ("combine-key", b"combine"),
- ("dispatch-key", b"dispatch"),
- ):
- (root / f"{key}.so").write_bytes(payload)
- evidence = namespace["_hybrid_jit_evidence"](root)
- self.assertEqual(
- [item["kernel_key"] for item in evidence],
- ["combine-key", "dispatch-key", "preprocess-key"],
- )
- self.assertNotIn(temporary, json.dumps(evidence))
- (root / "dispatch-key.so").write_bytes(b"changed")
- self.assertNotEqual(evidence, namespace["_hybrid_jit_evidence"](root))
- (root / "extra-key.so").write_bytes(b"extra")
- with self.assertRaisesRegex(RuntimeError, "expected 3"):
- namespace["_hybrid_jit_evidence"](root)
- (root / "extra-key.so").unlink()
- (root / "bad key.so").write_bytes(b"bad")
- with self.assertRaisesRegex(RuntimeError, "kernel key"):
- namespace["_hybrid_jit_evidence"](root)
- (root / "bad key.so").unlink()
- (root / "combine-key.so").unlink()
- (root / "combine-key.so").symlink_to(root / "dispatch-key.so")
- with self.assertRaisesRegex(RuntimeError, "regular file"):
- namespace["_hybrid_jit_evidence"](root)
- empty = root / "empty"
- empty.mkdir()
- with self.assertRaisesRegex(RuntimeError, "expected 3"):
- namespace["_hybrid_jit_evidence"](empty)
-
- def test_hybrid_uses_communication_domains_not_physical_hosts(self) -> None:
- path = HERE / "ep_deepep_hybrid.py"
- function = next(
- node for node in ast.parse(path.read_text(), str(path)).body
- if isinstance(node, ast.FunctionDef) and node.name == "_hybrid_topology"
- )
- namespace: dict[str, object] = {}
- exec(compile(ast.Module(body=[function], type_ignores=[]), str(path), "exec"), namespace)
- resolve = namespace["_hybrid_topology"]
- cases = (
- (8, 8, 8, "scale-up", "nvlink", "", 8, 1, 1),
- (16, 8, 8, "scale-out", "nvlink", "rdma", 8, 2, 2),
- (8, 4, 72, "scale-up", "mnnvl", "", 8, 1, 2),
- (16, 4, 72, "scale-up", "mnnvl", "", 16, 1, 4),
- )
- for world, gpn, domain, scope, up, out, ranks, domains, hosts in cases:
- with self.subTest(world=world, gpus_per_node=gpn, transport=up):
- topology = resolve(types.SimpleNamespace(
- gpus_per_node=gpn,
- scale_up_domain=domain,
- scope=scope,
- scale_up_transport=up,
- scale_out_transport=out,
- transport=up if not out else f"{up}-{out}",
- ), world)
- self.assertEqual(
- (topology["domain_ranks"], topology["communication_domains"],
- topology["physical_nodes"]),
- (ranks, domains, hosts),
- )
- with self.assertRaisesRegex(RuntimeError, "outside the fixed v1 matrix"):
- resolve(types.SimpleNamespace(
- gpus_per_node=8, scale_up_domain=8, scope="scale-up",
- scale_up_transport="nvlink", scale_out_transport="", transport="nvlink",
- ), 16)
-
- def test_mori_ep16_pins_upstream_internode_v1_resources(self) -> None:
- source = (HERE / "ep_mori.py").read_text()
- for fragment in (
- 'kernel_enum.InterNodeV1',
- 'self.block_num = self._block_target = 96',
- 'self.rdma_block_num = 64',
- 'self.dispatch_warps = self.combine_warps = 8',
- 'self.num_qps = 1',
- '"gpu_per_node": gpus_per_node',
- '"rdma_block_num": self.rdma_block_num',
- '"num_qp_per_pe": self.num_qps',
- '"use_external_inp_buf": self._external_input',
- 'os.environ["MORI_EP_LAUNCH_CONFIG_MODE"] = "MANUAL"',
- 'rdma_block_num=self.rdma_block_num',
- ):
- self.assertIn(fragment, source)
- self.assertGreaterEqual(source.count("rdma_block_num=self.rdma_block_num"), 2)
-
- def test_hybrid_deferred_provenance_wraps_before_conditioning_and_recaptures(self) -> None:
- path = HERE / "ep_deepep_hybrid.py"
- source = path.read_text()
- tree = ast.parse(source, str(path))
- backend = next(
- node for node in tree.body
- if isinstance(node, ast.ClassDef) and node.name == "DeepEPHybridBackend"
- )
- methods = {node.name for node in backend.body if isinstance(node, ast.FunctionDef)}
- self.assertIn("capture_deferred_provenance", methods)
- # Buffer construction moved into create_buffer (consumes make_inputs); the
- # cache-dir → HybridEPBuffer → wrapper-install ordering is pinned there now.
- constructor = next(node for node in backend.body if isinstance(node, ast.FunctionDef) and node.name == "create_buffer")
- buffer_call = next(
- node for node in ast.walk(constructor)
- if isinstance(node, ast.Call) and isinstance(node.func, ast.Name)
- and node.func.id == "HybridEPBuffer"
- )
- wrapper_install = next(
- node for node in ast.walk(constructor)
- if isinstance(node, ast.Assign)
- and any(
- isinstance(target, ast.Attribute)
- and target.attr == "update_template_config"
- for target in node.targets
- )
- )
- cache_line = source[:source.index('os.environ["HYBRID_EP_CACHE_DIR"]')].count("\n") + 1
- self.assertLess(cache_line, buffer_call.lineno)
- self.assertLess(buffer_call.lineno, wrapper_install.lineno)
-
- capture = next(
- node for node in backend.body
- if isinstance(node, ast.FunctionDef) and node.name == "capture_deferred_provenance"
- )
- called = {
- node.func.id if isinstance(node.func, ast.Name) else node.func.attr
- for node in ast.walk(capture) if isinstance(node, ast.Call)
- and isinstance(node.func, (ast.Name, ast.Attribute))
- }
- self.assertTrue({"_hybrid_jit_evidence", "_require_cross_rank_equal", "all_gather_object"} <= called)
- self.assertIn("changed after measurement", ast.get_source_segment(source, capture))
-
- artifacts = [[
- {"bytes": 1, "kernel_key": key, "sha256": digit * 64}
- for key, digit in (("a", "1"), ("b", "2"), ("c", "3"))
- ]]
-
- class FakeCuda:
- @staticmethod
- def synchronize() -> None:
- return None
-
- class FakeDist:
- @staticmethod
- def barrier() -> None:
- return None
-
- @staticmethod
- def get_world_size() -> int:
- return 2
-
- @staticmethod
- def all_gather_object(output, value) -> None:
- output[:] = [copy.deepcopy(value), copy.deepcopy(value)]
-
- namespace = {
- "torch": types.SimpleNamespace(cuda=FakeCuda),
- "dist": FakeDist,
- "_hybrid_jit_evidence": lambda _root: copy.deepcopy(artifacts[0]),
- "_require_cross_rank_equal": lambda _value, _label: None,
- }
- exec(compile(ast.Module(body=[capture], type_ignores=[]), str(path), "exec"), namespace)
- state = types.SimpleNamespace(
- _deferred_jit_diagnostics=None,
- _deferred_semantic_snapshot=None,
- _jit_root=Path("private-cache"),
- _realized_config=hybrid_realized_config(),
- backend_provenance={},
- )
- namespace["capture_deferred_provenance"](state)
- artifacts[0][0]["kernel_key"] = "changed"
- with self.assertRaisesRegex(RuntimeError, "kernel set changed"):
- namespace["capture_deferred_provenance"](state)
- artifacts[0][0]["kernel_key"] = "a"
- artifacts[0][0]["sha256"] = "f" * 64
- with self.assertRaisesRegex(RuntimeError, "artifacts changed"):
- namespace["capture_deferred_provenance"](state)
-
- harness = (HERE / "ep_harness.py").read_text()
- captures = [
- index for index in range(len(harness))
- if harness.startswith("capture_deferred_provenance()", index)
- ]
- self.assertEqual(len(captures), 2)
- self.assertLess(harness.index("for wt in conditioning_ladder:"), captures[0])
- self.assertLess(captures[0], harness.index("oracle = _run_expert_oracle("))
- self.assertLess(harness.index("trace_sig = hashlib.sha256"), captures[1])
-
- def test_hybrid_diagnostic_hashes_do_not_split_series_identity(self) -> None:
- keys, artifacts = hybrid_jit_provenance()
- provenance = {
- "deepep_tree": "b" * 40,
- "jit_kernel_keys": keys,
- "jit_shared_objects": artifacts,
- "loaded_libraries": [{
- "name": "hybrid_ep_cpp", "role": "deepep-hybrid-extension",
- "sha256": "a" * 64,
- }],
- "realized_config": hybrid_realized_config(),
- }
- baseline = ep_harness._series_provenance(provenance)
- changed = copy.deepcopy(provenance)
- changed["jit_shared_objects"][0]["rank_artifacts"][0]["sha256"] = "f" * 64
- self.assertEqual(ep_harness._series_provenance(changed), baseline)
- changed = copy.deepcopy(provenance)
- changed["loaded_libraries"][0]["sha256"] = "f" * 64
- self.assertEqual(ep_harness._series_provenance(changed), baseline)
- changed = copy.deepcopy(provenance)
- changed["jit_kernel_keys"][0] = "changed-key"
- self.assertNotEqual(ep_harness._series_provenance(changed), baseline)
- changed = copy.deepcopy(provenance)
- changed["realized_config"]["num_of_blocks_dispatch_api"] += 1
- self.assertNotEqual(ep_harness._series_provenance(changed), baseline)
- changed = copy.deepcopy(provenance)
- changed["deepep_tree"] = "c" * 40
- self.assertNotEqual(ep_harness._series_provenance(changed), baseline)
-
- def test_v2_series_identity_uses_source_and_sass_not_container_metadata(self) -> None:
- provenance = {
- "deepep_tree": "a" * 40,
- "loaded_libraries": [
- {"name": "deep_ep._C.so", "role": "deepep-extension", "sha256": "1" * 64},
- {"name": "libnccl.so.2", "role": "nccl", "sha256": "2" * 64},
- ],
- "jit_cubins": deepep_v2_jit_provenance(),
- "jit_random_seed": "collectivex-deepep-v2-fa8a9b1",
- }
- baseline = contracts.series_provenance(provenance)
- changed = copy.deepcopy(provenance)
- changed["loaded_libraries"][0]["sha256"] = "f" * 64
- changed["jit_cubins"][0]["cubin_sha256"] = "e" * 64
- self.assertEqual(contracts.series_provenance(changed), baseline)
- for mutate in (
- lambda item: item["loaded_libraries"][1].update(sha256="f" * 64),
- lambda item: item["jit_cubins"][0].update(source_sha256="f" * 64),
- lambda item: item["jit_cubins"][0].update(sass_sha256="f" * 64),
- lambda item: item.update(deepep_tree="f" * 40),
- ):
- changed = copy.deepcopy(provenance)
- mutate(changed)
- self.assertNotEqual(contracts.series_provenance(changed), baseline)
-
- def test_mnnvl_resolution_has_no_ambiguous_signature_fallback(self) -> None:
- self.assertEqual(
- contracts.resolve_deepep_mnnvl(
- requested=False, signature_parameters=(), deepep_commit=None,
- ),
- ({}, "not-requested"),
- )
- self.assertEqual(
- contracts.resolve_deepep_mnnvl(
- requested=True, signature_parameters=("allow_mnnvl",),
- deepep_commit="a" * 40,
- ),
- ({"allow_mnnvl": True}, "explicit-allow-mnnvl"),
- )
- with self.assertRaises(contracts.ContractError):
- contracts.resolve_deepep_mnnvl(
- requested=True, signature_parameters=(),
- deepep_commit="814e508537c6ffc775d59f6f1b9ba43f3a65968c",
- )
-
- def test_backend_provenance_requires_lineage_and_content_hashes(self) -> None:
- def record(role: str, name: str, digit: str) -> dict[str, str]:
- return {"role": role, "name": name, "sha256": digit * 64}
-
- hybrid_keys, hybrid_artifacts = hybrid_jit_provenance()
- v2 = {
- **contracts.DEEPEP_V2_V1_PROVENANCE,
- "api_signature_sha256": "c" * 64,
- "loaded_libraries": [
- record("deepep-extension", "deep_ep._C", "1"),
- record("nccl", "libnccl.so.2", "2"),
- record("nvshmem", "libnvshmem_host.so.3", "3"),
- ],
- "jit_cubins": deepep_v2_jit_provenance(),
- "jit_random_seed": "collectivex-deepep-v2-fa8a9b1",
- "deterministic": False,
- "num_experts": 256,
- "tuning_num_experts": 256,
- "allow_hybrid_mode": False,
- "gin_enabled": False,
- "communication_backend": "nccl-device-lsa",
- }
- deepep = {
- "deepep_version": "1.1.0", "deepep_commit": "a" * 40,
- "backend_lineage": "deepep-v1", "allow_mnnvl": False,
- "mnnvl_comm": "not-requested", "mode": "normal",
- "nvshmem_ibgda_nic_handler": "not-active",
- "num_nvl_bytes": 1024, "num_rdma_bytes": 0,
- }
- hybrid = {
- "deepep_commit": "a" * 40, "deepep_tree": "b" * 40,
- "branch": "hybrid-ep", "backend_lineage": "deepep-hybrid",
- "loaded_libraries": [
- record("deepep-extension", "deep_ep_cpp", "1"),
- record("deepep-hybrid-extension", "hybrid_ep_cpp", "2"),
- ],
- "jit_kernel_keys": hybrid_keys,
- "jit_shared_objects": hybrid_artifacts,
- "realized_config": hybrid_realized_config(),
- }
- uccl = {
- "uccl_version": "0.1.1", "uccl_commit": "pkg-0.1.1",
- "uccl_wrapper_commit": "c" * 40, "backend_lineage": "uccl",
- "uccl_dependency_versions": dict(contracts.UCCL_DEPENDENCY_VERSIONS),
- "loaded_libraries": [
- record("uccl-distribution", "uccl-0.1.1", "3"),
- record("uccl-wrapper", "uccl-deepep-wrapper", "4"),
- record("intervaltree-distribution", "intervaltree-3.1.0", "5"),
- record("sortedcontainers-distribution", "sortedcontainers-2.4.0", "6"),
- record("cuda-runtime", "nvidia-cuda-runtime-cu12-12.9.79", "7"),
- ],
- "mode": "normal", "num_nvl_bytes": 1024, "num_rdma_bytes": 0,
- }
- reference = {
- "nccl_version": "2.30.4", "collective_library": "nccl",
- "backend_lineage": "nccl",
- }
- for backend, provenance in (
- ("deepep", deepep), ("deepep-v2", v2), ("deepep-hybrid", hybrid),
- ("uccl", uccl), ("nccl-ep", reference),
- ):
- self.assertEqual(contracts.backend_provenance_issues(backend, provenance), [])
- changed = copy.deepcopy(provenance)
- if "loaded_libraries" in changed:
- changed["loaded_libraries"][0]["sha256"] = "invalid"
- expected = "loaded_libraries"
- else:
- changed["backend_lineage"] = "wrong"
- expected = "backend_lineage"
- self.assertIn(expected, contracts.backend_provenance_issues(backend, changed))
-
- for distribution_version in contracts.DEEPEP_V2_DISTRIBUTION_VERSIONS:
- changed = copy.deepcopy(v2)
- changed["deepep_distribution_version"] = distribution_version
- self.assertEqual(
- contracts.backend_provenance_issues("deepep-v2", changed), []
- )
- changed = copy.deepcopy(v2)
- changed["deepep_distribution_version"] = "2.0.0+unverified"
- self.assertIn(
- "deepep_distribution_version",
- contracts.backend_provenance_issues("deepep-v2", changed),
- )
-
- changed = copy.deepcopy(uccl)
- changed["uccl_dependency_versions"]["intervaltree"] = "3.2.0"
- self.assertIn(
- "uccl_dependency_versions",
- contracts.backend_provenance_issues("uccl", changed),
- )
- changed = copy.deepcopy(uccl)
- changed["loaded_libraries"] = [
- item
- for item in changed["loaded_libraries"]
- if item["role"] != "sortedcontainers-distribution"
- ]
- self.assertIn(
- "loaded_libraries", contracts.backend_provenance_issues("uccl", changed)
- )
-
- for field, mutate in (
- ("realized_config", lambda item: item["realized_config"].pop("hidden_dim")),
- ("jit_kernel_keys", lambda item: item["jit_kernel_keys"].reverse()),
- (
- "jit_shared_objects",
- lambda item: item["jit_shared_objects"][0]["rank_artifacts"][0].update(
- sha256="invalid"
- ),
- ),
- ):
- with self.subTest(hybrid_field=field):
- changed = copy.deepcopy(hybrid)
- mutate(changed)
- self.assertIn(
- field,
- contracts.backend_provenance_issues("deepep-hybrid", changed),
- )
-
- for field, value in (
- ("jit_cubins", [{"cache_key": "invalid", "cubin_sha256": "4" * 64}]),
- ("jit_random_seed", "different-seed"),
- ):
- with self.subTest(v2_field=field):
- changed = copy.deepcopy(v2)
- changed[field] = value
- self.assertIn(
- field,
- contracts.backend_provenance_issues("deepep-v2", changed),
- )
-
- changed = copy.deepcopy(v2)
- changed["gin_enabled"] = True
- self.assertIn("gin_enabled", contracts.backend_provenance_issues("deepep-v2", changed))
- changed = copy.deepcopy(v2)
- changed["communication_backend"] = "nccl-gin"
- self.assertIn(
- "communication_backend", contracts.backend_provenance_issues("deepep-v2", changed)
- )
- changed = copy.deepcopy(v2)
- changed.update(
- allow_hybrid_mode=True,
- gin_enabled=True,
- communication_backend="nccl-gin",
- )
- self.assertEqual(
- contracts.backend_provenance_issues("deepep-v2", changed),
- [],
- )
- changed["allow_hybrid_mode"] = False
- self.assertEqual(
- contracts.backend_provenance_issues("deepep-v2", changed),
- ["allow_hybrid_mode", "communication_backend", "gin_enabled"],
- )
- for field, expected in contracts.DEEPEP_V2_V1_PROVENANCE.items():
- with self.subTest(v2_pin_field=field):
- changed = copy.deepcopy(v2)
- changed[field] = not expected if type(expected) is bool else "wrong"
- self.assertIn(
- field,
- contracts.backend_provenance_issues("deepep-v2", changed),
- )
-
- schema = json.loads((ROOT / "schemas" / "raw-case-v1.schema.json").read_text())
- provenance_schema = schema["properties"]["implementation"]["properties"]["provenance"]
- self.assertEqual(
- provenance_schema["properties"]["realized_config"],
- {"$ref": "#/$defs/hybrid_realized_config"},
- )
- self.assertFalse(schema["$defs"]["hybrid_realized_config"]["additionalProperties"])
- self.assertEqual(provenance_schema["properties"]["jit_kernel_keys"]["minItems"], 3)
- self.assertEqual(provenance_schema["properties"]["jit_shared_objects"]["minItems"], 3)
-
- self.assertEqual(contracts.collective_kernel_generation("nccl"), "nccl")
- self.assertEqual(contracts.collective_kernel_generation("rccl"), "rccl")
- with self.assertRaises(contracts.ContractError):
- contracts.collective_kernel_generation("unknown")
-
- def test_transport_resource_provenance_is_exact(self) -> None:
- self.assertEqual(contracts.hybrid_communication_domains(8, 8), (8, 1))
- self.assertEqual(contracts.hybrid_communication_domains(16, 8), (8, 2))
- self.assertEqual(contracts.hybrid_communication_domains(8, 72), (8, 1))
- self.assertEqual(contracts.hybrid_communication_domains(16, 72), (16, 1))
-
- profile = contracts.project_resource_profile({
- "num_nvl_bytes": 1024, "num_rdma_bytes": 2048,
- "num_qps_per_rank": 32, "heap_size": "6G",
- })
- self.assertEqual(profile["persistent_bytes"], 3072)
- self.assertEqual(profile["qps_per_rank"], 32)
- self.assertEqual(
- contracts.project_resource_profile({
- "num_nvl_bytes": 0, "num_rdma_bytes": 0, "heap_size": "6G",
- })["persistent_bytes"],
- 0,
- )
- self.assertEqual(
- contracts.project_resource_profile({"heap_size": "6G"})[
- "persistent_bytes"
- ],
- "6G",
- )
-
- mori = {
- "mori_commit": "a" * 40, "kernel_type": "InterNodeV1",
- "block_num": 96, "rdma_block_num": 64,
- "dispatch_warps": 8, "combine_warps": 8, "num_qps": 1,
- "use_external_inp_buf": True, "gpus_per_node": 8,
- }
- self.assertEqual(contracts.backend_provenance_issues("mori", mori), [])
- for field in (
- "block_num", "rdma_block_num", "dispatch_warps", "combine_warps",
- "num_qps", "use_external_inp_buf", "gpus_per_node",
- ):
- changed = copy.deepcopy(mori)
- changed[field] = False if field == "use_external_inp_buf" else 0
- with self.subTest(mori_field=field):
- self.assertIn(
- field, contracts.backend_provenance_issues("mori", changed)
- )
-
- def test_routing_control_binds_binary_but_allows_treatment_configuration(self) -> None:
- hybrid_keys, hybrid_artifacts = hybrid_jit_provenance()
- implementation = {
- "kernel_generation": "hybrid",
- "name": "deepep-hybrid",
- "provenance": {
- "deepep_tree": "a" * 40,
- "loaded_libraries": [{
- "role": "deepep-extension", "name": "deep_ep_cpp", "sha256": "1" * 64,
- }],
- "local_experts": 32,
- "num_experts": 256,
- "num_sms": 24,
- "jit_cache_key": "case-one",
- "jit_cubins": [{"cache_key": "one", "cubin_sha256": "2" * 64}],
- "jit_kernel_keys": hybrid_keys,
- "jit_shared_objects": hybrid_artifacts,
- "realized_config": hybrid_realized_config(),
- },
- "resource_profile": {"configured_units": 24},
- }
- baseline = contracts.routing_implementation_control_sha256(implementation)
- treatment = copy.deepcopy(implementation)
- treatment["provenance"].update({
- "local_experts": 36,
- "num_experts": 288,
- "jit_cache_key": "case-two",
- "jit_cubins": [{"cache_key": "two", "cubin_sha256": "3" * 64}],
- "jit_kernel_keys": ["changed-a", "changed-b", "changed-c"],
- "jit_shared_objects": hybrid_jit_provenance(3)[1],
- "realized_config": {
- **hybrid_realized_config(),
- "num_of_experts_per_rank": 36,
- },
- })
- self.assertEqual(
- contracts.routing_implementation_control_sha256(treatment), baseline
- )
- changed = copy.deepcopy(implementation)
- changed["provenance"]["loaded_libraries"][0]["sha256"] = "4" * 64
- self.assertEqual(
- contracts.routing_implementation_control_sha256(changed), baseline
- )
- changed = copy.deepcopy(implementation)
- changed["provenance"]["deepep_tree"] = "b" * 40
- self.assertNotEqual(
- contracts.routing_implementation_control_sha256(changed), baseline
- )
- changed = copy.deepcopy(implementation)
- changed["provenance"]["num_sms"] = 20
- self.assertNotEqual(
- contracts.routing_implementation_control_sha256(changed), baseline
- )
-
- def test_runtime_pins_uccl_wheel_and_hybrid_source_tree(self) -> None:
- runtime = (ROOT / "runtime" / "run_in_container.sh").read_text()
- common = (ROOT / "runtime" / "common.sh").read_text()
- self.assertIn("cd /ix/experimental/CollectiveX", runtime)
- for launcher_name in ("launch_single-slurm.sh", "launch_gb-nv.sh"):
- launcher = (ROOT / "launchers" / launcher_name).read_text()
- self.assertIn("$MOUNT_SRC:/ix", launcher)
- self.assertIn("cx_prepare_backend_cache", launcher)
- self.assertNotIn('$(cx_prepare_backend_cache', launcher)
- self.assertIn("$CX_PREPARED_BACKEND_CACHE:/cx-cache", launcher)
- self.assertIn("CX_BACKEND_CACHE_ROOT=/cx-cache", launcher)
- self.assertIn("CX_BACKEND_SOURCE_ROOT=/ix/experimental/CollectiveX/.cx_sources", launcher)
- self.assertIn('|| [ "$CX_BENCH" = deepep-hybrid ]', launcher)
- self.assertIn("cx_prepare_backend_source", launcher)
- cache_block = launcher[launcher.index('if [ "$CX_BENCH" = deepep-v2 ]'):]
- self.assertLess(
- cache_block.index("cx_set_failure_stage backend-setup"),
- cache_block.index("cx_prepare_backend_cache"),
- )
- self.assertLess(
- cache_block.index("cx_prepare_backend_source"),
- cache_block.index("cx_set_failure_stage scheduler-allocation"),
- )
- self.assertIn("--frandom-seed=$seed", runtime)
- self.assertIn("DEEPEP_V2_JIT_RANDOM_SEED", runtime)
- persisted = runtime[runtime.index("cx_persist_backend_env()") :]
- self.assertIn("CUDA_HOME CPATH NVCC_PREPEND_FLAGS", persisted)
- self.assertIn(
- "390c1320918972206546e44d79b132988f2818ec07e23afcd0595f7183916cec",
- runtime,
- )
- self.assertIn("--require-hashes", runtime)
- self.assertIn('$PWD/.cx_backend/uccl-wrapper-node-$node_id', runtime)
- self.assertNotIn("/tmp/uccl_deepep_pkg", runtime)
- self.assertIn('export PYTHONPATH="$wrapper_root:${PYTHONPATH:-}"', runtime)
- self.assertIn("d77aeab7f1bb52b615666fe178d26ced41fae08e", common)
- self.assertIn("HEAD^{tree}", runtime)
- self.assertIn("$PWD/.cx_backend/deepep-hybrid-", runtime)
- self.assertIn("cx_materialize_backend_source deepep-hybrid", runtime)
- self.assertIn("cx_materialize_backend_source deepep-v2", runtime)
- self.assertIn("cx_deepep_hybrid_marker_content_sha256", runtime)
- self.assertIn("cx_deepep_hybrid_cache_is_valid", runtime)
- self.assertIn("cx_extension_pair_sha256", runtime)
- self.assertIn(".collectivex-complete.tmp.", runtime)
- self.assertNotIn("cx_fetch_revision", runtime)
- self.assertIn("cx_fetch_revision", common)
- self.assertIn("third-party/fmt", common)
- hybrid = runtime[
- runtime.index("cx_build_deepep_hybrid()"):
- runtime.index("# UCCL EP")
- ]
- configure = runtime[
- runtime.index("cx_configure_deepep_hybrid_build()"):
- runtime.index("cx_deepep_hybrid_marker_content_sha256()")
- ]
- self.assertIn("cx_prepare_cuda_cccl", hybrid)
- self.assertIn("unset NVSHMEM_DIR", hybrid)
- self.assertIn(
- "unset HYBRID_EP_MULTINODE USE_NIXL RDMA_CORE_HOME", configure
- )
- self.assertIn("cx_configure_deepep_hybrid_build || return 1", hybrid)
- self.assertIn('[ "$(uname -m)" = x86_64 ]', configure)
- self.assertIn('[ -n "${GLOO_SOCKET_IFNAME:-}" ]', configure)
- self.assertIn('[ -d "/sys/class/infiniband/$rdma_name" ]', configure)
- self.assertIn("command -v make", configure)
- self.assertIn("/usr/include/infiniband/verbs.h", configure)
- self.assertIn("export HYBRID_EP_MULTINODE=1 USE_NIXL=0", configure)
- self.assertNotIn("cx_prepare_deepep_toolchain", hybrid)
- toolchain = runtime[
- runtime.index("cx_prepare_deepep_toolchain()"):
- runtime.index("cx_probe_deepep()")
- ]
- self.assertIn('overlay="$root/nvshmem-overlay"', toolchain)
- self.assertIn("flock 8 || exit 1", toolchain)
- self.assertIn('mv "$temporary" "$overlay" || exit 1', toolchain)
- self.assertNotIn("/tmp/collectivex-nvshmem", toolchain)
- jit = runtime[
- runtime.index("cx_enable_deepep_v2_jit_reproducibility()"):
- runtime.index("cx_probe_deepep_v2()")
- ]
- self.assertIn('cccl="${CX_CUDA_CCCL:-}"', jit)
- self.assertNotIn("/usr/local/cuda*", jit)
- self.assertIn("deepep-v2-cache-v3|$cpu|sm${arch/./}", runtime)
- self.assertNotIn("deepep-v2-cache-v1|", runtime)
- self.assertIn('base="${CX_BACKEND_CACHE_ROOT:-}"', runtime)
- self.assertNotIn("${CX_BACKEND_CACHE_ROOT:-$PWD/.cx_backend}", runtime)
- self.assertIn(
- "recipe=aot-persistent-nvshmem-active-cuda-maxjobs16-v3", runtime
- )
- self.assertNotIn("recipe=aot-source-date-epoch-arch-maxjobs16-v1", runtime)
- self.assertNotIn("recipe=$source_sha", runtime)
- self.assertIn("pip=26.1.2|setuptools=82.0.1|wheel=0.47.0|ninja=1.13.0", runtime)
- self.assertIn("manual-unverified", runtime)
- self.assertIn("cx_deepep_v2_content_sha256", runtime)
- self.assertIn("DeepEP V2 cache validation failed", runtime)
- probe = runtime[
- runtime.index("cx_probe_deepep_v2()"):
- runtime.index("cx_deepep_v2_content_sha256()")
- ]
- self.assertNotIn("torch.cuda.nccl.version", probe)
- self.assertIn("ncclGetVersion", probe)
- self.assertIn("runtime_version.value == 23004", probe)
- self.assertIn("cx_nvidia_package_root nvidia-nccl-cu13 nccl", runtime)
- self.assertIn("cx_nvidia_package_root nvidia-nvshmem-cu12 nvshmem", runtime)
- self.assertNotIn("import os,nvidia.nccl", runtime)
- self.assertNotIn("import os,nvidia.nvshmem", runtime)
- self.assertIn(
- 'export EP_JIT_CACHE_DIR="/tmp/collectivex-deepep-v2-jit-$execution_id"',
- runtime,
- )
- self.assertIn('execution_id="${COLLECTIVEX_EXECUTION_ID:-manual}"', runtime)
- self.assertNotIn('stage_root="${CX_BACKEND_SOURCE_ROOT%/.cx_sources}"', runtime)
- self.assertNotIn('export EP_JIT_CACHE_DIR="$root/jit"', runtime)
- self.assertIn('EP_NVSHMEM_ROOT_DIR="$NVSHMEM_DIR"', runtime)
- reference = (HERE / "ep_nccl.py").read_text()
- self.assertIn("self.kernel_generation = contracts.collective_kernel_generation", reference)
- self.assertIn("scale-out NCCL network mode is not IB", reference)
- self.assertIn("scale-out RCCL must auto-select its HCA-pinned network", reference)
- self.assertIn("scale-out collective HCA allowlist is invalid", reference)
- self.assertIn('entries={len(entries)}, shaped={shaped}', reference)
- self.assertNotIn("scale-out collective transport is not pinned to RDMA", reference)
-
- def test_deepep_v2_cache_recovers_from_an_unpublished_partial_build(self) -> None:
- runtime = ROOT / "runtime" / "run_in_container.sh"
- with tempfile.TemporaryDirectory() as temporary:
- cache_key = "a" * 64
- content_hash = "b" * 64
- root = Path(temporary) / f"deepep-v2-{cache_key}"
- root.mkdir(mode=0o700)
- marker = root / ".collectivex-complete"
- stale = root / "stale-partial-build"
- stale.write_text("partial\n")
- command = r'''
- set -euo pipefail
- eval "$(sed -n '/^cx_build_deepep_v2()/,/^}/p' "$1")"
- cache_root="$2"; expected_revision="$3"; expected_tree="$4"; expected_fmt="$5"
- expected_content="$6"
- cx_log() { :; }
- cx_verify_backend_cache_mount() { return 0; }
- cx_cuda_arch() { printf '9.0'; }
- cx_deepep_v2_root() { printf '%s' "$cache_root"; }
- cx_activate_deepep_v2() { export DEEPEP_V2_COMMIT="$expected_revision"; }
- cx_prepare_deepep_toolchain() { export NVSHMEM_DIR=/tmp/cx-test-nvshmem; }
- cx_probe_deepep_v2() { return 0; }
- cx_deepep_v2_content_sha256() { printf '%s' "$expected_content"; }
- cx_deepep_v2_cache_is_valid() {
- test -f "$2" && test "$(wc -l < "$2" | tr -d ' ')" = 5
- }
- cx_enable_deepep_v2_jit_reproducibility() { return 0; }
- cx_materialize_backend_source() { mkdir -p "$2/third-party/fmt"; }
- flock() { return 0; }
- python3() {
- if [ "${1:-}" = -m ] && [ "${2:-}" = venv ]; then
- mkdir -p "$3/bin"
- printf '#!/bin/sh\nexit 0\n' > "$3/bin/python"
- chmod 700 "$3/bin/python"
- fi
- return 0
- }
- git() {
- case " $* " in
- *' third-party/fmt rev-parse HEAD '*) printf '%s\n' "$expected_fmt" ;;
- *' rev-parse HEAD^{tree} '*) printf '%s\n' "$expected_tree" ;;
- *' show -s --format=%ct HEAD '*) printf '1\n' ;;
- *) return 0 ;;
- esac
- }
- cx_git_in_tree() { shift; git "$@"; }
- cx_build_deepep_v2
- '''
- subprocess.run(
- [
- "bash", "-c", command, "_", str(runtime), str(root),
- COMMIT, TREE, FMT_COMMIT, content_hash,
- ],
- check=True,
- )
- self.assertFalse(stale.exists())
- self.assertEqual(
- marker.read_text(),
- f"{COMMIT}\n{TREE}\n{FMT_COMMIT}\n{cache_key}\n{content_hash}\n",
- )
- self.assertEqual(list(root.glob(".collectivex-complete.tmp.*")), [])
-
- def test_deepep_v2_published_cache_is_never_deleted_after_probe_failure(self) -> None:
- runtime = ROOT / "runtime" / "run_in_container.sh"
- with tempfile.TemporaryDirectory() as temporary:
- cache_key = "a" * 64
- root = Path(temporary) / f"deepep-v2-{cache_key}"
- root.mkdir(mode=0o700)
- marker = root / ".collectivex-complete"
- marker.write_text("published\n")
- sentinel = root / "active-reader"
- sentinel.write_text("active\n")
- command = r'''
- set -euo pipefail
- eval "$(sed -n '/^cx_build_deepep_v2()/,/^}/p' "$1")"
- cache_root="$2"
- cx_log() { :; }
- cx_verify_backend_cache_mount() { return 0; }
- cx_cuda_arch() { printf '9.0'; }
- cx_deepep_v2_root() { printf '%s' "$cache_root"; }
- cx_deepep_v2_cache_is_valid() { return 0; }
- cx_activate_deepep_v2() { return 0; }
- cx_prepare_deepep_toolchain() { return 0; }
- cx_enable_deepep_v2_jit_reproducibility() { return 0; }
- cx_probe_deepep_v2() { return 1; }
- ! cx_build_deepep_v2
- '''
- subprocess.run(
- ["bash", "-c", command, "_", str(runtime), str(root)],
- check=True,
- )
- self.assertEqual(sentinel.read_text(), "active\n")
- self.assertEqual(marker.read_text(), "published\n")
-
- def test_deepep_v2_corrupt_published_cache_fails_without_reset(self) -> None:
- runtime = ROOT / "runtime" / "run_in_container.sh"
- with tempfile.TemporaryDirectory() as temporary:
- cache_key = "a" * 64
- root = Path(temporary) / f"deepep-v2-{cache_key}"
- root.mkdir(mode=0o700)
- marker = root / ".collectivex-complete"
- marker.write_text("corrupt\n")
- sentinel = root / "active-reader"
- sentinel.write_text("active\n")
- command = r'''
- set -euo pipefail
- eval "$(sed -n '/^cx_build_deepep_v2()/,/^}/p' "$1")"
- cache_root="$2"
- cx_log() { :; }
- cx_verify_backend_cache_mount() { return 0; }
- cx_cuda_arch() { printf '9.0'; }
- cx_deepep_v2_root() { printf '%s' "$cache_root"; }
- cx_deepep_v2_cache_is_valid() { return 1; }
- flock() { return 0; }
- ! cx_build_deepep_v2
- '''
- subprocess.run(
- ["bash", "-c", command, "_", str(runtime), str(root)],
- check=True,
- )
- self.assertEqual(sentinel.read_text(), "active\n")
- self.assertEqual(marker.read_text(), "corrupt\n")
-
- def test_deepep_v2_marker_requires_private_owned_cache_objects(self) -> None:
- runtime = ROOT / "runtime" / "run_in_container.sh"
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary) / "cache"
- root.mkdir(mode=0o700)
- (root / "source").mkdir(mode=0o700)
- (root / "venv").mkdir(mode=0o700)
- marker = root / ".collectivex-complete"
- cache_key = "a" * 64
- content_hash = "b" * 64
- marker.write_text(
- f"{COMMIT}\n{TREE}\n{FMT_COMMIT}\n{cache_key}\n{content_hash}\n"
- )
- root.chmod(0o2700)
- marker.chmod(0o600)
- command = r'''
- set -euo pipefail
- eval "$(sed -n '/^cx_deepep_v2_marker_content_sha256()/,/^}/p' "$1")"
- cx_deepep_v2_marker_content_sha256 "$2" "$3" "$4" "$5" "$6" "$7"
- '''
- args = [
- "bash", "-c", command, "_", str(runtime), str(root), str(marker),
- COMMIT, TREE, FMT_COMMIT, cache_key,
- ]
- valid = subprocess.run(args, text=True, capture_output=True, check=True)
- self.assertEqual(valid.stdout, content_hash)
- marker.chmod(0o644)
- self.assertNotEqual(subprocess.run(args).returncode, 0)
-
- def test_deepep_hybrid_marker_requires_a_private_regular_file(self) -> None:
- runtime = ROOT / "runtime" / "run_in_container.sh"
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary) / "cache"
- root.mkdir(mode=0o700)
- marker = root / ".collectivex-complete"
- content_hash = "b" * 64
- marker.write_text(f"{COMMIT}\n{TREE}\n{content_hash}\n")
- root.chmod(0o2700)
- marker.chmod(0o600)
- command = r'''
- set -euo pipefail
- eval "$(sed -n '/^cx_deepep_hybrid_marker_content_sha256()/,/^}/p' "$1")"
- cx_deepep_hybrid_marker_content_sha256 "$2" "$3" "$4" "$5"
- '''
- args = [
- "bash", "-c", command, "_", str(runtime), str(root), str(marker),
- COMMIT, TREE,
- ]
- valid = subprocess.run(args, text=True, capture_output=True, check=True)
- self.assertEqual(valid.stdout, content_hash)
- marker_contract = runtime.read_text()
- marker_contract = marker_contract[
- marker_contract.index("cx_deepep_hybrid_marker_content_sha256()"):
- marker_contract.index("cx_deepep_hybrid_cache_is_valid()")
- ]
- self.assertIn("marker_item.st_uid != root_item.st_uid", marker_contract)
- self.assertNotIn("st_uid != os.getuid()", marker_contract)
- marker.chmod(0o644)
- self.assertNotEqual(subprocess.run(args).returncode, 0)
-
- def test_deepep_v2_installed_content_digest_binds_every_distribution_file(self) -> None:
- runtime = ROOT / "runtime" / "run_in_container.sh"
- with tempfile.TemporaryDirectory() as temporary:
- site = Path(temporary) / "venv" / "lib" / "python3.11" / "site-packages"
- package = site / "deep_ep"
- info = site / "deep_ep-2.0.0.dist-info"
- package.mkdir(parents=True)
- info.mkdir()
- (package / "__init__.py").write_text("__version__ = '2.0.0'\n")
- extension = package / "_C.so"
- extension.write_bytes(b"extension-one")
- (info / "METADATA").write_text(
- "Metadata-Version: 2.1\nName: deep_ep\nVersion: 2.0.0\n"
- )
- (info / "RECORD").write_text(
- "deep_ep/__init__.py,,\n"
- "deep_ep/_C.so,,\n"
- "deep_ep-2.0.0.dist-info/METADATA,,\n"
- "deep_ep-2.0.0.dist-info/RECORD,,\n"
- )
- command = r'''
- set -euo pipefail
- eval "$(sed -n '/^cx_deepep_v2_content_sha256()/,/^}/p' "$1")"
- cx_deepep_v2_content_sha256
- '''
- env = {
- **os.environ,
- "PYTHONPATH": str(site),
- "VIRTUAL_ENV": str(Path(temporary) / "venv"),
- }
- first = subprocess.run(
- ["bash", "-c", command, "_", str(runtime)],
- text=True, capture_output=True, check=True, env=env,
- ).stdout
- extension.write_bytes(b"extension-two")
- second = subprocess.run(
- ["bash", "-c", command, "_", str(runtime)],
- text=True, capture_output=True, check=True, env=env,
- ).stdout
- self.assertRegex(first, r"^[0-9a-f]{64}$")
- self.assertRegex(second, r"^[0-9a-f]{64}$")
- self.assertNotEqual(first, second)
- extension.unlink()
- outside = Path(temporary) / "outside.so"
- outside.write_bytes(b"outside")
- extension.symlink_to(outside)
- self.assertNotEqual(
- subprocess.run(
- ["bash", "-c", command, "_", str(runtime)], env=env,
- ).returncode,
- 0,
- )
-
- def test_uccl_content_identity_excludes_install_generated_files(self) -> None:
- keep = load_uccl_function(
- "_is_uccl_runtime_payload", {"PurePosixPath": PurePosixPath}
- )
- self.assertTrue(keep("uccl/ep.abi3.so"))
- self.assertTrue(keep("uccl.libs/libnuma.so"))
- self.assertFalse(keep("uccl/__pycache__/collective.cpython-312.pyc"))
- self.assertFalse(keep("uccl-0.1.1.dist-info/RECORD"))
-
- def test_uccl_dependency_versions_are_exact(self) -> None:
- installed = dict(contracts.UCCL_DEPENDENCY_VERSIONS)
- dependency_versions = load_uccl_function(
- "_uccl_dependency_versions",
- {
- "contracts": contracts,
- "metadata": types.SimpleNamespace(
- version=lambda package: installed[package]
- ),
- },
- )
- self.assertEqual(dependency_versions(), contracts.UCCL_DEPENDENCY_VERSIONS)
- installed["intervaltree"] = "3.2.0"
- with self.assertRaisesRegex(RuntimeError, "differ from the v1 contract"):
- dependency_versions()
-
- schema = json.loads((ROOT / "schemas" / "raw-case-v1.schema.json").read_text())
- dependency_schema = schema["properties"]["implementation"]["properties"][
- "provenance"
- ]["properties"]["uccl_dependency_versions"]
- self.assertFalse(dependency_schema["additionalProperties"])
- self.assertEqual(
- {
- package: definition["const"]
- for package, definition in dependency_schema["properties"].items()
- },
- contracts.UCCL_DEPENDENCY_VERSIONS,
- )
-
- def test_uccl_support_dependency_content_is_path_free(self) -> None:
- with tempfile.TemporaryDirectory() as directory:
- root = Path(directory)
- source_entry = PurePosixPath("intervaltree/__init__.py")
- cache_entry = PurePosixPath("intervaltree/__pycache__/__init__.pyc")
- metadata_entry = PurePosixPath("intervaltree-3.1.0.dist-info/RECORD")
- for entry in (source_entry, cache_entry, metadata_entry):
- path = root / entry
- path.parent.mkdir(parents=True, exist_ok=True)
- path.write_bytes(entry.as_posix().encode())
- distribution = types.SimpleNamespace(
- files=[source_entry, cache_entry, metadata_entry],
- locate_file=lambda item: root / item,
- )
- evidence_for = load_uccl_function(
- "_python_dependency_evidence",
- {
- "Path": Path,
- "PurePosixPath": PurePosixPath,
- "contracts": contracts,
- "metadata": types.SimpleNamespace(
- distribution=lambda package: distribution
- ),
- },
- )
- evidence = evidence_for("intervaltree", "3.1.0")
- self.assertEqual(
- evidence,
- contracts.content_manifest_evidence(
- role="intervaltree-distribution",
- name="intervaltree-3.1.0",
- files=[(source_entry.as_posix(), root / source_entry)],
- ),
- )
- self.assertNotIn(str(root), json.dumps(evidence))
-
- def test_uccl_hashes_the_mapped_pinned_libcudart_without_exposing_paths(
- self,
- ) -> None:
- with tempfile.TemporaryDirectory() as directory:
- root = Path(directory)
- entry = PurePosixPath("nvidia/cuda_runtime/lib/libcudart.so.12")
- library = root / entry
- library.parent.mkdir(parents=True)
- library.write_bytes(b"pinned CUDA 12 runtime")
- distribution = types.SimpleNamespace(
- files=[entry],
- locate_file=lambda item: root / item,
- )
- evidence_for = load_uccl_function(
- "_loaded_libcudart_evidence",
- {
- "Path": Path,
- "PurePosixPath": PurePosixPath,
- "contracts": contracts,
- "metadata": types.SimpleNamespace(
- distribution=lambda package: distribution
- ),
- },
- )
- maps = root / "maps"
- maps.write_text(f"7f00-7f10 r-xp 00000000 00:00 0 {library}\n")
- evidence = evidence_for("12.9.79", maps)
- self.assertEqual(
- evidence,
- contracts.content_manifest_evidence(
- role="cuda-runtime",
- name="nvidia-cuda-runtime-cu12-12.9.79",
- files=[("libcudart.so", library)],
- ),
- )
- self.assertNotIn(str(root), json.dumps(evidence))
-
- unowned = root / "unowned" / library.name
- unowned.parent.mkdir()
- unowned.write_bytes(library.read_bytes())
- maps.write_text(f"7f00-7f10 r-xp 00000000 00:00 0 {unowned}\n")
- with self.assertRaisesRegex(RuntimeError, "not owned") as raised:
- evidence_for("12.9.79", maps)
- self.assertNotIn(str(root), str(raised.exception))
-
- def test_private_runtime_logs_are_not_public_artifacts(self) -> None:
- path = subprocess.check_output(
- [
- "bash", "-c", 'source "$1"; cx_private_log_path test', "_",
- str(ROOT / "runtime" / "common.sh"),
- ],
- text=True,
- env={**os.environ, "COLLECTIVEX_EXECUTION_ID": "contract-test"},
- ).strip()
- try:
- log = Path(path)
- self.assertEqual(stat.S_IMODE(log.stat().st_mode), 0o600)
- self.assertEqual(stat.S_IMODE(log.parent.stat().st_mode), 0o700)
- self.assertFalse(log.is_relative_to(ROOT))
- finally:
- shutil.rmtree(Path(path).parent, ignore_errors=True)
-
- def test_canonical_amd_runtime_logs_use_the_isolated_control_tree(self) -> None:
- common = str(ROOT / "runtime" / "common.sh")
- with tempfile.TemporaryDirectory() as temporary:
- parent = Path(temporary) / "parent"
- parent.mkdir(mode=0o700)
- job = parent / f"inferencex-collectivex-123-1-test-{os.getpid()}"
- control = job / "control"
- control.mkdir(parents=True, mode=0o700)
- environment = {
- **os.environ,
- "COLLECTIVEX_CANONICAL_GHA": "1",
- "COLLECTIVEX_EXECUTION_ID": f"canonical-amd-{os.getpid()}",
- "CX_JOB_PARENT": str(parent),
- "CX_JOB_ROOT": str(job),
- }
- path = subprocess.check_output(
- ["bash", "-c", 'source "$1"; cx_private_log_path test', "_", common],
- text=True,
- env=environment,
- ).strip()
- log = Path(path)
- self.assertTrue(log.is_relative_to(control / "private-logs"))
- self.assertEqual(stat.S_IMODE(log.stat().st_mode), 0o600)
- subprocess.run(
- ["bash", "-c", 'source "$1"; cx_cleanup_private_logs 0', "_", common],
- check=True,
- env=environment,
- )
- self.assertFalse(log.parent.exists())
-
- def test_private_scheduler_diagnostic_is_value_free(self) -> None:
- common = str(ROOT / "runtime" / "common.sh")
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary) / "inferencex-collectivex-123-1-test"
- tag = "scheduler-diagnostic"
- directory = root / "control" / "private-logs" / tag
- directory.mkdir(parents=True, mode=0o700)
- log = directory / "scheduler-allocation.log"
- private_value = "private-node-123"
- log.write_text(f"salloc: Pending job allocation on {private_value}\n")
- log.chmod(0o600)
- result = subprocess.run(
- ["bash", "-c", 'source "$1"; cx_report_private_scheduler_failure', "_", common],
- text=True,
- capture_output=True,
- env={
- **os.environ,
- "COLLECTIVEX_CANONICAL_GHA": "1",
- "COLLECTIVEX_EXECUTION_ID": tag,
- "CX_JOB_ROOT": str(root),
- },
- )
- self.assertEqual(result.returncode, 0, result.stderr)
- self.assertIn("scheduler-diagnostic=pending", result.stderr)
- self.assertNotIn(private_value, result.stderr)
-
- log.write_text("salloc: error: Unable to contact Slurm controller\n")
- result = subprocess.run(
- ["bash", "-c", 'source "$1"; cx_report_private_scheduler_failure', "_", common],
- text=True,
- capture_output=True,
- env={
- **os.environ,
- "COLLECTIVEX_CANONICAL_GHA": "1",
- "COLLECTIVEX_EXECUTION_ID": tag,
- "CX_JOB_ROOT": str(root),
- },
- )
- self.assertEqual(result.returncode, 0, result.stderr)
- self.assertIn("scheduler-diagnostic=controller", result.stderr)
-
- log.write_text(
- "salloc: error: Job submit/allocate failed: Requested time limit is invalid\n"
- )
- result = subprocess.run(
- ["bash", "-c", 'source "$1"; cx_report_private_scheduler_failure', "_", common],
- text=True,
- capture_output=True,
- env={
- **os.environ,
- "COLLECTIVEX_CANONICAL_GHA": "1",
- "COLLECTIVEX_EXECUTION_ID": tag,
- "CX_JOB_ROOT": str(root),
- },
- )
- self.assertEqual(result.returncode, 0, result.stderr)
- self.assertIn("scheduler-diagnostic=resource-request", result.stderr)
-
- def test_private_runtime_logs_reject_traversal_and_symlinks(self) -> None:
- common = str(ROOT / "runtime" / "common.sh")
- for variable, value in (
- ("COLLECTIVEX_EXECUTION_ID", ".."),
- ("CX_TEST_LABEL", ".."),
- ):
- environment = {
- **os.environ,
- "COLLECTIVEX_EXECUTION_ID": "contract-adversarial",
- "CX_TEST_LABEL": "test",
- variable: value,
- }
- result = subprocess.run(
- ["bash", "-c", 'source "$1"; cx_private_log_path "$CX_TEST_LABEL"', "_", common],
- text=True,
- capture_output=True,
- env=environment,
- )
- self.assertNotEqual(result.returncode, 0)
- self.assertNotIn(value, result.stderr)
-
- private_root = Path(f"/tmp/inferencex-collectivex-{os.getuid()}")
- private_root.mkdir(mode=0o700, exist_ok=True)
- self.assertFalse(private_root.is_symlink())
- os.chmod(private_root, 0o700)
- with tempfile.TemporaryDirectory() as temporary:
- target = Path(temporary)
- tag = f"contract-symlink-{os.getpid()}"
- link = private_root / tag
- link.symlink_to(target, target_is_directory=True)
- try:
- result = subprocess.run(
- ["bash", "-c", 'source "$1"; cx_private_log_path test', "_", common],
- text=True,
- capture_output=True,
- env={**os.environ, "COLLECTIVEX_EXECUTION_ID": tag},
- )
- self.assertNotEqual(result.returncode, 0)
- self.assertEqual(list(target.iterdir()), [])
- finally:
- link.unlink(missing_ok=True)
-
- tag = f"contract-log-symlink-{os.getpid()}"
- directory = private_root / tag
- directory.mkdir(mode=0o700)
- target_file = target / "target"
- target_file.write_text("unchanged")
- log_link = directory / "test.log"
- log_link.symlink_to(target_file)
- try:
- result = subprocess.run(
- ["bash", "-c", 'source "$1"; cx_private_log_path test', "_", common],
- text=True,
- capture_output=True,
- env={**os.environ, "COLLECTIVEX_EXECUTION_ID": tag},
- )
- self.assertNotEqual(result.returncode, 0)
- self.assertEqual(target_file.read_text(), "unchanged")
- finally:
- log_link.unlink(missing_ok=True)
- directory.rmdir()
-
- def test_operator_config_failure_is_value_free(self) -> None:
- with tempfile.TemporaryDirectory() as temporary:
- config = Path(temporary) / "operator.env"
- config.write_text("printf 'private-config-token\\n' >&2\nfalse\n")
- config.chmod(0o600)
- result = subprocess.run(
- ["bash", "-c",
- 'export COLLECTIVEX_EXECUTION_ID="operator-failure-$$"; '
- "trap 'cx_cleanup_private_logs 0' EXIT; source \"$1\"; "
- "cx_load_operator_config", "_",
- str(ROOT / "runtime" / "common.sh")],
- text=True,
- capture_output=True,
- env={
- **os.environ,
- "CX_RUNNER": "h100-dgxc",
- "COLLECTIVEX_OPERATOR_CONFIG": str(config),
- },
- )
- self.assertNotEqual(result.returncode, 0)
- self.assertIn("runner-local configuration failed", result.stderr)
- self.assertNotIn("private-config-token", result.stderr)
-
- def test_ephemeral_operator_config_is_removed_after_source(self) -> None:
- with tempfile.TemporaryDirectory() as temporary:
- config = Path(temporary) / "operator.env"
- decoy = Path(temporary) / "decoy"
- decoy.write_text("keep")
- config.write_text(json.dumps(operator_config(Path(temporary) / "storage")))
- config.chmod(0o600)
- result = subprocess.run(
- [
- "bash", "-c",
- 'export COLLECTIVEX_EXECUTION_ID="operator-ephemeral-$$"; '
- "trap 'cx_cleanup_private_logs 0' EXIT; "
- 'config="$COLLECTIVEX_OPERATOR_CONFIG"; source "$1"; '
- 'cx_load_operator_config; test ! -e "$config"; '
- 'test "$CX_PARTITION" = test; '
- 'test -z "${COLLECTIVEX_OPERATOR_CONFIG+x}"',
- "_", str(ROOT / "runtime" / "common.sh"),
- ],
- text=True,
- capture_output=True,
- env={
- **os.environ,
- "CX_RUNNER": "h100-dgxc",
- "COLLECTIVEX_OPERATOR_CONFIG": str(config),
- "COLLECTIVEX_OPERATOR_CONFIG_EPHEMERAL": "1",
- },
- )
- self.assertEqual(result.returncode, 0, result.stderr)
- self.assertFalse(config.exists())
- self.assertEqual(decoy.read_text(), "keep")
-
- def test_operator_config_is_strict_per_runner_json(self) -> None:
- command = (
- 'source "$1"; export COLLECTIVEX_EXECUTION_ID="operator-config-$$"; '
- "trap 'cx_cleanup_private_logs 0' EXIT; cx_load_operator_config; "
- 'test "$CX_PARTITION" = test; '
- '[ "$CX_RUNNER" != b200-dgxc ] || test "$CX_QOS" = test-qos; '
- '[ "$CX_RUNNER" != b200-dgxc ] || test "$CX_NODELIST" = "test-[0-1]"; '
- 'test -z "${COLLECTIVEX_OPERATOR_CONFIG_CONTENT+x}"; '
- 'test -z "${ENROOT_CACHE_PATH+x}"'
- )
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary)
- document = operator_config(root / "storage")
- config = root / "operator.json"
- config.write_text(json.dumps(document))
- config.chmod(0o600)
- for runner in capability.PLATFORMS:
- with self.subTest(runner=runner):
- result = subprocess.run(
- ["bash", "-c", command, "_", str(ROOT / "runtime" / "common.sh")],
- text=True,
- capture_output=True,
- env={
- **os.environ,
- "CX_RUNNER": runner,
- "ENROOT_CACHE_PATH": "/private/stale-enroot-cache",
- "COLLECTIVEX_OPERATOR_CONFIG": str(config),
- },
- )
- self.assertEqual(result.returncode, 0, result.stderr)
-
- lock_dir = root / "amd-locks"
- document["runners"]["mi355x"]["lock_dir"] = str(lock_dir)
- config.write_text(json.dumps(document))
- config.chmod(0o600)
- canonical = subprocess.run(
- [
- "bash",
- "-c",
- 'source "$1"; export COLLECTIVEX_EXECUTION_ID="canonical-lock-$$"; '
- "trap 'cx_cleanup_private_logs 0' EXIT; cx_load_operator_config; "
- 'cx_lock_canonical_gha_env mi355x; test "$CX_LOCK_DIR" = "$2"',
- "_",
- str(ROOT / "runtime" / "common.sh"),
- str(lock_dir),
- ],
- text=True,
- capture_output=True,
- env={
- **os.environ,
- "CX_RUNNER": "mi355x",
- "CX_SHARD_FILE": ".shards/test.json",
- "CX_SHARD_SKU": "mi355x",
- "CX_NODES": "1",
- "CX_GPUS_PER_NODE": "8",
- "COLLECTIVEX_CANONICAL_GHA": "1",
- "COLLECTIVEX_OPERATOR_CONFIG": str(config),
- "COLLECTIVEX_SOURCE_SHA": "a" * 40,
- "GITHUB_ACTIONS": "true",
- "GITHUB_RUN_ATTEMPT": "1",
- "GITHUB_RUN_ID": "1",
- "GITHUB_WORKSPACE": str(root.resolve()),
- },
- )
- self.assertEqual(canonical.returncode, 0, canonical.stderr)
-
- selected_only = {
- "schema_version": 1,
- "runners": {"h100-dgxc": document["runners"]["h100-dgxc"]},
- }
- result = subprocess.run(
- [
- "bash", "-c", command + '; test "$CX_STAGE_DIR" = "$2"', "_",
- str(ROOT / "runtime" / "common.sh"), str(root / "storage"),
- ],
- text=True,
- capture_output=True,
- env={
- **os.environ,
- "CX_RUNNER": "h100-dgxc",
- "CX_STAGE_DIR": "/private/stale-stage",
- "ENROOT_CACHE_PATH": "/private/stale-enroot-cache",
- "COLLECTIVEX_OPERATOR_CONFIG_LOADED": "1",
- "COLLECTIVEX_OPERATOR_CONFIG_CONTENT": json.dumps(selected_only),
- "COLLECTIVEX_OPERATOR_CONFIG_REQUIRED": "1",
- },
- )
- self.assertEqual(result.returncode, 0, result.stderr)
-
- rejected = json.loads(json.dumps(document))
- rejected["runners"]["h100-dgxc"]["shell"] = "private-command"
- boolean_version = {**document, "schema_version": True}
- missing_socket = json.loads(json.dumps(document))
- del missing_socket["runners"]["h100-dgxc"]["socket_ifname"]
- missing_rdma = json.loads(json.dumps(document))
- del missing_rdma["runners"]["mi355x"]["rdma_devices"]
- missing_amd_stage = json.loads(json.dumps(document))
- del missing_amd_stage["runners"]["mi325x"]["stage_dir"]
- missing_h100_stage = json.loads(json.dumps(document))
- del missing_h100_stage["runners"]["h100-dgxc"]["stage_dir"]
- missing_b300_stage = json.loads(json.dumps(document))
- del missing_b300_stage["runners"]["b300"]["stage_dir"]
- missing_gb300_stage = json.loads(json.dumps(document))
- del missing_gb300_stage["runners"]["gb300"]["stage_dir"]
- missing_nvidia_account = json.loads(json.dumps(document))
- del missing_nvidia_account["runners"]["h100-dgxc"]["account"]
- for invalid in (rejected, boolean_version, missing_nvidia_account):
- config.write_text(json.dumps(invalid))
- config.chmod(0o600)
- result = subprocess.run(
- ["bash", "-c", command, "_", str(ROOT / "runtime" / "common.sh")],
- text=True,
- capture_output=True,
- env={
- **os.environ,
- "CX_RUNNER": "h100-dgxc",
- "COLLECTIVEX_OPERATOR_CONFIG": str(config),
- },
- )
- self.assertNotEqual(result.returncode, 0)
- self.assertNotIn("private-command", result.stderr)
-
- for valid, runner in (
- (missing_socket, "h100-dgxc"),
- (missing_rdma, "h100-dgxc"),
- (missing_amd_stage, "h100-dgxc"),
- (missing_h100_stage, "h100-dgxc"),
- (missing_b300_stage, "b300"),
- (missing_gb300_stage, "gb300"),
- (missing_amd_stage, "mi325x"),
- ):
- config.write_text(json.dumps(valid))
- config.chmod(0o600)
- result = subprocess.run(
- [
- "bash", "-c", command + "; cx_apply_network_profile 1 nvlink",
- "_", str(ROOT / "runtime" / "common.sh"),
- ],
- text=True,
- capture_output=True,
- env={
- **os.environ,
- "CX_RUNNER": runner,
- "COLLECTIVEX_OPERATOR_CONFIG": str(config),
- },
- )
- self.assertEqual(result.returncode, 0, result.stderr)
-
- config.write_text(json.dumps(missing_socket))
- config.chmod(0o600)
- scaleout = subprocess.run(
- [
- "bash", "-c", command + "; cx_apply_network_profile 2 nvlink-rdma",
- "_", str(ROOT / "runtime" / "common.sh"),
- ],
- text=True,
- capture_output=True,
- env={
- **os.environ,
- "CX_RUNNER": "h100-dgxc",
- "COLLECTIVEX_OPERATOR_CONFIG": str(config),
- },
- )
- self.assertEqual(scaleout.returncode, 0, scaleout.stderr)
-
- config.write_text(json.dumps(missing_amd_stage))
- config.chmod(0o600)
- selected_missing_stage = subprocess.run(
- ["bash", "-c", command, "_", str(ROOT / "runtime" / "common.sh")],
- text=True,
- capture_output=True,
- env={
- **os.environ,
- "CX_RUNNER": "mi325x",
- "COLLECTIVEX_OPERATOR_CONFIG": str(config),
- },
- )
- self.assertEqual(
- selected_missing_stage.returncode, 0, selected_missing_stage.stderr
- )
-
- config.write_text(json.dumps(document))
- config.chmod(0o644)
- result = subprocess.run(
- ["bash", "-c", command, "_", str(ROOT / "runtime" / "common.sh")],
- text=True,
- capture_output=True,
- env={
- **os.environ,
- "CX_RUNNER": "h100-dgxc",
- "COLLECTIVEX_OPERATOR_CONFIG": str(config),
- },
- )
- self.assertNotEqual(result.returncode, 0)
-
-
-if __name__ == "__main__":
- unittest.main()
diff --git a/experimental/CollectiveX/tests/test_ep_precision_adapters.py b/experimental/CollectiveX/tests/test_ep_precision_adapters.py
index d68b1e2571..d98de83c88 100644
--- a/experimental/CollectiveX/tests/test_ep_precision_adapters.py
+++ b/experimental/CollectiveX/tests/test_ep_precision_adapters.py
@@ -2,7 +2,6 @@
"""CPU-only contract tests for native EP precision adapter wiring."""
from __future__ import annotations
-import ast
import sys
import unittest
from pathlib import Path
@@ -77,61 +76,5 @@ def test_bf16_evidence_is_exact_and_has_no_scale_checks(self):
self.assertIsNone(evidence["scales_positive"])
-class NativeAdapterWiringTests(unittest.TestCase):
- @staticmethod
- def _tree(name: str) -> ast.Module:
- return ast.parse((TESTS / name).read_text(encoding="utf-8"))
-
- @staticmethod
- def _keywords(tree: ast.AST, attribute: str) -> list[set[str]]:
- result = []
- for node in ast.walk(tree):
- if not isinstance(node, ast.Call):
- continue
- function = node.func
- if (
- isinstance(function, ast.Attribute) and function.attr == attribute
- ) or (
- isinstance(function, ast.Name) and function.id == attribute
- ):
- result.append({keyword.arg for keyword in node.keywords})
- return result
-
- def test_deepep_and_uccl_wire_native_low_latency_controls(self):
- # DeepEP and UCCL share the low-latency dispatch/combine surface in the family base.
- tree = self._tree("ep_deepep_family.py")
- dispatch_calls = self._keywords(tree, "low_latency_dispatch")
- combine_calls = self._keywords(tree, "low_latency_combine")
- self.assertTrue(any("use_fp8" in call for call in dispatch_calls))
- self.assertTrue(any("use_logfmt" in call for call in combine_calls))
-
- def test_elastic_and_hybrid_constructors_enable_native_fp8(self):
- v2 = self._tree("ep_deepep_v2.py")
- hybrid = self._tree("ep_deepep_hybrid.py")
- self.assertTrue(
- any("use_fp8_dispatch" in call for call in self._keywords(v2, "ElasticBuffer"))
- )
- self.assertTrue(
- any("use_fp8" in call for call in self._keywords(hybrid, "HybridEPBuffer"))
- )
- self.assertTrue(
- any("scaling_factor" in call for call in self._keywords(hybrid, "dispatch"))
- )
-
- def test_mori_wires_dispatch_scales_and_direct_cast_config(self):
- source = (TESTS / "ep_mori.py").read_text(encoding="utf-8")
- harness = (TESTS / "ep_harness.py").read_text(encoding="utf-8")
- backend = (TESTS / "ep_backend.py").read_text(encoding="utf-8")
- precision = (TESTS / "ep_precision.py").read_text(encoding="utf-8")
- self.assertIn('"fp8_direct_cast" if self._direct_cast_combine', source)
- self.assertIn("p.scales,", source)
- self.assertIn("dispatch_scales=_scales", source)
- self.assertIn("dispatch_needs_combine_cleanup = True", source)
- # The dispatch-cleanup branch rule is encoded once in the base timing template.
- self.assertIn("post=finish_dispatch if dispatch_needs_cleanup else None", backend)
- self.assertEqual(harness.count("view.combine_input = transformed"), 2)
- self.assertIn("transformed = view.combine_input.float()", precision)
-
-
if __name__ == "__main__":
unittest.main()
diff --git a/experimental/CollectiveX/tests/test_precision_scheduling.py b/experimental/CollectiveX/tests/test_precision_scheduling.py
deleted file mode 100644
index 39abcb36a6..0000000000
--- a/experimental/CollectiveX/tests/test_precision_scheduling.py
+++ /dev/null
@@ -1,566 +0,0 @@
-#!/usr/bin/env python3
-"""CPU-only tests for CollectiveX communication-precision scheduling."""
-from __future__ import annotations
-
-import copy
-from pathlib import Path
-import sys
-import unittest
-from unittest import mock
-
-
-HERE = Path(__file__).resolve().parent
-ROOT = HERE.parent
-sys.path[:0] = [str(ROOT), str(HERE)]
-
-import capability # noqa: E402
-import identity # noqa: E402
-import probe_precision # noqa: E402
-import sweep_matrix # noqa: E402
-
-
-class PrecisionSchedulingTest(unittest.TestCase):
- @staticmethod
- def _synthetic_provisional_targets() -> list[dict[str, object]]:
- targets = []
- for ep in (8, 16):
- target = next(
- item for item in capability.precision_targets()
- if item["sku"] == "h100-dgxc" and item["backend"] == "deepep"
- and item["ep"] == ep and item["mode"] == "normal"
- ).copy()
- target.update(disposition="provisional", basis="test-provisional-cell")
- targets.append(target)
- return targets
-
- def test_precision_probe_inventory_is_exact_and_non_mutating(self) -> None:
- before = copy.deepcopy(capability.PRECISION_CAPABILITIES)
- targets = probe_precision.provisional_targets()
- key = lambda item: (
- item["sku"], item["backend"], item["ep"], item["mode"],
- item["precision_profile"],
- )
- self.assertEqual(targets, sorted(capability.provisional_precision_targets(), key=key))
- self.assertEqual(
- [
- (item["sku"], item["backend"], item["ep"], item["precision_profile"])
- for item in targets
- ],
- [],
- )
- self.assertEqual(capability.PRECISION_CAPABILITIES, before)
- self.assertEqual(
- len({
- (item["backend"], item["sku"], item["ep"], item["mode"],
- item["precision_profile"])
- for item in targets
- }),
- len(targets),
- )
-
- def test_precision_probe_selects_only_one_exact_provisional_cell(self) -> None:
- target = self._synthetic_provisional_targets()[0]
- with mock.patch.object(probe_precision, "provisional_targets", return_value=[target]):
- selected = probe_precision.select_target(
- backend=target["backend"], sku=target["sku"], ep=target["ep"],
- mode=target["mode"], precision_profile=target["precision_profile"],
- )
- self.assertEqual(selected, target)
- with self.assertRaisesRegex(probe_precision.ProbeError, "target-not-provisional"):
- probe_precision.select_target(
- backend=target["backend"], sku=target["sku"], ep=target["ep"],
- mode=target["mode"], precision_profile="d-bf16.c-bf16",
- )
-
- def test_precision_probe_workflow_plan_binds_exact_controls(self) -> None:
- targets = self._synthetic_provisional_targets()
- with mock.patch.object(probe_precision, "provisional_targets", return_value=targets):
- plan = probe_precision.workflow_plan(backend="deepep", only_sku="h100-dgxc")
- self.assertTrue(plan["include"])
- self.assertTrue(all(
- row["backend"] == "deepep" and row["sku"] == "h100-dgxc"
- for row in plan["include"]
- ))
- row = plan["include"][0]
- with mock.patch.object(probe_precision, "provisional_targets", return_value=targets):
- exact = probe_precision.workflow_plan(probe_id=row["id"])
- self.assertEqual(exact["include"], [row])
- with self.assertRaisesRegex(ValueError, "ID is invalid"):
- probe_precision.workflow_plan(probe_id="probe-invalid")
- with self.assertRaisesRegex(ValueError, "select no provisional"):
- probe_precision.workflow_plan(probe_id="probe-00000000000000000000")
- with mock.patch.object(probe_precision, "provisional_targets", return_value=targets):
- control = probe_precision.extract_control(
- plan, probe_id=row["id"], sku=row["sku"], backend=row["backend"],
- nodes=row["nodes"],
- )
- self.assertEqual(
- probe_precision.validate_control(
- control, sku=row["sku"], backend=row["backend"], nodes=row["nodes"],
- ),
- control,
- )
- with mock.patch.object(probe_precision, "provisional_targets", return_value=targets), \
- self.assertRaisesRegex(ValueError, "workflow matrix"):
- probe_precision.extract_control(
- plan, probe_id=row["id"], sku=row["sku"], backend=row["backend"],
- nodes=row["nodes"] + 1,
- )
- with self.assertRaisesRegex(ValueError, "select no provisional"):
- probe_precision.workflow_plan(backend="mori", only_sku="b200-dgxc")
- with mock.patch.object(probe_precision, "provisional_targets", return_value=targets):
- ep16 = probe_precision.workflow_plan(
- backend="deepep", only_sku="h100-dgxc", min_nodes=2,
- )
- self.assertTrue(ep16["include"])
- self.assertEqual({row["ep"] for row in ep16["include"]}, {16})
- with self.assertRaisesRegex(ValueError, "node filters"):
- probe_precision.workflow_plan(min_nodes=2, max_nodes=1)
-
- def test_precision_probe_manifest_is_sanitized_and_runtime_evidence_is_required(self) -> None:
- target = self._synthetic_provisional_targets()[0]
- topology = capability.topology_for(target["sku"], target["ep"])
- self.assertIsNotNone(topology)
- topology_record = probe_precision._topology_record(topology, False)
- document = probe_precision.build_manifest(
- target=target, topology=topology_record, disposition="unsupported",
- reason="unsupported-native-api", runtime_executed=True, evidence=None,
- )
- self.assertEqual(document["result"], {
- "disposition": "unsupported",
- "reason": "unsupported-native-api",
- "registry_mutation": False,
- "runtime_executed": True,
- "static_inspection_sufficient": False,
- })
- with self.assertRaises((TypeError, ValueError)):
- probe_precision.build_manifest(
- target=target, topology=topology_record, disposition="supported",
- reason=probe_precision.SUPPORTED_REASON, runtime_executed=True,
- evidence=None,
- )
-
- def test_precision_probe_aggregates_shapes_as_manifest_arrays(self) -> None:
- tensor = probe_precision._aggregate_tensor_summaries([
- {"finite": True, "rank": 2, "shape": [8, 7168], "storage_dtype": "bfloat16"},
- {"finite": True, "rank": 2, "shape": [16, 7168], "storage_dtype": "bfloat16"},
- ])
- self.assertEqual(tensor["shapes"], [[8, 7168], [16, 7168]])
- self.assertTrue(all(type(shape) is list for shape in tensor["shapes"]))
-
- scale = probe_precision._aggregate_scale_contracts([
- {
- "alignment": "hidden-block-128", "dtype": "f32", "finite": True,
- "group_size": 128, "layout": "per-token-hidden-block",
- "padding": "right-zero-pad-hidden-to-128", "positive": True,
- "runtime_shape": [8, 56], "runtime_storage_dtype": "float32",
- },
- {
- "alignment": "hidden-block-128", "dtype": "f32", "finite": True,
- "group_size": 128, "layout": "per-token-hidden-block",
- "padding": "right-zero-pad-hidden-to-128", "positive": True,
- "runtime_shape": [16, 56], "runtime_storage_dtype": "float32",
- },
- ])
- self.assertEqual(scale["runtime_shapes"], [[8, 56], [16, 56]])
- self.assertTrue(all(type(shape) is list for shape in scale["runtime_shapes"]))
-
- def test_precision_profiles_bind_exact_formats_and_timing_boundaries(self) -> None:
- scheduled = set(identity.V1_NORMAL_PRECISION_PROFILE_IDS) | set(
- identity.V1_LOW_LATENCY_PRECISION_PROFILE_IDS
- )
- # The scheduled subset plus the control must be a subset of the frozen
- # registry; the four normal profiles the v1 sweep no longer schedules
- # stay defined-but-unscheduled so their schema enums and capability
- # targets survive for republication of already-collected evidence.
- self.assertLessEqual(
- scheduled | {identity.V1_CONTROL_PRECISION_PROFILE},
- set(identity.V1_PRECISION_PROFILES),
- )
- self.assertEqual(
- set(identity.V1_PRECISION_PROFILES)
- - scheduled
- - {identity.V1_CONTROL_PRECISION_PROFILE},
- {
- "d-bf16.c-fp8-e4m3fn-direct-cast-noscale",
- "d-fp8-e4m3fn-b128-f32-prequantized.c-fp8-e4m3fn-direct-cast-noscale",
- "d-fp8-e4m3fnuz-b128-f32-prequantized.c-bf16",
- "d-bf16.c-fp8-e4m3fnuz-direct-cast-noscale",
- },
- )
- self.assertNotIn(identity.V1_CONTROL_PRECISION_PROFILE, scheduled)
- self.assertNotIn("fp4", repr(identity.V1_PRECISION_PROFILES).lower())
-
- required_axis_fields = {
- "api_input_dtype",
- "api_output_dtype",
- "communication_format",
- "scale_dtype",
- "scale_layout",
- "scale_group_size",
- "padding_contract",
- "alignment_contract",
- "quantization_origin",
- "conversion_boundary",
- }
- for name in identity.V1_PRECISION_PROFILES:
- with self.subTest(profile=name):
- profile = identity.precision_profile(name)
- self.assertEqual(profile["profile_id"], name)
- self.assertEqual(set(profile["dispatch"]), required_axis_fields)
- self.assertEqual(set(profile["combine"]), required_axis_fields)
- self.assertRegex(name, r"^[a-z0-9][a-z0-9.-]*$")
-
- prequantized = identity.precision_profile(
- "d-fp8-e4m3fn-b128-f32-prequantized.c-bf16"
- )["dispatch"]
- fused = identity.precision_profile(
- "d-fp8-e4m3fn-b128-f32-fused.c-bf16"
- )["dispatch"]
- self.assertEqual(prequantized["conversion_boundary"], "before-dispatch-timing")
- self.assertEqual(fused["conversion_boundary"], "inside-dispatch-timing")
- self.assertEqual(prequantized["scale_group_size"], 128)
-
- self.assertEqual(
- identity.combine_oracle_tolerances(
- identity.precision_profile(identity.V1_CONTROL_PRECISION_PROFILE)
- ),
- {"atol": 2e-2, "rtol": 5e-2},
- )
- self.assertEqual(
- identity.combine_oracle_tolerances(identity.precision_profile(
- "d-bf16.c-fp8-e4m3fn-direct-cast-noscale"
- )),
- {"atol": 4e-2, "rtol": 8e-2},
- )
-
- base = {"mode": "normal"}
- precision_case = {
- **base,
- "precision_profile": "d-fp8-e4m3fn-b128-f32-prequantized.c-bf16",
- }
- self.assertIs(identity.profile_for_case(base), identity.V1_NORMAL_CASE_PROFILE)
- self.assertIn("communication_precision", identity.profile_for_case(precision_case))
- self.assertNotEqual(
- identity.digest("case", identity.profile_for_case(base)),
- identity.digest("case", identity.profile_for_case(precision_case)),
- )
-
- def test_capability_registry_uses_exact_native_targets(self) -> None:
- targets = capability.precision_targets()
- self.assertTrue(targets)
- self.assertEqual(
- {item["disposition"] for item in targets},
- {"supported", "unsupported"},
- )
- self.assertEqual(
- len(targets), 62
- )
- self.assertEqual(
- sum(item["disposition"] == "supported" for item in targets), 33
- )
- self.assertEqual(
- sum(item["disposition"] == "unsupported" for item in targets), 29
- )
- self.assertEqual(len(capability.provisional_precision_targets()), 0)
- keys = {
- (
- item["precision_profile"],
- item["backend"],
- item["sku"],
- item["ep"],
- item["mode"],
- )
- for item in targets
- }
- self.assertEqual(len(keys), len(targets))
-
- normal = "d-fp8-e4m3fn-b128-f32-prequantized.c-bf16"
- direct = "d-bf16.c-fp8-e4m3fn-direct-cast-noscale"
- low_latency = "d-fp8-e4m3fn-b128-f32-fused.c-bf16"
- cases = (
- (("h200-dgxc", "deepep-v2", 8, "normal", normal), "unsupported"),
- (("h100-dgxc", "deepep-v2", 8, "normal", normal), "not-applicable"),
- (("h200-dgxc", "nccl-ep", 8, "normal", normal), "not-applicable"),
- (("mi355x", "mori", 8, "normal", direct), "unsupported"),
- (("mi355x", "mori", 16, "normal", direct), "not-applicable"),
- (("h200-dgxc", "deepep", 8, "low-latency", low_latency), "supported"),
- (("h100-dgxc", "deepep", 16, "low-latency", low_latency), "supported"),
- (("h200-dgxc", "deepep-hybrid", 8, "low-latency", low_latency),
- "not-applicable"),
- )
- for (sku, backend, ep, mode, profile), expected in cases:
- with self.subTest(sku=sku, backend=backend, profile=profile):
- topology = capability.topology_for(sku, ep)
- self.assertIsNotNone(topology)
- disposition, _ = capability.resolve_disposition(
- sku,
- backend,
- ep=ep,
- nodes=topology["nodes"], # type: ignore[index]
- mode=mode,
- precision_profile=profile,
- )
- self.assertEqual(disposition, expected)
-
- control, _ = capability.resolve_disposition(
- "h200-dgxc",
- "deepep",
- ep=8,
- nodes=1,
- precision_profile=identity.V1_CONTROL_PRECISION_PROFILE,
- )
- self.assertEqual(control, "supported")
- h100_ep16, reason = capability.resolve_disposition(
- "h100-dgxc", "nccl-ep", ep=16, nodes=2,
- precision_profile=identity.V1_CONTROL_PRECISION_PROFILE,
- )
- self.assertEqual(h100_ep16, "supported")
- self.assertEqual(reason, "ok")
-
- # B300 EP16 runs over the RoCE GPU-fabric: nccl-ep and the DeepEP Hybrid
- # control path (native DOCA transport) execute there; DeepEP V1/V2 cannot --
- # their internode path is NVSHMEM-IBGDA, which needs GDRCopy /dev/gdrdrv, and
- # that char device is unprovisioned on B300 hosts.
- for backend in ("deepep-hybrid", "nccl-ep"):
- disposition, reason = capability.resolve_disposition(
- "b300", backend, ep=16, nodes=2,
- precision_profile=identity.V1_CONTROL_PRECISION_PROFILE,
- )
- self.assertEqual(disposition, "supported")
- self.assertEqual(reason, "ok")
- for backend, version in (("deepep", "V1"), ("deepep-v2", "V2")):
- disposition, reason = capability.resolve_disposition(
- "b300", backend, ep=16, nodes=2,
- precision_profile=identity.V1_CONTROL_PRECISION_PROFILE,
- )
- self.assertEqual(disposition, "unsupported")
- self.assertEqual(
- reason,
- f"DeepEP {version} EP16 requires GDRCopy /dev/gdrdrv for "
- "NVSHMEM-IBGDA, unprovisioned on B300 hosts",
- )
- b300_ep8, reason = capability.resolve_disposition(
- "b300", "nccl-ep", ep=8, nodes=1,
- precision_profile=identity.V1_CONTROL_PRECISION_PROFILE,
- )
- self.assertEqual(b300_ep8, "supported")
- self.assertEqual(reason, "ok")
-
- disposition, reason = capability.resolve_disposition(
- "b200-dgxc", "deepep", ep=16, nodes=2,
- precision_profile=identity.V1_CONTROL_PRECISION_PROFILE,
- )
- self.assertEqual(disposition, "unsupported")
- self.assertEqual(
- reason,
- "DeepEP V1 EP16 requires unavailable GPU doorbell mapping or GDRCopy on B200",
- )
- self.assertEqual(
- capability.DEEPEP_V1_IBGDA_NIC_HANDLERS["b200-dgxc"],
- "cpu",
- )
-
- disposition, reason = capability.resolve_disposition(
- "b200-dgxc", "deepep-hybrid", ep=16, nodes=2,
- precision_profile=identity.V1_CONTROL_PRECISION_PROFILE,
- )
- self.assertEqual(disposition, "unsupported")
- self.assertEqual(
- reason,
- "DeepEP Hybrid EP16 requires unavailable GPU-to-NIC doorbell/UAR mappings on B200",
- )
- b200_reference, reason = capability.resolve_disposition(
- "b200-dgxc", "nccl-ep", ep=16, nodes=2,
- precision_profile=identity.V1_CONTROL_PRECISION_PROFILE,
- )
- self.assertEqual(b200_reference, "supported")
- self.assertEqual(reason, "ok")
-
- for ep in (8, 16):
- disposition, reason = capability.resolve_disposition(
- "mi355x", "mori", ep=ep, nodes=ep // 8,
- precision_profile=identity.V1_CONTROL_PRECISION_PROFILE,
- )
- self.assertEqual(disposition, "unsupported")
- self.assertEqual(
- reason,
- f"Pinned MoRI backend construction does not complete on MI355X EP{ep}",
- )
-
- disposition, reason = capability.resolve_disposition(
- "mi300x", "mori", ep=16, nodes=2,
- precision_profile=identity.V1_CONTROL_PRECISION_PROFILE,
- )
- self.assertEqual(disposition, "unsupported")
- self.assertEqual(
- reason,
- "Pinned MoRI distributed initialization does not complete on MI300X EP16",
- )
-
- for sku in ("b200-dgxc", "h200-dgxc"):
- disposition, reason = capability.resolve_disposition(
- sku, "deepep-v2", ep=16, nodes=2,
- precision_profile=identity.V1_CONTROL_PRECISION_PROFILE,
- )
- self.assertEqual(disposition, "supported")
- self.assertEqual(reason, "ok")
-
- def test_split_suites_track_provisional_state_and_do_not_duplicate_bf16(self) -> None:
- suites = sweep_matrix._load("suites.yaml")
- workloads = sweep_matrix._load("workloads.yaml")
- sweep_matrix.validate_config_documents(suites, workloads)
- normal = suites["suites"]["ep-precision-normal-v1"]
- low_latency = suites["suites"]["ep-precision-low-latency-v1"]
- self.assertEqual(
- (
- normal["mode"],
- normal["phases"],
- normal["token_points_decode"],
- normal["token_points_prefill"],
- ),
- ("normal", ["decode", "prefill"], [128], [512]),
- )
- self.assertEqual(
- (
- low_latency["mode"],
- low_latency["phases"],
- low_latency["token_points_decode"],
- ),
- ("low-latency", ["decode"], [128]),
- )
- self.assertFalse(normal["provisional"])
- self.assertFalse(low_latency["provisional"])
- self.assertEqual(
- normal["required_publication"], "comparable-experimental"
- )
- self.assertEqual(
- low_latency["required_publication"], "comparable-experimental"
- )
- listed = normal["precision_profiles"] + low_latency["precision_profiles"]
- self.assertNotIn(identity.V1_CONTROL_PRECISION_PROFILE, listed)
- self.assertEqual(len(listed), len(set(listed)))
-
- matrix = sweep_matrix.validate_matrix_document(
- sweep_matrix.resolve_matrix(backends="all")
- )
- scheduled_profiles = {
- item["case"].get("precision_profile")
- for item in matrix["requested_cases"]
- if "precision_profile" in item["case"]
- }
- self.assertEqual(
- scheduled_profiles,
- set(normal["precision_profiles"]) | set(low_latency["precision_profiles"]),
- )
- self.assertEqual(
- {item["precision_profile"] for item in capability.provisional_precision_targets()},
- set(),
- )
-
- stale = copy.deepcopy(suites)
- stale["suites"]["ep-precision-normal-v1"]["provisional"] = True
- with self.assertRaisesRegex(SystemExit, "must track unresolved"):
- sweep_matrix.validate_config_documents(stale, workloads)
-
- def test_resolved_profiles_schedule_without_cartesian_fill(self) -> None:
- suites = sweep_matrix._load("suites.yaml")
- workloads = sweep_matrix._load("workloads.yaml")
- promoted = copy.deepcopy(capability.PRECISION_CAPABILITIES)
- for rules in promoted.values():
- for rule in rules:
- rule["disposition"] = "supported"
- normal_profile = "d-fp8-e4m3fn-b128-f32-prequantized.c-bf16"
- promoted[normal_profile][0]["disposition"] = "unsupported"
- resolved_suites = copy.deepcopy(suites)
- for name in ("ep-precision-normal-v1", "ep-precision-low-latency-v1"):
- resolved_suites["suites"][name]["provisional"] = False
-
- def load_config(name: str) -> dict[str, object]:
- if name == "suites.yaml":
- return resolved_suites
- if name == "workloads.yaml":
- return workloads
- raise AssertionError(name)
-
- suite_names = "ep-precision-normal-v1,ep-precision-low-latency-v1"
- scheduled_ids = list(identity.V1_NORMAL_PRECISION_PROFILE_IDS) + list(
- identity.V1_LOW_LATENCY_PRECISION_PROFILE_IDS
- )
- expected_cases = sum(
- len(resolved_suites["suites"][
- "ep-precision-normal-v1"
- if target["mode"] == "normal"
- else "ep-precision-low-latency-v1"
- ]["phases"])
- for target in capability.precision_targets(scheduled_ids)
- )
- with mock.patch.object(capability, "PRECISION_CAPABILITIES", promoted):
- sweep_matrix.validate_config_documents(resolved_suites, workloads)
- with mock.patch.object(sweep_matrix, "_load", side_effect=load_config):
- matrix = sweep_matrix.validate_matrix_document(
- sweep_matrix.resolve_matrix(suites=suite_names, backends="all")
- )
- expected_unsupported_cases = sum(
- len(resolved_suites["suites"][
- "ep-precision-normal-v1"
- if target["mode"] == "normal"
- else "ep-precision-low-latency-v1"
- ]["phases"])
- for target in capability.precision_targets(scheduled_ids)
- if capability.resolve_disposition(
- target["sku"], target["backend"], ep=target["ep"],
- nodes=capability.topology_for(target["sku"], target["ep"])["nodes"],
- mode=target["mode"], precision_profile=target["precision_profile"],
- )[0] != "supported"
- )
-
- unsupported = [
- item for item in matrix["requested_cases"]
- if item["disposition"] == "unsupported"
- ]
- self.assertEqual(
- len(unsupported),
- expected_unsupported_cases,
- )
- self.assertTrue(all(
- item["reason"] == "precision-profile-unsupported" for item in unsupported
- ))
- self.assertTrue(any(
- item["disposition"] == "runnable" for item in matrix["requested_cases"]
- ))
-
- self.assertEqual(len(matrix["requested_cases"]), expected_cases)
- self.assertEqual(
- {item["case"]["precision_profile"] for item in matrix["requested_cases"]},
- set(identity.V1_NORMAL_PRECISION_PROFILE_IDS)
- | set(identity.V1_LOW_LATENCY_PRECISION_PROFILE_IDS),
- )
- self.assertFalse(any(
- item["case"]["precision_profile"] == identity.V1_CONTROL_PRECISION_PROFILE
- for item in matrix["requested_cases"]
- ))
- direct_cases = [
- item for item in matrix["requested_cases"]
- if "direct-cast" in item["case"]["precision_profile"]
- ]
- self.assertTrue(direct_cases)
- self.assertEqual({item["case"]["ep"] for item in direct_cases}, {8})
-
- def test_probe_runtime_diagnostics_are_closed_stage_codes(self) -> None:
- probe = (ROOT / "tests" / "probe_precision.py").read_text()
- run_ep = (ROOT / "tests" / "run_ep.py").read_text()
- for stage in (
- "distributed-init", "runtime-context", "construction-consensus",
- "operation-consensus", "evidence-aggregation",
- ):
- self.assertIn(f'"{stage}"', probe)
- self.assertIn("precision-probe-diagnostic={diagnostic_stage}-exception", probe)
- self.assertNotIn("precision-probe-diagnostic={exc}", probe)
- self.assertNotIn("cpu:gloo,cuda:nccl", probe + run_ep)
- self.assertIn('backend="nccl", rank=rank, world_size=world_size, device_id=device', probe)
- self.assertIn('backend="nccl",', run_ep)
-
-
-if __name__ == "__main__":
- unittest.main()
diff --git a/experimental/CollectiveX/tests/test_publisher.py b/experimental/CollectiveX/tests/test_publisher.py
deleted file mode 100644
index 904b38b088..0000000000
--- a/experimental/CollectiveX/tests/test_publisher.py
+++ /dev/null
@@ -1,2958 +0,0 @@
-#!/usr/bin/env python3
-"""Focused end-to-end tests for the isolated CollectiveX publisher."""
-from __future__ import annotations
-
-import copy
-import hashlib
-import itertools
-import json
-import os
-from pathlib import Path
-import subprocess
-import sys
-import tempfile
-import types
-import unittest
-from unittest import mock
-import zipfile
-
-HERE = Path(__file__).resolve().parent
-ROOT = HERE.parent
-sys.path[:0] = [str(ROOT), str(HERE)]
-
-import contracts # noqa: E402
-import identity # noqa: E402
-import publisher # noqa: E402
-import summarize # noqa: E402
-import sweep_matrix # noqa: E402
-
-
-RUN = {
- "repository": "SemiAnalysisAI/InferenceX",
- "run_id": "12345",
- "run_attempt": 1,
- "qualification_index": 1,
- "source_sha": "a" * 40,
-}
-
-
-def _unsupported_delivery(
- root: Path, ordinals: tuple[int, ...] = (1,), run: dict = RUN,
-) -> tuple[Path, Path]:
- matrix = sweep_matrix.resolve_matrix(backends="all")
- wrapper = next(item for item in matrix["requested_cases"] if item["disposition"] == "unsupported")
- matrix = {
- "format": "collectivex.matrix.v1",
- "schema_version": 1,
- "requested_cases": [wrapper],
- "include": [],
- }
- case = {key: value for key, value in wrapper["case"].items() if key != "case_id"}
- artifact_name = f"cxunsupported-{run['run_id']}-{run['run_attempt']}"
- git_run = {
- "artifact": artifact_name,
- "job": "setup",
- "ref": "collectivex",
- "repo": run["repository"],
- "qualification_index": run["qualification_index"],
- "run_attempt": str(run["run_attempt"]),
- "run_id": run["run_id"],
- "source_sha": run["source_sha"],
- }
- allocation = {
- "artifact": artifact_name,
- "execution_id": f"{run['run_id']}_{run['run_attempt']}_unsupported",
- "job": "setup",
- "qualification_index": run["qualification_index"],
- "repo": run["repository"],
- "run_attempt": str(run["run_attempt"]),
- "run_id": run["run_id"],
- "runner": "capability-resolver",
- "source_sha": run["source_sha"],
- }
- matrix_path = root / "matrix.json"
- artifact = root / artifact_name
- artifact.mkdir()
- matrix_path.write_text(json.dumps(matrix))
- control_sha256 = hashlib.sha256(matrix_path.read_bytes()).hexdigest()
- for ordinal in ordinals:
- terminal = contracts.make_terminal_document(
- allocation_factors=allocation, attempt_ordinal=ordinal, case=case,
- case_factors={"case": case, "profile": identity.V1_CASE_PROFILE,
- "sku": wrapper["sku"]},
- control_sha256=control_sha256, failure_mode="capability",
- generated_at="2026-07-04T00:00:00Z", git_run=git_run,
- reason=wrapper["reason"], return_code=5, source="matrix-capability-resolver",
- status="unsupported", expected_case_id=wrapper["case"]["case_id"],
- )
- (artifact / f"unsupported-{ordinal}.json").write_text(json.dumps(terminal))
- return matrix_path, artifact
-
-
-def _args(
- store: Path, matrix: Path, artifact: Path, run: dict = RUN
-) -> types.SimpleNamespace:
- return types.SimpleNamespace(
- store_root=str(store),
- matrix=str(matrix),
- artifact=[str(artifact)],
- repository=run["repository"],
- run_id=run["run_id"],
- run_attempt=run["run_attempt"],
- qualification_index=run["qualification_index"],
- source_sha=run["source_sha"],
- )
-
-
-def _ids(seed: str) -> tuple[str, str, str, str, str, str]:
- case = identity.digest("case", {"seed": seed})
- allocation = identity.allocation_id({"seed": seed})
- attempt = identity.attempt_id(allocation=allocation, case=case, ordinal=1)
- series = identity.series_id({"seed": seed})
- point = identity.point_id(series=series, tokens_per_rank=8)
- evidence = identity.evidence_id(
- point=point, allocation=allocation, attempt=attempt, sample_sha256="b" * 64
- )
- return case, allocation, attempt, series, point, evidence
-
-
-def _component(scale: float = 1.0) -> dict:
- latency = {"p50": 10.0 * scale, "p90": 12.0 * scale,
- "p95": 14.0 * scale, "p99": 20.0 * scale}
- byte_provenance = {
- "accounting_contract": "activation-data-plus-scales-v1",
- "activation_data_bytes": 100_000,
- "scale_bytes": 0,
- "total_logical_bytes": 100_000,
- }
- return {
- "origin": "measured",
- "latency_us": latency,
- "byte_provenance": byte_provenance,
- "activation_data_rate_gbps_at_latency_percentile": {
- name: byte_provenance["activation_data_bytes"] / (value * 1000.0)
- for name, value in latency.items()
- },
- "total_logical_data_rate_gbps_at_latency_percentile": {
- name: byte_provenance["total_logical_bytes"] / (value * 1000.0)
- for name, value in latency.items()
- },
- "sample_count": 512,
- }
-
-
-def _precision_axis_evidence() -> dict:
- return {
- "encoded_payload_valid": True,
- "scales_finite": None,
- "scales_positive": None,
- "dequantized_semantics": True,
- "saturation_count": 0,
- "saturation_rate": 0.0,
- "max_abs_error": 0.0,
- "max_rel_error": 0.0,
- "passed": True,
- }
-
-
-def _precision_evidence(
- profile_id: str = identity.V1_CONTROL_PRECISION_PROFILE,
-) -> dict:
- axis = _precision_axis_evidence()
- return {
- "profile_id": profile_id,
- "dispatch": axis,
- "combine": copy.deepcopy(axis),
- "passed": True,
- }
-
-
-def _hybrid_provenance(ep_size: int = 1) -> dict:
- realized = {field: 1 for field in contracts.HYBRID_REALIZED_CONFIG_FIELDS}
- for field in contracts.HYBRID_REALIZED_BOOL_FIELDS:
- realized[field] = True
- realized.update({
- "num_of_experts_per_rank": 1,
- "num_of_nodes": 1,
- "num_of_ranks_per_node": ep_size,
- "token_data_type": "UINT16",
- })
- kernel_keys = ["combine-key", "dispatch-key", "preprocess-key"]
- return {
- "backend_lineage": "deepep-hybrid", "branch": "hybrid-ep",
- "deepep_commit": "a" * 40, "deepep_tree": "b" * 40,
- "device_sms": 1,
- "jit_kernel_keys": kernel_keys,
- "jit_shared_objects": [
- {
- "kernel_key": key,
- "rank_artifacts": [
- {"bytes": 1, "rank": rank, "sha256": f"{index + 1:x}" * 64}
- for rank in range(ep_size)
- ],
- }
- for index, key in enumerate(kernel_keys)
- ],
- "loaded_libraries": [
- {"name": "deep_ep_cpp", "role": "deepep-extension", "sha256": "4" * 64},
- {"name": "hybrid_ep_cpp", "role": "deepep-hybrid-extension", "sha256": "5" * 64},
- ],
- "realized_config": realized,
- "resource_mode": "fixed-profile",
- "tuned_source": "deepep-hybrid-configurer-autotune-v1",
- }
-
-
-def _native_fixture(backend: str = "nccl-ep") -> tuple[dict, dict]:
- def digest(value: object) -> str:
- return hashlib.sha256(contracts.canonical_json_bytes(value)).hexdigest()
-
- scheduled = {
- "backend": backend, "canonical": True, "eplb": False, "ep": 1,
- "experts": 1, "gpus_per_node": 1, "hidden": 1, "ladder": "1", "nodes": 1,
- "mode": "normal", "phase": "decode", "required_publication": "official",
- "routing": "uniform", "samples_per_point": 512,
- "scale_out_transport": None, "scale_up_domain": 1,
- "scale_up_transport": "nvlink", "scope": "scale-up", "suite": "ep-core-v1",
- "timing": "8:64:32", "topk": 1,
- "topology_class": "fixture", "transport": "nvlink",
- "warmup_semantics": "full-roundtrip-before-each-component-trial-point-v1",
- "workload": "deepseek-v3-v1",
- }
- communication_precision = identity.precision_profile(
- identity.V1_CONTROL_PRECISION_PROFILE
- )
- case_factors = {
- "case": scheduled, "profile": identity.V1_NORMAL_CASE_PROFILE, "sku": "fixture"
- }
- case_id = identity.digest("case", case_factors)
- git_run = {
- "artifact": "cxshard-fixture-999-1", "job": "sweep", "ref": "collectivex",
- "repo": RUN["repository"], "qualification_index": 1,
- "run_attempt": "1", "run_id": "999",
- "source_sha": RUN["source_sha"],
- }
- allocation_factors = {
- "artifact": git_run["artifact"], "execution_id": "999_1_fixture",
- "job": git_run["job"], "qualification_index": 1,
- "repo": git_run["repo"], "run_attempt": "1",
- "run_id": "999", "runner": "fixture", "source_sha": git_run["source_sha"],
- }
- allocation_id = identity.allocation_id(allocation_factors)
- attempt_id = identity.attempt_id(allocation=allocation_id, case=case_id, ordinal=1)
- member_id, member_checksums, routing_hash, routing_rows, routing_weights = (
- contracts._expected_canonical_trace(
- "uniform", hidden=1, topk=1, logical_experts=1, physical_experts=1,
- ep_size=1, tokens_per_rank=1, seed=67, eplb_enabled=False,
- reference_tokens_per_rank=2048,
- )
- )
- workload_id = identity.workload_id({
- "members": [{"checksums": member_checksums, "workload_id": member_id}]
- })
- runtime = {
- "accelerator_runtime": {"kind": "cuda", "version": "13.0"},
- "collective_library": {"kind": "nccl", "version": "2.30.4"},
- "device": {
- "arch": "sm100", "compute_units": 1, "memory_bytes": 1,
- "product": "Fixture GPU", "warp_size": 32,
- },
- "driver_version": "1", "framework": {"kind": "torch", "version": "2.10.0"},
- "machine": "fixture", "python_version": "3.12", "vendor": "nvidia",
- }
- implementation_provenance = (
- {
- "backend": "nccl-ep", "backend_lineage": "nccl",
- "collective_library": "nccl", "nccl_version": "2.30.4",
- "reference_semantics": "fixture-v1",
- }
- if backend == "nccl-ep"
- else _hybrid_provenance()
- )
- kernel_generation = "nccl" if backend == "nccl-ep" else "hybrid"
- implementation = {
- "kernel_generation": kernel_generation,
- "name": backend,
- "provenance": implementation_provenance,
- "resource_profile": contracts.project_resource_profile(implementation_provenance),
- }
- public_config = contracts.public_series_config(
- kernel_generation=implementation["kernel_generation"],
- provenance=implementation_provenance,
- resource_profile=implementation["resource_profile"],
- resource_mode="fixed-profile",
- device_product=runtime["device"]["product"],
- )
- series_factors = {
- "backend": backend, "case_id": case_id,
- "image_digest": "sha256:" + "d" * 64,
- "implementation_contract_sha256": digest({
- **implementation,
- "provenance": contracts.series_provenance(implementation_provenance),
- }),
- "public_config_sha256": contracts.public_series_config_sha256(public_config),
- "routing_control_sha256": contracts.routing_implementation_control_sha256(
- implementation
- ),
- "runtime_fingerprint_sha256": digest(runtime),
- "source_sha": RUN["source_sha"], "squash_sha256": "e" * 64,
- "workload_id": workload_id,
- }
- series_id = identity.series_id(series_factors)
- point_id = identity.point_id(series=series_id, tokens_per_rank=1)
- sample_components = {
- name: {
- "availability": "measured", "sample_count": 512,
- "trials": [[latency] * 8 for _ in range(64)],
- }
- for name, latency in (("combine", 20.0), ("dispatch", 10.0), ("roundtrip", 40.0))
- }
- sample_components["stage"] = {
- "availability": "unavailable", "sample_count": 0, "trials": None,
- }
- sample_sha = digest({"components": sample_components, "tokens_per_rank": 1})
- evidence_id = identity.evidence_id(
- point=point_id, allocation=allocation_id, attempt=attempt_id,
- sample_sha256=sample_sha,
- )
- samples = {
- "allocation_id": allocation_id, "attempt_id": attempt_id, "case_id": case_id,
- "format": contracts.SAMPLES_FORMAT,
- "points": [{
- "components": sample_components, "evidence_id": evidence_id,
- "point_id": point_id, "sample_sha256": sample_sha, "tokens_per_rank": 1,
- }],
- "sampling": {
- "iterations_per_trial": 8, "reduction": "cross-rank-max-per-iteration",
- "trials": 64,
- },
- "qualification_index": 1, "schema_version": 1, "series_id": series_id,
- }
- sample_bytes = contracts.canonical_json_bytes(samples)
- oracle = {
- "atol": 0.02,
- "checks": {name: True for name in (
- "combine_values", "counts", "metadata", "multiplicity", "payload",
- "source_set", "weights",
- )},
- "combine_weight_semantics": "unweighted-rank-sum",
- "contract": "expert-specific-transform-v1", "dispatch_sha256": "1" * 64,
- "max_absolute_error": 0.0, "max_elementwise_relative_error": 0.0,
- "max_relative_error": 0.0, "max_weight_error": 0.0,
- "order_sha256": "2" * 64, "ordering_contract": "fixture-order-v1",
- "passed": True, "receive_count": 1, "rtol": 0.05,
- }
- def pct(value: float) -> dict[str, float]:
- return {name: value for name in ("p50", "p90", "p95", "p99")}
-
- def measured(value: float) -> dict:
- return {
- "availability": "measured", "origin": "measured",
- "percentiles_us": pct(value), "sample_count": 512,
- }
- row = {
- "anomalies": [],
- "components": {
- "combine": measured(20.0), "dispatch": measured(10.0),
- "isolated_sum": {
- "availability": "derived", "origin": "derived-percentile-sum",
- "percentiles_us": pct(30.0), "sample_count": 0,
- },
- "roundtrip": measured(40.0),
- "stage": {
- "availability": "unavailable", "origin": None,
- "percentiles_us": None, "sample_count": 0,
- },
- },
- "correctness": {
- "contract": "expert-specific-transform-v1", "max_relative_error": 0.0,
- "passed": True, "precision": _precision_evidence(),
- "rank_evidence": [{
- "input_unchanged": True, "order_stable": True,
- "post_timing": copy.deepcopy(oracle), "pre_timing": copy.deepcopy(oracle),
- "rank": 0,
- }],
- "scope": "dispatch-metadata-and-transformed-combine",
- },
- "evidence_id": evidence_id, "global_tokens": 1,
- "byte_provenance": {
- "combine": {
- "accounting_contract": "activation-data-plus-scales-v1",
- "activation_data_bytes": 2, "scale_bytes": 0,
- "total_logical_bytes": 2,
- },
- "dispatch": {
- "accounting_contract": "activation-data-plus-scales-v1",
- "activation_data_bytes": 2, "scale_bytes": 0,
- "total_logical_bytes": 2,
- },
- "roundtrip": {
- "accounting_contract": "activation-data-plus-scales-v1",
- "activation_data_bytes": 4, "scale_bytes": 0,
- "total_logical_bytes": 4,
- },
- "stage": {
- "accounting_contract": "activation-data-plus-scales-v1",
- "activation_data_bytes": 0, "scale_bytes": 0,
- "total_logical_bytes": 0,
- },
- },
- "point_id": point_id,
- "receive": {"max": 1, "mean": 1.0, "min": 1, "total": 1},
- "routing": contracts._expected_routing_summary(
- routing_rows,
- routing_weights,
- physical_experts=1,
- ep_size=1,
- tokens_per_rank=1,
- gpus_per_node=1,
- scale_up_domain=1,
- ),
- "sample_histograms": {
- name: contracts._expected_histogram([value] * 512)
- for name, value in (("combine", 20.0), ("dispatch", 10.0), ("roundtrip", 40.0))
- },
- "sample_sha256": sample_sha,
- "token_rate_at_latency_percentile": pct(25_000.0), "tokens_per_rank": 1,
- }
- row["sample_histograms"]["stage"] = None
- raw = {
- "case": {
- "attempt_ordinal": 1, "backend": backend,
- "eplb": {
- "calibration_token_offset": None, "calibration_trace_sha256": None,
- "calibration_window": None, "calibration_workload_id": None,
- "enabled": False, "imbalance_after": None, "imbalance_before": None,
- "mapping_hash": None, "max_replicas": None, "num_logical_experts": 1,
- "num_physical_experts": 1, "num_redundant": 0, "planner": None,
- "reference_tokens_per_rank": None, "replicated_experts": 0,
- },
- "ep_size": 1, "mode": "normal", "phase": "decode",
- "required_publication": "official", "resource_mode": "fixed-profile", "runner": "fixture",
- "shape": {
- "activation_profile": "canonical-counter-source-v4",
- "precision_profile": identity.V1_CONTROL_PRECISION_PROFILE,
- "dispatch_precision": communication_precision["dispatch"],
- "combine_precision": communication_precision["combine"],
- "eplb": False, "experts": 1, "experts_per_rank": 1, "hidden": 1,
- "kernel_gen": kernel_generation, "num_logical_experts": 1,
- "routing": "uniform", "topk": 1,
- },
- "suite": "ep-core-v1", "workload_name": "deepseek-v3-v1",
- },
- "format": contracts.RAW_FORMAT, "generated_at": "2026-07-04T00:00:00Z",
- "identity": {
- "allocation_factors": allocation_factors, "allocation_id": allocation_id,
- "attempt_id": attempt_id, "attempt_ordinal": 1, "case_factors": case_factors,
- "case_id": case_id, "series_factors": series_factors, "series_id": series_id,
- },
- "implementation": implementation,
- "measurement": {
- "component_order_contract": "qualification-hash-rotated-components-v1",
- "conditioning": {
- "contract": "fixed-phase-ramp-8-roundtrips-v1",
- "ladder": [1, 2, 4, 8, 16, 32, 64, 128],
- "roundtrips_per_shape": 8,
- },
- "contract": "layout-and-dispatch-v1",
- "execution_order_sha256": "9" * 64,
- "qualification_index": 1,
- "rows": [row],
- "sampling": {
- "contract": "fixed-512-v1", "iterations_per_trial": 8,
- "percentile_method": "nearest-rank",
- "reduction": "cross-rank-max-per-iteration", "samples_per_component": 512,
- "trials": 64, "warmup_iterations": 32,
- "warmup_semantics": "full-roundtrip-before-each-component-trial-point-v1",
- },
- "source_allocation": "even",
- },
- "outcome": {
- "publication_status": "diagnostic", "reasons": [], "status": "success",
- "validity": {
- "anomaly_free": True, "execution_status": "complete",
- "measurement_conformance": "conformant", "provenance_complete": True,
- "resource_conformance": implementation["resource_profile"]["conformance_class"],
- "sampling_conformance": "conformant",
- "semantic_correctness": "pass",
- "workload_identity": "consistent-across-ranks",
- "workload_source": "canonical-serialized",
- },
- },
- "provenance": {
- "allocation_stratum_sha256": "f" * 64,
- "command": "run_ep", "distributed_launcher": "torchrun", "git_run": git_run,
- "image": {
- "arch": "amd64", "digest": "sha256:" + "d" * 64,
- "digest_verified": True, "reference": "fixture:1", "squash_sha256": "e" * 64,
- },
- "redaction": "sanitized-v1",
- },
- "record_type": "case-attempt",
- "runtime_fingerprint": runtime,
- "sample_artifact": {
- "bytes": len(sample_bytes), "format": contracts.SAMPLES_FORMAT,
- "path": "samples.json", "sha256": hashlib.sha256(sample_bytes).hexdigest(),
- },
- "schema_version": 1,
- "topology": {
- "device_count": 1, "device_product": "Fixture GPU", "gpus_per_node": 1,
- "nodes": 1, "placement": "packed",
- "realized_placement": {
- "gpus_per_node": 1, "nodes": 1, "ranks_per_node": 1,
- "unique_local_ranks": True, "valid": True,
- },
- "scale_out_transport": None, "scale_up_domain": 1,
- "scale_up_transport": "nvlink", "scope": "scale-up",
- "topology_class": "fixture", "transport": "nvlink",
- "world_size": 1,
- },
- "workload": {
- "activation_generator": "collectivex-activation-counter-v4",
- "activation_identity": hashlib.sha256(
- b"counter|seed=67|hidden=1|gen=collectivex-activation-counter-v4"
- ).hexdigest(),
- "activation_profile": "canonical-counter-source-v4", "cross_rank_consistent": True,
- "manifest_checksums": {member_id: member_checksums}, "members": [member_id],
- "routing_generator": "collectivex-routing-counter-v3", "source": "canonical-serialized",
- "trace_hashes": [routing_hash],
- "trace_signature": hashlib.sha256(routing_hash.encode()).hexdigest(),
- "workload_id": workload_id,
- },
- }
- return raw, samples
-
-
-def _series(seed: str, backend: str, *, decision_grade: bool = False) -> tuple[dict, dict]:
- case, allocation, attempt, series_id, point_id, evidence = _ids(seed)
- allocations = [identity.allocation_id({"seed": seed, "run": run}) for run in range(1)]
- eligibility = publisher._eligibility_record(
- allocations if decision_grade else [allocation],
- complete=decision_grade,
- correct=True,
- measured=True,
- stable_ordering=True,
- )
- component = _component(1.0 if backend == "deepep" else 1.2)
- qualification_indices = [1]
- communication_precision = identity.precision_profile(
- identity.V1_CONTROL_PRECISION_PROFILE
- )
- item = {
- "series_id": series_id,
- "label": f"H100 / {backend}",
- "status": "decision-grade" if decision_grade else "diagnostic",
- "case_ids": [case],
- "allocation_ids": allocations if decision_grade else [allocation],
- "model": "deepseek-v3-v1",
- "suite": "ep-core-v1",
- "mode": "normal",
- "phase": "decode",
- "publication_tier": "official",
- "backend": {
- "id": backend, "label": publisher.BACKEND_LABELS[backend],
- "role": "reference" if backend == "nccl-ep" else "library",
- "generation": "nccl" if backend == "nccl-ep" else None,
- "version": "1.0"},
- "build": {
- "implementation_contract_sha256": hashlib.sha256(backend.encode()).hexdigest(),
- "public_config_sha256": "0" * 64,
- "routing_control_sha256": hashlib.sha256(backend.encode()).hexdigest(),
- "runtime_fingerprint_sha256": "3" * 64,
- "image_digest": "sha256:" + "1" * 64,
- "source_sha": "a" * 40,
- "squash_sha256": "2" * 64,
- },
- "system": {
- "sku": "h100-dgxc", "label": "NVIDIA H100", "vendor": "nvidia",
- "topology_class": "h100-nvlink-island", "transport": "nvlink",
- "scale_up_transport": "nvlink", "scale_out_transport": None,
- "scope": "scale-up", "nodes": 1, "gpus_per_node": 8,
- "scale_up_domain": 8,
- "world_size": 8, "ep_size": 8, "placement": "packed",
- },
- "workload": {
- "workload_id": identity.workload_id({"shape": "shared"}),
- "hidden": 7168, "top_k": 8, "experts": 256,
- "routing": "uniform", "eplb": False,
- "precision_profile": identity.V1_CONTROL_PRECISION_PROFILE,
- "dispatch_precision": communication_precision["dispatch"],
- "combine_precision": communication_precision["combine"],
- "activation_profile": "canonical-counter-source-v4",
- },
- "eplb": {
- "enabled": False,
- "calibration_workload_id": None, "calibration_trace_sha256": None,
- "calibration_window": None, "calibration_token_offset": None,
- "planner": None, "mapping_sha256": None,
- "logical_experts": 256, "physical_experts": 256,
- "redundant_experts": 0, "reference_tokens_per_rank": None,
- "replicated_experts": 0, "max_replicas": None,
- "imbalance_before": None, "imbalance_after": None,
- },
- "resource": {"mode": "fixed-profile", "profile": "profile-1", "comm_units_kind": "sm", "configured_units": 24},
- "measurement": {
- "contract": "layout-and-dispatch-v1",
- "component_order_contract": "qualification-hash-rotated-components-v1",
- "combine_semantics": "activation-only", "payload_unit": "token-rank",
- "sampling_contract": "fixed-512-v1",
- "iters": 8, "trials": 64, "warmups": 32, "samples_per_component": 512,
- "qualification_indices": qualification_indices,
- "headline_component": "roundtrip", "headline_percentile": "p99",
- },
- "points": [{
- "point_id": point_id, "tokens_per_rank": 8, "global_tokens": 64,
- "anomalies": [],
- "correctness": {"semantic_pass": True, "precision": _precision_evidence()},
- "trial_diagnostics": {
- "flagged": False,
- "reasons": [],
- "components": {
- "dispatch": None,
- "stage": None,
- "combine": None,
- "roundtrip": {
- "drift_flagged": False,
- "first_last_median_ratio": 1.0,
- "outlier_flagged": False,
- "robust_outlier_fraction": 0.0,
- "trial_count": 64,
- },
- },
- },
- "routing": {
- "fanout_mean": 4.0, "recv_tokens_max": 64,
- "expert_load_cv": 0.5, "payload_rank_cv": 0.25,
- "hotspot_ratio": 2.0, "empty_expert_count": 0,
- "empty_rank_count": 0, "routed_copies": 256,
- },
- "components": {"dispatch": None, "stage": None, "combine": None,
- "roundtrip": component, "isolated_sum": None},
- "roundtrip_token_rate_at_latency_percentile": {
- name: 64 / (latency * 1e-6)
- for name, latency in component["latency_us"].items()
- },
- "evidence_ids": [evidence],
- }],
- "eligibility": eligibility,
- }
- item["build"]["public_config_sha256"] = contracts.public_series_config_sha256(
- publisher._public_series_config(item)
- )
- case = identity.digest("case", publisher._public_case_factors(item))
- item["case_ids"] = [case]
- build = item["build"]
- series_id = identity.series_id({
- "backend": item["backend"]["id"],
- "case_id": case,
- "image_digest": build["image_digest"],
- "implementation_contract_sha256": build["implementation_contract_sha256"],
- "public_config_sha256": build["public_config_sha256"],
- "routing_control_sha256": build["routing_control_sha256"],
- "runtime_fingerprint_sha256": build["runtime_fingerprint_sha256"],
- "source_sha": build["source_sha"],
- "squash_sha256": build["squash_sha256"],
- "workload_id": item["workload"]["workload_id"],
- })
- item["series_id"] = series_id
- point_id = identity.point_id(series=series_id, tokens_per_rank=8)
- item["points"][0]["point_id"] = point_id
- attempt = identity.attempt_id(allocation=allocation, case=case, ordinal=1)
- evidence = identity.evidence_id(
- point=point_id, allocation=allocation, attempt=attempt,
- sample_sha256=hashlib.sha256(seed.encode()).hexdigest(),
- )
- item["points"][0]["evidence_ids"] = [evidence]
- runs = {
- str(run): {8: {
- "latency_us": {
- statistic: component["latency_us"][statistic] * (1 + run / 100)
- for statistic in ("p50", "p99")
- },
- "activation_data_rate_gbps_at_latency_percentile": {
- statistic: component["activation_data_rate_gbps_at_latency_percentile"][statistic] / (1 + run / 100)
- for statistic in ("p50", "p99")
- },
- "total_logical_data_rate_gbps_at_latency_percentile": {
- statistic: component["total_logical_data_rate_gbps_at_latency_percentile"][statistic] / (1 + run / 100)
- for statistic in ("p50", "p99")
- },
- }}
- for run in range(1)
- }
- trial_blocks = {
- run_id: {8: {
- "dispatch": None,
- "stage": None,
- "combine": None,
- "roundtrip": tuple(
- tuple(metrics[8]["latency_us"]["p99"] for _ in range(8))
- for _ in range(64)
- ),
- }}
- for run_id, metrics in runs.items()
- }
- internal = {"run_metrics": runs, "trial_blocks": trial_blocks}
- return item, internal
-
-
-def _precision_series(
- seed: str,
- precision_profile: str,
- *,
- tokens: tuple[int, ...] = (128,),
-) -> tuple[dict, dict]:
- item, internal = _series(seed, "deepep", decision_grade=True)
- precision = identity.precision_profile(precision_profile)
- if precision_profile != identity.V1_CONTROL_PRECISION_PROFILE:
- item["suite"] = "ep-precision-normal-v1"
- item["publication_tier"] = "comparable-experimental"
- item["workload"].update({
- "precision_profile": precision_profile,
- "dispatch_precision": precision["dispatch"],
- "combine_precision": precision["combine"],
- })
- template = item["points"][0]
- old_token = template["tokens_per_rank"]
- old_metrics = {
- run_id: metrics[old_token]
- for run_id, metrics in internal["run_metrics"].items()
- }
- old_trials = {
- run_id: metrics[old_token]
- for run_id, metrics in internal["trial_blocks"].items()
- }
- item["points"] = []
- for token in tokens:
- point = copy.deepcopy(template)
- point["tokens_per_rank"] = token
- point["global_tokens"] = token * item["system"]["ep_size"]
- point["point_id"] = identity.point_id(
- series=item["series_id"], tokens_per_rank=token
- )
- point["correctness"]["precision"] = _precision_evidence(precision_profile)
- point["roundtrip_token_rate_at_latency_percentile"] = {
- name: point["global_tokens"] / (latency * 1e-6)
- for name, latency in point["components"]["roundtrip"]["latency_us"].items()
- }
- item["points"].append(point)
- internal["run_metrics"] = {
- run_id: {token: copy.deepcopy(metrics) for token in tokens}
- for run_id, metrics in old_metrics.items()
- }
- internal["trial_blocks"] = {
- run_id: {token: copy.deepcopy(metrics) for token in tokens}
- for run_id, metrics in old_trials.items()
- }
- return item, internal
-
-
-def _dataset() -> dict:
- item, _ = _series("one", "deepep")
- case = item["case_ids"][0]
- allocation = item["allocation_ids"][0]
- attempt = identity.attempt_id(allocation=allocation, case=case, ordinal=1)
- evidence = item["points"][0]["evidence_ids"][0]
- return {
- "format": "collectivex.public.v1", "schema_version": 1,
- "generated_at": "2026-07-04T00:00:00Z", "source_bundle_ids": ["c" * 64],
- "promotion": {
- "status": "diagnostic", "reason": None, "matrix_id": "d" * 64,
- "allocation_ids": [allocation], "required_allocations": 1,
- "qualification_indices": [1],
- "requested_cases": 1, "terminal_cases": 1,
- "measured_cases": 1, "unsupported_cases": 0,
- "requested_points": 1, "terminal_points": 1,
- "measured_points": 1, "unsupported_points": 0,
- "policy": "collectivex-decision-grade-v1",
- },
- "coverage": [{
- "case_id": case, "label": "case", "required": True, "sku": "h100-dgxc",
- "suite": item["suite"], "workload": item["model"],
- "publication_tier": item["publication_tier"],
- "backend": "deepep", "backend_generation": item["backend"]["generation"],
- "mode": "normal", "phase": "decode",
- "routing": item["workload"]["routing"], "eplb": False,
- "precision_profile": item["workload"]["precision_profile"],
- "dispatch_precision": item["workload"]["dispatch_precision"],
- "combine_precision": item["workload"]["combine_precision"],
- "resource": item["resource"],
- "topology": publisher._coverage_topology(item["system"]),
- "points": [{
- "point_id": item["points"][0]["point_id"],
- "series_id": item["series_id"],
- "tokens_per_rank": 8, "global_tokens": 64,
- "terminal_status": "measured", "reason": None,
- }],
- "disposition": "runnable",
- "selected_attempt_id": attempt,
- "outcome": "success", "failure_mode": None, "reason": None,
- "attempt_ids": [attempt],
- }],
- "attempts": [{
- "attempt_id": attempt,
- "evidence": [{"evidence_id": evidence,
- "point_id": item["points"][0]["point_id"]}],
- "case_id": case,
- "allocation_id": allocation, "run_id": "1", "run_attempt": 1,
- "qualification_index": 1,
- "attempt_index": 1,
- "selected": True, "outcome": "success", "failure_mode": None, "reason": None,
- "series_id": item["series_id"],
- "completed_at": "2026-07-04T00:00:00Z",
- }],
- "series": [item], "cohorts": [], "rankings": [], "recommendations": [],
- "sensitivities": [],
- }
-
-
-def _promoted_dataset(*, precision_profiles: tuple[str, ...] = ()) -> dict:
- specifications = [
- ("library-fast", "deepep", None, False, None),
- ("library-slow", "uccl", None, False, None),
- ("chip-peer", "deepep", "h200-dgxc", False, None),
- ("system-one", "nccl-ep", None, True, None),
- ("system-two", "nccl-ep", "h200-dgxc", True, None),
- ]
- specifications.extend(
- (f"precision-{index}", "deepep", None, False, precision_profile)
- for index, precision_profile in enumerate(precision_profiles)
- )
- series = []
- internals = {}
- attempts = []
- coverage = []
- for seed, backend, peer_sku, reference, precision_profile in specifications:
- item, internal = _series(seed, backend, decision_grade=True)
- if peer_sku:
- platform = publisher.capability.PLATFORMS[peer_sku]
- item["system"].update({
- "sku": peer_sku,
- "label": f"NVIDIA {platform['product'].upper()}",
- "topology_class": platform["topology_class"],
- "transport": platform["transport"],
- })
- if reference:
- item["backend"]["role"] = "reference"
- if precision_profile is not None:
- precision = identity.precision_profile(precision_profile)
- item["suite"] = "ep-precision-normal-v1"
- item["publication_tier"] = "comparable-experimental"
- item["workload"].update({
- "precision_profile": precision_profile,
- "dispatch_precision": precision["dispatch"],
- "combine_precision": precision["combine"],
- })
- item["points"][0]["correctness"]["precision"] = _precision_evidence(
- precision_profile
- )
- case_id = identity.digest("case", publisher._public_case_factors(item))
- item["case_ids"] = [case_id]
- build = item["build"]
- build["public_config_sha256"] = contracts.public_series_config_sha256(
- publisher._public_series_config(item)
- )
- item["series_id"] = identity.series_id({
- "backend": item["backend"]["id"],
- "case_id": case_id,
- "image_digest": build["image_digest"],
- "implementation_contract_sha256": build["implementation_contract_sha256"],
- "public_config_sha256": build["public_config_sha256"],
- "routing_control_sha256": build["routing_control_sha256"],
- "runtime_fingerprint_sha256": build["runtime_fingerprint_sha256"],
- "source_sha": build["source_sha"],
- "squash_sha256": build["squash_sha256"],
- "workload_id": item["workload"]["workload_id"],
- })
- point = item["points"][0]
- point["point_id"] = identity.point_id(
- series=item["series_id"], tokens_per_rank=point["tokens_per_rank"]
- )
- case_attempts = []
- evidence_ids = []
- for run_id, allocation_id in enumerate(item["allocation_ids"], 1):
- attempt_id = identity.attempt_id(
- allocation=allocation_id, case=case_id, ordinal=1
- )
- evidence_id = identity.evidence_id(
- point=point["point_id"], allocation=allocation_id,
- attempt=attempt_id,
- sample_sha256=hashlib.sha256(f"{seed}-{run_id}".encode()).hexdigest(),
- )
- attempts.append({
- "attempt_id": attempt_id,
- "evidence": [{"evidence_id": evidence_id, "point_id": point["point_id"]}],
- "case_id": case_id, "allocation_id": allocation_id,
- "run_id": str(run_id), "run_attempt": 1,
- "qualification_index": run_id,
- "attempt_index": 1, "selected": True,
- "outcome": "success", "failure_mode": None, "reason": None,
- "series_id": item["series_id"],
- "completed_at": "2026-07-04T00:00:00Z",
- })
- case_attempts.append(attempt_id)
- evidence_ids.append(evidence_id)
- point["evidence_ids"] = evidence_ids
- coverage.append({
- "case_id": case_id, "label": seed, "required": True,
- "sku": item["system"]["sku"], "suite": item["suite"],
- "workload": item["model"], "publication_tier": item["publication_tier"],
- "backend": backend, "backend_generation": item["backend"]["generation"],
- "mode": item["mode"], "phase": item["phase"],
- "routing": item["workload"]["routing"], "eplb": item["workload"]["eplb"],
- "precision_profile": item["workload"]["precision_profile"],
- "dispatch_precision": item["workload"]["dispatch_precision"],
- "combine_precision": item["workload"]["combine_precision"],
- "resource": item["resource"], "disposition": "runnable",
- "topology": publisher._coverage_topology(item["system"]),
- "points": [{
- "point_id": point["point_id"], "series_id": item["series_id"],
- "tokens_per_rank": point["tokens_per_rank"],
- "global_tokens": point["global_tokens"],
- "terminal_status": "measured", "reason": None,
- }],
- "selected_attempt_id": case_attempts[-1], "outcome": "success",
- "failure_mode": None, "reason": None, "attempt_ids": case_attempts,
- })
- series.append(item)
- internals[item["series_id"]] = internal
-
- unsupported_case, unsupported = next(
- (case_id, case)
- for case_id, case in publisher._canonical_coverage_cases().items()
- if case["sku"] == "mi355x" and case["backend"] == "deepep"
- and case["phase"] == "decode" and case["routing"] == "uniform"
- and not case["eplb"] and case["ep"] == 8
- )
- unsupported_attempts = []
- for run_id in range(1, 2):
- allocation_id = identity.allocation_id(
- {"seed": "planned-unsupported", "run": run_id}
- )
- attempt_id = identity.attempt_id(
- allocation=allocation_id, case=unsupported_case, ordinal=1
- )
- attempts.append({
- "attempt_id": attempt_id, "evidence": [], "case_id": unsupported_case,
- "allocation_id": allocation_id, "run_id": str(run_id),
- "run_attempt": 1, "qualification_index": run_id,
- "attempt_index": 1, "selected": True, "outcome": "unsupported",
- "failure_mode": "capability", "reason": "backend-platform-unsupported",
- "series_id": None, "completed_at": "2026-07-04T00:00:00Z",
- })
- unsupported_attempts.append(attempt_id)
- coverage.append({
- "case_id": unsupported_case, "label": "planned unsupported", "required": True,
- "sku": unsupported["sku"], "suite": unsupported["suite"],
- "workload": unsupported["workload"],
- "publication_tier": unsupported["required_publication"],
- "backend": unsupported["backend"], "backend_generation": None,
- "mode": unsupported["mode"], "phase": unsupported["phase"],
- "routing": unsupported["routing"], "eplb": unsupported["eplb"],
- "precision_profile": identity.V1_CONTROL_PRECISION_PROFILE,
- "dispatch_precision": identity.precision_profile(
- identity.V1_CONTROL_PRECISION_PROFILE
- )["dispatch"],
- "combine_precision": identity.precision_profile(
- identity.V1_CONTROL_PRECISION_PROFILE
- )["combine"],
- "resource": {
- "mode": "fixed-profile", "profile": None,
- "comm_units_kind": None, "configured_units": None,
- },
- "topology": publisher._coverage_topology(unsupported),
- "points": [{
- "point_id": None, "series_id": None,
- "tokens_per_rank": token, "global_tokens": token * unsupported["ep"],
- "terminal_status": "unsupported",
- "reason": "backend-platform-unsupported",
- } for token in map(int, unsupported["ladder"].split())],
- "disposition": "unsupported", "selected_attempt_id": unsupported_attempts[-1],
- "outcome": "unsupported", "failure_mode": "capability",
- "reason": "backend-platform-unsupported", "attempt_ids": unsupported_attempts,
- })
- cohorts, rankings, recommendations, sensitivities = publisher.build_decisions(
- series, internals
- )
- return {
- "format": "collectivex.public.v1", "schema_version": 1,
- "generated_at": "2026-07-04T00:00:00Z",
- "source_bundle_ids": ["a" * 64],
- "promotion": {
- "status": "promoted", "reason": None,
- "matrix_id": publisher.CANONICAL_FULL_V1_MATRIX_SHA256,
- "allocation_ids": sorted({item["allocation_id"] for item in attempts}),
- "required_allocations": 1, "qualification_indices": [1],
- "requested_cases": len(coverage), "terminal_cases": len(coverage),
- "measured_cases": len(coverage) - 1, "unsupported_cases": 1,
- "requested_points": sum(len(item["points"]) for item in coverage),
- "terminal_points": sum(len(item["points"]) for item in coverage),
- "measured_points": sum(
- point["terminal_status"] == "measured"
- for item in coverage for point in item["points"]
- ),
- "unsupported_points": sum(
- point["terminal_status"] == "unsupported"
- for item in coverage for point in item["points"]
- ),
- "policy": "collectivex-decision-grade-v1",
- },
- "coverage": sorted(coverage, key=lambda item: item["case_id"]),
- "attempts": sorted(attempts, key=lambda item: item["attempt_id"]),
- "series": sorted(series, key=lambda item: item["series_id"]),
- "cohorts": cohorts, "rankings": rankings,
- "recommendations": recommendations, "sensitivities": sensitivities,
- }
-
-
-def _cohort_counts(dataset: dict) -> dict[str, int]:
- return {
- kind: sum(item["kind"] == kind for item in dataset["cohorts"])
- for kind in ("library", "system")
- }
-
-
-def _canonical_entry_from_coverage(item: dict) -> dict:
- """Build a canonical-catalog entry that a coverage row projects onto.
-
- Used to monkeypatch ``_canonical_coverage_cases`` so a partial promotion
- fixture whose case ids are synthetic can still exercise the partial-scope
- membership + disposition contract.
- """
- return {
- "sku": item["sku"],
- "suite": item["suite"],
- "workload": item["workload"],
- "required_publication": item["publication_tier"],
- "backend": item["backend"],
- "mode": item["mode"],
- "phase": item["phase"],
- "routing": item["routing"],
- "eplb": item["eplb"],
- "precision_profile": item["precision_profile"],
- "dispatch_precision": item["dispatch_precision"],
- "combine_precision": item["combine_precision"],
- "disposition": item["disposition"],
- "reason": item["reason"],
- "ladder": " ".join(str(point["tokens_per_rank"]) for point in item["points"]),
- **item["topology"],
- }
-
-
-class PublisherTest(unittest.TestCase):
- def test_trial_diagnostics_flag_drift_and_robust_outliers(self) -> None:
- def runs() -> dict[str, dict[int, dict[str, object]]]:
- return {
- str(index): {
- 8: {
- "dispatch": tuple(tuple([10.0] * 8) for _ in range(64)),
- "stage": None,
- "combine": tuple(tuple([10.0] * 8) for _ in range(64)),
- "roundtrip": tuple(tuple([10.0] * 8) for _ in range(64)),
- }
- }
- for index in range(1, 2)
- }
-
- stable = publisher._trial_diagnostics(runs(), 8)
- self.assertFalse(stable["flagged"])
-
- drift = runs()
- drift["1"][8]["roundtrip"] = tuple(
- tuple([10.0 + 2.0 * trial / 63.0] * 8) for trial in range(64)
- )
- self.assertEqual(publisher._trial_diagnostics(drift, 8)["reasons"], ["trial-drift"])
-
- outliers = runs()
- outliers["1"][8]["roundtrip"] = tuple(
- tuple([100.0 if 20 <= trial < 36 else 10.0] * 8) for trial in range(64)
- )
- summary = publisher._trial_diagnostics(outliers, 8)
- self.assertEqual(summary["reasons"], ["trial-outliers"])
- self.assertGreater(
- summary["components"]["roundtrip"]["robust_outlier_fraction"],
- publisher.TRIAL_OUTLIER_FRACTION_LIMIT,
- )
-
- def test_terminal_allocation_and_source_status_are_bound(self) -> None:
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary).resolve()
- matrix, artifact = _unsupported_delivery(root)
- path = next(artifact.glob("*.json"))
- terminal = contracts.strict_load(path)
- self.assertIs(contracts.validate_terminal_document(terminal), terminal)
- self.assertEqual(
- contracts.validate_delivery(
- [str(path)], str(matrix), disposition="unsupported"
- ),
- 1,
- )
-
- for control_sha256 in (None, "0" * 64):
- broken = copy.deepcopy(terminal)
- broken["provenance"]["control_sha256"] = control_sha256
- path.write_text(json.dumps(broken))
- with self.assertRaisesRegex(contracts.ContractError, "exact control document"):
- contracts.validate_delivery(
- [str(path)], str(matrix), disposition="unsupported"
- )
- path.write_text(json.dumps(terminal))
-
- for field in (
- "artifact", "job", "repo", "run_attempt", "run_id", "source_sha", "runner"
- ):
- broken = copy.deepcopy(terminal)
- broken["identity"]["allocation_factors"][field] = f"forged-{field}"
- allocation_id = identity.allocation_id(
- broken["identity"]["allocation_factors"]
- )
- broken["identity"]["allocation_id"] = allocation_id
- broken["identity"]["attempt_id"] = identity.attempt_id(
- allocation=allocation_id,
- case=broken["identity"]["case_id"],
- ordinal=broken["identity"]["attempt_ordinal"],
- )
- with self.assertRaisesRegex(
- contracts.ContractError, "allocation factors differ"
- ):
- contracts.validate_terminal_document(broken)
-
- broken = copy.deepcopy(terminal)
- broken["outcome"]["status"] = "failed"
- with self.assertRaisesRegex(contracts.ContractError, "source and outcome"):
- contracts.validate_terminal_document(broken)
- broken = copy.deepcopy(terminal)
- broken["provenance"]["source"] = "runtime-emitter"
- with self.assertRaisesRegex(contracts.ContractError, "source and outcome"):
- contracts.validate_terminal_document(broken)
-
- for path_parts, replacement in (
- (("provenance", "source"), "unregistered-producer"),
- (("outcome", "failure_mode"), "unsupported-capability"),
- (("outcome", "reason"), "unregistered-capability"),
- ):
- with self.subTest(path=path_parts):
- broken = copy.deepcopy(terminal)
- broken[path_parts[0]][path_parts[1]] = replacement
- with self.assertRaises(publisher.PublisherError):
- publisher._schema("terminal-outcome-v1.schema.json", broken)
- with self.assertRaises(contracts.ContractError):
- contracts.validate_terminal_document(broken)
-
- runtime_allocation = copy.deepcopy(
- terminal["identity"]["allocation_factors"]
- )
- runtime_allocation["runner"] = terminal["identity"]["case_factors"]["sku"]
- runtime = contracts.make_terminal_document(
- allocation_factors=runtime_allocation,
- attempt_ordinal=1,
- case=terminal["case"],
- case_factors=terminal["identity"]["case_factors"],
- control_sha256=terminal["provenance"]["control_sha256"],
- failure_mode="setup",
- generated_at=terminal["generated_at"],
- git_run=terminal["provenance"]["git_run"],
- reason="launcher-setup-failed",
- return_code=1,
- source="runtime-emitter",
- status="failed",
- expected_case_id=terminal["identity"]["case_id"],
- )
- publisher._schema("terminal-outcome-v1.schema.json", runtime)
- broken = copy.deepcopy(runtime)
- broken["outcome"]["reason"] = "backend-setup-failed"
- with self.assertRaises(publisher.PublisherError):
- publisher._schema("terminal-outcome-v1.schema.json", broken)
- with self.assertRaises(contracts.ContractError):
- contracts.validate_terminal_document(broken)
-
- def test_post_emit_demotion_uses_closed_failure_taxonomy(self) -> None:
- raw, _ = _native_fixture()
- expected = {
- 5: "runtime-identity",
- 6: "execution",
- 124: "timeout",
- 137: "timeout",
- 134: "execution",
- 9: "execution",
- }
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary)
- for return_code, failure_mode in expected.items():
- with self.subTest(return_code=return_code):
- path = root / f"attempt-{return_code}.json"
- path.write_text(json.dumps(raw))
- terminal = contracts.demote_raw_attempt(path, return_code)
- self.assertEqual(
- terminal["outcome"],
- {
- "failure_mode": failure_mode,
- "reason": "post-emit-distributed-command-failed",
- "return_code": return_code,
- "status": "failed",
- },
- )
- self.assertEqual(terminal["provenance"]["source"], "post-emit-command")
- publisher._schema("terminal-outcome-v1.schema.json", terminal)
-
- broken = copy.deepcopy(terminal)
- broken["outcome"]["reason"] = "distributed-command-failed"
- with self.assertRaises(publisher.PublisherError):
- publisher._schema("terminal-outcome-v1.schema.json", broken)
- with self.assertRaises(contracts.ContractError):
- contracts.validate_terminal_document(broken)
-
- def test_artifact_safety_accepts_current_v1_fixtures(self) -> None:
- raw, samples = _native_fixture()
- publisher.artifact_safety.assert_publication_safe([
- sweep_matrix.resolve_matrix(backends="all"),
- raw,
- samples,
- _dataset(),
- _promoted_dataset(),
- ])
-
- def test_native_raw_and_sample_schema_match_semantic_validator(self) -> None:
- raw, samples = _native_fixture()
- publisher._schema("samples-v1.schema.json", samples)
- publisher._schema("raw-case-v1.schema.json", raw)
- self.assertIs(contracts.validate_raw_document(raw, samples), raw)
- provenance = raw["provenance"]
- image = provenance["image"]
- self.assertTrue(contracts.provenance_complete(
- raw["implementation"]["provenance"], raw["case"]["backend"],
- provenance["git_run"],
- allocation_stratum_sha256=provenance["allocation_stratum_sha256"],
- image_digest=image["digest"], image_verified=image["digest_verified"],
- squash_sha256=image["squash_sha256"],
- ))
- self.assertFalse(contracts.provenance_complete(
- raw["implementation"]["provenance"], raw["case"]["backend"],
- provenance["git_run"], allocation_stratum_sha256=None,
- image_digest=image["digest"], image_verified=image["digest_verified"],
- squash_sha256=image["squash_sha256"],
- ))
- missing_stratum = copy.deepcopy(raw)
- missing_stratum["provenance"]["allocation_stratum_sha256"] = None
- with self.assertRaises(publisher.PublisherError):
- publisher._schema("raw-case-v1.schema.json", missing_stratum)
- with self.assertRaisesRegex(contracts.ContractError, "allocation stratum"):
- contracts.validate_raw_document(missing_stratum, samples)
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary).resolve()
- (root / "samples.json").write_bytes(contracts.canonical_json_bytes(samples))
- (root / "raw.json").write_bytes(contracts.canonical_json_bytes(raw))
- self.assertEqual(contracts.load_raw_attempt(root / "raw.json"), raw)
- for target in ("raw", "samples"):
- broken_raw, broken_samples = copy.deepcopy((raw, samples))
- broken = broken_raw if target == "raw" else broken_samples
- broken["unexpected"] = True
- with self.assertRaises(publisher.PublisherError):
- publisher._schema(
- "raw-case-v1.schema.json" if target == "raw" else "samples-v1.schema.json",
- broken,
- )
- with self.assertRaises(contracts.ContractError):
- contracts.validate_raw_document(broken_raw, broken_samples)
- tampered = copy.deepcopy(raw)
- tampered["measurement"]["rows"][0]["token_rate_at_latency_percentile"]["p50"] *= 2
- with self.assertRaisesRegex(contracts.ContractError, "token_rate_at_latency_percentile"):
- contracts.validate_raw_document(tampered, samples)
- tampered = copy.deepcopy(raw)
- tampered["case"]["shape"]["hidden"] = 2
- with self.assertRaises(contracts.ContractError):
- contracts.validate_raw_document(tampered, samples)
- tampered = copy.deepcopy(raw)
- configured = tampered["implementation"]["resource_profile"]["configured_units"]
- tampered["implementation"]["resource_profile"]["configured_units"] = (
- 1 if configured is None else configured + 1
- )
- with self.assertRaisesRegex(contracts.ContractError, "resource profile"):
- contracts.validate_raw_document(tampered, samples)
- tampered = copy.deepcopy(raw)
- oracle = tampered["measurement"]["rows"][0]["correctness"]["rank_evidence"][0]
- oracle["pre_timing"]["checks"]["combine_values"] = False
- with self.assertRaisesRegex(contracts.ContractError, "passed differs"):
- contracts.validate_raw_document(tampered, samples)
-
- def test_hybrid_raw_binds_realized_config_and_every_rank_artifact(self) -> None:
- raw, samples = _native_fixture("deepep-hybrid")
- publisher._schema("raw-case-v1.schema.json", raw)
- self.assertIs(contracts.validate_raw_document(raw, samples), raw)
-
- mutations = {
- "hidden_dim": lambda provenance: provenance["realized_config"].update(
- hidden_dim=2
- ),
- "experts_per_rank": lambda provenance: provenance["realized_config"].update(
- num_of_experts_per_rank=2
- ),
- "ranks_per_node": lambda provenance: provenance["realized_config"].update(
- num_of_ranks_per_node=2
- ),
- "num_nodes": lambda provenance: provenance["realized_config"].update(
- num_of_nodes=2
- ),
- "token_data_type": lambda provenance: provenance["realized_config"].update(
- token_data_type="UINT8"
- ),
- "rank_coverage": lambda provenance: [
- artifact["rank_artifacts"].append({
- "bytes": 1, "rank": 1, "sha256": "9" * 64,
- })
- for artifact in provenance["jit_shared_objects"]
- ],
- }
- for name, mutate in mutations.items():
- with self.subTest(name=name):
- changed = copy.deepcopy(raw)
- mutate(changed["implementation"]["provenance"])
- with self.assertRaisesRegex(
- contracts.ContractError,
- "DeepEP Hybrid realized config/JIT evidence differs",
- ):
- contracts.validate_raw_document(changed, samples)
-
- def test_native_contract_recomputes_routing_receive_histograms_and_anomalies(self) -> None:
- raw, samples = _native_fixture()
-
- tampered = copy.deepcopy(raw)
- changed = tampered["measurement"]["rows"][0]
- changed["routing"]["routed_copies"] *= 2
- for name in ("combine", "dispatch", "roundtrip"):
- byte_provenance = changed["byte_provenance"][name]
- byte_provenance["activation_data_bytes"] *= 2
- byte_provenance["total_logical_bytes"] *= 2
- with self.assertRaisesRegex(contracts.ContractError, "routing.routed_copies"):
- contracts.validate_raw_document(tampered, samples)
-
- tampered = copy.deepcopy(raw)
- changed = tampered["measurement"]["rows"][0]
- changed["routing"]["payload_copies_per_rank"] = [2]
- changed["receive"] = {"max": 2, "mean": 2.0, "min": 2, "total": 2}
- with self.assertRaisesRegex(contracts.ContractError, "payload_copies_per_rank"):
- contracts.validate_raw_document(tampered, samples)
-
- tampered = copy.deepcopy(raw)
- tampered["measurement"]["rows"][0]["sample_histograms"]["roundtrip"][
- "counts"
- ] = [511]
- with self.assertRaisesRegex(contracts.ContractError, "sample_histograms"):
- contracts.validate_raw_document(tampered, samples)
-
- tampered = copy.deepcopy(raw)
- tampered["measurement"]["rows"][0]["anomalies"] = [{
- "type": "roundtrip_gt_isolated_sum",
- "T": 1,
- "roundtrip_p99": 40.0,
- "isolated_sum_p99": 30.0,
- "ratio": 1.33,
- "threshold": 3.0,
- }]
- tampered["outcome"]["validity"]["anomaly_free"] = False
- with self.assertRaisesRegex(contracts.ContractError, "anomalies"):
- contracts.validate_raw_document(tampered, samples)
-
- anomalous_raw, anomalous_samples = copy.deepcopy((raw, samples))
- sample_point = anomalous_samples["points"][0]
- sample_point["components"]["roundtrip"]["trials"] = [
- [100.0] * 8 for _ in range(64)
- ]
- sample_core = {
- "components": sample_point["components"],
- "tokens_per_rank": sample_point["tokens_per_rank"],
- }
- sample_sha = hashlib.sha256(
- contracts.canonical_json_bytes(sample_core)
- ).hexdigest()
- point_id = sample_point["point_id"]
- evidence_id = identity.evidence_id(
- point=point_id,
- allocation=anomalous_raw["identity"]["allocation_id"],
- attempt=anomalous_raw["identity"]["attempt_id"],
- sample_sha256=sample_sha,
- )
- sample_point.update({"sample_sha256": sample_sha, "evidence_id": evidence_id})
- changed = anomalous_raw["measurement"]["rows"][0]
- changed["sample_sha256"] = sample_sha
- changed["evidence_id"] = evidence_id
- changed["components"]["roundtrip"]["percentiles_us"] = {
- name: 100.0 for name in ("p50", "p90", "p95", "p99")
- }
- changed["token_rate_at_latency_percentile"] = {
- name: 10_000.0 for name in ("p50", "p90", "p95", "p99")
- }
- changed["sample_histograms"]["roundtrip"] = contracts._expected_histogram(
- [100.0] * 512
- )
- changed["anomalies"] = contracts._expected_anomalies(1, changed["components"])
- anomalous_raw["outcome"]["validity"]["anomaly_free"] = False
- sample_bytes = contracts.canonical_json_bytes(anomalous_samples)
- anomalous_raw["sample_artifact"].update({
- "bytes": len(sample_bytes),
- "sha256": hashlib.sha256(sample_bytes).hexdigest(),
- })
- self.assertIs(
- contracts.validate_raw_document(anomalous_raw, anomalous_samples),
- anomalous_raw,
- )
- changed["anomalies"] = []
- anomalous_raw["outcome"]["validity"]["anomaly_free"] = True
- with self.assertRaisesRegex(contracts.ContractError, "anomalies"):
- contracts.validate_raw_document(anomalous_raw, anomalous_samples)
-
- def test_native_contract_rejects_every_schema_only_nested_mutation(self) -> None:
- raw, samples = _native_fixture()
- self.assertIs(contracts.validate_raw_document(raw, samples), raw)
-
- def locate(document: object, path: tuple[object, ...]) -> object:
- value = document
- for part in path:
- value = value[part] # type: ignore[index]
- return value
-
- def reject_raw(document: dict) -> None:
- with self.assertRaises(publisher.PublisherError):
- publisher._schema("raw-case-v1.schema.json", document)
- with self.assertRaises(contracts.ContractError):
- contracts.validate_raw_document(document, samples)
-
- required_fields = (
- (("measurement", "rows", 0, "receive"), "total"),
- (("measurement", "rows", 0, "routing"), "fanout_mean"),
- (("measurement", "rows", 0, "routing", "source_token_stats"), "ranks"),
- (("measurement", "rows", 0, "sample_histograms"), "roundtrip"),
- (("measurement", "rows", 0, "sample_histograms", "roundtrip"), "n"),
- (("runtime_fingerprint", "accelerator_runtime"), "kind"),
- (("runtime_fingerprint", "collective_library"), "kind"),
- (("runtime_fingerprint", "framework"), "kind"),
- )
- for path, required in required_fields:
- with self.subTest(path=path, mutation="missing"):
- broken = copy.deepcopy(raw)
- del locate(broken, path)[required] # type: ignore[index]
- reject_raw(broken)
- with self.subTest(path=path, mutation="extra"):
- broken = copy.deepcopy(raw)
- locate(broken, path)["unexpected"] = None # type: ignore[index]
- reject_raw(broken)
-
- invalid_values = (
- (("measurement", "rows", 0, "receive", "mean"), "one"),
- (("measurement", "rows", 0, "routing", "fanout_mean"), "one"),
- (("measurement", "rows", 0, "sample_histograms", "roundtrip", "bins"), 0),
- (("provenance", "image", "arch"), "AMD64"),
- (("runtime_fingerprint", "accelerator_runtime", "kind"), "rocm"),
- )
- for path, invalid in invalid_values:
- with self.subTest(path=path, mutation="value"):
- broken = copy.deepcopy(raw)
- parent = locate(broken, path[:-1])
- parent[path[-1]] = invalid # type: ignore[index]
- reject_raw(broken)
-
- def reject_samples(document: dict) -> None:
- with self.assertRaises(publisher.PublisherError):
- publisher._schema("samples-v1.schema.json", document)
- with self.assertRaises(contracts.ContractError):
- contracts.validate_samples_document(document)
-
- for path, required in (
- (("points", 0), "evidence_id"),
- (("points", 0, "components"), "roundtrip"),
- (("points", 0, "components", "roundtrip"), "trials"),
- (("sampling",), "reduction"),
- ):
- with self.subTest(path=path, artifact="samples"):
- broken = copy.deepcopy(samples)
- del locate(broken, path)[required] # type: ignore[index]
- reject_samples(broken)
-
- def test_terminal_contract_and_schema_reject_the_same_shape_gaps(self) -> None:
- with tempfile.TemporaryDirectory() as temporary:
- _, artifact = _unsupported_delivery(Path(temporary).resolve())
- terminal = contracts.strict_load(next(artifact.glob("*.json")))
- publisher._schema("terminal-outcome-v1.schema.json", terminal)
- self.assertIs(contracts.validate_terminal_document(terminal), terminal)
-
- def reject(document: dict) -> None:
- with self.assertRaises(publisher.PublisherError):
- publisher._schema("terminal-outcome-v1.schema.json", document)
- with self.assertRaises(contracts.ContractError):
- contracts.validate_terminal_document(document)
-
- for path, invalid in (
- (("outcome", "failure_mode"), "Not Safe"),
- (("outcome", "reason"), "x" * 241),
- (("provenance", "source"), "Not Safe"),
- (("provenance", "git_run", "ref"), ""),
- ):
- with self.subTest(path=path):
- broken = copy.deepcopy(terminal)
- parent = broken
- for part in path[:-1]:
- parent = parent[part]
- parent[path[-1]] = invalid
- reject(broken)
-
- def test_invalid_retry_is_quarantined_before_valid_retry_upload(self) -> None:
- raw, samples = _native_fixture()
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary).resolve()
- sample_bytes = contracts.canonical_json_bytes(samples)
- bad = copy.deepcopy(raw)
- bad["sample_artifact"].update({
- "path": "a01.samples.json", "bytes": len(sample_bytes),
- "sha256": hashlib.sha256(sample_bytes).hexdigest(),
- })
- bad["measurement"]["rows"][0]["token_rate_at_latency_percentile"]["p50"] *= 2
- (root / "a01.samples.json").write_bytes(sample_bytes)
- (root / "a01.json").write_bytes(contracts.canonical_json_bytes(bad))
- self.assertTrue(contracts.quarantine_invalid_attempt(root / "a01.json"))
- valid = copy.deepcopy(raw)
- valid["sample_artifact"].update({
- "path": "a02.samples.json", "bytes": len(sample_bytes),
- "sha256": hashlib.sha256(sample_bytes).hexdigest(),
- })
- (root / "a02.samples.json").write_bytes(sample_bytes)
- (root / "a02.json").write_bytes(contracts.canonical_json_bytes(valid))
- paths = sorted(str(path) for path in root.glob("*.json"))
- self.assertEqual(contracts.validate_attempt_paths(paths), 1)
- self.assertTrue((root / "a01.json.quarantine").is_file())
- self.assertTrue((root / "a01.samples.json.quarantine").is_file())
-
- def test_ingest_archives_without_publishing_a_channel(self) -> None:
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary).resolve()
- matrix, artifact = _unsupported_delivery(root)
- self.assertEqual(len(summarize.load_results(str(artifact), None, None)), 1)
- result = publisher.ingest_command(_args(root / "store", matrix, artifact))
- store = publisher.Store(root / "store")
- self.assertEqual(result["status"], "accepted")
- self.assertTrue((store.incoming / result["incoming_id"] / "COMPLETE").is_file())
- self.assertTrue((store.bundles / result["bundle_id"] / "COMPLETE").is_file())
- self.assertEqual(list(store.channels.iterdir()), [])
- self.assertEqual(list(store.datasets.iterdir()), [])
- self.assertEqual(os.stat(store.private).st_mode & 0o777, 0o700)
- self.assertEqual(os.stat(store.public).st_mode & 0o777, 0o755)
- self.assertEqual(os.stat(store.bundles / result["bundle_id"]).st_mode & 0o777, 0o500)
-
- def test_repeated_ingest_is_content_idempotent(self) -> None:
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary).resolve()
- matrix, artifact = _unsupported_delivery(root)
- args = _args(root / "store", matrix, artifact)
- first = publisher.ingest_command(args)
- store = publisher.Store(root / "store")
- second = publisher.ingest_command(args)
- self.assertEqual(second, first)
- self.assertEqual(len(list(store.incoming.iterdir())), 1)
- self.assertEqual(len(list(store.bundles.iterdir())), 1)
- self.assertEqual(len(list(store.datasets.iterdir())), 0)
- self.assertEqual(len(list(store.channels.iterdir())), 0)
- bundle = publisher.strict_load(
- store.bundles / first["bundle_id"] / "bundle.json"
- )
- terminal = publisher.strict_load(next(artifact.glob("*.json")))
- self.assertEqual(bundle["created_at"], terminal["generated_at"])
-
- def test_dataset_is_invariant_to_bundle_argument_order(self) -> None:
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary).resolve()
- store_root = root / "store"
- bundle_ids = []
- for run_id in (9, 11, 10):
- run = {**RUN, "run_id": str(run_id)}
- delivery = root / f"run-{run_id}"
- delivery.mkdir()
- matrix, artifact = _unsupported_delivery(delivery, run=run)
- result = publisher.ingest_command(
- _args(store_root, matrix, artifact, run=run)
- )
- bundle_ids.append(result["bundle_id"])
- datasets = [
- publisher.build_dataset(
- publisher.Store(store_root), order, promote=False,
- )
- for order in itertools.permutations(bundle_ids)
- ]
- self.assertTrue(all(dataset == datasets[0] for dataset in datasets[1:]))
- self.assertEqual(datasets[0]["generated_at"], "2026-07-04T00:00:00Z")
- selected = datasets[0]["coverage"][0]["selected_attempt_id"]
- selected_attempt = next(
- item for item in datasets[0]["attempts"]
- if item["attempt_id"] == selected
- )
- self.assertEqual(selected_attempt["run_id"], "11")
-
- def test_diagnostic_dataset_orders_reruns_by_run_attempt(self) -> None:
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary).resolve()
- store_root = root / "store"
- bundle_ids = []
- for run_attempt in (1, 2):
- run = {**RUN, "run_attempt": run_attempt}
- delivery = root / f"attempt-{run_attempt}"
- delivery.mkdir()
- matrix, artifact = _unsupported_delivery(delivery, run=run)
- result = publisher.ingest_command(
- _args(store_root, matrix, artifact, run=run)
- )
- bundle_ids.append(result["bundle_id"])
- dataset = publisher.build_dataset(
- publisher.Store(store_root), bundle_ids, promote=False
- )
- selected_id = dataset["coverage"][0]["selected_attempt_id"]
- selected = next(
- item for item in dataset["attempts"]
- if item["attempt_id"] == selected_id
- )
- self.assertEqual(selected["run_attempt"], 2)
-
- def test_promotion_requires_every_runnable_case_to_succeed_in_every_bundle(self) -> None:
- cases = {
- "runnable": {"_disposition": "runnable"},
- "planned-unsupported": {"_disposition": "unsupported"},
- }
- bundles = []
- for _ in range(3):
- runnable = {
- "identity": {"case_id": "runnable"},
- "outcome": {"status": "success"},
- }
- unsupported = {
- "identity": {"case_id": "planned-unsupported"},
- "outcome": {"status": "unsupported"},
- }
- bundles.append({
- "selected": {"runnable": runnable, "planned-unsupported": unsupported},
- "documents": {"runnable": runnable, "planned-unsupported": unsupported},
- })
- publisher._require_runnable_promotion_success(bundles, cases)
-
- for status in ("failed", "invalid", "unsupported", "diagnostic"):
- with self.subTest(status=status):
- broken = copy.deepcopy(bundles)
- broken[1]["selected"]["runnable"]["outcome"]["status"] = status
- with self.assertRaisesRegex(
- publisher.PublisherError, "every runnable matrix case"
- ):
- publisher._require_runnable_promotion_success(broken, cases)
-
- broken = copy.deepcopy(bundles)
- broken[1]["documents"]["retry"] = {
- "identity": {"case_id": "runnable"},
- "outcome": {"status": "failed"},
- }
- with self.assertRaisesRegex(publisher.PublisherError, "rejects runnable cases"):
- publisher._require_runnable_promotion_success(broken, cases)
-
- def test_promoted_public_dataset_rejects_failed_retry_history(self) -> None:
- dataset = _promoted_dataset()
- successful = next(
- item for item in dataset["attempts"]
- if item["outcome"] == "success"
- )
- failed = copy.deepcopy(successful)
- old_attempt_id = successful["attempt_id"]
- successful["attempt_index"] = 2
- successful["attempt_id"] = identity.attempt_id(
- allocation=successful["allocation_id"], case=successful["case_id"], ordinal=2
- )
- failed.update({
- "attempt_id": old_attempt_id,
- "attempt_index": 1,
- "outcome": "failed",
- "failure_mode": "execution",
- "reason": "execution-failed",
- "series_id": None,
- "selected": False,
- "evidence": [],
- })
- dataset["attempts"].append(failed)
- dataset["attempts"].sort(key=lambda item: item["attempt_id"])
- coverage = next(
- item for item in dataset["coverage"]
- if item["case_id"] == failed["case_id"]
- )
- coverage["attempt_ids"] = [
- successful["attempt_id"] if value == old_attempt_id else value
- for value in coverage["attempt_ids"]
- ]
- coverage["attempt_ids"].append(failed["attempt_id"])
- coverage["attempt_ids"].sort()
- if coverage["selected_attempt_id"] == old_attempt_id:
- coverage["selected_attempt_id"] = successful["attempt_id"]
-
- fixture_catalog = publisher._case_disposition_catalog_sha256(dataset["coverage"])
- with mock.patch.object(
- publisher, "CANONICAL_FULL_V1_CASE_CATALOG_SHA256", fixture_catalog
- ), self.assertRaisesRegex(publisher.PublisherError, "rejects runnable cases"):
- publisher.validate_public_dataset(dataset)
-
- def test_unselected_success_does_not_reference_an_unpublished_series(self) -> None:
- raw, _ = _native_fixture()
- retained = publisher._public_attempt(raw, selected=False)
- selected = publisher._public_attempt(raw, selected=True)
- self.assertEqual(retained["outcome"], "success")
- self.assertIsNone(retained["series_id"])
- self.assertEqual(selected["series_id"], raw["identity"]["series_id"])
-
- def test_public_dataset_selects_latest_derived_retry(self) -> None:
- dataset = _dataset()
- first = dataset["attempts"][0]
- second = copy.deepcopy(first)
- second.update({
- "attempt_id": identity.attempt_id(
- allocation=first["allocation_id"], case=first["case_id"], ordinal=2
- ),
- "attempt_index": 2,
- "selected": False,
- "series_id": None,
- "evidence": [],
- })
- dataset["attempts"].append(second)
- dataset["attempts"].sort(key=lambda item: item["attempt_id"])
- dataset["coverage"][0]["attempt_ids"].append(second["attempt_id"])
- dataset["coverage"][0]["attempt_ids"].sort()
- with self.assertRaisesRegex(publisher.PublisherError, "select the latest retry"):
- publisher.validate_public_dataset(dataset)
-
- second["attempt_id"] = identity.digest("attempt", {"not": "derived"})
- dataset["attempts"].sort(key=lambda item: item["attempt_id"])
- dataset["coverage"][0]["attempt_ids"] = [
- item["attempt_id"] for item in dataset["attempts"]
- ]
- with self.assertRaisesRegex(publisher.PublisherError, "retry identity differs"):
- publisher.validate_public_dataset(dataset)
-
- def test_promotion_requires_a_cohort_for_every_comparison_kind(self) -> None:
- stable_fast, stable_fast_internal = _series(
- "stable-fast", "deepep", decision_grade=True
- )
- stable_slow, stable_slow_internal = _series(
- "stable-slow", "uccl", decision_grade=True
- )
- unstable_fast, unstable_fast_internal = _series(
- "unstable-fast", "deepep", decision_grade=True
- )
- unstable_slow, unstable_slow_internal = _series(
- "unstable-slow", "uccl", decision_grade=True
- )
- unstable_fast["phase"] = unstable_slow["phase"] = "prefill"
- unstable_fast["series_id"] = identity.series_id({"test": "unstable-fast"})
- unstable_slow["series_id"] = identity.series_id({"test": "unstable-slow"})
- # A single-allocation cohort has no cross-run ordering to disagree, so the
- # prefill pair is made ineligible through missing trial evidence instead.
- del unstable_slow_internal["trial_blocks"]
- series = [stable_fast, stable_slow, unstable_fast, unstable_slow]
- internals = {
- stable_fast["series_id"]: stable_fast_internal,
- stable_slow["series_id"]: stable_slow_internal,
- unstable_fast["series_id"]: unstable_fast_internal,
- unstable_slow["series_id"]: unstable_slow_internal,
- }
- cohorts, _, _, _ = publisher.build_decisions(series, internals)
- eligible = [item for item in cohorts if item["eligibility"]["decision_grade"]]
- ineligible = [item for item in cohorts if not item["eligibility"]["decision_grade"]]
- self.assertEqual({item["kind"] for item in eligible}, {"library"})
- self.assertTrue(ineligible)
- anchor_series = [
- {
- "series_id": "uniform",
- "workload": {"routing": "uniform", "eplb": False},
- "build": {"implementation_contract_sha256": "1" * 64},
- }
- ]
- required = eligible + [
- {
- "kind": kind,
- "eligibility": {"decision_grade": True},
- }
- for kind in publisher.REQUIRED_COHORT_KINDS
- if kind != "library"
- ]
- with mock.patch.object(
- publisher, "REQUIRED_PROMOTION_COHORT_COUNTS", {}
- ), mock.patch.object(
- publisher, "_expected_chip_cohort_count", return_value=1
- ):
- publisher._require_promotion_cohorts(
- required + ineligible, anchor_series
- )
- # Promotion checks *structural* presence: a required comparison kind
- # must appear in the catalogued shape, regardless of its per-cohort
- # verdict. A kind is "missing" only when no cohort of that kind exists
- # at all — a diagnostic (non-decision-grade) cohort of the kind still
- # satisfies presence and ships with its verdict. So the missing-kind
- # error only fires when every cohort of the kind is removed.
- for kind in publisher.REQUIRED_COHORT_KINDS:
- with self.subTest(missing_kind=kind), self.assertRaisesRegex(
- publisher.PublisherError, rf"cohort kinds:.*{kind}"
- ):
- publisher._require_promotion_cohorts([
- item for item in required + ineligible
- if item["kind"] != kind
- ], anchor_series)
- # A kind present only as a diagnostic (non-decision-grade) cohort must
- # NOT veto promotion — it ships labelled as diagnostic. (The outer
- # patches already pin the fixed counts to {} and the chip count to 1.)
- diagnostic_only = [
- {"kind": kind, "eligibility": {"decision_grade": False}}
- for kind in publisher.REQUIRED_COHORT_KINDS
- ]
- publisher._require_promotion_cohorts(diagnostic_only, anchor_series)
-
- def test_promotion_requires_exact_counts(self) -> None:
- dataset = _promoted_dataset()
- counts = _cohort_counts(dataset)
- with mock.patch.object(
- publisher, "REQUIRED_PROMOTION_COHORT_COUNTS", counts
- ):
- publisher._require_promotion_cohorts(
- dataset["cohorts"], dataset["series"]
- )
-
- wrong_counts = {**counts, "library": counts["library"] + 1}
- with mock.patch.object(
- publisher, "REQUIRED_PROMOTION_COHORT_COUNTS", wrong_counts
- ), self.assertRaisesRegex(publisher.PublisherError, "exactly"):
- publisher._require_promotion_cohorts(
- dataset["cohorts"], dataset["series"]
- )
-
- def test_promotion_requires_every_derived_chip_cohort_present(self) -> None:
- # Promotion checks that the published shape carries *every* derived chip
- # cohort the series imply (a structural completeness check), but it no
- # longer requires each chip cohort to be decision-grade — a diagnostic
- # chip cohort ships labelled with its verdict, it does not veto the run.
- dataset = _promoted_dataset()
- chip = next(item for item in dataset["cohorts"] if item["kind"] == "chip")
- self.assertEqual(
- publisher._expected_chip_cohort_count(dataset["series"]),
- sum(item["kind"] == "chip" for item in dataset["cohorts"]),
- )
- with mock.patch.object(
- publisher, "REQUIRED_PROMOTION_COHORT_COUNTS", _cohort_counts(dataset)
- ):
- # Dropping a derived chip cohort leaves the shape structurally
- # incomplete → rejected.
- missing = [item for item in dataset["cohorts"] if item is not chip]
- with self.assertRaisesRegex(publisher.PublisherError, "derived chip cohorts"):
- publisher._require_promotion_cohorts(missing, dataset["series"])
-
- # A non-decision-grade (diagnostic) chip cohort keeps the shape
- # complete → promotion accepts it; the verdict rides along for the
- # consumer to surface or hide.
- chip["eligibility"]["decision_grade"] = False
- publisher._require_promotion_cohorts(
- dataset["cohorts"], dataset["series"]
- )
-
- def test_promotion_rejects_more_than_one_bundle(self) -> None:
- bundles = {
- str(run_id): {
- "id": str(run_id), "cases": [],
- "manifest": {
- "matrix": {"sha256": publisher.CANONICAL_FULL_V1_MATRIX_SHA256},
- "run": {
- "run_id": str(run_id), "run_attempt": 1,
- "qualification_index": 1,
- },
- },
- }
- for run_id in range(1, 3)
- }
- with mock.patch.object(
- publisher, "load_bundle", side_effect=lambda _, bundle_id: bundles[bundle_id]
- ), self.assertRaisesRegex(publisher.PublisherError, "qualification index 1"):
- publisher.build_dataset(object(), list(bundles), promote=True)
-
- dataset = _promoted_dataset()
- dataset["source_bundle_ids"].append("d" * 64)
- counts = _cohort_counts(dataset)
- with mock.patch.object(
- publisher,
- "CANONICAL_FULL_V1_CASE_CATALOG_SHA256",
- publisher._case_disposition_catalog_sha256(dataset["coverage"]),
- ), mock.patch.object(
- publisher, "REQUIRED_PROMOTION_COHORT_COUNTS", counts
- ), self.assertRaisesRegex(publisher.PublisherError, "complete coverage"):
- publisher.validate_public_dataset(dataset)
-
- def test_promotion_accepts_a_non_first_attempt_run(self) -> None:
- # Publication is no longer pinned to run attempt 1: a re-run whose legs
- # re-emit at a single consistent attempt (e.g. attempt 2) is eligible to
- # promote. The single-run gate must not reject a lone qualification-index-1
- # bundle merely because its run_attempt is 2 — it fails later on coverage,
- # not on the attempt.
- bundle = {
- "id": "1", "cases": [],
- "manifest": {
- "matrix": {"sha256": publisher.CANONICAL_FULL_V1_MATRIX_SHA256},
- "run": {
- "run_id": "1", "run_attempt": 2,
- "qualification_index": 1,
- },
- },
- }
- # The lone bundle passes the single-run/attempt gate and only fails
- # further downstream (the fixture omits documents/coverage), so the
- # failure is any post-gate error — never the attempt rejection.
- with mock.patch.object(
- publisher, "load_bundle", side_effect=lambda _, bundle_id: bundle
- ), self.assertRaises(Exception) as ctx:
- publisher.build_dataset(object(), ["1"], promote=True)
- self.assertNotIn("single versioned run", str(ctx.exception))
- self.assertNotIn("first-attempt", str(ctx.exception))
-
- def test_standalone_promotion_binds_matrix_and_requested_dispositions(self) -> None:
- dataset = _promoted_dataset()
- fixture_catalog = publisher._case_disposition_catalog_sha256(dataset["coverage"])
- with self.assertRaisesRegex(
- publisher.PublisherError, "canonical case/disposition catalog"
- ):
- publisher.validate_public_dataset(dataset)
- with mock.patch.object(
- publisher, "CANONICAL_FULL_V1_CASE_CATALOG_SHA256", fixture_catalog
- ), mock.patch.object(
- publisher,
- "REQUIRED_PROMOTION_COHORT_COUNTS",
- _cohort_counts(dataset),
- ):
- publisher.validate_public_dataset(dataset)
-
- # A diagnostic (non-decision-grade) series does NOT veto promotion — the
- # published dataset ships it labelled with its verdict, and the consumer
- # decides whether to surface it. Validation succeeds as long as at least
- # one decision-grade series remains (the honesty floor asserted below).
- diagnostic = copy.deepcopy(dataset)
- item = diagnostic["series"][0]
- item["status"] = "diagnostic"
- item["eligibility"].update({
- "decision_grade": False,
- "reasons": ["insufficient-allocations"],
- })
- with mock.patch.object(
- publisher, "CANONICAL_FULL_V1_CASE_CATALOG_SHA256", fixture_catalog
- ), mock.patch.object(
- publisher,
- "REQUIRED_PROMOTION_COHORT_COUNTS",
- _cohort_counts(dataset),
- ):
- validated = publisher.validate_public_dataset(diagnostic)
- shipped = next(
- series for series in validated["series"]
- if series["series_id"] == item["series_id"]
- )
- self.assertEqual(shipped["status"], "diagnostic")
- self.assertFalse(shipped["eligibility"]["decision_grade"])
- self.assertIn(
- "insufficient-allocations", shipped["eligibility"]["reasons"]
- )
-
- # Honesty floor: a dataset with NO decision-grade series is still rejected
- # — a promoted "v1" must carry at least one cleanly measured series.
- all_diagnostic = copy.deepcopy(dataset)
- for series in all_diagnostic["series"]:
- series["status"] = "diagnostic"
- series["eligibility"].update({
- "decision_grade": False,
- "reasons": ["insufficient-allocations"],
- })
- with mock.patch.object(
- publisher, "CANONICAL_FULL_V1_CASE_CATALOG_SHA256", fixture_catalog
- ), mock.patch.object(
- publisher,
- "REQUIRED_PROMOTION_COHORT_COUNTS",
- _cohort_counts(dataset),
- ), self.assertRaisesRegex(
- publisher.PublisherError, "at least one decision-grade series"
- ):
- publisher.validate_public_dataset(all_diagnostic)
-
- for original, replacement in (("runnable", "unsupported"),
- ("unsupported", "runnable")):
- with self.subTest(original=original):
- broken = copy.deepcopy(dataset)
- item = next(
- coverage for coverage in broken["coverage"]
- if coverage["disposition"] == original
- )
- item["disposition"] = replacement
- with mock.patch.object(
- publisher,
- "CANONICAL_FULL_V1_CASE_CATALOG_SHA256",
- publisher._case_disposition_catalog_sha256(broken["coverage"]),
- ), self.assertRaisesRegex(
- publisher.PublisherError,
- "requested dispositions" if original == "runnable"
- else "coverage dimensions",
- ):
- publisher.validate_public_dataset(broken)
-
- def test_workflow_matrix_and_catalog_digests_do_not_drift(self) -> None:
- with tempfile.TemporaryDirectory() as temporary:
- matrix_path = Path(temporary) / "matrix_full.json"
- result = subprocess.run(
- [
- sys.executable, str(ROOT / "sweep_matrix.py"),
- "--suites", "all", "--max-cases", "128",
- "--backends", "all", "--out", str(matrix_path),
- ],
- text=True,
- capture_output=True,
- )
- self.assertEqual(result.returncode, 0, result.stderr)
- if publisher.capability.provisional_precision_targets():
- workflow = (
- ROOT.parent.parent / ".github" / "workflows" / "collectivex-sweep.yml"
- ).read_text()
- self.assertIn(
- "V1 sweeps require every precision capability cell to be resolved",
- workflow,
- )
- return
- self.assertEqual(
- hashlib.sha256(matrix_path.read_bytes()).hexdigest(),
- publisher.CANONICAL_FULL_V1_MATRIX_SHA256,
- )
- matrix = contracts.strict_load(matrix_path)
- coverage = [
- {
- "case_id": item["case"]["case_id"],
- "disposition": item["disposition"],
- }
- for item in matrix["requested_cases"]
- ]
- self.assertEqual(
- publisher._case_disposition_catalog_sha256(coverage),
- publisher.CANONICAL_FULL_V1_CASE_CATALOG_SHA256,
- )
- self.assertEqual(
- (
- len(matrix["include"]), len(coverage),
- sum(item["disposition"] == "runnable" for item in coverage),
- sum(item["disposition"] == "unsupported" for item in coverage),
- sum(
- len(item["case"]["ladder"].split())
- for item in matrix["requested_cases"]
- ),
- sum(
- len(item["case"]["ladder"].split())
- for item in matrix["requested_cases"]
- if item["disposition"] == "runnable"
- ),
- sum(
- len(item["case"]["ladder"].split())
- for item in matrix["requested_cases"]
- if item["disposition"] == "unsupported"
- ),
- ),
- (51, 322, 167, 155, 1314, 673, 641),
- )
- library: dict[tuple, set[str]] = {}
- system: dict[tuple, set[str]] = {}
- for requested in matrix["requested_cases"]:
- if requested["disposition"] != "runnable":
- continue
- case = requested["case"]
- shape = tuple(
- case[field]
- for field in (
- "workload", "mode", "hidden", "topk", "experts", "ep", "phase"
- )
- )
- route = (case["routing"], case["eplb"])
- if case["backend"] != "nccl-ep":
- library.setdefault((requested["sku"], shape, route), set()).add(
- case["backend"]
- )
- else:
- system.setdefault((shape, route), set()).add(requested["sku"])
- self.assertEqual(
- {
- "library": sum(len(variants) >= 2 for variants in library.values()),
- "system": sum(len(variants) >= 2 for variants in system.values()),
- },
- publisher.REQUIRED_PROMOTION_COHORT_COUNTS,
- )
-
- def test_partial_promotion_binds_canonical_membership(self) -> None:
- # A "v1 tag + success" promotion may cover a subset of the canonical
- # matrix, but every covered case must be a real canonical case with the
- # canonical disposition, covered_skus must agree with the coverage, and
- # the subset must be a proper subset of the full catalog.
- dataset = _promoted_dataset()
- fixture_catalog = publisher._case_disposition_catalog_sha256(dataset["coverage"])
- cohort_counts = _cohort_counts(dataset)
-
- # covered_skus is checked for both scopes, before the scope branch.
- inconsistent = copy.deepcopy(dataset)
- inconsistent["promotion"]["covered_skus"] = ["nvidia-nonexistent"]
- with mock.patch.object(
- publisher, "CANONICAL_FULL_V1_CASE_CATALOG_SHA256", fixture_catalog
- ), mock.patch.object(
- publisher, "REQUIRED_PROMOTION_COHORT_COUNTS", cohort_counts
- ), self.assertRaisesRegex(
- publisher.PublisherError, "covered_skus is inconsistent"
- ):
- publisher.validate_public_dataset(inconsistent)
-
- # A partial promotion whose cases are not in the canonical catalog is
- # rejected (the synthetic fixture case ids are not canonical).
- partial = copy.deepcopy(dataset)
- partial["promotion"]["coverage_scope"] = "partial"
- partial["promotion"]["covered_skus"] = sorted(
- {item["sku"] for item in partial["coverage"]}
- )
- with self.assertRaisesRegex(
- publisher.PublisherError, "cases outside the canonical v1 catalog"
- ):
- publisher.validate_public_dataset(partial)
-
- # With the canonical catalog monkeypatched to a proper superset of the
- # covered cases (matching identities + dispositions), a partial
- # promotion validates without the full-matrix catalog pin or the
- # cohort-count requirement that gate a full promotion — and without
- # monkeypatching CANONICAL_FULL_V1_CASE_CATALOG_SHA256 or
- # REQUIRED_PROMOTION_COHORT_COUNTS at all.
- canonical = {
- item["case_id"]: _canonical_entry_from_coverage(item)
- for item in dataset["coverage"]
- }
- superset = dict(
- canonical,
- **{"canonical-case-not-in-this-run": dict(
- next(iter(canonical.values())), disposition="unsupported"
- )},
- )
- happy = copy.deepcopy(dataset)
- happy["promotion"]["coverage_scope"] = "partial"
- happy["promotion"]["covered_skus"] = sorted(
- {item["sku"] for item in happy["coverage"]}
- )
- happy["promotion"]["matrix_id"] = "e" * 64 # partial: matrix not pinned
- with mock.patch.object(
- publisher, "_canonical_coverage_cases", lambda: superset
- ):
- self.assertIs(publisher.validate_public_dataset(happy), happy)
-
- # A "partial" promotion that in fact covers the entire canonical catalog
- # must go through the full-coverage contract instead.
- entire = copy.deepcopy(happy)
- with mock.patch.object(
- publisher, "_canonical_coverage_cases", lambda: canonical
- ), self.assertRaisesRegex(
- publisher.PublisherError, "must not cover the entire canonical catalog"
- ):
- publisher.validate_public_dataset(entire)
-
- def test_rejection_is_quarantined_without_updating_dev_latest(self) -> None:
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary).resolve()
- matrix, artifact = _unsupported_delivery(root)
- store = publisher.Store(root / "store")
- sentinel = b"existing-promoted-pointer\n"
- (store.channels / "dev-latest.json").write_bytes(sentinel)
- (artifact / "unknown.json").write_text('{"format":"unknown"}')
- with self.assertRaises(publisher.PublisherError):
- publisher.ingest_command(_args(store.root, matrix, artifact))
- self.assertEqual((store.channels / "dev-latest.json").read_bytes(), sentinel)
- self.assertFalse((store.channels / "latest-attempt.json").exists())
- self.assertEqual(list(store.datasets.iterdir()), [])
- self.assertTrue(any(store.quarantine.iterdir()))
-
- def test_repeated_rejection_is_content_idempotent(self) -> None:
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary).resolve()
- matrix, artifact = _unsupported_delivery(root)
- store = publisher.Store(root / "store")
- (artifact / "unknown.json").write_text('{"format":"unknown"}')
- with self.assertRaises(publisher.PublisherError):
- publisher.ingest_command(_args(store.root, matrix, artifact))
- counts = tuple(
- len(list(path.iterdir()))
- for path in (store.incoming, store.quarantine, store.datasets, store.channels)
- )
- with self.assertRaises(publisher.PublisherError):
- publisher.ingest_command(_args(store.root, matrix, artifact))
- self.assertEqual(
- tuple(
- len(list(path.iterdir()))
- for path in (
- store.incoming, store.quarantine, store.datasets, store.channels
- )
- ),
- counts,
- )
-
- def test_distinct_rejections_create_distinct_quarantine_objects(self) -> None:
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary).resolve()
- matrix, artifact = _unsupported_delivery(root)
- store = publisher.Store(root / "store")
- unknown = artifact / "unknown.json"
- unknown.write_text('{"format":"unknown-one"}')
- with self.assertRaises(publisher.PublisherError):
- publisher.ingest_command(_args(store.root, matrix, artifact))
- first = {path.name for path in store.quarantine.iterdir()}
- unknown.write_text('{"format":"unknown-two"}')
- with self.assertRaises(publisher.PublisherError):
- publisher.ingest_command(_args(store.root, matrix, artifact))
- second = {path.name for path in store.quarantine.iterdir()}
- self.assertNotEqual(second, first)
- self.assertEqual(len(second), 2)
- self.assertEqual(list(store.datasets.iterdir()), [])
- self.assertEqual(list(store.channels.iterdir()), [])
-
- def test_zip_traversal_is_rejected(self) -> None:
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary).resolve()
- archive = root / "bad.zip"
- with zipfile.ZipFile(archive, "w") as handle:
- handle.writestr("../escape.json", "{}")
- with self.assertRaisesRegex(publisher.PublisherError, "escapes"):
- publisher.extract_archive(archive, root / "out")
-
- def test_store_and_directory_archive_reject_symlinks(self) -> None:
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary).resolve()
- real = root / "real"
- real.mkdir()
- alias = root / "alias"
- alias.symlink_to(real, target_is_directory=True)
- with self.assertRaisesRegex(publisher.PublisherError, "symlinked parent"):
- publisher.Store(alias / "store")
- self.assertFalse((real / "store").exists())
- artifact = root / f"cxunsupported-{RUN['run_id']}-{RUN['run_attempt']}"
- artifact.mkdir()
- target = root / "target.json"
- target.write_text("{}")
- (artifact / "linked.json").symlink_to(target)
- with self.assertRaisesRegex(publisher.PublisherError, "symlink"):
- publisher._archive_download_directory(artifact, root / "artifact.zip")
-
- def test_offline_caller_metadata_is_validated_before_store_creation(self) -> None:
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary).resolve()
- matrix, artifact = _unsupported_delivery(root)
- store_root = root / "store"
- args = _args(store_root, matrix, artifact)
- args.run_id = "0"
- with self.assertRaisesRegex(publisher.PublisherError, "run-id"):
- publisher.ingest_command(args)
- self.assertFalse(store_root.exists())
-
- promote = types.SimpleNamespace(
- store_root=str(store_root), bundle=["not-a-digest"]
- )
- with self.assertRaisesRegex(publisher.PublisherError, "bundle IDs"):
- publisher.promote_command(promote)
- self.assertFalse(store_root.exists())
- with self.assertRaisesRegex(publisher.PublisherError, "absolute path"):
- publisher._store_from_args(types.SimpleNamespace(store_root="relative-store"))
-
- def test_store_rejects_group_or_world_writable_root(self) -> None:
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary).resolve() / "unsafe-store"
- root.mkdir()
- root.chmod(0o772)
- with self.assertRaisesRegex(publisher.PublisherError, "group/world writable"):
- publisher.Store(root)
-
- def test_retry_ordinals_must_be_contiguous_from_one(self) -> None:
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary).resolve()
- matrix, artifact = _unsupported_delivery(root, (1, 3))
- with self.assertRaisesRegex(publisher.PublisherError, "contiguous ordinals"):
- publisher.ingest_command(_args(root / "store", matrix, artifact))
-
- def test_delivery_rejects_extra_archive_and_non_native_member(self) -> None:
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary).resolve()
- matrix, artifact = _unsupported_delivery(root)
- extra = root / f"cxshard-extra-{RUN['run_id']}-{RUN['run_attempt']}"
- extra.mkdir()
- (extra / "extra.json").write_text("{}")
- args = _args(root / "store-extra", matrix, artifact)
- args.artifact.append(str(extra))
- with self.assertRaisesRegex(publisher.PublisherError, "archive set"):
- publisher.ingest_command(args)
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary).resolve()
- matrix, artifact = _unsupported_delivery(root)
- (artifact / "notes.txt").write_text("not native evidence")
- with self.assertRaisesRegex(publisher.PublisherError, "unconsumed"):
- publisher.ingest_command(_args(root / "store-member", matrix, artifact))
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary).resolve()
- matrix, artifact = _unsupported_delivery(root)
- path = next(artifact.glob("*.json"))
- terminal = json.loads(path.read_text())
- terminal["outcome"]["reason"] = next(
- reason for reason in contracts.CAPABILITY_FAILURE_REASONS
- if reason != terminal["outcome"]["reason"]
- )
- path.write_text(json.dumps(terminal))
- with self.assertRaisesRegex(publisher.PublisherError, "reason differs"):
- publisher.ingest_command(_args(root / "store-reason", matrix, artifact))
-
- def test_rates_invert_latency_and_global_tokens_use_ep_size(self) -> None:
- dataset = _dataset()
- publisher.validate_public_dataset(dataset)
- rates = dataset["series"][0]["points"][0]["components"]["roundtrip"][
- "activation_data_rate_gbps_at_latency_percentile"
- ]
- self.assertGreater(rates["p50"], rates["p99"])
- broken = copy.deepcopy(dataset)
- broken["series"][0]["points"][0]["global_tokens"] = 128
- with self.assertRaisesRegex(publisher.PublisherError, "EP size"):
- publisher.validate_public_dataset(broken)
- broken = copy.deepcopy(dataset)
- broken["series"][0]["points"][0]["roundtrip_token_rate_at_latency_percentile"]["p99"] *= 2
- with self.assertRaisesRegex(publisher.PublisherError, "token throughput"):
- publisher.validate_public_dataset(broken)
- broken = copy.deepcopy(dataset)
- broken["attempts"][0]["evidence"][0]["point_id"] = identity.point_id(
- series=broken["series"][0]["series_id"], tokens_per_rank=16
- )
- with self.assertRaisesRegex(publisher.PublisherError, "point evidence"):
- publisher.validate_public_dataset(broken)
- broken = copy.deepcopy(dataset)
- broken["attempts"][0]["series_id"] = None
- with self.assertRaisesRegex(publisher.PublisherError, "present exactly for selected success"):
- publisher.validate_public_dataset(broken)
- broken = copy.deepcopy(dataset)
- component = broken["series"][0]["points"][0]["components"]["roundtrip"]
- component["activation_data_rate_gbps_at_latency_percentile"] = None
- with self.assertRaisesRegex(publisher.PublisherError, "measured data rates are missing"):
- publisher.validate_public_dataset(broken)
-
- for mutate in (
- lambda item: item.update({"model": "different-model"}),
- lambda item: item["workload"].update({"hidden": 4096}),
- lambda item: item["workload"].update({"top_k": 4}),
- lambda item: item["workload"].update({"experts": 128}),
- ):
- broken = copy.deepcopy(dataset)
- mutate(broken["series"][0])
- with self.assertRaisesRegex(publisher.PublisherError, "frozen v1"):
- publisher.validate_public_dataset(broken)
-
- broken = copy.deepcopy(dataset)
- broken["series"][0]["eplb"]["mapping_sha256"] = "f" * 64
- with self.assertRaisesRegex(publisher.PublisherError, "claims a plan"):
- publisher.validate_public_dataset(broken)
-
- broken = copy.deepcopy(dataset)
- broken["series"][0]["backend"].update({
- "id": "nccl-ep", "label": publisher.BACKEND_LABELS["nccl-ep"],
- "role": "reference", "generation": "rccl",
- })
- broken["coverage"][0]["backend"] = "nccl-ep"
- with self.assertRaisesRegex(publisher.PublisherError, "configuration"):
- publisher.validate_public_dataset(broken)
-
- def test_public_coverage_binds_exact_topology_and_case_identity(self) -> None:
- dataset = _promoted_dataset()
- dataset["promotion"]["status"] = "diagnostic"
- self.assertEqual(
- {item["disposition"] for item in dataset["coverage"]},
- {"runnable", "unsupported"},
- )
- for item in dataset["coverage"]:
- self.assertEqual(
- tuple(item["topology"]), publisher.COVERAGE_TOPOLOGY_FIELDS
- )
- publisher.validate_public_dataset(dataset)
-
- broken = copy.deepcopy(dataset)
- unsupported = next(
- item for item in broken["coverage"]
- if item["disposition"] == "unsupported"
- )
- unsupported["topology"]["nodes"] = 2
- with self.assertRaisesRegex(publisher.PublisherError, "capability registry"):
- publisher.validate_public_dataset(broken)
-
- broken = copy.deepcopy(dataset)
- unsupported = next(
- item for item in broken["coverage"]
- if item["disposition"] == "unsupported"
- )
- unsupported["sku"] = "mi325x"
- topology = publisher.capability.topology_for("mi325x", 8)
- self.assertIsNotNone(topology)
- unsupported["topology"] = publisher._coverage_topology({
- "ep_size": 8, **topology,
- })
- with self.assertRaisesRegex(publisher.PublisherError, "case identity"):
- publisher.validate_public_dataset(broken)
-
- def test_cohort_contract_and_labels_name_mode_explicitly(self) -> None:
- dataset = _promoted_dataset()
- dataset["promotion"]["status"] = "diagnostic"
- publisher.validate_public_dataset(dataset)
- for cohort in dataset["cohorts"]:
- self.assertIn("mode", cohort["controlled_factors"])
- self.assertIn("/ normal /", cohort["label"])
-
- broken = copy.deepcopy(dataset)
- cohort = broken["cohorts"][0]
- cohort["controlled_factors"].remove("mode")
- cohort["cohort_id"] = publisher._derived_id("cxcohort-v1-", {
- "kind": cohort["kind"], "series_ids": cohort["series_ids"],
- "controlled_factors": cohort["controlled_factors"],
- "varying_factors": cohort["varying_factors"],
- })
- broken["cohorts"].sort(key=lambda item: item["cohort_id"])
- with self.assertRaisesRegex(publisher.PublisherError, "cohort factors"):
- publisher.validate_public_dataset(broken)
-
- def test_routing_and_eplb_facts_must_match_across_repeats(self) -> None:
- raw, _ = _native_fixture()
- descriptor = publisher._eplb_descriptor(raw)
- facts = publisher._routing_facts(raw["measurement"]["rows"][0])
- self.assertEqual(
- publisher._exact_repeat_value([descriptor, copy.deepcopy(descriptor)], "EPLB"),
- descriptor,
- )
- self.assertEqual(
- publisher._exact_repeat_value([facts, copy.deepcopy(facts)], "routing"),
- facts,
- )
- changed = copy.deepcopy(facts)
- changed["hotspot_ratio"] += 0.1
- with self.assertRaisesRegex(publisher.PublisherError, "routing differs"):
- publisher._exact_repeat_value([facts, changed], "routing")
-
- def test_publisher_owns_stable_rankings_and_recommendations(self) -> None:
- fast, fast_internal = _series("fast", "deepep", decision_grade=True)
- slow, slow_internal = _series("slow", "uccl", decision_grade=True)
- reference, reference_internal = _series("reference", "nccl-ep", decision_grade=True)
- reference_peer, reference_peer_internal = _series(
- "reference-peer", "nccl-ep", decision_grade=True
- )
- reference["backend"]["role"] = "reference"
- reference_peer["backend"]["role"] = "reference"
- reference_peer["system"].update({"sku": "h200-dgxc", "label": "NVIDIA H200"})
- cohorts, rankings, recommendations, _ = publisher.build_decisions(
- [fast, slow, reference, reference_peer], {
- fast["series_id"]: fast_internal,
- slow["series_id"]: slow_internal,
- reference["series_id"]: reference_internal,
- reference_peer["series_id"]: reference_peer_internal,
- }
- )
- library = next(item for item in cohorts if item["kind"] == "library")
- ranking = next(item for item in rankings if item["cohort_id"] == library["cohort_id"]
- and item["metric"]["measure"] == "latency_us"
- and item["metric"]["statistic"] == "p99")
- self.assertTrue(library["eligibility"]["decision_grade"])
- self.assertEqual(ranking["entries"][0]["series_id"], fast["series_id"])
- self.assertTrue(any(item["series_id"] == fast["series_id"] for item in recommendations))
- self.assertFalse(any(
- entry["series_id"] == reference["series_id"]
- for item in rankings if item["cohort_id"] == library["cohort_id"]
- for entry in item["entries"]
- ))
- self.assertTrue(any(
- item["kind"] == "system" and reference["series_id"] in item["series_ids"]
- for item in cohorts
- ))
-
- def test_promoted_series_fields_are_bound_to_case_and_series_identities(self) -> None:
- dataset = _promoted_dataset()
- changed = copy.deepcopy(dataset)
- series = next(
- item for item in changed["series"]
- if item["system"]["sku"] == "h100-dgxc"
- )
- series["system"].update({
- "sku": "h200-dgxc", "label": "NVIDIA H200",
- "topology_class": "h200-nvlink-island",
- })
- for case_id in series["case_ids"]:
- coverage = next(
- item for item in changed["coverage"] if item["case_id"] == case_id
- )
- coverage["sku"] = "h200-dgxc"
- coverage["topology"] = publisher._coverage_topology(series["system"])
- with self.assertRaisesRegex(publisher.PublisherError, "configuration|case identity"):
- publisher.validate_public_dataset(changed)
-
- for field, value in (
- ("source_sha", "b" * 40),
- ("image_digest", "sha256:" + "4" * 64),
- ("squash_sha256", "5" * 64),
- ("runtime_fingerprint_sha256", "6" * 64),
- ("implementation_contract_sha256", "7" * 64),
- ("public_config_sha256", "9" * 64),
- ("routing_control_sha256", "8" * 64),
- ):
- changed = copy.deepcopy(dataset)
- changed["series"][0]["build"][field] = value
- with self.subTest(build_field=field), self.assertRaisesRegex(
- publisher.PublisherError, "commit"
- ):
- publisher.validate_public_dataset(changed)
- changed = copy.deepcopy(dataset)
- changed["series"][0]["workload"]["workload_id"] = identity.workload_id(
- {"changed": True}
- )
- with self.assertRaisesRegex(publisher.PublisherError, "committed factors"):
- publisher.validate_public_dataset(changed)
-
- for mutate, message in (
- (lambda item: item["backend"].update({
- "generation": "fabricated", "version": "fabricated-999",
- }), "configuration"),
- (lambda item: item["resource"].update({
- "profile": "profile-fabricated", "configured_units": 99,
- }), "configuration"),
- (lambda item: item["system"].update({"label": "Fabricated H100"}), "projection|commit"),
- ):
- changed = copy.deepcopy(dataset)
- mutate(changed["series"][0])
- with self.assertRaisesRegex(publisher.PublisherError, message):
- publisher.validate_public_dataset(changed)
-
- diagnostic = _dataset()
- diagnostic["series"][0]["build"]["source_sha"] = "b" * 40
- with self.assertRaisesRegex(publisher.PublisherError, "committed factors"):
- publisher.validate_public_dataset(diagnostic)
-
- def test_all_decision_metrics_require_stable_repeat_ordering(self) -> None:
- fast, fast_internal = _series("ordering-fast", "deepep", decision_grade=True)
- slow, slow_internal = _series("ordering-slow", "uccl", decision_grade=True)
- internals = {
- fast["series_id"]: fast_internal,
- slow["series_id"]: slow_internal,
- }
-
- cohorts, rankings, recommendations, _ = publisher.build_decisions(
- [fast, slow], internals
- )
- library = next(item for item in cohorts if item["kind"] == "library")
- self.assertTrue(library["eligibility"]["decision_grade"])
- self.assertTrue(library["eligibility"]["stable_ordering"])
- self.assertEqual(
- len([item for item in rankings if item["cohort_id"] == library["cohort_id"]]),
- 6,
- )
- self.assertEqual(
- len([
- item for item in recommendations
- if item["cohort_id"] == library["cohort_id"]
- ]),
- 1,
- )
-
- # A cohort whose single run is missing trial evidence cannot be ranked.
- del slow_internal["trial_blocks"]
- cohorts, rankings, recommendations, _ = publisher.build_decisions(
- [fast, slow], internals
- )
- library = next(item for item in cohorts if item["kind"] == "library")
- self.assertFalse(library["eligibility"]["decision_grade"])
- self.assertIn("missing-trial-blocks", library["eligibility"]["reasons"])
- self.assertFalse(any(
- item["cohort_id"] == library["cohort_id"] for item in rankings
- ))
- self.assertFalse(any(
- item["cohort_id"] == library["cohort_id"] for item in recommendations
- ))
-
- def test_p99_bootstrap_is_deterministic_and_dataset_bound(self) -> None:
- fast, fast_internal = _series("bootstrap-fast", "deepep", decision_grade=True)
- slow, slow_internal = _series("bootstrap-slow", "uccl", decision_grade=True)
- internals = {
- fast["series_id"]: fast_internal,
- slow["series_id"]: slow_internal,
- }
-
- first = publisher._hierarchical_p99_ratio(
- fast["series_id"], slow["series_id"], 8, internals, "a" * 64
- )
- repeated = publisher._hierarchical_p99_ratio(
- fast["series_id"], slow["series_id"], 8, internals, "a" * 64
- )
- rebound = publisher._hierarchical_p99_ratio(
- fast["series_id"], slow["series_id"], 8, internals, "b" * 64
- )
-
- self.assertEqual(first, repeated)
- self.assertEqual(first["resamples"], 10_000)
- self.assertEqual(first["confidence"], 0.95)
- self.assertEqual(first["equivalence_band"], 0.05)
- self.assertTrue(first["all_runs_agree"])
- self.assertTrue(first["baseline_wins"])
- self.assertGreater(first["ci95"][0], 1.05)
- self.assertNotEqual(first["seed_sha256"], rebound["seed_sha256"])
-
- def test_p99_equivalence_band_emits_competition_tie_without_recommendation(self) -> None:
- fast, fast_internal = _series("tie-fast", "deepep", decision_grade=True)
- near, near_internal = _series("tie-near", "uccl", decision_grade=True)
- fast_point = fast["points"][0]
- near_point = near["points"][0]
- fast_component = fast_point["components"]["roundtrip"]
- near_component = near_point["components"]["roundtrip"]
- for statistic, latency in fast_component["latency_us"].items():
- near_latency = latency * 1.03
- near_component["latency_us"][statistic] = near_latency
- for field, byte_field in (
- ("activation_data_rate_gbps_at_latency_percentile", "activation_data_bytes"),
- ("total_logical_data_rate_gbps_at_latency_percentile", "total_logical_bytes"),
- ):
- near_component[field][statistic] = (
- near_component["byte_provenance"][byte_field]
- / (near_latency * 1000.0)
- )
- near_point["roundtrip_token_rate_at_latency_percentile"][statistic] = (
- near_point["global_tokens"] / (near_latency * 1e-6)
- )
- for run_id, fast_metrics in fast_internal["run_metrics"].items():
- for statistic in ("p50", "p99"):
- latency = fast_metrics[8]["latency_us"][statistic] * 1.03
- near_internal["run_metrics"][run_id][8]["latency_us"][statistic] = latency
- for field, byte_field in (
- ("activation_data_rate_gbps_at_latency_percentile", "activation_data_bytes"),
- ("total_logical_data_rate_gbps_at_latency_percentile", "total_logical_bytes"),
- ):
- near_internal["run_metrics"][run_id][8][field][statistic] = (
- near_component["byte_provenance"][byte_field]
- / (latency * 1000.0)
- )
- near_internal["trial_blocks"][run_id][8]["roundtrip"] = tuple(
- tuple(sample * 1.03 for sample in block)
- for block in fast_internal["trial_blocks"][run_id][8]["roundtrip"]
- )
- internals = {
- fast["series_id"]: fast_internal,
- near["series_id"]: near_internal,
- }
-
- cohorts, rankings, recommendations, _ = publisher.build_decisions(
- [fast, near], internals, dataset_binding="c" * 64
- )
- library = next(item for item in cohorts if item["kind"] == "library")
- ranking = next(
- item for item in rankings
- if item["cohort_id"] == library["cohort_id"]
- and item["metric"]["measure"] == "latency_us"
- and item["metric"]["statistic"] == "p99"
- )
- self.assertEqual([entry["rank"] for entry in ranking["entries"]], [1, 1])
- self.assertFalse(any(
- item["cohort_id"] == library["cohort_id"]
- for item in recommendations
- ))
- self.assertNotIn(
- "trial_blocks", json.dumps({"series": [fast, near], "rankings": rankings})
- )
-
- def test_p99_winner_requires_every_run_to_agree(self) -> None:
- fast, fast_internal = _series("run-fast", "deepep", decision_grade=True)
- slow, slow_internal = _series("run-slow", "uccl", decision_grade=True)
- ratios = {"0": 0.98}
- for run_id, ratio in ratios.items():
- slow_internal["trial_blocks"][run_id][8]["roundtrip"] = tuple(
- tuple(sample * ratio for sample in block)
- for block in fast_internal["trial_blocks"][run_id][8]["roundtrip"]
- )
- result = publisher._hierarchical_p99_ratio(
- fast["series_id"], slow["series_id"], 8,
- {
- fast["series_id"]: fast_internal,
- slow["series_id"]: slow_internal,
- },
- "d" * 64,
- )
- self.assertFalse(result["all_runs_agree"])
- self.assertFalse(result["baseline_wins"])
- self.assertTrue(result["tie"])
-
- def test_p99_tie_set_does_not_include_decisively_beaten_middle_series(self) -> None:
- fast, fast_internal = _series("tie-fast", "deepep", decision_grade=True)
- middle, middle_internal = _series("tie-middle", "uccl", decision_grade=True)
- uncertain, uncertain_internal = _series(
- "tie-uncertain", "nccl-ep", decision_grade=True
- )
- for index, item in enumerate((fast, middle, uncertain), 1):
- item["points"][0]["components"]["roundtrip"]["latency_us"]["p99"] = float(index)
- outcomes = {
- middle["series_id"]: {"baseline_wins": True},
- uncertain["series_id"]: {"baseline_wins": False},
- }
- with mock.patch.object(
- publisher,
- "_hierarchical_p99_ratio",
- side_effect=lambda _baseline, candidate, *_args: outcomes[candidate],
- ):
- tie_ids = publisher._p99_top_tie_ids(
- [fast, middle, uncertain],
- {
- fast["series_id"]: fast_internal,
- middle["series_id"]: middle_internal,
- uncertain["series_id"]: uncertain_internal,
- },
- 8,
- "d" * 64,
- publisher._derived_id(
- "cxcohort-v1-", {"fixture": "noncontiguous-tie"}
- ),
- )
- self.assertEqual(tie_ids, {fast["series_id"], uncertain["series_id"]})
-
- def test_precision_cohorts_isolate_axes_and_never_recommend(self) -> None:
- profiles = (
- identity.V1_CONTROL_PRECISION_PROFILE,
- "d-fp8-e4m3fn-b128-f32-prequantized.c-bf16",
- "d-bf16.c-fp8-e4m3fn-direct-cast-noscale",
- "d-fp8-e4m3fn-b128-f32-prequantized.c-fp8-e4m3fn-direct-cast-noscale",
- )
- series = []
- internals = {}
- for index, profile_id in enumerate(profiles):
- item, internal = _series(
- f"precision-{index}", "deepep", decision_grade=True
- )
- precision = identity.precision_profile(profile_id)
- item["suite"] = (
- "ep-core-v1"
- if index == 0
- else "ep-precision-normal-v1"
- )
- item["mode"] = "normal"
- item["publication_tier"] = (
- "official" if index == 0 else "comparable-experimental"
- )
- item["workload"].update({
- "precision_profile": profile_id,
- "dispatch_precision": precision["dispatch"],
- "combine_precision": precision["combine"],
- })
- item["series_id"] = identity.series_id({"precision-fixture": profile_id})
- series.append(item)
- internals[item["series_id"]] = internal
-
- cohorts, rankings, recommendations, sensitivities = publisher.build_decisions(
- series, internals, dataset_binding="e" * 64
- )
- precision_cohorts = [
- cohort for cohort in cohorts
- if cohort["kind"] in publisher.PRECISION_COHORT_KINDS
- ]
- self.assertEqual(
- {kind: sum(cohort["kind"] == kind for cohort in precision_cohorts)
- for kind in publisher.PRECISION_COHORT_KINDS},
- {"dispatch-precision": 2, "combine-precision": 2, "precision-pair": 1},
- )
- self.assertTrue(all(
- cohort["publication_tier"] == "comparable-experimental"
- and cohort["eligibility"]["decision_grade"]
- for cohort in precision_cohorts
- ))
- self.assertEqual(len(rankings), 24)
- self.assertEqual(len(sensitivities), 24)
- self.assertEqual(recommendations, [])
- pair = next(
- cohort for cohort in precision_cohorts
- if cohort["kind"] == "precision-pair"
- )
- self.assertEqual(
- pair["varying_factors"],
- [
- "dispatch-precision", "combine-precision", "precision-profile",
- "resource",
- ],
- )
- self.assertNotIn("resource", pair["controlled_factors"])
- self.assertFalse(any(
- sensitivity["cohort_id"] == pair["cohort_id"]
- for sensitivity in sensitivities
- ))
- self.assertFalse(any(
- ranking["cohort_id"] == pair["cohort_id"]
- for ranking in rankings
- ))
-
- def test_private_trial_copy_is_component_extensible(self) -> None:
- blocks = [[float(trial + iteration + 1) for iteration in range(8)]
- for trial in range(64)]
- copied = publisher._private_trial_components({
- "points": [{
- "tokens_per_rank": 8,
- "components": {
- "roundtrip": {"availability": "measured", "trials": blocks},
- "stage": {"availability": "measured", "trials": blocks},
- "combine": {"availability": "not-applicable", "trials": None},
- },
- }],
- })
- self.assertEqual(set(copied[8]), {"roundtrip", "stage", "combine"})
- self.assertEqual(len(copied[8]["stage"]), 64)
- self.assertIsNone(copied[8]["combine"])
-
- def test_missing_private_trials_blocks_decision_grade(self) -> None:
- fast, fast_internal = _series("trials-fast", "deepep", decision_grade=True)
- slow, slow_internal = _series("trials-slow", "uccl", decision_grade=True)
- del slow_internal["trial_blocks"]
- cohorts, rankings, recommendations, sensitivities = publisher.build_decisions(
- [fast, slow], {
- fast["series_id"]: fast_internal,
- slow["series_id"]: slow_internal,
- }
- )
- library = next(item for item in cohorts if item["kind"] == "library")
- self.assertFalse(library["eligibility"]["decision_grade"])
- self.assertIn("missing-trial-blocks", library["eligibility"]["reasons"])
- self.assertEqual((rankings, recommendations, sensitivities), ([], [], []))
-
- def test_extra_eligibility_reason_blocks_decision_grade(self) -> None:
- allocations = [identity.allocation_id({"run": run}) for run in range(1)]
- eligibility = publisher._eligibility_record(
- allocations, complete=True, correct=True, measured=True,
- stable_ordering=True,
- extra_reasons=["incomplete-provenance"],
- )
- self.assertFalse(eligibility["decision_grade"])
- self.assertEqual(eligibility["reasons"], ["incomplete-provenance"])
- self.assertIs(publisher._eligibility(eligibility, "fixture"), eligibility)
- broken = {**eligibility, "decision_grade": True}
- with self.assertRaisesRegex(publisher.PublisherError, "promotion gates"):
- publisher._eligibility(broken, "fixture")
-
- def test_schema_is_strict_and_channel_target_must_be_complete(self) -> None:
- dataset = _dataset()
- dataset["unexpected"] = True
- with self.assertRaises(publisher.PublisherError):
- publisher.validate_public_dataset(dataset)
- with mock.patch.object(publisher, "MAX_PUBLIC_DATASET_BYTES", 1), self.assertRaisesRegex(
- publisher.PublisherError, "serving size limit"
- ):
- publisher.validate_public_dataset(_dataset())
- with tempfile.TemporaryDirectory() as temporary:
- store = publisher.Store(Path(temporary).resolve())
- dataset = _promoted_dataset()
- with mock.patch.object(
- publisher, "CANONICAL_FULL_V1_CASE_CATALOG_SHA256",
- publisher._case_disposition_catalog_sha256(dataset["coverage"]),
- ), mock.patch.object(
- publisher, "REQUIRED_PROMOTION_COHORT_COUNTS", _cohort_counts(dataset),
- ):
- digest, size = store.install_dataset(dataset)
- store.update_channel("dev-latest", digest, size, dataset["generated_at"])
- self.assertEqual(
- store.verify_channel("dev-latest")["dataset"]["sha256"], digest
- )
- channel_path = store.channels / "dev-latest.json"
- pointer = publisher.strict_load(channel_path)
- pointer["generated_at"] = "2099-01-01T00:00:00Z"
- channel_path.write_bytes(contracts.canonical_json_bytes(pointer))
- with self.assertRaisesRegex(publisher.PublisherError, "metadata differs"):
- store.verify_channel("dev-latest")
- store.update_channel("dev-latest", digest, size, dataset["generated_at"])
- with self.assertRaisesRegex(publisher.PublisherError, "metadata differs"):
- store.update_channel(
- "dev-latest", digest, size + 1, dataset["generated_at"]
- )
- with self.assertRaisesRegex(publisher.PublisherError, "metadata differs"):
- store.update_channel(
- "dev-latest", digest, size, "2026-07-05T00:00:00Z"
- )
- os.chmod(channel_path, 0o666)
- with self.assertRaisesRegex(publisher.PublisherError, "regular 644"):
- store.verify_channel("dev-latest")
- os.chmod(channel_path, 0o644)
- dataset_dir = store.datasets / digest
- os.chmod(dataset_dir, 0o755)
- with self.assertRaisesRegex(publisher.PublisherError, "mode differs"):
- store.verify_channel("dev-latest")
- os.chmod(dataset_dir, 0o555)
- os.chmod(dataset_dir / "dataset.json", 0o644)
- with self.assertRaisesRegex(publisher.PublisherError, "mode differs"):
- store.verify_channel("dev-latest")
- os.chmod(dataset_dir / "dataset.json", 0o444)
- os.chmod(dataset_dir, 0o755)
- (dataset_dir / "COMPLETE").unlink()
- os.chmod(dataset_dir, 0o555)
- with self.assertRaisesRegex(publisher.PublisherError, "incomplete"):
- store.verify_channel("dev-latest")
-
- def test_store_modes_do_not_depend_on_process_umask(self) -> None:
- with tempfile.TemporaryDirectory() as temporary:
- previous = os.umask(0o077)
- try:
- store = publisher.Store(Path(temporary).resolve())
- dataset = _promoted_dataset()
- with mock.patch.object(
- publisher, "CANONICAL_FULL_V1_CASE_CATALOG_SHA256",
- publisher._case_disposition_catalog_sha256(dataset["coverage"]),
- ), mock.patch.object(
- publisher, "REQUIRED_PROMOTION_COHORT_COUNTS",
- _cohort_counts(dataset),
- ):
- digest, size = store.install_dataset(dataset)
- store.update_channel(
- "dev-latest", digest, size, dataset["generated_at"]
- )
- with store.locked():
- pass
- finally:
- os.umask(previous)
- self.assertEqual(
- store.root.stat().st_mode & 0o777,
- 0o750,
- )
- self.assertEqual(
- (store.channels / "dev-latest.json").stat().st_mode & 0o777,
- 0o644,
- )
- self.assertEqual(
- (store.datasets / digest / "dataset.json").stat().st_mode & 0o777,
- 0o444,
- )
- self.assertEqual(
- (store.locks / "publisher.lock").stat().st_mode & 0o777,
- 0o600,
- )
-
- def test_verify_requires_a_promoted_dev_latest_channel(self) -> None:
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary).resolve()
- args = types.SimpleNamespace(
- store_root=str(root / "store"), channel=None, bundle=[]
- )
- with self.assertRaises(publisher.PublisherError):
- publisher.verify_command(args)
- store = publisher.Store(args.store_root)
- dataset = _promoted_dataset()
- with mock.patch.object(
- publisher, "CANONICAL_FULL_V1_CASE_CATALOG_SHA256",
- publisher._case_disposition_catalog_sha256(dataset["coverage"]),
- ), mock.patch.object(
- publisher, "REQUIRED_PROMOTION_COHORT_COUNTS", _cohort_counts(dataset),
- ):
- digest, size = store.install_dataset(dataset)
- store.update_channel(
- "dev-latest", digest, size, dataset["generated_at"]
- )
- result = publisher.verify_command(args)
- self.assertEqual(set(result["channels"]), {"dev-latest"})
- explicit = types.SimpleNamespace(
- store_root=args.store_root, channel=["dev-latest"], bundle=[]
- )
- self.assertEqual(
- publisher.verify_command(explicit)["channels"], result["channels"]
- )
- unknown = types.SimpleNamespace(
- store_root=args.store_root, channel=["latest-attempt"], bundle=[]
- )
- with self.assertRaisesRegex(publisher.PublisherError, "unknown channel"):
- publisher.verify_command(unknown)
-
-
-if __name__ == "__main__":
- unittest.main()
diff --git a/experimental/CollectiveX/tests/test_sampling_contract.py b/experimental/CollectiveX/tests/test_sampling_contract.py
deleted file mode 100644
index b64f65ad60..0000000000
--- a/experimental/CollectiveX/tests/test_sampling_contract.py
+++ /dev/null
@@ -1,4434 +0,0 @@
-#!/usr/bin/env python3
-"""CPU-only behavioral tests for the CollectiveX v1 execution contract."""
-from __future__ import annotations
-
-import argparse
-import ast
-import copy
-from collections import Counter
-import hashlib
-import io
-import json
-import os
-from pathlib import Path
-import re
-import stat
-import subprocess
-import sys
-import tarfile
-import tempfile
-import types
-import unittest
-from unittest import mock
-
-import numpy as np
-
-HERE = Path(__file__).resolve().parent
-ROOT = HERE.parent
-sys.path[:0] = [str(ROOT), str(HERE)]
-
-import artifact_safety # noqa: E402
-import capability # noqa: E402
-import contracts # noqa: E402
-import eplb # noqa: E402
-import ep_harness # noqa: E402
-import identity # noqa: E402
-import run_ep # noqa: E402
-import source_archive # noqa: E402
-import summarize # noqa: E402
-import sweep_matrix # noqa: E402
-import workload # noqa: E402
-
-
-class SamplingContractTest(unittest.TestCase):
- def test_identity_and_fixed_sampling_profile(self) -> None:
- identity.verify_test_vector()
- self.assertTrue(identity.is_typed_id(identity.IDENTITY_TEST_VECTOR["series_id"], "series"))
- self.assertEqual(ep_harness.SAMPLING_CONTRACT, "fixed-512-v1")
- self.assertEqual(
- (
- ep_harness.TIMED_ITERS_PER_TRIAL,
- ep_harness.TRIALS_PER_POINT,
- ep_harness.TIMED_SAMPLES_PER_POINT,
- ep_harness.WARMUP_ITERS_PER_TRIAL,
- ),
- (8, 64, 512, 32),
- )
- self.assertEqual(identity.V1_CASE_PROFILE["activation_profile"], "canonical-counter-source-v4")
- self.assertEqual(
- identity.V1_CASE_PROFILE["activation_generator"],
- "collectivex-activation-counter-v4",
- )
- self.assertEqual(identity.V1_CASE_PROFILE["sampling_contract"], "fixed-512-v1")
- self.assertEqual(identity.V1_CASE_PROFILE["percentile_method"], "nearest-rank")
- self.assertEqual(
- identity.V1_CASE_PROFILE["rank_reduction"],
- "cross-rank-max-per-iteration",
- )
- self.assertEqual(
- identity.V1_CASE_PROFILE["oracle_contract"],
- "expert-specific-transform-v1",
- )
- self.assertEqual(
- set(identity.V1_CASE_PROFILES), {"normal", "low-latency"}
- )
- self.assertEqual(
- identity.V1_LOW_LATENCY_CASE_PROFILE["payload_unit"], "token-expert"
- )
- self.assertNotEqual(
- identity.digest("case", identity.V1_NORMAL_CASE_PROFILE),
- identity.digest("case", identity.V1_LOW_LATENCY_CASE_PROFILE),
- )
- parser = argparse.ArgumentParser()
- ep_harness.add_common_args(parser)
- args = parser.parse_args(
- [
- "--runner", "test", "--topology-class", "test",
- "--scope", "scale-up", "--scale-up-transport", "nvlink",
- "--out", "result.json",
- ]
- )
- self.assertEqual((args.iters, args.trials, args.warmup), (8, 64, 32))
- self.assertEqual(args.qualification_index, 1)
- with mock.patch.dict(os.environ, {"CX_QUALIFICATION_INDEX": "3"}):
- parser = argparse.ArgumentParser()
- ep_harness.add_common_args(parser)
- env_args = parser.parse_args(
- [
- "--runner", "test", "--topology-class", "test",
- "--scope", "scale-up", "--scale-up-transport", "nvlink",
- "--out", "result.json",
- ]
- )
- self.assertEqual(env_args.qualification_index, 3)
- for profile in ((8, 64, 32), (128, 4, 32), (8, 1, 4), (0, 64, 32)):
- with self.subTest(profile=profile):
- self.assertEqual(
- ep_harness.sampling_contract_error(*profile) is None,
- profile == (8, 64, 32),
- )
-
- def test_nearest_rank_percentiles_use_all_512_samples(self) -> None:
- samples = list(range(1, 513))
- self.assertEqual(ep_harness.percentile(samples, 50), 256)
- self.assertEqual(ep_harness.percentile(samples, 99), 507)
-
- def test_qualification_order_is_deterministic_and_position_balanced(self) -> None:
- values = [1, 2, 4, 8, 16, 32, 64, 128]
- for qualification_index in range(1, ep_harness.QUALIFICATION_RUNS + 1):
- orders = [
- ep_harness.qualification_order(values, qualification_index, trial)
- for trial in range(64)
- ]
- self.assertEqual(
- orders,
- [
- ep_harness.qualification_order(values, qualification_index, trial)
- for trial in range(64)
- ],
- )
- self.assertTrue(all(sorted(order) == values for order in orders))
- for position in range(len(values)):
- self.assertEqual(
- {value: sum(order[position] == value for order in orders) for value in values},
- {value: 8 for value in values},
- )
- with self.assertRaises(ValueError):
- ep_harness.qualification_order(values, 0, 0)
- with self.assertRaises(ValueError):
- ep_harness.qualification_order([1, 1], 1, 0)
-
- def test_sample_evidence_preserves_exact_trial_blocks(self) -> None:
- trials = [
- [float(trial * 8 + sample) for sample in range(8)]
- for trial in range(64)
- ]
- evidence = ep_harness.sampled_component_evidence(trials)
- self.assertEqual(evidence["availability"], "measured")
- self.assertEqual(evidence["sample_count"], 512)
- self.assertEqual(evidence["trials"], trials)
- self.assertIsNot(evidence["trials"], trials)
- self.assertEqual(
- ep_harness.sampled_component_evidence([]),
- {"availability": "unavailable", "sample_count": 0, "trials": None},
- )
- for malformed in (trials[:-1], [*trials[:-1], trials[-1][:-1]]):
- with self.assertRaises(ValueError):
- ep_harness.sampled_component_evidence(malformed)
- invalid = copy.deepcopy(trials)
- invalid[0][0] = float("nan")
- with self.assertRaises(ValueError):
- ep_harness.sampled_component_evidence(invalid)
-
- def test_terminal_summary_uses_bound_sku_and_route(self) -> None:
- terminal = {
- "format": contracts.TERMINAL_FORMAT,
- "case": {
- "backend": "deepep", "phase": "prefill", "ep": 8,
- "suite": "ep-precision-normal-v1", "routing": "uniform", "eplb": False,
- "required_publication": "comparable-experimental",
- },
- "identity": {"case_factors": {"sku": "h100-dgxc"}},
- }
- self.assertEqual(
- summarize._identity(terminal),
- (
- "h100-dgxc", "ep-precision-normal-v1", "uniform", "prefill", False,
- "comparable-experimental", 8,
- ),
- )
-
- def test_matrix_cases_and_shards_are_identity_bound(self) -> None:
- matrix = sweep_matrix.validate_matrix_document(
- sweep_matrix.resolve_matrix(backends="all")
- )
- requested = {item["case"]["case_id"]: item for item in matrix["requested_cases"]}
- assigned = [case_id for shard in matrix["include"] for case_id in shard["case_ids"]]
- runnable = {
- case_id for case_id, item in requested.items()
- if item["disposition"] == "runnable"
- }
- runnable_cases = [
- item for item in matrix["requested_cases"]
- if item["disposition"] == "runnable"
- ]
- unsupported_cases = [
- item for item in matrix["requested_cases"]
- if item["disposition"] == "unsupported"
- ]
- self.assertEqual(
- (
- len(matrix["include"]),
- len(matrix["requested_cases"]),
- len(runnable_cases),
- len(unsupported_cases),
- sum(
- len(item["case"]["ladder"].split())
- for item in matrix["requested_cases"]
- ),
- sum(len(item["case"]["ladder"].split()) for item in runnable_cases),
- sum(len(item["case"]["ladder"].split()) for item in unsupported_cases),
- ),
- (51, 322, 167, 155, 1314, 673, 641),
- )
- # B300 EP16 runs over RoCE: only nccl-ep and the DeepEP Hybrid control path
- # execute. DeepEP V1/V2 are walled (NVSHMEM-IBGDA needs GDRCopy /dev/gdrdrv,
- # unprovisioned on B300 hosts) so they must NOT appear among runnable cases.
- b300_ep16 = [
- item for item in runnable_cases
- if item["sku"] == "b300" and item["case"]["ep"] == 16
- and item["case"]["backend"] in {
- "deepep", "deepep-v2", "deepep-hybrid", "nccl-ep",
- }
- ]
- self.assertTrue(b300_ep16)
- self.assertEqual(
- {item["case"]["backend"] for item in b300_ep16},
- {"deepep-hybrid", "nccl-ep"},
- )
- expected_topologies = {}
- for sku, product in (
- ("h100-dgxc", "h100"), ("h200-dgxc", "h200"),
- ("b200-dgxc", "b200"), ("b300", "b300"),
- ):
- expected_topologies[sku, 8] = (
- 1, 8, 8, "scale-up", "nvlink", None, "nvlink",
- f"{product}-nvlink-island",
- )
- expected_topologies[sku, 16] = (
- 2, 8, 8, "scale-out", "nvlink", "rdma", "nvlink-rdma",
- f"{product}-nvlink-rdma",
- )
- for sku in ("gb200", "gb300"):
- topology_class = f"{sku}-nvl72-mnnvl"
- expected_topologies[sku, 8] = (
- 2, 4, 72, "scale-up", "mnnvl", None, "mnnvl", topology_class,
- )
- expected_topologies[sku, 16] = (
- 4, 4, 72, "scale-up", "mnnvl", None, "mnnvl", topology_class,
- )
- for sku in ("mi300x", "mi355x"):
- expected_topologies[sku, 8] = (
- 1, 8, 8, "scale-up", "xgmi", None, "xgmi", f"{sku}-xgmi",
- )
- expected_topologies[sku, 16] = (
- 2, 8, 8, "scale-out", "xgmi", "rdma", "xgmi-rdma",
- f"{sku}-xgmi-rdma",
- )
- topology_fields = sweep_matrix.TOPOLOGY_FIELDS
- observed_topologies: dict[tuple[str, int], set[tuple[object, ...]]] = {}
- for item in matrix["requested_cases"]:
- case = item["case"]
- observed_topologies.setdefault((item["sku"], case["ep"]), set()).add(
- tuple(case[field] for field in topology_fields)
- )
- self.assertEqual(
- {key: next(iter(values)) for key, values in observed_topologies.items()},
- expected_topologies,
- )
- self.assertTrue(all(len(values) == 1 for values in observed_topologies.values()))
- self.assertEqual(
- {
- (sku, ep): tuple(topology[field] for field in topology_fields)
- for sku, platform in capability.PLATFORMS.items()
- if sku != "mi325x"
- for ep, topology in platform["topologies"].items()
- },
- expected_topologies,
- )
- self.assertIsNotNone(capability.topology_for("mi325x", 8))
- self.assertEqual(
- Counter(shard["n"] for shard in matrix["include"]),
- Counter({2: 32, 6: 13, 4: 5, 5: 1}),
- )
- ll_cases = [
- item for item in matrix["requested_cases"]
- if item["case"]["mode"] == "low-latency"
- ]
- self.assertEqual(len(ll_cases), 48)
- self.assertTrue(all(
- item["case"]["backend"] in {"deepep", "uccl"}
- and item["case"]["phase"] == "decode"
- and item["case"]["routing"] == "uniform"
- and not item["case"]["eplb"]
- and (
- (
- item["case"]["suite"] == "ep-low-latency-v1"
- and item["case"]["ladder"] == "1 2 4 8 16 32 64 128"
- )
- or (
- item["case"]["suite"] == "ep-precision-low-latency-v1"
- and item["case"]["ladder"] == "128"
- )
- )
- for item in ll_cases
- ))
- for shard in matrix["include"]:
- ep = next(
- requested[case_id]["case"]["ep"] for case_id in shard["case_ids"]
- )
- self.assertEqual(
- tuple(shard[field] for field in topology_fields),
- expected_topologies[shard["sku"], ep],
- )
- skus = sorted({shard["sku"] for shard in matrix["include"]})
- self.assertEqual(
- [shard["sku"] for shard in matrix["include"][:len(skus)]],
- skus,
- )
- self.assertEqual(set(assigned), runnable)
- self.assertEqual(len(assigned), len(set(assigned)))
- self.assertEqual({item["case"]["ep"] for item in matrix["requested_cases"]}, {8, 16})
- self.assertFalse(capability.resolve("gb200", "deepep", ep=8, nodes=1)[0])
- excluded = {
- "uccl": {"b200-dgxc", "b300"},
- }
- for backend, skus in excluded.items():
- for sku in skus:
- with self.subTest(backend=backend, sku=sku):
- self.assertFalse(capability.resolve(sku, backend)[0])
- for case_id, item in requested.items():
- case = {key: value for key, value in item["case"].items() if key != "case_id"}
- self.assertEqual(
- case_id,
- identity.case_id(
- sku=item["sku"], profile=identity.profile_for_case(case), case=case
- ),
- )
- self.assertEqual(case["timing"], "8:64:32")
- self.assertEqual(case["samples_per_point"], 512)
-
- bad_matrix = copy.deepcopy(matrix)
- bad_matrix["schema_version"] = True
- with self.assertRaises(sweep_matrix.MatrixError):
- sweep_matrix.validate_matrix_document(bad_matrix)
-
- bad_catalog = copy.deepcopy(matrix)
- wrapper = next(
- item for item in bad_catalog["requested_cases"]
- if item["disposition"] == "runnable"
- )
- old_id = wrapper["case"]["case_id"]
- wrapper["case"]["hidden"] = 1
- factors = {key: value for key, value in wrapper["case"].items() if key != "case_id"}
- new_id = identity.case_id(
- sku=wrapper["sku"], profile=identity.V1_CASE_PROFILE, case=factors
- )
- wrapper["case"]["case_id"] = new_id
- for shard in bad_catalog["include"]:
- shard["case_ids"] = [new_id if value == old_id else value for value in shard["case_ids"]]
- with self.assertRaisesRegex(sweep_matrix.MatrixError, "frozen v1"):
- sweep_matrix.validate_matrix_document(bad_catalog)
-
- bad_topology = copy.deepcopy(matrix)
- wrapper = next(
- item for item in bad_topology["requested_cases"]
- if item["disposition"] == "runnable"
- )
- old_id = wrapper["case"]["case_id"]
- wrapper["case"]["transport"] = "incorrect-transport"
- factors = {key: value for key, value in wrapper["case"].items() if key != "case_id"}
- new_id = identity.case_id(
- sku=wrapper["sku"], profile=identity.V1_CASE_PROFILE, case=factors
- )
- wrapper["case"]["case_id"] = new_id
- for shard in bad_topology["include"]:
- shard["case_ids"] = [new_id if value == old_id else value for value in shard["case_ids"]]
- with self.assertRaisesRegex(sweep_matrix.MatrixError, "platform registry"):
- sweep_matrix.validate_matrix_document(bad_topology)
-
- shard_meta = matrix["include"][0]
- requested_cases = {item["case"]["case_id"]: item["case"] for item in matrix["requested_cases"]}
- shard = {
- "schema_version": True,
- "id": shard_meta["id"],
- "sku": shard_meta["sku"],
- "backend": shard_meta["backend"],
- "nodes": shard_meta["nodes"],
- "n": shard_meta["n"],
- "cases": [requested_cases[value] for value in shard_meta["case_ids"]],
- }
- with self.assertRaises(sweep_matrix.MatrixError):
- sweep_matrix.validate_shard_control(
- shard, sku=shard_meta["sku"], backend=shard_meta["backend"],
- nodes=shard_meta["nodes"],
- )
-
- def test_matrix_yaml_and_config_validation_are_strict(self) -> None:
- suites = sweep_matrix._load("suites.yaml")
- workloads = sweep_matrix._load("workloads.yaml")
- self.assertEqual(
- {tuple(suite["ep_degrees"]) for suite in suites["suites"].values()},
- {(8, 16)},
- )
- invalid = (
- ("unknown top", lambda s, _w: s.update({"typo": True})),
- (
- "unknown suite field",
- lambda s, _w: s["suites"]["ep-core-v1"].update({"modes": ["normal"]}),
- ),
- (
- "unknown workload field",
- lambda _s, w: w["model_derived"]["deepseek-v3-v1"].update({"unused": 1}),
- ),
- (
- "string phases",
- lambda s, _w: s["suites"]["ep-core-v1"].update({"phases": "decode"}),
- ),
- (
- "unknown routing",
- lambda s, _w: s["suites"]["ep-core-v1"].update({"routings": ["random"]}),
- ),
- (
- "integer EPLB",
- lambda s, _w: s["suites"]["ep-core-v1"].update({"eplb": [0, 1]}),
- ),
- (
- "duplicate platform",
- lambda s, _w: s["suites"]["ep-core-v1"]["platforms"].append("h100-dgxc"),
- ),
- (
- "missing EP degrees",
- lambda s, _w: s["suites"]["ep-core-v1"].pop("ep_degrees"),
- ),
- (
- "non-v1 EP degrees",
- lambda s, _w: s["suites"]["ep-core-v1"].update({"ep_degrees": [8]}),
- ),
- ("missing top field", lambda s, _w: s.pop("schema_version")),
- (
- "string dimension",
- lambda _s, w: w["model_derived"]["deepseek-v3-v1"].update({"hidden": "7168"}),
- ),
- (
- "unreachable phase ladder",
- lambda s, _w: s["suites"]["ep-core-v1"].update({"phases": ["decode"]}),
- ),
- )
- for label, mutate in invalid:
- with self.subTest(label=label), self.assertRaises(SystemExit):
- bad_suites, bad_workloads = copy.deepcopy(suites), copy.deepcopy(workloads)
- mutate(bad_suites, bad_workloads)
- sweep_matrix.validate_config_documents(bad_suites, bad_workloads)
-
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary)
- (root / "configs").mkdir()
- (root / "configs" / "duplicate.yaml").write_text(
- "schema_version: 1\nsuites:\n same: 1\n same: 2\n"
- )
- with mock.patch.object(sweep_matrix, "HERE", root), self.assertRaisesRegex(
- SystemExit, "duplicate YAML key"
- ):
- sweep_matrix._load("duplicate.yaml")
-
- def test_semantically_duplicate_suite_points_are_rejected(self) -> None:
- matrix = sweep_matrix.resolve_matrix()
- with mock.patch.object(
- sweep_matrix, "_semantic_points", return_value=["duplicate"]
- ), self.assertRaisesRegex(
- sweep_matrix.MatrixError, "duplicates a semantic token point"
- ):
- sweep_matrix.validate_matrix_document(matrix)
-
- def test_only_three_shared_launchers_are_registered(self) -> None:
- expected = {
- "launch_single-slurm.sh",
- "launch_gb-nv.sh",
- "launch_mi-amds.sh",
- }
- self.assertEqual({path.name for path in (ROOT / "launchers").glob("launch_*.sh")}, expected)
- self.assertEqual(
- {platform["launcher"] for platform in capability.PLATFORMS.values()},
- {"single-slurm", "gb-nv", "mi-amds"},
- )
- for platform in capability.PLATFORMS.values():
- launcher = ROOT / "launchers" / f"launch_{platform['launcher']}.sh"
- self.assertTrue(launcher.is_file())
- source = launcher.read_text()
- self.assertNotIn("RUNNER_NAME", source)
- self.assertIn("cx_preflight_allocation", source)
- lock_environment = 'cx_lock_canonical_gha_env "$RUNNER"'
- self.assertIn(lock_environment, source)
- self.assertLess(
- source.index("cx_load_operator_config"),
- source.index(lock_environment),
- )
- validate = 'cx_validate_shard_control "$CX_DIR"'
- stage = 'MOUNT_SRC="$(cx_stage_path '
- self.assertIn(validate, source)
- self.assertLess(source.index(validate), source.index(stage))
- self.assertLess(source.index(stage), source.index('cx_stage_repo "$REPO_ROOT"'))
- self.assertLess(source.index(validate), source.index("cx_require_vars"))
- if platform["launcher"] in {"single-slurm", "mi-amds"}:
- network = "cx_validate_network_profile_on_job"
- self.assertIn(network, source)
- self.assertLess(source.index("cx_salloc_jobid"), source.index(network))
- self.assertLess(source.index(network), source.index("cx_preflight_allocation"))
- if platform["launcher"] == "single-slurm":
- self.assertLess(
- source.index(network),
- source.index("CX_ENROOT_LOCAL_IMPORT=1 cx_ensure_squash"),
- )
-
- common = (ROOT / "runtime" / "common.sh").read_text()
- workflow = (ROOT.parent.parent / ".github" / "workflows" / "collectivex-sweep.yml").read_text()
- amd_launcher = (ROOT / "launchers" / "launch_mi-amds.sh").read_text()
- self.assertIn('if [ "$RUNNER" = mi355x ]; then\n allocation+=(--exclusive)', amd_launcher)
- self.assertNotIn('--gres=gpu:"$GPN" --exclusive', amd_launcher)
- self.assertNotIn("RUNNER_NAME", common)
- self.assertNotIn("RUNNER_NAME:", workflow)
- self.assertNotIn("flashinfer", capability.BACKENDS)
- self.assertFalse((HERE / "ep_flashinfer.py").exists())
-
- def test_canonical_operator_config_requires_a_private_audit_salt(self) -> None:
- salt = "a" * 64
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary).resolve()
- config = root / "operator.json"
- document = {
- "schema_version": 1,
- "audit_salt": salt,
- "runners": {
- "h100-dgxc": {
- "partition": "test", "account": "test",
- "squash_dir": str(root), "stage_dir": str(root),
- "ib_gid_index": "3", "rdma_service_level": "2",
- "rdma_traffic_class": "104",
- },
- },
- }
- command = (
- 'source "$1"; export COLLECTIVEX_EXECUTION_ID="audit-config-$$"; '
- "trap 'cx_cleanup_private_logs 0' EXIT; cx_load_operator_config; "
- 'test "$CX_AUDIT_SALT" = "$EXPECTED_AUDIT_SALT"; '
- 'test "$CX_IB_GID_INDEX:$CX_RDMA_SERVICE_LEVEL:$CX_RDMA_TRAFFIC_CLASS" = 3:2:104'
- )
-
- def invoke(
- value: dict, *, canonical: bool, expect_salt: bool = True,
- audit_override: str | None = None,
- ) -> subprocess.CompletedProcess[str]:
- config.write_text(json.dumps(value))
- config.chmod(0o600)
- environment = {
- **os.environ,
- "CX_RUNNER": "h100-dgxc",
- "COLLECTIVEX_OPERATOR_CONFIG": str(config),
- "EXPECTED_AUDIT_SALT": salt,
- }
- if canonical:
- environment["COLLECTIVEX_CANONICAL_GHA"] = "1"
- if audit_override is not None:
- environment["COLLECTIVEX_OPERATOR_AUDIT_SALT"] = audit_override
- invocation = command if expect_salt else (
- 'source "$1"; export COLLECTIVEX_EXECUTION_ID="audit-config-$$"; '
- "trap 'cx_cleanup_private_logs 0' EXIT; cx_load_operator_config; "
- 'test -z "${CX_AUDIT_SALT+x}"'
- )
- return subprocess.run(
- ["bash", "-c", invocation, "_", str(ROOT / "runtime" / "common.sh")],
- text=True,
- capture_output=True,
- env=environment,
- )
-
- accepted = invoke(document, canonical=True)
- self.assertEqual(accepted.returncode, 0, accepted.stderr)
- self.assertNotIn(salt, accepted.stdout + accepted.stderr)
-
- missing = copy.deepcopy(document)
- del missing["audit_salt"]
- rejected = invoke(missing, canonical=True)
- self.assertNotEqual(rejected.returncode, 0)
- self.assertRegex(rejected.stderr, r"validation-line-[0-9]+")
- self.assertNotIn(salt, rejected.stdout + rejected.stderr)
-
- separate = invoke(missing, canonical=True, audit_override=salt)
- self.assertEqual(separate.returncode, 0, separate.stderr)
- self.assertNotIn(salt, separate.stdout + separate.stderr)
-
- malformed_override = invoke(missing, canonical=True, audit_override="A" * 64)
- self.assertNotEqual(malformed_override.returncode, 0)
- self.assertNotIn("A" * 64, malformed_override.stdout + malformed_override.stderr)
-
- manual = invoke(missing, canonical=False, expect_salt=False)
- self.assertEqual(manual.returncode, 0, manual.stderr)
- self.assertNotIn(salt, manual.stdout + manual.stderr)
-
- malformed = copy.deepcopy(document)
- malformed["audit_salt"] = "A" * 64
- rejected = invoke(malformed, canonical=False)
- self.assertNotEqual(rejected.returncode, 0)
- self.assertNotIn("A" * 64, rejected.stdout + rejected.stderr)
-
- missing_field = copy.deepcopy(document)
- del missing_field["runners"]["h100-dgxc"]["account"]
- rejected = invoke(missing_field, canonical=True)
- self.assertNotEqual(rejected.returncode, 0)
- self.assertIn("validation-missing-required-account", rejected.stderr)
-
- for field, value in (
- ("ib_gid_index", "03"),
- ("rdma_service_level", "16"),
- ("rdma_traffic_class", "256"),
- ):
- invalid_selector = copy.deepcopy(document)
- invalid_selector["runners"]["h100-dgxc"][field] = value
- rejected = invoke(invalid_selector, canonical=True)
- self.assertNotEqual(rejected.returncode, 0)
-
- def test_scaleout_network_profile_is_explicit_and_allowlisted(self) -> None:
- command = r'''
- set -euo pipefail
- source "$1"
- test "$(cx_nccl_hca_device_name '=mlx5_0:1')" = mlx5_0
- test "$(cx_nccl_hca_device_name 'mlx5_1')" = mlx5_1
- ! (unset CX_SOCKET_IFNAME CX_RDMA_DEVICES; cx_apply_network_profile 2 nvlink-rdma)
- ! (export CX_SOCKET_IFNAME=eth0; unset CX_RDMA_DEVICES; cx_apply_network_profile 2 nvlink-rdma)
- export CX_SOCKET_IFNAME=ib0 CX_RDMA_DEVICES=mlx5_0:1,mlx5_1:1
- export NCCL_NET=Socket NCCL_IB_HCA=stale NVSHMEM_HCA_LIST=stale
- export NVSHMEM_HCA_PE_MAPPING=stale NVSHMEM_REMOTE_TRANSPORT=stale
- export CX_SHARD_SKU=b300
- cx_apply_network_profile 1 nvlink
- test "$NVSHMEM_DISABLE_IB" = 1
- test -z "${NVSHMEM_HCA_PE_MAPPING+x}${NVSHMEM_REMOTE_TRANSPORT+x}"
- export CX_BENCH=deepep CX_MODE=low-latency CX_IB_GID_INDEX=3
- cx_apply_network_profile 1 nvlink
- test -z "${NVSHMEM_DISABLE_IB+x}${NCCL_NET+x}${NCCL_IB_HCA+x}"
- test -z "${NVSHMEM_IB_GID_INDEX+x}"
- cx_export_gid_index_for_link_layer roce 0
- test "$NVSHMEM_HCA_LIST:$NVSHMEM_IB_GID_INDEX" = mlx5_0:1,mlx5_1:1:3
- test "$NVSHMEM_ENABLE_NIC_PE_MAPPING" = 1
- test "$NVSHMEM_IB_ENABLE_IBGDA:$NVSHMEM_IBGDA_NIC_HANDLER" = 1:gpu
- unset CX_BENCH CX_MODE CX_IB_GID_INDEX
- unset CX_SHARD_SKU
- cx_apply_network_profile 1 nvlink
- test -z "${NCCL_NET+x}${NCCL_IB_HCA+x}${NVSHMEM_DISABLE_IB+x}${NVSHMEM_HCA_LIST+x}"
- cx_apply_network_profile 4 mnnvl
- test -z "${NCCL_NET+x}${NCCL_IB_HCA+x}${NVSHMEM_HCA_LIST+x}"
- export CX_IB_GID_INDEX=3 CX_RDMA_SERVICE_LEVEL=2 CX_RDMA_TRAFFIC_CLASS=104
- cx_apply_network_profile 2 nvlink-rdma
- test "$NCCL_SOCKET_IFNAME:$GLOO_SOCKET_IFNAME:$UCCL_SOCKET_IFNAME" = ib0:ib0:ib0
- test "$NCCL_NET:$NCCL_IB_HCA" = 'IB:=mlx5_0:1,mlx5_1:1'
- NCCL_IB_HCA=mlx5_0:1,mlx5_1:1
- export NCCL_IB_HCA CX_NODES=2 CX_TRANSPORT=nvlink-rdma
- cx_restore_exact_hca_selector
- test "$NCCL_IB_HCA" = '=mlx5_0:1,mlx5_1:1'
- wrapper="$(cx_slurm_rank_wrapper)"
- bash -n <<< "$wrapper"
- grep -Fq '. /ix/experimental/CollectiveX/runtime/common.sh' <<< "$wrapper"
- grep -Fq 'cx_apply_network_profile "$CX_NODES" "$CX_TRANSPORT" || exit 68' <<< "$wrapper"
- grep -Fq 'cx_write_runtime_stage execution || exit 68' <<< "$wrapper"
- test "$NVSHMEM_HCA_LIST" = mlx5_0:1,mlx5_1:1
- test "$NVSHMEM_ENABLE_NIC_PE_MAPPING" = 1
- test "$MORI_RDMA_DEVICES:$EP_NIC_NAME" = mlx5_0,mlx5_1:mlx5_0
- test "$MORI_RDMA_TC:$MORI_IO_TC:$MORI_RDMA_SL:$MORI_IO_SL" = 104:104:2:2
- test -z "${NCCL_IB_GID_INDEX+x}${NVSHMEM_IB_GID_INDEX+x}${UCCL_IB_GID_INDEX+x}"
- cx_export_gid_index_for_link_layer roce 1
- test "$NCCL_IB_GID_INDEX:$NCCL_IB_SL" = 3:2
- test "$NVSHMEM_IB_ENABLE_IBGDA:$NVSHMEM_IBGDA_NIC_HANDLER" = 1:gpu
- export CX_SHARD_SKU=b200-dgxc CX_BENCH=deepep
- cx_apply_network_profile 2 nvlink-rdma
- test "$NVSHMEM_IB_ENABLE_IBGDA:$NVSHMEM_IBGDA_NIC_HANDLER" = 1:cpu
- unset CX_SHARD_SKU CX_BENCH
- cx_apply_network_profile 2 nvlink-rdma
- cx_export_gid_index_for_link_layer infiniband 1
- test -z "${NVSHMEM_IB_GID_INDEX+x}${NCCL_IB_GID_INDEX+x}${UCCL_IB_GID_INDEX+x}"
- test "$NVSHMEM_IB_SL:$NCCL_IB_SL:$UCCL_IB_SL" = 2:2:2
- export CX_RDMA_LINK_LAYER=roce
- cx_apply_network_profile 2 nvlink-rdma
- test "$NVSHMEM_IB_GID_INDEX:$NCCL_IB_GID_INDEX:$UCCL_IB_GID_INDEX" = 3:3:3
- '''
- subprocess.run(
- ["bash", "-c", command, "_", str(ROOT / "runtime" / "common.sh")],
- check=True,
- env={**os.environ, "COLLECTIVEX_OPERATOR_CONFIG": "/dev/null"},
- )
-
- v2_adapter = (ROOT / "tests" / "ep_deepep_v2.py").read_text()
- container_runtime = (ROOT / "runtime" / "run_in_container.sh").read_text()
- for version in ("2.0.0+fa8a9b1", "2.0.0+local"):
- self.assertIn(version, v2_adapter)
- self.assertIn(version, container_runtime)
-
- def test_network_mode_is_loaded_from_validated_control(self) -> None:
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary)
- mixed = root / "mixed.json"
- mixed.write_text(json.dumps({"cases": [
- {"mode": "normal"}, {"mode": "low-latency"},
- ]}))
- normal = root / "normal.json"
- normal.write_text(json.dumps({"cases": [{"mode": "normal"}]}))
- probe = root / "probe.json"
- probe.write_text(json.dumps({
- "id": "probe-id",
- "target": {
- "backend": "deepep", "sku": "b300", "ep": 8,
- "mode": "low-latency", "precision_profile": "profile",
- },
- }))
- command = r'''
- set -euo pipefail
- source "$1"
- export CX_SHARD_FILE="$2" CX_PRECISION_PROBE="$3"
- cx_load_network_control_mode "$4"
- printf '%s' "$CX_MODE"
- '''
- for path, precision, expected in (
- (mixed, "0", "low-latency"),
- (normal, "0", "normal"),
- (probe, "1", "low-latency"),
- ):
- result = subprocess.run(
- ["bash", "-c", command, "_", str(ROOT / "runtime" / "common.sh"),
- str(path), precision, str(ROOT)],
- text=True, capture_output=True, check=True,
- env={**os.environ, "COLLECTIVEX_OPERATOR_CONFIG": "/dev/null"},
- )
- self.assertEqual(result.stdout, expected)
-
- def test_network_profile_validation_is_private_and_all_node(self) -> None:
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary).resolve()
- binary = root / "srun"
- arguments = root / "arguments"
- script = root / "script"
- binary.write_text(
- "#!/usr/bin/env bash\n"
- "printf '%s\\n' \"$@\" > \"$CAPTURE_ARGS\"\n"
- "cat > \"$CAPTURE_SCRIPT\"\n"
- "tasks=1\n"
- "for arg in \"$@\"; do\n"
- " case \"$arg\" in --ntasks=*) tasks=\"${arg#--ntasks=}\" ;; esac\n"
- "done\n"
- "if [ \"${SRUN_RC:-0}\" = 0 ]; then\n"
- " for ((task=0; task None:
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary).resolve()
- for name, payload in (
- ("squeue", "#!/bin/sh\nprintf '%s\\n' 'rack-[1,3]'\n"),
- ("scontrol", "#!/bin/sh\nprintf '%s\\n' rack-1 rack-3\n"),
- ):
- binary = root / name
- binary.write_text(payload)
- binary.chmod(0o700)
- result = subprocess.run(
- [
- "bash", "-c",
- 'source "$1"; cx_allocation_nodes_csv 42',
- "_", str(ROOT / "runtime" / "common.sh"),
- ],
- text=True,
- capture_output=True,
- env={
- **os.environ,
- "PATH": f"{root}:{os.environ['PATH']}",
- "COLLECTIVEX_OPERATOR_CONFIG": "/dev/null",
- },
- )
- self.assertEqual(result.returncode, 0, result.stderr)
- self.assertEqual(result.stdout, "rack-1,rack-3")
-
- (root / "squeue").write_text("#!/bin/sh\nprintf 'rack;unsafe\\n'\n")
- (root / "squeue").chmod(0o700)
- rejected = subprocess.run(
- [
- "bash", "-c",
- 'source "$1"; cx_allocation_nodes_csv 42',
- "_", str(ROOT / "runtime" / "common.sh"),
- ],
- env={
- **os.environ,
- "PATH": f"{root}:{os.environ['PATH']}",
- "COLLECTIVEX_OPERATOR_CONFIG": "/dev/null",
- },
- )
- self.assertNotEqual(rejected.returncode, 0)
-
- def test_allocation_preflight_proves_shared_write_visibility(self) -> None:
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary).resolve()
- mount = root / "mount"
- runtime = mount / "experimental" / "CollectiveX" / "runtime"
- runtime.mkdir(parents=True)
- (runtime / "run_in_container.sh").write_text("#!/bin/sh\n")
- squash = root / "image.sqsh"
- squash.write_bytes(b"squash")
- binary = root / "bin"
- binary.mkdir()
- (binary / "unsquashfs").write_text("#!/bin/sh\nexit 0\n")
- (binary / "unsquashfs").chmod(0o700)
- (binary / "srun").write_text(
- "#!/usr/bin/env bash\n"
- "set -euo pipefail\n"
- "case \" $* \" in *' --input=all '*) ;; *) exit 97 ;; esac\n"
- "worker=\"$FAKE_ROOT/worker.sh\"\n"
- "cat > \"$worker\"\n"
- "args=(\"$@\")\n"
- "start=0\n"
- "for ((i=0; i<${#args[@]}; i++)); do\n"
- " [ \"${args[$i]}\" != -- ] || start=$((i + 1))\n"
- "done\n"
- "[ \"$start\" -gt 0 ]\n"
- "worker_args=(\"${args[@]:$start}\")\n"
- "probe=\"${worker_args[4]}\"\n"
- "case \"${FAKE_MODE:-success}\" in\n"
- " missing-source) rm -f -- \"$probe/source\" ;;\n"
- " readonly) chmod 500 \"$probe\" ;;\n"
- "esac\n"
- "for ((node=0; node None:
- runtime = (ROOT / "runtime" / "run_in_container.sh").read_text()
- probe = runtime[runtime.index("cx_probe_deepep()"):
- runtime.index("cx_activate_deepep_v2()")]
- self.assertIn('expected_version="1.2.1"', probe)
- self.assertIn('expected_version="1.1.0+814e508"', probe)
- self.assertNotIn("pip install", probe)
- self.assertNotIn("cx_fetch_revision", probe)
- self.assertIn("Path(deep_ep.__file__).resolve() in recorded_files", probe)
- self.assertIn("Path(buffer_module.__file__).resolve() in recorded_files", probe)
-
- harness = (HERE / "ep_harness.py").read_text()
- pass_one = harness[harness.index("# ---- Pass 1"):
- harness.index("# ---- Pass 2")]
- self.assertLess(
- pass_one.index("input_snapshots[T] ="),
- pass_one.index("oracle = _run_expert_oracle"),
- )
- self.assertIn("pre_input_unchanged", pass_one)
- # The combine timing template reuses one dispatched+staged handle (no per-iter
- # re-dispatch) for stateless-combine backends; it lives in the base class now.
- backend = (HERE / "ep_backend.py").read_text()
- self.assertIn(
- "hh = prep_combine()\n torch.cuda.synchronize()",
- backend,
- )
-
- def test_squash_imports_are_reproducible_and_use_a_fresh_cache_key(self) -> None:
- common = (ROOT / "runtime" / "common.sh").read_text()
- amd = (ROOT / "launchers" / "launch_mi-amds.sh").read_text()
- self.assertIn('CX_SQUASH_FORMAT_VERSION="repro-v1"', common)
- self.assertIn("SOURCE_DATE_EPOCH=\"$CX_SQUASH_SOURCE_DATE_EPOCH\"", common)
- self.assertIn("${COLLECTIVEX_IMAGE_DIGEST#sha256:}", common)
- self.assertIn("cx_ensure_squash_on_job", amd)
- self.assertIn('"${CX_LOCK_DIR:-}"', amd)
- self.assertNotIn('"${CX_LOCK_DIR:-/tmp}"', amd)
- self.assertIn('[ -n "$lock_dir" ] || lock_dir="$squash_dir/.locks"', common)
- self.assertGreaterEqual(common.count("--chdir=/tmp"), 2)
- self.assertGreaterEqual(amd.count("--chdir=/tmp"), 2)
- self.assertIn('ENROOT_CACHE_PATH="$compute_home/enroot-cache"', common)
- self.assertIn('ENROOT_RUNTIME_PATH="$compute_home/enroot-run"', common)
- self.assertEqual(common.count('cx_reverify_registry_image "$image"'), 2)
- result = subprocess.run(
- [
- "bash",
- "-c",
- f'source "{ROOT / "runtime" / "common.sh"}"; '
- 'COLLECTIVEX_IMAGE_DIGEST="sha256:$(printf b%.0s {1..64})"; '
- 'CX_IMAGE_PLATFORM=linux/amd64; cx_squash_path /cache repo/image:tag; '
- 'printf "\\n"; CX_IMAGE_PLATFORM=linux/arm64; '
- 'cx_squash_path /cache repo/image:tag',
- ],
- text=True,
- capture_output=True,
- )
- self.assertEqual(result.returncode, 0, result.stderr)
- digest = "b" * 64
- self.assertEqual(
- result.stdout.splitlines(),
- [
- f"/cache/repro-v1_{digest}_repo_image_tag.sqsh",
- f"/cache/repro-v1_linux_arm64_{digest}_repo_image_tag.sqsh",
- ],
- )
-
- def test_launchers_preserve_platform_specific_runtime_requirements(self) -> None:
- single = (ROOT / "launchers" / "launch_single-slurm.sh").read_text()
- gb = (ROOT / "launchers" / "launch_gb-nv.sh").read_text()
- amd = (ROOT / "launchers" / "launch_mi-amds.sh").read_text()
- common = (ROOT / "runtime" / "common.sh").read_text()
- self.assertIn("ALLOC_EXTRA=(--mem=0)", single)
- self.assertIn("ALLOC_EXTRA=(-N 1 --mem=0)", single)
- self.assertIn("SRUN_EXTRA=(--mpi=none --container-remap-root)", single)
- self.assertIn("CX_ENROOT_LOCAL_IMPORT=1", single)
- self.assertIn('PRODUCT="${CX_SHARD_SKU:-${CX_GB_PRODUCT:-', gb)
- self.assertIn("cx_ensure_squash_on_job", gb)
- self.assertIn("--mem=0 --cpus-per-task=35", gb)
- self.assertIn("--container-writable", gb)
- self.assertIn("--container-remap-root", gb)
- workload_stage = common[
- common.index("workload_args=("):
- common.index("workload_log=", common.index("workload_args=("))
- ]
- self.assertNotIn("--workload", workload_stage)
- self.assertIn('cd -P -- "$(dirname "${BASH_SOURCE[0]}")"', amd)
- self.assertIn("mi325x) CPUS_PER_TASK=256", amd)
- self.assertIn("/dev/kfd:/dev/kfd,/dev/dri:/dev/dri", amd)
- self.assertIn("--container-writable --container-remap-root", amd)
- self.assertIn(
- "CX_DISTRIBUTED_CONTAINER_ARGS=(--container-writable --container-remap-root)",
- amd,
- )
- collect = common[common.index("cx_collect_results()"):
- common.index("cx_cleanup_stage()")]
- cleanup = common[common.index("cx_launcher_cleanup()"):
- common.index("cx_install_launcher_fail_safe()")]
- self.assertNotIn("cx_cleanup_stage", collect)
- self.assertLess(cleanup.index("cx_cancel_job"), cleanup.index("cx_cleanup_stage"))
- runtime = (ROOT / "runtime" / "run_in_container.sh").read_text()
- self.assertIn('distribution.read_text("direct_url.json")', runtime)
- self.assertIn("6548e9c504a12b2471af4b7f4d9546321210a57a456b5dc55bd4a8dad0f932ac", runtime)
- self.assertIn("2671cff7baf8c2c214ff4bac721af875d513130670bec57601998bd1aae82882", runtime)
- stage_cleanup = common[
- common.index("cx_cleanup_stage()"):
- common.index("cx_has_result_doc()")
- ]
- self.assertIn('allow_uid_mapped_root = sys.argv[3] == "1"', stage_cleanup)
- self.assertIn("base_metadata.st_uid == 0", stage_cleanup)
- self.assertIn("stat.S_IMODE(base_metadata.st_mode) == 0o700", stage_cleanup)
- self.assertIn("parent_metadata.st_uid != 0", stage_cleanup)
- self.assertIn("stat.S_IWGRP | stat.S_IWOTH", stage_cleanup)
-
- def test_deferred_backend_provenance_resolves_before_measurement(self) -> None:
- harness = (ROOT / "tests" / "ep_harness.py").read_text()
- conditioning = harness.index("for wt in conditioning_ladder")
- provenance = harness.index("# Setup may materialize deferred provenance")
- measurement = harness.index("# ---- Pass 1: build each deterministic problem")
- self.assertLess(conditioning, provenance)
- self.assertLess(provenance, measurement)
-
- def test_backend_specific_routing_contracts_are_explicit(self) -> None:
- hybrid = (ROOT / "tests" / "ep_deepep_hybrid.py").read_text()
- self.assertIn("self.domain_rank = int(self.buffer.local_rank)", hybrid)
- self.assertIn(
- "probability_columns = self.domain_rank * self.local_experts + local_expert_ids",
- hybrid,
- )
- self.assertIn("h.recv_probs[:count][rows, probability_columns]", hybrid)
-
- mori = (ROOT / "tests" / "ep_mori.py").read_text()
- self.assertIn("topk_idx=indices", mori)
- self.assertIn("indices=indices", mori)
- self.assertIn(
- "combine_indices = p.indices if self._async_ll else h.dispatch_indices",
- mori,
- )
- self.assertIn("h.combine_input,\n None,\n combine_indices", mori)
- self.assertIn('"use_external_inp_buf": self._external_input', mori)
- self.assertIn("self.block_num = self._block_target = 64", mori)
- self.assertIn('config_kwargs["block_num"] = self.block_num', mori)
- self.assertIn(
- 'config_kwargs["warp_num_per_block"] = self.dispatch_warps', mori
- )
- self.assertIn("count > tensor.size(0)", mori)
- self.assertIn("return combined[:p.T]", mori)
- self.assertNotIn("return combined\n", mori)
- self.assertIn(
- "raw_expert_ids < local_start + experts_per_rank",
- mori,
- )
- self.assertNotIn("MoRI returned a non-local expert", mori)
- harness = (ROOT / "tests" / "ep_harness.py").read_text()
- self.assertIn("problem.recv_tokens = backend.recv_tokens(handle)", harness)
-
- def test_mori_masks_global_topk_metadata_to_the_local_rank(self) -> None:
- path = HERE / "ep_mori.py"
- tree = ast.parse(path.read_text(), str(path))
- helper = next(
- node
- for node in tree.body
- if isinstance(node, ast.FunctionDef) and node.name == "_project_local_metadata"
- )
- namespace: dict[str, object] = {}
- exec(compile(ast.Module(body=[helper], type_ignores=[]), str(path), "exec"), namespace)
- raw_ids = np.array([[0, 32, 63, -1], [64, 95, 7, 96]], dtype=np.int64)
- raw_weights = np.arange(8, dtype=np.float32).reshape(2, 4)
- torch_module = types.SimpleNamespace(
- where=np.where,
- full_like=np.full_like,
- zeros_like=np.zeros_like,
- )
- ids, weights, local_ids = namespace["_project_local_metadata"](
- torch_module, raw_ids, raw_weights, 1, 32
- )
- np.testing.assert_array_equal(
- ids,
- np.array([[-1, 32, 63, -1], [-1, -1, -1, -1]], dtype=np.int64),
- )
- np.testing.assert_array_equal(
- weights,
- np.array([[0, 1, 2, 0], [0, 0, 0, 0]], dtype=np.float32),
- )
- counts = np.bincount(local_ids, minlength=32)
- self.assertEqual((counts[0], counts[31], int(counts.sum())), (1, 1, 2))
- commit_helper = next(
- node for node in tree.body
- if isinstance(node, ast.FunctionDef) and node.name == "_mori_source_commit"
- )
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary)
- module = root / "python" / "mori" / "__init__.py"
- module.parent.mkdir(parents=True)
- module.touch()
- git = root / ".git"
- git.mkdir()
- (git / "HEAD").write_text("a" * 40 + "\n")
- commit_namespace = {
- "Path": Path,
- "re": re,
- "mori": types.SimpleNamespace(__file__=str(module)),
- }
- exec(
- compile(ast.Module(body=[commit_helper], type_ignores=[]), str(path), "exec"),
- commit_namespace,
- )
- self.assertEqual(commit_namespace["_mori_source_commit"](), "a" * 40)
- (git / "HEAD").write_text("ref: refs/heads/main\n")
- with self.assertRaisesRegex(RuntimeError, "detached commit"):
- commit_namespace["_mori_source_commit"]()
-
- profile = contracts.project_resource_profile(
- {
- "block_num": 64,
- "device_cus": 304,
- "kernel_type": "AsyncLL",
- "tuned_source": "upstream-asyncll-64x8-external-input",
- }
- )
- self.assertIsNone(profile["comm_units_kind"])
- self.assertIsNone(profile["configured_units"])
-
- def test_mori_zipf_capacity_covers_worst_rank_skew(self) -> None:
- path = HERE / "ep_mori.py"
- tree = ast.parse(path.read_text(), str(path))
- helper = next(
- node
- for node in tree.body
- if isinstance(node, ast.FunctionDef) and node.name == "_mori_input_capacity"
- )
- namespace: dict[str, object] = {}
- exec(
- compile(ast.Module(body=[helper], type_ignores=[]), str(path), "exec"),
- namespace,
- )
- capacity = namespace["_mori_input_capacity"]
- self.assertEqual(capacity(types.SimpleNamespace(routing="uniform"), 8), 512)
- self.assertEqual(
- capacity(
- types.SimpleNamespace(
- routing="zipf", tokens_ladder="512", phase="prefill"
- ),
- 8,
- ),
- 4096,
- )
- self.assertEqual(
- capacity(
- types.SimpleNamespace(
- routing="zipf", tokens_ladder="128", phase="decode"
- ),
- 8,
- ),
- 1024,
- )
-
- def test_squash_identity_rehashes_instead_of_trusting_a_sidecar(self) -> None:
- with tempfile.TemporaryDirectory() as temporary:
- image = Path(temporary) / "image.sqsh"
- image.write_bytes(b"current squash bytes")
- sidecar = Path(f"{image}.sha256")
- sidecar.write_text("a" * 64)
- os.utime(sidecar, (image.stat().st_mtime + 10, image.stat().st_mtime + 10))
- result = subprocess.run(
- [
- "bash", "-c",
- 'source "$1"; COLLECTIVEX_EXECUTION_ID="squash-hash-$$"; '
- 'cx_export_squash_identity "$2"; cx_cleanup_private_logs 0; '
- 'printf "%s" "$COLLECTIVEX_SQUASH_SHA256"',
- "_", str(ROOT / "runtime" / "common.sh"), str(image),
- ],
- text=True,
- capture_output=True,
- )
- self.assertEqual(result.returncode, 0, result.stderr)
- self.assertEqual(result.stdout, hashlib.sha256(image.read_bytes()).hexdigest())
-
- def _run_salloc_scenario(
- self, salloc_body: str, squeue_body: str, *, cleanup: bool
- ) -> dict[str, object]:
- prefix = f"inferencex-collectivex-{os.getpid()}-1-"
- with tempfile.TemporaryDirectory(prefix=prefix, dir="/tmp") as temporary:
- root = Path(temporary)
- command_dir = root / "bin"
- repo = root / "repo"
- command_dir.mkdir()
- repo.mkdir()
- paths = {
- name: root / name
- for name in ("arguments", "squeue-calls", "sleep-calls", "scancel-calls")
- }
- scripts = {
- "salloc": (
- "printf '%s\\n' \"$@\" > \"$CX_TEST_SALLOC_ARGUMENTS\"\n"
- + salloc_body
- ),
- "squeue": (
- "printf '%s\\n' \"$*\" >> \"$CX_TEST_SQUEUE_CALLS\"\n"
- + squeue_body
- ),
- "sleep": "printf '%s\\n' \"$1\" >> \"$CX_TEST_SLEEP_CALLS\"\n",
- "scancel": (
- "printf '%s\\n' \"$*\" >> \"$CX_TEST_SCANCEL_CALLS\"\n"
- ),
- }
- for name, body in scripts.items():
- path = command_dir / name
- path.write_text(f"#!/usr/bin/env bash\n{body}\n")
- path.chmod(0o700)
- execution_id = f"scheduler-{root.name}"
- expected_name = "cx-" + hashlib.sha256(
- execution_id.encode()
- ).hexdigest()[:24]
- command = r'''
- source "$1"
- JOB_ID=""
- set +e
- cx_salloc_jobid --partition=compute
- run_rc=$?
- set -e
- printf '%s:%s:%s\n' \
- "$run_rc" "$JOB_ID" "$CX_ALLOCATION_UNCERTAIN"
- cx_cleanup_private_logs 0
- if [ "$3" = cleanup ]; then
- export CX_JOB_ROOT="$2" REPO_ROOT="$2/repo" MOUNT_SRC="$2/repo"
- export COLLECTIVEX_CANONICAL_GHA=1
- cx_clear_allocation_jobid() { rm -f -- "$CX_JOB_ROOT/allocation-job-id"; }
- cx_write_cleanup_guard() {
- rm -f -- "$CX_JOB_ROOT/cleanup-safe" "$CX_JOB_ROOT/cleanup-unsafe"
- : > "$CX_JOB_ROOT/cleanup-$1"
- }
- unset CX_BENCH
- cx_launcher_cleanup "$run_rc"
- fi
- exit "$run_rc"
- '''
- result = subprocess.run(
- [
- "bash", "-c", command, "_", str(ROOT / "runtime" / "common.sh"),
- str(root), "cleanup" if cleanup else "no-cleanup",
- ],
- text=True,
- capture_output=True,
- env={
- **os.environ,
- "PATH": f"{command_dir}:{os.environ['PATH']}",
- "COLLECTIVEX_EXECUTION_ID": execution_id,
- "CX_TEST_SALLOC_ARGUMENTS": str(paths["arguments"]),
- "CX_TEST_SQUEUE_CALLS": str(paths["squeue-calls"]),
- "CX_TEST_SLEEP_CALLS": str(paths["sleep-calls"]),
- "CX_TEST_SCANCEL_CALLS": str(paths["scancel-calls"]),
- },
- )
- return {
- "result": result,
- "job_name": expected_name,
- "arguments": paths["arguments"].read_text().splitlines(),
- "squeue_calls": (
- paths["squeue-calls"].read_text().splitlines()
- if paths["squeue-calls"].exists() else []
- ),
- "sleep_calls": (
- paths["sleep-calls"].read_text().splitlines()
- if paths["sleep-calls"].exists() else []
- ),
- "scancel_calls": (
- paths["scancel-calls"].read_text().splitlines()
- if paths["scancel-calls"].exists() else []
- ),
- "cleanup_safe": (root / "cleanup-safe").is_file(),
- "cleanup_unsafe": (root / "cleanup-unsafe").is_file(),
- }
-
- def test_salloc_job_id_parser_uses_the_portable_grant_message(self) -> None:
- scenario = self._run_salloc_scenario(
- "printf 'salloc: Granted job allocation 4242\\n' >&2",
- "exit 2",
- cleanup=False,
- )
- result = scenario["result"]
- self.assertIsInstance(result, subprocess.CompletedProcess)
- self.assertEqual(result.returncode, 0, result.stderr)
- self.assertEqual(
- result.stdout, "0:4242:0\n"
- )
- self.assertEqual(
- scenario["arguments"],
- [
- "--partition=compute",
- f"--job-name={scenario['job_name']}",
- "--no-shell",
- ],
- )
- self.assertIn("scheduler-request=submit", result.stderr)
- self.assertEqual(scenario["squeue_calls"], [])
-
- def test_salloc_job_id_parser_ignores_duplicate_wrapper_output(self) -> None:
- scenario = self._run_salloc_scenario(
- "for _ in {1..10000}; do "
- "printf 'salloc: Granted job allocation 4242\\n' >&2; done",
- "exit 2",
- cleanup=False,
- )
- result = scenario["result"]
- self.assertEqual(result.returncode, 0, result.stderr)
- self.assertEqual(result.stdout, "0:4242:0\n")
- self.assertNotIn("grant-parse", result.stderr)
-
- def test_salloc_contains_a_shell_wrapper_that_calls_exit(self) -> None:
- common = ROOT / "runtime" / "common.sh"
- with tempfile.TemporaryDirectory() as temporary:
- log = Path(temporary) / "scheduler.log"
- command = r'''
- set -euo pipefail
- source "$1"
- cx_private_log_path() { : > "$CX_TEST_LOG"; printf '%s' "$CX_TEST_LOG"; }
- salloc() { printf 'salloc: Granted job allocation 4242\n' >&2; exit 0; }
- export COLLECTIVEX_EXECUTION_ID=wrapper-exit
- JOB_ID=""
- cx_salloc_jobid --partition=compute
- test "$JOB_ID" = 4242
- '''
- result = subprocess.run(
- ["bash", "-c", command, "_", str(common), str(log)],
- text=True,
- capture_output=True,
- env={**os.environ, "CX_TEST_LOG": str(log)},
- )
- self.assertEqual(result.returncode, 0, result.stderr)
-
- def test_salloc_accepts_a_live_grant_despite_wrapper_failure(self) -> None:
- scenario = self._run_salloc_scenario(
- "printf 'salloc: Granted job allocation 4242\\n' >&2; exit 1",
- r'''
- case " $* " in
- *" -j 4242 "*) printf '4242\n' ;;
- *) exit 2 ;;
- esac
- ''',
- cleanup=False,
- )
- result = scenario["result"]
- self.assertEqual(result.returncode, 0, result.stderr)
- self.assertEqual(result.stdout, "0:4242:0\n")
- self.assertIn("scheduler-request=verified-grant", result.stderr)
- self.assertEqual(scenario["squeue_calls"], ["-h -j 4242 -o %A"])
-
- def test_salloc_rejects_a_vanished_grant_after_wrapper_failure(self) -> None:
- scenario = self._run_salloc_scenario(
- "printf 'salloc: Granted job allocation 4242\\n' >&2; exit 1",
- "exit 0",
- cleanup=False,
- )
- result = scenario["result"]
- self.assertEqual(result.returncode, 1)
- self.assertEqual(result.stdout, "1:4242:0\n")
- self.assertIn("scheduler-request=rejected", result.stderr)
-
- def test_salloc_verified_rejection_is_cleanup_safe(self) -> None:
- scenario = self._run_salloc_scenario("exit 1", "exit 0", cleanup=True)
- result = scenario["result"]
- self.assertEqual(result.returncode, 1)
- self.assertEqual(result.stdout, "1::0\n")
- self.assertIn("scheduler-request=rejected", result.stderr)
- self.assertIn(
- "failure-class=scheduler-allocation diagnostic=empty-log",
- result.stderr,
- )
- self.assertEqual(len(scenario["squeue_calls"]), 3)
- scheduler_user = subprocess.check_output(["id", "-un"], text=True).strip()
- self.assertTrue(all(
- f"--name={scenario['job_name']}" in call
- and f"--user={scheduler_user}" in call
- for call in scenario["squeue_calls"]
- ))
- self.assertEqual(scenario["sleep_calls"], ["1", "2"])
- self.assertTrue(scenario["cleanup_safe"])
- self.assertFalse(scenario["cleanup_unsafe"])
-
- def test_salloc_recovers_and_cancels_one_matching_allocation(self) -> None:
- scenario = self._run_salloc_scenario(
- "exit 1",
- r'''
- case " $* " in
- *" --name="*) printf '5151\n' ;;
- *" -j 5151 "*) exit 0 ;;
- *) exit 2 ;;
- esac
- ''',
- cleanup=True,
- )
- result = scenario["result"]
- self.assertEqual(result.returncode, 1)
- self.assertEqual(result.stdout, "1:5151:0\n")
- self.assertEqual(scenario["scancel_calls"], ["5151"])
- self.assertTrue(scenario["cleanup_safe"])
- self.assertFalse(scenario["cleanup_unsafe"])
-
- def test_salloc_ambiguous_lookup_remains_cleanup_unsafe(self) -> None:
- scenario = self._run_salloc_scenario(
- "exit 1", "printf '5151\\n5152\\n'", cleanup=True
- )
- result = scenario["result"]
- self.assertEqual(result.returncode, 1)
- self.assertEqual(result.stdout, "1::1\n")
- self.assertEqual(scenario["scancel_calls"], [])
- self.assertFalse(scenario["cleanup_safe"])
- self.assertTrue(scenario["cleanup_unsafe"])
-
- def test_salloc_query_failure_and_interruption_remain_cleanup_unsafe(self) -> None:
- query_failure = self._run_salloc_scenario("exit 1", "exit 2", cleanup=True)
- self.assertEqual(query_failure["result"].returncode, 1)
- self.assertEqual(len(query_failure["squeue_calls"]), 1)
- self.assertFalse(query_failure["cleanup_safe"])
- self.assertTrue(query_failure["cleanup_unsafe"])
-
- interrupted = self._run_salloc_scenario("exit 130", "exit 0", cleanup=True)
- self.assertEqual(interrupted["result"].returncode, 1)
- self.assertEqual(interrupted["squeue_calls"], [])
- self.assertFalse(interrupted["cleanup_safe"])
- self.assertTrue(interrupted["cleanup_unsafe"])
-
- def test_allocation_cleanup_fails_closed_when_scheduler_queries_fail(self) -> None:
- common = (ROOT / "runtime" / "common.sh").read_text()
- self.assertIn("for delay in 1 2 4 8 16 32 64; do", common)
- with tempfile.TemporaryDirectory() as temporary:
- directory = Path(temporary)
- for name, body in {
- "scancel": "exit 0",
- "squeue": "exit 2",
- "sleep": "exit 0",
- }.items():
- command = directory / name
- command.write_text(f"#!/usr/bin/env bash\n{body}\n")
- command.chmod(0o700)
- result = subprocess.run(
- [
- "bash", "-c",
- 'source "$1"; cx_cancel_job 4242',
- "_", str(ROOT / "runtime" / "common.sh"),
- ],
- text=True,
- capture_output=True,
- env={**os.environ, "PATH": f"{directory}:{os.environ['PATH']}"},
- )
- self.assertNotEqual(result.returncode, 0)
- self.assertIn("did not terminate", result.stderr)
-
- def test_launcher_signal_traps_cancel_the_allocation(self) -> None:
- prefix = f"inferencex-collectivex-{os.getpid()}-1-"
- with tempfile.TemporaryDirectory(prefix=prefix, dir="/tmp") as temporary:
- root = Path(temporary)
- binary = root / "bin"
- repo = root / "repo"
- calls = root / "scancel-calls"
- binary.mkdir()
- repo.mkdir()
- for name, body in {
- "scancel": 'printf "%s\\n" "$*" >> "$CX_TEST_SCANCEL_CALLS"',
- "squeue": "exit 0",
- }.items():
- path = binary / name
- path.write_text(f"#!/usr/bin/env bash\n{body}\n")
- path.chmod(0o700)
- command = r'''
- set -euo pipefail
- source "$1"
- export JOB_ID=5151 CX_JOB_ROOT="$2" REPO_ROOT="$2/repo" MOUNT_SRC="$2/repo"
- export COLLECTIVEX_CANONICAL_GHA=1
- cx_clear_allocation_jobid() { rm -f -- "$CX_JOB_ROOT/allocation-job-id"; }
- cx_write_cleanup_guard() {
- rm -f -- "$CX_JOB_ROOT/cleanup-safe" "$CX_JOB_ROOT/cleanup-unsafe"
- : > "$CX_JOB_ROOT/cleanup-$1"
- }
- cx_install_launcher_fail_safe
- kill -TERM $$
- exit 99
- '''
- result = subprocess.run(
- ["bash", "-c", command, "_", str(ROOT / "runtime" / "common.sh"), str(root)],
- text=True,
- capture_output=True,
- env={
- **os.environ,
- "PATH": f"{binary}:{os.environ['PATH']}",
- "CX_TEST_SCANCEL_CALLS": str(calls),
- },
- )
- self.assertEqual(result.returncode, 143, result.stderr)
- self.assertEqual(calls.read_text().splitlines(), ["5151"])
- self.assertTrue((root / "cleanup-safe").is_file())
- self.assertFalse((root / "cleanup-unsafe").exists())
-
- workflow = (ROOT.parent.parent / ".github" / "workflows" / "collectivex-sweep.yml").read_text()
- self.assertIn("cleanup-unsafe", workflow)
- self.assertIn("cleanup-safe", workflow)
- self.assertIn("Reconcile allocation cleanup", workflow)
- self.assertIn("Confirm allocation cleanup", workflow)
- self.assertLess(
- workflow.index("Reconcile allocation cleanup"),
- workflow.index("Confirm allocation cleanup"),
- )
- self.assertIn("Prepare pinned backend source archive", workflow)
- self.assertIn("Install pinned backend source seed", workflow)
- self.assertIn("CX_BACKEND_SOURCE_SEED_ROOT", workflow)
- self.assertIn("steps.gen.outputs.source_backends", workflow)
- self.assertIn("secrets.COLLECTIVEX_NETWORK_CONFIG_V1", workflow)
- self.assertIn("secrets.COLLECTIVEX_B200_CONFIG_V1", workflow)
- self.assertIn("secrets.COLLECTIVEX_MI300_CONFIG_V1", workflow)
- self.assertIn('set(overlay["runners"]) != {"b200-dgxc"}', workflow)
- self.assertIn('set(overlay["runners"]) != {"mi300x"}', workflow)
- self.assertIn("matrix.sku == 'mi300x' && '180' || '300'", workflow)
- self.assertIn('overlay_name != "COLLECTIVEX_MI300_CONFIG_CONTENT"', workflow)
- self.assertIn('"partition", "squash_dir", "stage_dir", "exclude_nodes"', workflow)
- self.assertIn("set(fields) - private_fields", workflow)
- self.assertIn('private_fields.add("exclude_nodes")', workflow)
- self.assertIn('invalid H100 overlay runner', workflow)
- self.assertIn('private_fields.update({"nodelist", "qos"})', workflow)
- self.assertIn('[ -z "${CX_QOS:-}" ] || allocation+=(--qos="$CX_QOS")', (
- ROOT / "launchers" / "launch_single-slurm.sh"
- ).read_text())
- self.assertIn('[ -z "${CX_NODELIST:-}" ] || allocation+=(--nodelist="$CX_NODELIST")', (
- ROOT / "launchers" / "launch_single-slurm.sh"
- ).read_text())
- self.assertIn("COLLECTIVEX_OPERATOR_CONFIG_EPHEMERAL=1", workflow)
- self.assertLess(
- workflow.index("unset COLLECTIVEX_OPERATOR_CONFIG_REQUIRED"),
- workflow.index('export COLLECTIVEX_OPERATOR_CONFIG="$operator_config"'),
- )
-
- def test_workflow_reconciles_a_recorded_allocation_after_launcher_loss(self) -> None:
- workflow = (
- ROOT.parent.parent / ".github" / "workflows" / "collectivex-sweep.yml"
- ).read_text()
- prefix = f"inferencex-collectivex-{os.getpid()}-1-"
- with tempfile.TemporaryDirectory(prefix=prefix, dir="/tmp") as temporary:
- root = Path(temporary)
- binary = root / "bin"
- calls = root / "scancel-calls"
- binary.mkdir()
- for name, body in {
- "scancel": 'printf "%s\\n" "$*" >> "$CX_TEST_SCANCEL_CALLS"',
- "squeue": "exit 0",
- "stat": (
- 'case "$3" in */allocation-job-id) mode=600 ;; *) mode=700 ;; esac\n'
- 'printf "%s:%s\\n" "$CX_TEST_UID" "$mode"'
- ),
- }.items():
- path = binary / name
- path.write_text(f"#!/usr/bin/env bash\n{body}\n")
- path.chmod(0o700)
- command = r'''
- set -euo pipefail
- source "$1"
- export CX_JOB_ROOT="$2"
- : > "$CX_JOB_ROOT/cleanup-unsafe"
- cx_record_allocation_jobid 6262
- cx_reconcile_recorded_allocation "$CX_JOB_ROOT"
- '''
- result = subprocess.run(
- ["bash", "-c", command, "_", str(ROOT / "runtime" / "common.sh"), str(root)],
- text=True,
- capture_output=True,
- env={
- **os.environ,
- "PATH": f"{binary}:{os.environ['PATH']}",
- "CX_TEST_SCANCEL_CALLS": str(calls),
- "CX_TEST_UID": str(os.getuid()),
- },
- )
- self.assertEqual(result.returncode, 0, result.stderr)
- self.assertEqual(calls.read_text().splitlines(), ["6262"])
- self.assertTrue((root / "cleanup-safe").is_file())
- self.assertFalse((root / "cleanup-unsafe").exists())
- self.assertFalse((root / "allocation-job-id").exists())
-
- prepare_start = workflow.index("- name: Prepare pinned backend source archive")
- source_archive_step = workflow[
- prepare_start:workflow.index("- uses: actions/upload-artifact", prepare_start)
- ]
- self.assertIn('--no-recursion -C "$work/experimental/CollectiveX"', source_archive_step)
- self.assertIn('members+=(".cx_sources/$source_basename")', source_archive_step)
- self.assertIn('-rf "$archive" "$member"', source_archive_step)
- self.assertIn('python3 "$destination/source_archive.py"', workflow)
- install_source = workflow[workflow.index("- name: Install pinned backend source seed"):]
- self.assertIn('chmod 600 -- "$archive"', install_source)
- self.assertLess(
- install_source.index('chmod 600 -- "$archive"'),
- install_source.index('python3 "$destination/source_archive.py"'),
- )
- artifact_validation = workflow[workflow.index("- name: Validate shard artifact safety"):]
- self.assertIn("steps.allocation_cleanup.outcome == 'success'", artifact_validation)
- self.assertIn(
- "inputs.operation != 'probe-precision' || steps.sweep_shard.outcome == 'success'",
- artifact_validation,
- )
- cleanup_function = (ROOT / "runtime" / "common.sh").read_text()
- self.assertIn('[ "${CX_PRECISION_PROBE:-0}" != 1 ]', cleanup_function)
- self.assertIn("precision-probe-failure-class=", cleanup_function)
- sweep_workflow = workflow[workflow.index(" sweep:"):]
- self.assertNotIn("GITHUB_WORKSPACE", sweep_workflow)
- self.assertNotIn("RUNNER_WORKSPACE", sweep_workflow)
- self.assertNotIn("github.workspace", sweep_workflow)
- self.assertIn("matrix.launcher == 'mi-amds'", sweep_workflow)
- self.assertIn("CX_JOB_PARENT:", sweep_workflow)
- self.assertIn("/tmp/inferencex-collectivex-parent-", sweep_workflow)
- self.assertIn("|| '/tmp'", sweep_workflow)
- source_step = sweep_workflow[:sweep_workflow.index("- uses: actions/download-artifact")]
- self.assertNotIn("unsafe_guards=", source_step)
- self.assertIn('shared_parent = os.path.join(os.getcwd(), ".collectivex-jobs")', source_step)
- self.assertIn("os.lstat(shared_parent)", source_step)
- self.assertIn("os.symlink(shared_parent, parent)", source_step)
- self.assertIn("for entry in os.scandir(scan_parent)", source_step)
- self.assertIn("cutoff = time.time() - 86400", source_step)
- self.assertIn("stat.S_IMODE(metadata.st_mode) != 0o700", source_step)
- self.assertIn('for marker_name in ("cleanup-safe", "cleanup-unsafe")', source_step)
- self.assertIn("stat.S_IMODE(marker.st_mode) == 0o600", source_step)
- self.assertIn("shutil.rmtree(entry.path)", source_step)
- self.assertLess(
- source_step.index('rev-parse HEAD'),
- source_step.index("echo 'prepared=true'"),
- )
- upload = workflow[workflow.index("- name: Stage shard artifact"):]
- self.assertIn("id: stage_artifact", upload)
- self.assertIn("id: upload_artifact", upload)
- self.assertIn("steps.stage_artifact.outcome == 'success'", upload)
- cleanup = workflow[workflow.index("- name: Cleanup isolated workspace"):]
- self.assertIn("CollectiveX cleanup parent is invalid", cleanup)
- self.assertIn('[ "${CX_JOB_ROOT%/*}" = "$CX_JOB_PARENT" ]', cleanup)
- self.assertIn('^inferencex-collectivex-', cleanup)
- self.assertIn('rm -f -- "$CX_JOB_PARENT"', cleanup)
- self.assertIn("steps.sweep_shard.outcome }}' != success", cleanup)
- self.assertIn('[ "$CX_JOB_PARENT" != /tmp ]', cleanup)
- self.assertIn("- name: Classify private scheduler failure", workflow)
- self.assertIn("cx_report_private_scheduler_failure", workflow)
- for step in (
- "sweep_shard", "allocation_cleanup", "artifact_safety",
- "delivery_contracts", "stage_artifact", "upload_artifact",
- ):
- self.assertIn(f"steps.{step}.outcome", cleanup)
- self.assertLess(
- cleanup.index('cleanup-safe" ]'),
- cleanup.index('rm -rf -- "$CX_JOB_ROOT"'),
- )
-
- def test_allocation_record_accepts_only_matching_amd_nested_roots(self) -> None:
- common = str(ROOT / "runtime" / "common.sh")
- tag = f"{os.getpid()}-1-mi300x-test"
- parent = Path("/tmp") / f"inferencex-collectivex-parent-{tag}"
- shared = Path(tempfile.mkdtemp(prefix="collectivex-amd-shared-"))
- root = shared / f"inferencex-collectivex-{tag}"
- root.mkdir(mode=0o700)
- try:
- parent.symlink_to(shared, target_is_directory=True)
- nested_root = parent / root.name
- result = subprocess.run(
- [
- "bash", "-c",
- 'source "$1"; '
- 'stat() { case "$3" in */allocation-job-id) mode=600 ;; *) mode=700 ;; esac; '
- 'printf "%s:%s\\n" "$(id -u)" "$mode"; }; '
- 'export CX_JOB_ROOT="$2"; '
- 'cx_record_allocation_jobid 6262; cat "$2/allocation-job-id"; '
- 'cx_clear_allocation_jobid',
- "_", common, str(nested_root),
- ],
- text=True,
- capture_output=True,
- )
- self.assertEqual(result.returncode, 0, result.stderr)
- self.assertEqual(result.stdout, "6262\n")
- self.assertFalse((root / "allocation-job-id").exists())
-
- mismatched = parent / f"inferencex-collectivex-{os.getpid()}-2-mi300x-test"
- result = subprocess.run(
- [
- "bash", "-c", 'source "$1"; cx_job_root_is_safe "$2"',
- "_", common, str(mismatched),
- ],
- )
- self.assertNotEqual(result.returncode, 0)
- finally:
- parent.unlink(missing_ok=True)
- root.rmdir()
- shared.rmdir()
-
- def test_v1_publication_requires_explicit_release_markers(self) -> None:
- workflows = ROOT.parent.parent / ".github" / "workflows"
- sweep = (workflows / "collectivex-sweep.yml").read_text()
- self.assertFalse((workflows / "collectivex-publish.yml").exists())
-
- self.assertIn("options: [sweep, probe-precision, publish, refresh]", sweep)
- self.assertIn("${{ inputs.only_sku }}-${{ inputs.probe_id }}", sweep)
- self.assertIn("cancel-in-progress: false", sweep)
- self.assertIn("COLLECTIVEX_MI355_CONFIG_V1", sweep)
- self.assertIn("COLLECTIVEX_MI300_CONFIG_V1", sweep)
- self.assertIn('set(overlay["runners"]) != {"mi300x"}', sweep)
- self.assertIn('set(overlay["runners"]) != {"mi355x"}', sweep)
- self.assertIn("collectivex.precision-probe-plan.v1", (ROOT / "tests" / "probe_precision.py").read_text())
- self.assertIn("cxprecision-probes-${{ github.run_id }}-${{ github.run_attempt }}", sweep)
- self.assertIn("--validate-bundle", sweep)
-
- # The release version is a numeric, incrementable field (1, 2, 3, ...),
- # never a hardcoded "v1" tag. 'unversioned' stays diagnostic.
- self.assertIn("release_tag:", sweep)
- self.assertIn("default: unversioned", sweep)
- self.assertNotIn("options: [unversioned, v1]", sweep)
- self.assertIn("or a positive integer N to publish as version N", sweep)
- self.assertIn("qualification_index:", sweep)
- self.assertIn("inputs.release_tag != 'unversioned'", sweep)
- self.assertNotIn("inputs.release_tag == 'v1'", sweep)
- self.assertIn(
- 'release_tag must be "unversioned" or a positive integer version N', sweep
- )
- # collectivex.release-tag.v1 is a frozen data-format id, NOT the release version.
- self.assertIn("collectivex.release-tag.v1", sweep)
- # The ep-size filter is a dispatch-time narrower (like max_nodes): it lets a
- # comprehensive run schedule EP8 across every SKU without any EP16 leg, and it
- # only applies to the sweep branch (not precision probes).
- self.assertIn("ep_sizes:", sweep)
- self.assertIn("INPUT_EP_SIZES: ${{ inputs.ep_sizes }}", sweep)
- self.assertIn(
- '[ -n "$INPUT_EP_SIZES" ] && args+=(--ep-sizes "$INPUT_EP_SIZES")', sweep
- )
- # A versioned run may cover the full canonical matrix or a filtered subset;
- # the release marker records the coverage scope AND the numeric version.
- self.assertNotIn("V1 sweeps require the exact unfiltered full matrix", sweep)
- self.assertNotIn("V1 release tag requires the locked full matrix", sweep)
- self.assertIn("a full release version requires the locked execution plan", sweep)
- self.assertIn('"coverage_scope": coverage_scope', sweep)
- self.assertIn('"version": version', sweep)
- self.assertIn(
- '"full" if matrix_sha256 == os.environ["EXPECTED_MATRIX_SHA256"] else "partial"',
- sweep,
- )
- self.assertIn("EXPECTED_MATRIX_SHA256", sweep)
- self.assertIn(
- "cxrelease-${{ inputs.release_tag }}-${{ github.run_id }}-${{ github.run_attempt }}",
- sweep,
- )
-
- self.assertIn("publish_run_id must be a single decimal run ID", sweep)
- self.assertIn("source run must be qualification index 1 exactly once", sweep)
- self.assertIn("and .qualification_index == 1", sweep)
- self.assertIn(
- '"coverage_scope","execution_plan_sha256","format","matrix_sha256"', sweep
- )
- # The marker key set now carries a numeric version, not a release_tag string.
- self.assertIn('"run_id","source_sha","version"]', sweep)
- self.assertIn('(.coverage_scope == "full" or .coverage_scope == "partial")', sweep)
- self.assertIn("cxrelease-[1-9][0-9]*-$run_id-$attempt/release", sweep)
- self.assertIn("run $run_id is not tagged for a release version", sweep)
- self.assertIn("ref: ${{ steps.runs.outputs.source_sha }}", sweep)
- self.assertIn("[[ \"$attempt\" =~ ^[1-9][0-9]*$ ]]", sweep)
- self.assertIn(
- "cxpublication-${{ steps.build.outputs.version }}-${{ github.run_id }}-${{ github.run_attempt }}",
- sweep,
- )
- self.assertIn("refresh source bytes differ from their requested digest", sweep)
- self.assertIn("retention-days: 90", sweep)
- self.assertNotIn("workflow_run:", sweep)
-
- # A successful versioned sweep (full OR partial) self-publishes its own run so
- # the frontend can discover many runs of a version JIT, without a separate
- # manual publish dispatch. Publication is no longer pinned to the first attempt:
- # a re-run whose legs re-emit at a single consistent attempt can also publish.
- # The artifact name embeds the numeric version so the frontend can group runs.
- self.assertIn("autopublish:", sweep)
- self.assertIn("inputs.qualification_index == '1'", sweep)
- self.assertIn(
- "inputs.qualification_index == '1' && needs.sweep.result == 'success'", sweep
- )
- self.assertNotIn("github.run_attempt == '1' &&", sweep)
- self.assertIn("CollectiveX publication (auto) — version", sweep)
- self.assertIn(
- "cxpublication-${{ inputs.release_tag }}-${{ github.run_id }}-${{ github.run_attempt }}",
- sweep,
- )
-
- def test_source_archive_preserves_only_contained_leaf_symlinks(self) -> None:
- selected = "deepep-hybrid-pinned"
- other = "deepep-v2-pinned"
-
- def directory(name: str) -> tarfile.TarInfo:
- member = tarfile.TarInfo(name)
- member.type = tarfile.DIRTYPE
- member.mode = 0o755
- return member
-
- def regular(
- name: str, payload: bytes, mode: int = 0o644
- ) -> tuple[tarfile.TarInfo, io.BytesIO]:
- member = tarfile.TarInfo(name)
- member.size = len(payload)
- member.mode = mode
- return member, io.BytesIO(payload)
-
- def symbolic(name: str, target: str) -> tarfile.TarInfo:
- member = tarfile.TarInfo(name)
- member.type = tarfile.SYMTYPE
- member.linkname = target
- member.mode = 0o777
- return member
-
- def write_archive(path: Path, extras: list[tarfile.TarInfo] | None = None) -> None:
- root = f".cx_sources/{selected}"
- with tarfile.open(path, "w") as archive:
- for name in (
- ".cx_sources", root, f"{root}/third-party",
- f"{root}/third-party/nccl", f"{root}/third-party/nccl/pkg",
- f"{root}/third-party/nccl/pkg/debian",
- f".cx_sources/{other}",
- ):
- archive.addfile(directory(name))
- member, stream = regular(
- f"{root}/third-party/nccl/LICENSE.txt", b"license\n"
- )
- archive.addfile(member, stream)
- member, stream = regular(f".cx_sources/{other}/sentinel", b"other\n")
- archive.addfile(member, stream)
- member, stream = regular(f"{root}/group-executable", b"exec\n", 0o010)
- archive.addfile(member, stream)
- archive.addfile(symbolic(
- f"{root}/third-party/nccl/pkg/debian/copyright",
- "../../LICENSE.txt",
- ))
- for member in extras or []:
- archive.addfile(member)
- path.chmod(0o600)
-
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary).resolve()
- archive = root / "source.tar"
- destination = root / "destination"
- destination.mkdir(mode=0o700)
- write_archive(archive)
- source_archive.extract_source_archive(archive, destination, selected)
- link = (
- destination / ".cx_sources" / selected / "third-party" / "nccl"
- / "pkg" / "debian" / "copyright"
- )
- self.assertTrue(link.is_symlink())
- self.assertEqual(os.readlink(link), "../../LICENSE.txt")
- self.assertEqual(link.read_text(), "license\n")
- self.assertFalse((destination / ".cx_sources" / other).exists())
- extracted = destination / ".cx_sources" / selected
- self.assertEqual(
- stat.S_IMODE((extracted / "group-executable").stat().st_mode), 0o700
- )
- self.assertEqual(
- stat.S_IMODE(
- (extracted / "third-party" / "nccl" / "LICENSE.txt").stat().st_mode
- ),
- 0o600,
- )
-
- invalid: dict[str, list[tarfile.TarInfo]] = {
- "absolute member": [directory("/outside")],
- "traversal member": [directory(".cx_sources/../outside")],
- "duplicate member": [directory(f".cx_sources/{selected}")],
- "absolute link": [symbolic(f".cx_sources/{selected}/absolute", "/tmp/x")],
- "escaping link": [symbolic(f".cx_sources/{selected}/escape", "../x")],
- "cross-root link": [
- symbolic(f".cx_sources/{selected}/cross", f"../{other}/sentinel")
- ],
- "missing target": [symbolic(f".cx_sources/{selected}/missing", "none")],
- }
- hardlink = tarfile.TarInfo(f".cx_sources/{selected}/hard")
- hardlink.type = tarfile.LNKTYPE
- hardlink.linkname = f".cx_sources/{selected}/third-party/nccl/LICENSE.txt"
- invalid["hardlink"] = [hardlink]
- fifo = tarfile.TarInfo(f".cx_sources/{selected}/fifo")
- fifo.type = tarfile.FIFOTYPE
- invalid["fifo"] = [fifo]
- character = tarfile.TarInfo(f".cx_sources/{selected}/character")
- character.type = tarfile.CHRTYPE
- invalid["character device"] = [character]
- block = tarfile.TarInfo(f".cx_sources/{selected}/block")
- block.type = tarfile.BLKTYPE
- invalid["block device"] = [block]
- unknown = tarfile.TarInfo(f".cx_sources/{selected}/unknown")
- unknown.type = b"Z"
- invalid["unknown type"] = [unknown]
- invalid["unsafe unselected root"] = [
- symbolic(f".cx_sources/{other}/escape", f"../{selected}/group-executable")
- ]
- chain_target = symbolic(
- f".cx_sources/{selected}/chain-target", "third-party/nccl/LICENSE.txt"
- )
- invalid["symlink chain"] = [
- chain_target, symbolic(f".cx_sources/{selected}/chain", "chain-target")
- ]
- linked_child = tarfile.TarInfo(f".cx_sources/{selected}/linked-file/child")
- invalid["symlink parent"] = [
- symbolic(
- f".cx_sources/{selected}/linked-file",
- "third-party/nccl/LICENSE.txt",
- ),
- linked_child,
- ]
- for label, extras in invalid.items():
- with self.subTest(label=label), tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary).resolve()
- archive = root / "source.tar"
- destination = root / "destination"
- destination.mkdir(mode=0o700)
- write_archive(archive, extras)
- with self.assertRaises(source_archive.SourceArchiveError):
- source_archive.extract_source_archive(archive, destination, selected)
- self.assertFalse((destination / ".cx_sources").exists())
-
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary).resolve()
- archive = root / "source.tar"
- destination = root / "destination"
- destination.mkdir(mode=0o700)
- existing = destination / ".cx_sources"
- existing.mkdir(mode=0o700)
- marker = existing / "marker"
- marker.write_text("existing\n")
- write_archive(archive)
- with self.assertRaises(source_archive.SourceArchiveError):
- source_archive.extract_source_archive(archive, destination, selected)
- self.assertEqual(marker.read_text(), "existing\n")
-
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary).resolve()
- archive = root / "source.tar"
- write_archive(archive)
- real_destination = root / "real-destination"
- real_destination.mkdir(mode=0o700)
- linked_destination = root / "linked-destination"
- linked_destination.symlink_to(real_destination, target_is_directory=True)
- with self.assertRaises((OSError, source_archive.SourceArchiveError)):
- source_archive.extract_source_archive(archive, linked_destination, selected)
- self.assertFalse((real_destination / ".cx_sources").exists())
-
- unsafe_destination = root / "unsafe-destination"
- unsafe_destination.mkdir(mode=0o700)
- unsafe_destination.chmod(0o755)
- with self.assertRaises(source_archive.SourceArchiveError):
- source_archive.extract_source_archive(archive, unsafe_destination, selected)
- self.assertFalse((unsafe_destination / ".cx_sources").exists())
-
- for limit, value in (
- ("MAX_ARCHIVE_MEMBERS", 1),
- ("MAX_MEMBER_BYTES", 1),
- ("MAX_EXPANDED_BYTES", 1),
- ("MAX_ARCHIVE_BYTES", 1),
- ("MAX_ARCHIVE_HEADERS", 1),
- ):
- with self.subTest(limit=limit), tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary).resolve()
- archive = root / "source.tar"
- destination = root / "destination"
- destination.mkdir(mode=0o700)
- write_archive(archive)
- with mock.patch.object(source_archive, limit, value):
- with self.assertRaises(source_archive.SourceArchiveError):
- source_archive.extract_source_archive(archive, destination, selected)
- self.assertFalse((destination / ".cx_sources").exists())
-
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary).resolve()
- archive = root / "source.tar"
- destination = root / "destination"
- destination.mkdir(mode=0o700)
- write_archive(archive)
- long_name = f".cx_sources/{selected}/long-name-result\0".encode()
- with tarfile.open(archive, "a") as handle:
- for _ in range(3):
- extension = tarfile.TarInfo("././@LongLink")
- extension.type = tarfile.GNUTYPE_LONGNAME
- extension.size = len(long_name)
- handle.addfile(extension, io.BytesIO(long_name))
- member, stream = regular("placeholder", b"payload\n")
- handle.addfile(member, stream)
- archive.chmod(0o600)
- for limit, value in (
- ("MAX_EXTENSION_CHAIN", 1),
- ("MAX_EXTENSION_MEMBER_BYTES", 1),
- ("MAX_EXTENSION_BYTES", len(long_name) * 2),
- ):
- with self.subTest(limit=limit), mock.patch.object(
- source_archive, limit, value
- ):
- with self.assertRaises(source_archive.SourceArchiveError):
- source_archive.extract_source_archive(
- archive, destination, selected
- )
- self.assertFalse((destination / ".cx_sources").exists())
-
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary).resolve()
- archive = root / "source.tar"
- destination = root / "destination"
- destination.mkdir(mode=0o700)
- write_archive(archive)
- with tarfile.open(archive, "a", format=tarfile.PAX_FORMAT) as handle:
- member, stream = regular(
- f".cx_sources/{selected}/sparse-v1", b"1\n0\n1\n"
- )
- member.pax_headers = {
- "GNU.sparse.major": "1",
- "GNU.sparse.minor": "0",
- "GNU.sparse.realsize": "1",
- }
- handle.addfile(member, stream)
- archive.chmod(0o600)
- with self.assertRaises(source_archive.SourceArchiveError):
- source_archive.extract_source_archive(archive, destination, selected)
- self.assertFalse((destination / ".cx_sources").exists())
-
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary).resolve()
- archive = root / "source.tar"
- destination = root / "destination"
- destination.mkdir(mode=0o700)
- write_archive(archive)
- original_next = tarfile.TarFile.next
-
- def sparse_next(handle: tarfile.TarFile) -> tarfile.TarInfo | None:
- member = original_next(handle)
- if member is not None and member.isfile():
- member.sparse = [(0, 1)]
- return member
-
- with mock.patch.object(tarfile.TarFile, "next", sparse_next):
- with self.assertRaises(source_archive.SourceArchiveError):
- source_archive.extract_source_archive(archive, destination, selected)
- self.assertFalse((destination / ".cx_sources").exists())
-
- def test_runtime_identity_and_realized_placement_are_behavioral(self) -> None:
- self.assertFalse(capability.runtime_identity_issues(
- "mi325x", vendor="amd", arch="gfx942", machine="amd64",
- device_name="AMD Instinct MI325X", device_count=8, world_size=8,
- ))
- self.assertTrue(capability.runtime_identity_issues(
- "mi355x", vendor="amd", arch="gfx942", machine="amd64",
- device_name="AMD Instinct MI325X", device_count=8, world_size=8,
- ))
- records = [("private-a", rank) for rank in range(4)] + [
- ("private-b", rank) for rank in range(4)
- ]
- self.assertEqual(
- run_ep._summarize_realized_placement(
- records, expected_nodes=2, expected_gpus_per_node=4, expected_world_size=8
- ),
- {
- "gpus_per_node": 4,
- "nodes": 2,
- "ranks_per_node": 4,
- "unique_local_ranks": True,
- "valid": True,
- },
- )
- with self.assertRaises(ValueError):
- run_ep._summarize_realized_placement(
- records[:-1] + [("private-b", 2)],
- expected_nodes=2,
- expected_gpus_per_node=4,
- expected_world_size=8,
- )
-
- def test_private_allocation_stratum_is_salted_ordered_and_rank_consistent(self) -> None:
- salt = "a" * 64
- hosts = ["private-node-b", "private-node-a", "private-node-a"]
- selectors = {
- "ib_gid_index": "3",
- "rdma_devices": "private-hca0:1,private-hca1:1",
- "rdma_service_level": "2",
- "rdma_traffic_class": "104",
- "socket_ifname": "private-if0",
- }
- digest = run_ep._allocation_stratum_sha256(
- hosts, audit_salt=salt, fabric_selectors=selectors, required=True
- )
- self.assertRegex(digest or "", r"^[0-9a-f]{64}$")
- self.assertEqual(
- digest,
- run_ep._allocation_stratum_sha256(
- list(reversed(hosts)),
- audit_salt=salt,
- fabric_selectors=selectors,
- required=True,
- ),
- )
- for changed_hosts, changed_salt, changed_selectors in (
- (hosts + ["private-node-c"], salt, selectors),
- (hosts, "b" * 64, selectors),
- (hosts, salt, {**selectors, "ib_gid_index": "4"}),
- ):
- self.assertNotEqual(
- digest,
- run_ep._allocation_stratum_sha256(
- changed_hosts,
- audit_salt=changed_salt,
- fabric_selectors=changed_selectors,
- required=True,
- ),
- )
- serialized = json.dumps({"allocation_stratum_sha256": digest})
- private_literals = [
- salt,
- *hosts,
- selectors["rdma_devices"],
- selectors["socket_ifname"],
- ]
- self.assertFalse(any(value in serialized for value in private_literals))
- self.assertNotIn("physical_hosts", serialized)
- self.assertNotIn("fabric_selectors", serialized)
- self.assertEqual(
- run_ep._common_allocation_stratum([digest, digest, digest], required=True),
- digest,
- )
- with self.assertRaisesRegex(ValueError, "differs across distributed ranks"):
- run_ep._common_allocation_stratum([digest, "b" * 64], required=True)
- with self.assertRaisesRegex(ValueError, "requires"):
- run_ep._allocation_stratum_sha256(
- hosts, audit_salt=None, fabric_selectors=selectors, required=True
- )
- self.assertIsNone(run_ep._allocation_stratum_sha256(
- hosts, audit_salt=None, fabric_selectors=selectors, required=False
- ))
-
- def test_collective_version_and_rccl_fingerprint_are_normalized(self) -> None:
- self.assertEqual(ep_harness.format_collective_version(23004), "2.30.4")
- self.assertEqual(ep_harness.format_collective_version(21805), "2.18.5")
- self.assertEqual(ep_harness.format_collective_version((2, 21, 5)), "2.21.5")
-
- properties = types.SimpleNamespace(
- multi_processor_count=304, total_memory=1024, warp_size=64
- )
- fake = types.SimpleNamespace(
- __version__="2.9.0",
- version=types.SimpleNamespace(cuda=None, hip="7.2"),
- cuda=types.SimpleNamespace(
- get_device_properties=lambda _device: properties,
- get_device_name=lambda _device: "AMD Instinct MI325X",
- nccl=types.SimpleNamespace(version=lambda: 21805),
- ),
- )
- with mock.patch.object(
- run_ep, "_loaded_collective_version", return_value="2.18.5"
- ):
- fingerprint = run_ep._runtime_fingerprint(
- fake, "device", machine="amd64", vendor="amd", arch="gfx942"
- )
- self.assertEqual(fingerprint["collective_library"], {"kind": "rccl", "version": "2.18.5"})
- self.assertEqual(fingerprint["accelerator_runtime"], {"kind": "hip", "version": "7.2"})
-
- class FakeCollective:
- @staticmethod
- def ncclGetVersion(pointer) -> int:
- pointer._obj.value = 23004
- return 0
-
- maps = "0-1 r-xp 0 00:00 0 /runtime/libnccl.so.2\n"
- with (
- mock.patch("builtins.open", return_value=io.StringIO(maps)),
- mock.patch.object(run_ep.os.path, "isfile", return_value=True),
- mock.patch.object(
- run_ep.os.path, "realpath", return_value="/runtime/libnccl.so.2"
- ),
- mock.patch.object(run_ep.ctypes, "CDLL", return_value=FakeCollective()),
- ):
- self.assertEqual(run_ep._loaded_collective_version(), "2.30.4")
-
- path = HERE / "ep_nccl.py"
- tree = ast.parse(path.read_text(), str(path))
- helper = next(
- node for node in tree.body
- if isinstance(node, ast.FunctionDef) and node.name == "_runtime_collective"
- )
- namespace = {"re": re}
- exec(compile(ast.Module(body=[helper], type_ignores=[]), str(path), "exec"), namespace)
- args = types.SimpleNamespace(
- runtime_fingerprint={
- "collective_library": {"kind": "nccl", "version": "2.30.4"}
- }
- )
- cuda = types.SimpleNamespace(version=types.SimpleNamespace(hip=None))
- self.assertEqual(namespace["_runtime_collective"](args, cuda), ("nccl", "2.30.4"))
- args.runtime_fingerprint["collective_library"]["version"] = None
- with self.assertRaisesRegex(RuntimeError, "runtime identity is unavailable"):
- namespace["_runtime_collective"](args, cuda)
- self.assertNotIn("torch.cuda.nccl.version", path.read_text())
-
- def test_workloads_bind_generator_activation_and_trace(self) -> None:
- args = ("uniform", 7168, 8, 256, 8, 64, 67)
- first = workload.compute_workload_id(*args)
- self.assertTrue(identity.is_typed_id(first, "workload"))
- self.assertEqual(first, workload.compute_workload_id(*args))
- self.assertNotEqual(first, workload.compute_workload_id(*args[:-1], 68))
- self.assertNotEqual(
- first,
- workload.compute_workload_id(*args, trace_checksum="a" * 64),
- )
- _, _, manifest = workload.build_workload(8, 2, 4, "uniform", 4, 67, 2)
- member, checksums, _, _ = workload.canonical_member(
- "uniform", 8, 2, 4, 2, 2, 67
- )
- self.assertEqual(member, manifest["workload_id"])
- self.assertEqual(checksums, manifest["checksums"])
-
- def test_canonical_members_are_bound_to_each_scheduled_row(self) -> None:
- case = {
- "routing": "uniform", "hidden": 8, "topk": 2, "experts": 4, "ep": 2,
- "mode": "normal",
- }
- eplb_record = {
- "enabled": False, "mapping_hash": None, "num_physical_experts": 4,
- }
-
- def expected(
- *, tokens: int = 1, hidden: int = 8
- ) -> tuple[str, dict[str, str], str]:
- member, checksums, row_hash, _, _ = contracts._expected_canonical_trace(
- "uniform", hidden, 2, 4, 4, 2, tokens, 67, False, 2048
- )
- return member, checksums, row_hash
-
- member, checksums, row_hash = expected()
- rows = [{"tokens_per_rank": 1, "routing": {"hash": row_hash}}]
- proof = {
- "manifest_checksums": {member: checksums},
- "members": [member],
- "workload_id": identity.workload_id({
- "members": [{"checksums": checksums, "workload_id": member}]
- }),
- }
- contracts._validate_canonical_workload(proof, case, rows, eplb_record)
-
- def replace_member(document: dict, replacement: tuple[str, dict[str, str], str]) -> None:
- replacement_id, replacement_checksums, _ = replacement
- document["members"] = [replacement_id]
- document["manifest_checksums"] = {replacement_id: replacement_checksums}
- document["workload_id"] = identity.workload_id({
- "members": [{
- "checksums": replacement_checksums,
- "workload_id": replacement_id,
- }]
- })
-
- mutations = {
- "wrong member token": lambda document, mutated_rows: replace_member(
- document, expected(tokens=2)
- ),
- "wrong member dimensions": lambda document, mutated_rows: replace_member(
- document, expected(hidden=16)
- ),
- "wrong member checksum": lambda document, mutated_rows: replace_member(
- document,
- (
- member,
- {**checksums, "topk_idx": "0" * 64},
- row_hash,
- ),
- ),
- "row hash unrelated to member": lambda document, mutated_rows: mutated_rows[0][
- "routing"
- ].update({"hash": "f" * 64}),
- }
- for label, mutate in mutations.items():
- with self.subTest(label=label), self.assertRaises(contracts.ContractError):
- bad_proof, bad_rows = copy.deepcopy(proof), copy.deepcopy(rows)
- mutate(bad_proof, bad_rows)
- contracts._validate_canonical_workload(
- bad_proof, case, bad_rows, eplb_record
- )
-
- def test_oracle_pass_cannot_ignore_combined_value_failure(self) -> None:
- oracle = {
- "atol": ep_harness.ORACLE_ATOL,
- "checks": {
- "combine_values": True,
- "counts": True,
- "metadata": True,
- "multiplicity": True,
- "payload": True,
- "source_set": True,
- "weights": True,
- },
- "combine_weight_semantics": "unweighted-rank-sum",
- "contract": ep_harness.ORACLE_CONTRACT,
- "dispatch_sha256": "a" * 64,
- "max_absolute_error": 0.0,
- "max_elementwise_relative_error": 0.0,
- "max_relative_error": 0.0,
- "max_weight_error": 0.0,
- "order_sha256": "b" * 64,
- "ordering_contract": "stable-v1",
- "passed": True,
- "receive_count": 1,
- "rtol": ep_harness.ORACLE_RTOL,
- }
- contracts._validate_oracle(oracle, "oracle")
- weighted = copy.deepcopy(oracle)
- weighted["combine_weight_semantics"] = "native-gate-weighted"
- with self.assertRaisesRegex(contracts.ContractError, "differs from v1"):
- contracts._validate_oracle(weighted, "oracle")
- tampered = copy.deepcopy(oracle)
- tampered["checks"]["combine_values"] = False
- with self.assertRaises(contracts.ContractError):
- contracts._validate_oracle(tampered, "oracle")
-
- for profile_id, expected in (
- ("d-bf16.c-fp8-e4m3fn-direct-cast-noscale", {"atol": 4e-2, "rtol": 8e-2}),
- ):
- with self.subTest(profile_id=profile_id):
- precision = identity.precision_profile(profile_id)
- codec_oracle = copy.deepcopy(oracle)
- codec_oracle.update(expected)
- contracts._validate_oracle(
- codec_oracle, "oracle", communication_precision=precision
- )
- codec_oracle["rtol"] = 5e-2
- with self.assertRaisesRegex(contracts.ContractError, "rtol"):
- contracts._validate_oracle(
- codec_oracle, "oracle", communication_precision=precision
- )
-
- def test_precision_evidence_rejects_direct_cast_saturation(self) -> None:
- profile_id = "d-bf16.c-fp8-e4m3fn-direct-cast-noscale"
- communication_precision = identity.precision_profile(profile_id)
- axis = {
- "dequantized_semantics": True,
- "encoded_payload_valid": True,
- "max_abs_error": 0.0,
- "max_rel_error": 0.0,
- "passed": True,
- "saturation_count": 0,
- "saturation_rate": 0.0,
- "scales_finite": None,
- "scales_positive": None,
- }
- evidence = {
- "combine": copy.deepcopy(axis),
- "dispatch": copy.deepcopy(axis),
- "passed": True,
- "profile_id": profile_id,
- }
- contracts._validate_precision_evidence(
- evidence, profile_id, communication_precision, "precision"
- )
- saturated = copy.deepcopy(evidence)
- saturated["combine"].update({"saturation_count": 1, "saturation_rate": 0.01})
- with self.assertRaisesRegex(contracts.ContractError, "passed differs"):
- contracts._validate_precision_evidence(
- saturated, profile_id, communication_precision, "precision"
- )
- saturated["combine"]["passed"] = False
- saturated["passed"] = False
- contracts._validate_precision_evidence(
- saturated, profile_id, communication_precision, "precision"
- )
-
- invented_scales = copy.deepcopy(evidence)
- invented_scales["combine"].update({
- "scales_finite": True,
- "scales_positive": True,
- })
- with self.assertRaisesRegex(contracts.ContractError, "must be null"):
- contracts._validate_precision_evidence(
- invented_scales, profile_id, communication_precision, "precision"
- )
-
- def test_oracle_stability_canonicalizes_native_receive_order(self) -> None:
- source = (HERE / "ep_harness.py").read_text()
- begin = source.index("canonical_order = torch.argsort")
- canonical = source[begin:source.index("problem.recv_tokens = receive_count", begin)]
- self.assertIn("canonical_sources", canonical)
- self.assertIn("canonical_ids", canonical)
- self.assertIn("canonical_weights", canonical)
- self.assertNotIn("_tensor_sha256(source_ids", canonical)
- mori = (HERE / "ep_mori.py").read_text()
- self.assertIn('"inter-node-v1" if self._inter_node', mori)
- self.assertIn('else "async-ll" if self._async_ll', mori)
- backend = types.SimpleNamespace(name="mori", kernel_generation="async-ll")
- self.assertEqual(ep_harness.kernel_generation(backend), "async-ll")
- backend.kernel_generation = "inter-node-v1"
- self.assertEqual(ep_harness.kernel_generation(backend), "inter-node-v1")
-
- def test_terminal_fail_safe_fills_only_missing_shard_cases(self) -> None:
- matrix = sweep_matrix.resolve_matrix(backends="all", max_cases=128)
- shard = next(item for item in matrix["include"] if item["n"] >= 2)
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary)
- matrix_path = root / "matrix.json"
- control_path = root / "control.json"
- out_dir = root / "results"
- matrix_path.write_text(json.dumps(matrix))
- control = sweep_matrix.extract_shard(
- matrix_path, shard["id"], control_path,
- sku=shard["sku"], backend=shard["backend"], nodes=shard["nodes"],
- )
- control["cases"] = control["cases"][:2]
- control["n"] = 2
- control_path.write_text(json.dumps(control))
- first = {key: value for key, value in control["cases"][0].items() if key != "case_id"}
- git_run = {
- "artifact": "artifact", "job": "job", "ref": "collectivex",
- "repo": "SemiAnalysisAI/InferenceX", "run_attempt": "1",
- "run_id": "123", "source_sha": "a" * 40,
- "qualification_index": 1,
- }
- allocation = {
- "artifact": "artifact", "execution_id": "execution", "job": "job",
- "repo": "SemiAnalysisAI/InferenceX", "run_attempt": "1", "run_id": "123",
- "runner": shard["sku"], "source_sha": "a" * 40,
- "qualification_index": 1,
- }
- out_dir.mkdir()
- existing = contracts.make_terminal_document(
- allocation_factors=allocation, attempt_ordinal=1, case=first,
- case_factors={
- "case": first,
- "profile": identity.profile_for_case(first),
- "sku": shard["sku"],
- },
- control_sha256=hashlib.sha256(control_path.read_bytes()).hexdigest(),
- failure_mode="setup", generated_at="2026-07-04T00:00:00Z", git_run=git_run,
- reason="launcher-setup-failed", return_code=7, source="runtime-emitter",
- status="failed",
- expected_case_id=control["cases"][0]["case_id"],
- )
- (out_dir / "existing.json").write_text(json.dumps(existing))
- (out_dir / "partial.json").write_text(json.dumps({
- "format": contracts.RAW_FORMAT,
- "identity": {"case_id": control["cases"][1]["case_id"]},
- "sample_artifact": {"path": "partial.samples.json"},
- }))
- (out_dir / "partial.samples.json").write_text("{broken")
- environment = {
- **os.environ,
- "COLLECTIVEX_OPERATOR_CONFIG": "/dev/null",
- "CX_SHARD_FILE": str(control_path),
- "CX_SHARD_SKU": shard["sku"],
- "CX_RUNNER": shard["sku"],
- "CX_BENCH": shard["backend"],
- "CX_NODES": str(shard["nodes"]),
- "COLLECTIVEX_EXECUTION_ID": "execution",
- "COLLECTIVEX_ARTIFACT_NAME": "artifact",
- "GITHUB_JOB": "job", "GITHUB_REF_NAME": "collectivex",
- "GITHUB_REPOSITORY": "SemiAnalysisAI/InferenceX",
- "GITHUB_RUN_ATTEMPT": "1", "GITHUB_RUN_ID": "123",
- "GITHUB_SHA": "a" * 40,
- "CX_QUALIFICATION_INDEX": "1",
- }
- subprocess.run(
- [
- "bash", "-c",
- 'source "$1"; cx_emit_setup_failures "$2" "$3" "$4" 7',
- "_", str(ROOT / "runtime" / "common.sh"), str(ROOT),
- str(out_dir), shard["backend"],
- ],
- check=True,
- env=environment,
- )
- attempts = [contracts.strict_load(path) for path in out_dir.glob("*.json")]
- self.assertEqual(len(attempts), 2)
- self.assertEqual(
- contracts.validate_attempt_paths([str(path) for path in out_dir.glob("*.json")]),
- 2,
- )
- delivery = [str(path) for path in out_dir.glob("*.json")]
- self.assertEqual(contracts.validate_delivery(delivery, str(control_path)), 2)
- with self.assertRaises(contracts.ContractError):
- contracts.validate_delivery(delivery[:1], str(control_path))
- self.assertEqual(
- {attempt["identity"]["case_id"] for attempt in attempts},
- {case["case_id"] for case in control["cases"]},
- )
- self.assertTrue((out_dir / "partial.json.quarantine").is_file())
- self.assertTrue((out_dir / "partial.samples.json.quarantine").is_file())
-
- preallocation = root / "preallocation"
- preallocation_results = preallocation / "experimental" / "CollectiveX" / "results"
- preallocation_results.mkdir(parents=True)
- failed = subprocess.run(
- [
- "bash", "-c",
- 'source "$1"; REPO_ROOT="$2"; export REPO_ROOT; '
- 'cx_install_launcher_fail_safe; cx_load_operator_config',
- "_", str(ROOT / "runtime" / "common.sh"), str(preallocation),
- ],
- env={**environment, "COLLECTIVEX_OPERATOR_CONFIG_REQUIRED": "1"},
- )
- self.assertNotEqual(failed.returncode, 0)
- preallocation_attempts = [
- contracts.validate_terminal_document(contracts.strict_load(path))
- for path in preallocation_results.glob("*.json")
- ]
- self.assertEqual(
- {attempt["identity"]["case_id"] for attempt in preallocation_attempts},
- {case["case_id"] for case in control["cases"]},
- )
-
- def test_runtime_identity_mismatch_is_failed_not_unsupported(self) -> None:
- wrapper = next(
- item for item in sweep_matrix.resolve_matrix()["requested_cases"]
- if item["disposition"] == "runnable"
- )
- case = wrapper["case"]
- environment = {
- "CX_RUNNER": wrapper["sku"], "CX_CASE_ID": case["case_id"],
- "CX_SUITE": case["suite"], "CX_WORKLOAD_NAME": case["workload"],
- "CX_REQUIRED_PUBLICATION": case["required_publication"],
- "CX_ROUTING": case["routing"], "CX_EPLB": "1" if case["eplb"] else "",
- "CX_EP": str(case["ep"]), "CX_NGPUS": str(case["ep"]),
- "CX_HIDDEN": str(case["hidden"]), "CX_TOPK": str(case["topk"]),
- "CX_EXPERTS": str(case["experts"]), "CX_NODES": str(case["nodes"]),
- "CX_GPUS_PER_NODE": str(case["gpus_per_node"]),
- "CX_SCALE_UP_DOMAIN": str(case["scale_up_domain"]),
- "CX_MODE": case["mode"], "CX_SCOPE": case["scope"],
- "CX_TOPO": case["topology_class"], "CX_TRANSPORT": case["transport"],
- "CX_SCALE_UP_TRANSPORT": case["scale_up_transport"],
- "CX_SCALE_OUT_TRANSPORT": case["scale_out_transport"] or "",
- "CX_TOKENS_LADDER": case["ladder"], "CX_CANONICAL": "1",
- "CX_ITERS": "8", "CX_TRIALS": "64", "CX_WARMUP": "32",
- "CX_SAMPLES_PER_POINT": "512", "GITHUB_RUN_ID": "123",
- "GITHUB_RUN_ATTEMPT": "1", "GITHUB_REF_NAME": "collectivex",
- "GITHUB_SHA": "a" * 40, "GITHUB_REPOSITORY": "SemiAnalysisAI/InferenceX",
- "GITHUB_JOB": "sweep", "COLLECTIVEX_ARTIFACT_NAME": "artifact",
- "COLLECTIVEX_EXECUTION_ID": "execution",
- }
- with mock.patch.dict(os.environ, environment, clear=False):
- terminal = contracts.make_terminal_from_environment(
- backend=case["backend"], phase=case["phase"], return_code=5
- )
- self.assertEqual(terminal["identity"]["case_id"], case["case_id"])
- self.assertEqual(
- terminal["outcome"],
- {
- "failure_mode": "runtime-identity",
- "reason": "runtime-identity-mismatch",
- "return_code": 5,
- "status": "failed",
- },
- )
- for mode, reason in contracts.RUNTIME_FAILURE_REASONS.items():
- with self.subTest(mode=mode), mock.patch.dict(os.environ, environment, clear=False):
- staged = contracts.make_terminal_from_environment(
- backend=case["backend"], phase=case["phase"], return_code=1,
- failure_mode=mode,
- )
- self.assertEqual(staged["outcome"]["reason"], reason)
- mismatched = copy.deepcopy(staged)
- mismatched["outcome"]["reason"] = "distributed-command-failed"
- if reason == "distributed-command-failed":
- mismatched["outcome"]["reason"] = "backend-setup-failed"
- with self.assertRaisesRegex(
- contracts.ContractError, "source and outcome are not registered"
- ):
- contracts.validate_terminal_document(mismatched)
- with mock.patch.dict(os.environ, environment, clear=False):
- with self.assertRaisesRegex(
- contracts.ContractError, "runtime failure mode is not registered"
- ) as raised:
- contracts.make_terminal_from_environment(
- backend=case["backend"], phase=case["phase"], return_code=1,
- failure_mode="raw-private-error",
- )
- self.assertNotIn("raw-private-error", str(raised.exception))
- with mock.patch.dict(os.environ, environment, clear=False):
- generic = contracts.make_terminal_from_environment(
- backend=case["backend"], phase=case["phase"], return_code=6,
- )
- self.assertEqual(
- generic["outcome"],
- {
- "failure_mode": "execution",
- "reason": "distributed-command-failed",
- "return_code": 6,
- "status": "failed",
- },
- )
- manual_environment = {
- "CX_RUNNER": "manual-runner",
- "COLLECTIVEX_EXECUTION_ID": "manual-execution",
- }
- with mock.patch.dict(os.environ, manual_environment, clear=True):
- manual = contracts.make_terminal_from_environment(
- backend="nccl-ep", phase="decode", return_code=6,
- )
- self.assertIsNone(manual["provenance"]["git_run"])
- self.assertEqual(
- {
- field: manual["case"][field]
- for field in ("suite", "workload", "canonical", "required_publication")
- },
- {
- "suite": "manual", "workload": "manual", "canonical": False,
- "required_publication": "diagnostic",
- },
- )
- self.assertEqual(
- manual["identity"]["allocation_factors"],
- {
- "artifact": None, "execution_id": "manual-execution", "job": None,
- "qualification_index": 1, "repo": None,
- "run_attempt": None, "run_id": None,
- "runner": "manual-runner", "source_sha": None,
- },
- )
- broken = copy.deepcopy(manual)
- broken["identity"]["allocation_factors"]["artifact"] = "forged-artifact"
- allocation_id = identity.allocation_id(
- broken["identity"]["allocation_factors"]
- )
- broken["identity"]["allocation_id"] = allocation_id
- broken["identity"]["attempt_id"] = identity.attempt_id(
- allocation=allocation_id,
- case=broken["identity"]["case_id"],
- ordinal=broken["identity"]["attempt_ordinal"],
- )
- with self.assertRaisesRegex(
- contracts.ContractError, "allocation factors differ"
- ):
- contracts.validate_terminal_document(broken)
-
- def test_launchers_use_private_logs_and_allowlisted_failure_stages(self) -> None:
- expected = {
- "launch_single-slurm.sh": {
- "setup", "registry-verification", "container-import", "container-hash",
- "repository-stage", "scheduler-allocation", "container-launch",
- "artifact-collection",
- },
- "launch_gb-nv.sh": {
- "setup", "registry-verification", "container-import", "container-hash",
- "repository-stage", "scheduler-allocation", "container-launch", "backend-setup",
- "execution", "artifact-collection",
- },
- "launch_mi-amds.sh": {
- "setup", "repository-stage", "registry-verification", "scheduler-allocation",
- "container-import", "container-hash", "container-launch", "artifact-collection",
- },
- }
- common = (ROOT / "runtime" / "common.sh").read_text()
- for name, stages in expected.items():
- launcher = (ROOT / "launchers" / name).read_text()
- stage_source = launcher + common if name == "launch_gb-nv.sh" else launcher
- self.assertNotIn("--export=ALL", launcher)
- if name == "launch_gb-nv.sh":
- self.assertIn("cx_run_distributed_shard", launcher)
- else:
- self.assertIn("cx_container_exports", launcher)
- self.assertIn("collect_rc=0", launcher)
- for stage in stages:
- with self.subTest(launcher=name, stage=stage):
- self.assertIn(f"cx_set_failure_stage {stage}", stage_source)
- amd = (ROOT / "launchers" / "launch_mi-amds.sh").read_text()
- self.assertIn("cx_ensure_squash_on_job", amd)
- self.assertIn("for allocation_attempt in 1 2 3", amd)
- self.assertIn("allocated nodes failed container import; retrying elsewhere", amd)
- self.assertIn('rejected_nodes="$(cx_allocation_nodes_csv "$JOB_ID")"', amd)
- self.assertIn("cx_fail_stage container-hash", amd)
- self.assertNotIn('cat "$import_log"', amd)
- self.assertIn('bash -s -- "$sq" "$lock" "$image"', common)
- self.assertIn("> \"$log\" 2>&1 <<'BASH'", common)
- self.assertIn("cx_fail_stage container-import", common)
- run_ep = (ROOT / "tests" / "run_ep.py").read_text()
- self.assertIn("def _all_gather_json(", run_ep)
- self.assertIn("dist_module.all_gather(gathered, encoded)", run_ep)
- self.assertNotIn("dist.all_gather_object", run_ep)
- self.assertIn('args.backend == "deepep-v2"', run_ep)
- self.assertNotIn('args.backend in {"deepep-v2", "nccl-ep"}', run_ep)
- self.assertIn('dist.init_process_group("nccl", device_id=device)', run_ep)
- nccl = (ROOT / "tests" / "ep_nccl.py").read_text()
- self.assertIn('if _library == "nccl" and network_selection != "IB"', nccl)
- self.assertIn('if _library == "rccl" and network_selection:', nccl)
- runtime = (ROOT / "runtime" / "run_in_container.sh").read_text()
- export_start = common.index("\ncx_container_exports() {")
- exports = common[export_start:common.index("\n}", export_start)]
- export_names = {
- name
- for payload in re.findall(r"printf '%s' '([^']*)'", exports)
- for name in payload.split(",") if name
- }
- for private_name in (
- "COLLECTIVEX_OPERATOR_CONFIG", "GITHUB_TOKEN", "GITHUB_WORKSPACE", "HOME",
- "CX_PARTITION", "CX_ACCOUNT", "CX_SQUASH_DIR", "CX_STAGE_DIR",
- ):
- self.assertNotIn(private_name, export_names)
- self.assertIn("CX_BACKEND_CACHE_ROOT", export_names)
- self.assertIn("CX_BACKEND_CACHE_SENTINEL_SHA256", export_names)
- self.assertNotIn("CX_PREPARED_BACKEND_CACHE", export_names)
- self.assertIn("MORI_COMMIT", export_names)
- self.assertIn("cx_write_runtime_stage backend-setup", runtime)
- self.assertIn("cx_write_runtime_stage execution", runtime)
- distributed = common[common.index("cx_run_distributed_shard()") :]
- self.assertIn("cx_private_log_path shard-summary", distributed)
- self.assertIn("cx_fail_stage execution", distributed)
- self.assertIn('cx_fail_stage execution "$runtime_log"', distributed)
- self.assertIn("precision probe timed out rc=%s limit=%ss", distributed)
- self.assertIn('export CX_MODE="$mode" CX_PHASE="$ph"', distributed)
- rank_wrapper = common[
- common.index("cx_slurm_rank_wrapper()") : common.index(
- "cx_validate_shard_control()"
- )
- ]
- self.assertLess(
- rank_wrapper.index(". /ix/experimental/CollectiveX/runtime/common.sh"),
- rank_wrapper.index('if [ "${CX_NODES:-1}" -gt 1 ]'),
- )
- self.assertEqual(
- distributed.count(
- '--container-name="$container_name" --container-image="$SQUASH_FILE"'
- ),
- 5,
- )
- shard_runtime = runtime[runtime.index('elif [ -n "${CX_SHARD_FILE:-}" ]') :]
- self.assertIn('"CX_PRECISION_PROFILE": g("precision_profile")', shard_runtime)
- self.assertIn("rdma-port-%s=inactive", common)
- self.assertIn("rdma-device-%s=missing", common)
- single_slurm = (ROOT / "launchers" / "launch_single-slurm.sh").read_text()
- self.assertIn("for allocation_attempt in 1 2 3", single_slurm)
- self.assertIn('RUNNER:$validation_failure', single_slurm)
- self.assertIn('h100-dgxc:network', single_slurm)
- self.assertIn('b300:cuda-context', single_slurm)
- self.assertIn('cx_validate_cuda_context_on_job "$JOB_ID" "$NODES" "$GPN"', single_slurm)
- self.assertIn('rejected_nodes="$(cx_allocation_nodes_csv "$JOB_ID")"', single_slurm)
- self.assertIn('export CX_SALLOC_ATTEMPT="$allocation_attempt"', single_slurm)
- self.assertIn('export CX_NETWORK_VALIDATION_ATTEMPT="$allocation_attempt"', single_slurm)
- self.assertIn('test -c /dev/gdrdrv', single_slurm)
- self.assertIn('/dev/gdrdrv:/dev/gdrdrv', single_slurm)
- self.assertIn(
- '[ "$RUNNER" = b200-dgxc ] && [ "$CX_BENCH" = deepep ] && [ "$NODES" -gt 1 ]',
- single_slurm,
- )
- self.assertIn('log_label+="-a${CX_SALLOC_ATTEMPT}"', common)
- self.assertIn('log_label+="-a${CX_NETWORK_VALIDATION_ATTEMPT}"', common)
- self.assertIn('cx_validate_cuda_context_on_job()', common)
- self.assertIn('cuDevicePrimaryCtxRetain', common)
- self.assertIn('diagnostic="accelerator-unavailable"', common)
-
- def test_case_failure_diagnostic_precedes_normal_srun_footer(self) -> None:
- with tempfile.TemporaryDirectory() as temporary:
- log = Path(temporary) / "runtime.log"
- log.write_text(
- "WARN: deepep decode run failed rc=1 (CX_RUN_TIMEOUT=900s)\n"
- "SHARD done: 6/6 case(s) failed\n"
- "srun: error: task exited 1\n"
- )
- result = subprocess.run(
- [
- "bash", "-c",
- 'source "$1"; cx_fail_stage execution "$2"',
- "_", str(ROOT / "runtime" / "common.sh"), str(log),
- ],
- text=True,
- capture_output=True,
- )
- self.assertEqual(result.returncode, 1)
- self.assertIn("diagnostic=benchmark-case-failure", result.stderr)
-
- def test_non_timeout_failure_warning_is_classified_as_case_failure(self) -> None:
- with tempfile.TemporaryDirectory() as temporary:
- log = Path(temporary) / "runtime.log"
- log.write_text("WARN: deepep decode run failed rc=1\nsrun: task exited 1\n")
- result = subprocess.run(
- [
- "bash", "-c",
- 'source "$1"; cx_fail_stage execution "$2"',
- "_", str(ROOT / "runtime" / "common.sh"), str(log),
- ],
- text=True,
- capture_output=True,
- )
- self.assertEqual(result.returncode, 1)
- self.assertNotIn("diagnostic=network-or-timeout", result.stderr)
- self.assertIn("diagnostic=benchmark-case-failure", result.stderr)
-
- def test_backend_import_failure_precedes_container_cleanup_noise(self) -> None:
- with tempfile.TemporaryDirectory() as temporary:
- log = Path(temporary) / "runtime.log"
- log.write_text(
- "ERROR: MoRI backend import failed\n"
- "pyxis: failed to mount container cleanup path: invalid argument\n"
- )
- result = subprocess.run(
- [
- "bash", "-c",
- 'source "$1"; cx_fail_stage backend-setup "$2"',
- "_", str(ROOT / "runtime" / "common.sh"), str(log),
- ],
- text=True,
- capture_output=True,
- )
- self.assertEqual(result.returncode, 1)
- self.assertIn("diagnostic=backend-build", result.stderr)
- self.assertNotIn("diagnostic=container-runtime", result.stderr)
-
- def test_all_ranks_validate_raw_evidence_before_rank_zero_writes(self) -> None:
- harness = (ROOT / "tests" / "ep_harness.py").read_text()
- validation = harness.index("contracts.validate_raw_document(doc, samples_document)")
- rank_zero_write = harness.index("if rank == 0:", validation)
- self.assertLess(validation, rank_zero_write)
- self.assertNotIn("if rank != 0:", harness[harness.index("def run_sweep("):validation])
-
- def test_precision_probe_timeout_reports_only_the_last_closed_stage(self) -> None:
- with tempfile.TemporaryDirectory() as temporary:
- log = Path(temporary) / "runtime.log"
- log.write_text(
- "[collectivex] precision-probe-stage=distributed-init\n"
- "[collectivex] precision-probe-stage=backend-construction\n"
- "[collectivex] precision-probe-stage=native-operation\n"
- "[collectivex] precision probe timed out rc=124 limit=900s\n"
- )
- result = subprocess.run(
- [
- "bash", "-c",
- 'source "$1"; cx_fail_stage execution "$2"',
- "_", str(ROOT / "runtime" / "common.sh"), str(log),
- ],
- text=True,
- capture_output=True,
- )
- self.assertEqual(result.returncode, 1)
- self.assertIn("diagnostic=native-operation-timeout", result.stderr)
- self.assertNotIn("distributed-init", result.stderr)
-
- def test_precision_probe_memory_failure_reports_the_last_closed_stage(self) -> None:
- with tempfile.TemporaryDirectory() as temporary:
- log = Path(temporary) / "runtime.log"
- log.write_text(
- "[collectivex] precision-probe-stage=backend-construction\n"
- "CUDA out of memory\n"
- )
- result = subprocess.run(
- [
- "bash", "-c",
- 'source "$1"; cx_fail_stage execution "$2"',
- "_", str(ROOT / "runtime" / "common.sh"), str(log),
- ],
- text=True,
- capture_output=True,
- )
- self.assertEqual(result.returncode, 1)
- self.assertIn(
- "diagnostic=backend-construction-accelerator-memory", result.stderr
- )
-
- def test_precision_probe_unknown_failure_reports_only_the_last_closed_stage(self) -> None:
- with tempfile.TemporaryDirectory() as temporary:
- log = Path(temporary) / "runtime.log"
- log.write_text(
- "[collectivex] precision-probe-stage=native-operation\n"
- "private backend failure details\n"
- )
- result = subprocess.run(
- [
- "bash", "-c",
- 'source "$1"; cx_fail_stage execution "$2"',
- "_", str(ROOT / "runtime" / "common.sh"), str(log),
- ],
- text=True,
- capture_output=True,
- )
- self.assertEqual(result.returncode, 1)
- self.assertIn("diagnostic=native-operation-failed", result.stderr)
- self.assertNotIn("private backend", result.stderr)
-
- def test_precision_probe_traceback_prefers_the_last_closed_stage(self) -> None:
- with tempfile.TemporaryDirectory() as temporary:
- log = Path(temporary) / "runtime.log"
- log.write_text(
- "[collectivex] precision-probe-stage=native-operation\n"
- "Traceback (most recent call last):\nRuntimeError: private details\n"
- )
- result = subprocess.run(
- [
- "bash", "-c",
- 'source "$1"; cx_fail_stage execution "$2"',
- "_", str(ROOT / "runtime" / "common.sh"), str(log),
- ],
- text=True,
- capture_output=True,
- )
- self.assertEqual(result.returncode, 1)
- self.assertIn("diagnostic=native-operation-failed", result.stderr)
- self.assertNotIn("python-exception", result.stderr)
-
- def test_private_runtime_failure_signatures_override_case_footer(self) -> None:
- signatures = {
- "DeepEP V2 no-GIN run is outside one realized LSA domain":
- "accelerator-topology",
- "NCCL exception (/src/nccl.cu:111): 3": "accelerator-topology",
- "NCCL exception (/src/nccl.cu:112): 3": "accelerator-topology",
- "CUDA error: call requires newer driver": "accelerator-driver",
- "NCCL failure in ncclCommWindowRegister": "nccl-device-api",
- "Communicator does not support symmetric memory": "nccl-device-api",
- "NCCL exception (/src/nccl.cu:106): 5": "nccl-device-api",
- "NCCL exception (/src/nccl.cu:127): 5": "nccl-device-api",
- "NCCL exception (/src/nccl.cu:128): 5": "nccl-device-api",
- "NCCL exception (/src/nccl.cu:129): 5": "nccl-device-api",
- "NCCL exception (/src/nccl.cu:135): 5": "nccl-device-api",
- "NVCC compilation failed": "jit-toolchain",
- "CUDA out of memory": "accelerator-memory",
- "torch rendezvous timed out": "network-or-timeout",
- "torch.distributed.DistBackendError: ncclRemoteError": "collective-remote",
- "torch.distributed.DistBackendError: ncclSystemError": "collective-system",
- "torch.distributed.DistBackendError: ncclInternalError": "collective-internal",
- "torch.distributed.DistBackendError: ncclInvalidUsage":
- "collective-invalid-usage",
- "dist.init_process_group('nccl')\nDistBackendError: ncclInvalidUsage":
- "collective-init-invalid-usage",
- "dist.all_gather_object(records, local)\nDistBackendError: ncclInvalidUsage":
- "collective-consensus-invalid-usage",
- "dist.all_to_all_single(output, input)\nDistBackendError: ncclInvalidUsage":
- "collective-alltoall-invalid-usage",
- "torch.distributed.DistBackendError: unknown collective failure":
- "collective-backend",
- "ModuleNotFoundError: missing module": "python-import",
- "AttributeError: backend has no attribute 'probe'": "backend-api",
- "PrecisionError: FP8 dispatch payload is missing block-128 scales":
- "precision-dispatch-scales-missing",
- "PrecisionError: native FP8 dispatch payload has an invalid dtype or shape":
- "precision-dispatch-payload-shape",
- "PrecisionError: native FP8 dispatch scales have an invalid dtype or shape":
- "precision-dispatch-scale-shape",
- "PrecisionError: expert-packed FP8 receive count exceeds capacity":
- "precision-receive-capacity",
- "PrecisionError: expert-packed FP8 receive counts has an invalid shape":
- "precision-receive-shape",
- "PrecisionError: active torch build does not expose torch.float8_e4m3fnuz":
- "precision-runtime-dtype",
- "PrecisionError: pinned MoRI API omits EpDispatchCombineQuantType.Fp8DirectCast":
- "precision-combine-api",
- "PrecisionError: MoRI dispatch FP8 format differs from the pinned GPU architecture":
- "precision-architecture-format",
- "PrecisionError: MoRI native FP8 dispatch requires hidden divisible by 128":
- "precision-hidden-alignment",
- "PrecisionError: unsupported precision profile": "precision-contract",
- "AssertionError: probe invariant": "python-assertion",
- "RuntimeError: probe execution failed": "python-runtime",
- "ValueError: probe fields differ from collectivex.precision-probe.v1":
- "probe-schema-value",
- "ValueError: probe target is not a provisional native adapter cell":
- "probe-target-value",
- "ValueError: probe result reason is invalid": "probe-result-value",
- "ValueError: probe privacy contract differs": "probe-privacy-value",
- "ValueError: probe API calls are empty": "probe-api-value",
- "ValueError: probe completion contract differs": "probe-completion-value",
- "ValueError: probe image digest is invalid": "probe-identity-value",
- "ValueError: probe precision correctness did not pass":
- "probe-correctness-failed",
- "ValueError: probe scale shapes are invalid": "probe-scale-shape",
- "ValueError: probe precision dispatch input is not finite": "probe-nonfinite",
- "ValueError: probe precision combine output shape is invalid":
- "probe-tensor-shape",
- "ValueError: probe transport fallback is present": "probe-transport-value",
- "tests/probe_precision.py\nValueError: probe argument": "probe-manifest-value",
- "tests/ep_harness.py\nValueError: harness argument": "harness-value",
- "tests/workload.py\nValueError: workload argument": "workload-value",
- "tests/run_ep.py\nValueError: runner argument": "runner-value",
- "tests/ep_deepep.py\nValueError: adapter argument": "deepep-adapter-value",
- "site-packages/torch/module.py\nValueError: dependency argument": "dependency-value",
- "ValueError: probe argument": "python-value",
- "KeyError: 'probe-key'": "python-key",
- "OSError: probe path": "python-os",
- "NotImplementedError: probe API": "python-system",
- "CalledProcessError: probe command": "python-subprocess",
- "Traceback (most recent call last):": "python-exception",
- }
- with tempfile.TemporaryDirectory() as temporary:
- log = Path(temporary) / "runtime.log"
- for signature, diagnostic in signatures.items():
- log.write_text(f"{signature}\nSHARD done: 6/6 case(s) failed\n")
- result = subprocess.run(
- [
- "bash", "-c",
- 'source "$1"; cx_fail_stage execution "$2"',
- "_", str(ROOT / "runtime" / "common.sh"), str(log),
- ],
- text=True,
- capture_output=True,
- env={**os.environ, "CX_BENCH": "deepep-v2"},
- )
- self.assertEqual(result.returncode, 1)
- self.assertIn(f"diagnostic={diagnostic}", result.stderr)
-
- log.write_text(
- "NCCL exception (/src/nccl.cu:106): 5\n"
- "SHARD done: 6/6 case(s) failed\n"
- )
- result = subprocess.run(
- [
- "bash", "-c", 'source "$1"; cx_fail_stage execution "$2"',
- "_", str(ROOT / "runtime" / "common.sh"), str(log),
- ],
- text=True, capture_output=True,
- env={**os.environ, "CX_BENCH": "deepep"},
- )
- self.assertIn("diagnostic=benchmark-case-failure", result.stderr)
-
- def test_runtime_stage_marker_distinguishes_launch_from_execution(self) -> None:
- with tempfile.TemporaryDirectory() as temporary:
- mount = Path(temporary)
- root = mount / "experimental" / "CollectiveX"
- root.mkdir(parents=True)
- command = r'''
- set -euo pipefail
- source "$1"
- export COLLECTIVEX_EXECUTION_ID=test_1_shard CX_TS=test
- cx_set_failure_stage container-launch
- cx_prepare_runtime_marker "$2"
- (cd "$2/experimental/CollectiveX"; cx_write_runtime_stage backend-setup)
- cx_adopt_runtime_stage "$2"
- test "$CX_FAILSAFE_MODE" = backend-setup
- (cd "$2/experimental/CollectiveX"; cx_write_runtime_stage execution)
- cx_adopt_runtime_stage "$2"
- test "$CX_FAILSAFE_MODE" = execution
- '''
- subprocess.run(
- ["bash", "-c", command, "_", str(ROOT / "runtime" / "common.sh"),
- str(mount)],
- check=True,
- )
-
- def test_canonical_gha_environment_is_locked_but_manual_overrides_survive(self) -> None:
- common = ROOT / "runtime" / "common.sh"
- command = r'''
- set -euo pipefail
- source "$1"
- export COLLECTIVEX_CANONICAL_GHA=1 GITHUB_ACTIONS=true
- export GITHUB_RUN_ID=123 GITHUB_RUN_ATTEMPT=1
- export COLLECTIVEX_SOURCE_SHA=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
- export CX_SHARD_FILE=.shards/test.json CX_SHARD_SKU=mi325x
- export CX_NODES=1 CX_GPUS_PER_NODE=8
- export CX_IMAGE=untrusted CX_IMAGE_DIGEST=untrusted CX_NGPUS=99
- export CX_NCCL_HOME=/untrusted CX_LOCK_DIR=/tmp CX_SQUASH_DIR=/shared/containers
- export CX_STAGE_DIR=/private/stale-stage
- export CX_MORI_KERNEL_TYPE=intranode MORI_ENABLE_SDMA=0
- export NCCL_MNNVL_ENABLE=1 MC_FORCE_MNNVL=1 CX_DRYRUN=1
- export CX_BACKEND_CACHE_ROOT=/untrusted CX_BACKEND_CACHE_SENTINEL_SHA256=bad
- export CX_PREPARED_BACKEND_CACHE=/untrusted CX_BACKEND_SOURCE_ROOT=/untrusted
- ! (cx_lock_canonical_gha_env mi325x)
- export COLLECTIVEX_OPERATOR_CONFIG_LOADED=$$
- export CX_STAGE_DIR="$GITHUB_WORKSPACE" CX_AUDIT_SALT="$(printf 'a%.0s' {1..64})"
- unset CX_LOCK_DIR
- cx_lock_canonical_gha_env mi325x
- test "$CX_IMAGE" = "$CX_IMAGE_AMD_MORI_MI325"
- test "$CX_IMAGE_DIGEST" = "$CX_IMAGE_AMD_MORI_MI325_DIGEST"
- test "$CX_NGPUS:$CX_SEED:$CX_RUN_TIMEOUT" = 8:67:1800
- test "$CX_MORI_KERNEL_TYPE:$MORI_DISABLE_AUTO_XGMI:$MORI_ENABLE_SDMA" = asyncll:0:1
- test "$MORI_COMMIT" = "$CX_MORI_COMMIT_MI325"
- test "$MORI_APP_LOG_LEVEL:$MORI_SHMEM_LOG_LEVEL:$MORI_IO_LOG_LEVEL" = info:info:info
- test "$CX_STAGE_DIR" = "$GITHUB_WORKSPACE"
- test -z "${CX_NCCL_HOME+x}${CX_LOCK_DIR+x}${NCCL_MNNVL_ENABLE+x}${MC_FORCE_MNNVL+x}"
- test -z "${CX_BACKEND_CACHE_ROOT+x}${CX_BACKEND_CACHE_SENTINEL_SHA256+x}"
- test -z "${CX_PREPARED_BACKEND_CACHE+x}${CX_BACKEND_SOURCE_ROOT+x}"
- test -z "${CX_DRYRUN+x}"
-
- export COLLECTIVEX_OPERATOR_CONFIG_LOADED=$$
- export CX_SHARD_SKU=mi300x CX_NODES=2 CX_GPUS_PER_NODE=8
- export CX_STAGE_DIR="$TEST_STAGE_ALIAS" CX_AUDIT_SALT="$(printf 'a%.0s' {1..64})"
- cx_lock_canonical_gha_env mi300x
- test "$CX_STAGE_DIR" = "$TEST_STAGE_TARGET"
-
- cx_prepare_implicit_stage_base() {
- if [ -n "${1:-}" ]; then test "$1" = "$TEST_SHARED_PARENT"; fi
- if [ -n "${2:-}" ]; then test "$2" = 123; fi
- printf '%s' "$TEST_IMPLICIT_STAGE"
- }
- export COLLECTIVEX_OPERATOR_CONFIG_LOADED=$$
- export CX_SHARD_SKU=h100-dgxc CX_NODES=1 CX_GPUS_PER_NODE=8
- export CX_SQUASH_DIR="$TEST_SHARED_PARENT/containers"
- unset CX_STAGE_DIR
- cx_lock_canonical_gha_env h100-dgxc
- test "$CX_STAGE_DIR" = "$TEST_IMPLICIT_STAGE"
-
- export COLLECTIVEX_OPERATOR_CONFIG_LOADED=$$
- export CX_SHARD_SKU=b200-dgxc CX_NODES=1 CX_GPUS_PER_NODE=8
- unset CX_STAGE_DIR
- cx_lock_canonical_gha_env b200-dgxc
- test "$CX_STAGE_DIR" = "$TEST_IMPLICIT_STAGE"
-
- export COLLECTIVEX_OPERATOR_CONFIG_LOADED=$$
- export CX_SHARD_SKU=b300 CX_NODES=1 CX_GPUS_PER_NODE=8
- export CX_STAGE_DIR=/legacy/root-owned-stage
- cx_lock_canonical_gha_env b300
- test "$CX_STAGE_DIR" = "$TEST_IMPLICIT_STAGE"
- test "$CX_STAGE_PARENT_OWNER_OK" = 1
- test -z "${NVSHMEM_DISABLE_IB+x}"
-
- export COLLECTIVEX_OPERATOR_CONFIG_LOADED=$$
- export CX_STAGE_DIR=/legacy/group-writable-stage
- export CX_SHARD_SKU=gb300 CX_NODES=2 CX_GPUS_PER_NODE=4
- export CX_IMAGE=untrusted CX_NGPUS=1 CX_MORI_KERNEL_TYPE=untrusted
- export MORI_ENABLE_SDMA=0 CX_NCCL_HOME=/untrusted CX_MASTER_PORT=1
- cx_lock_canonical_gha_env gb300
- test "$CX_IMAGE" = "$CX_IMAGE_MULTIARCH"
- test "$CX_IMAGE_DIGEST" = "$CX_IMAGE_MULTIARCH_DIGEST"
- test "$CX_NGPUS:$CX_SEED:$CX_RUN_TIMEOUT" = 8:67:900
- test "$CX_NCCL_HOME:$CX_MASTER_PORT" = /usr:29551
- test "$CX_STAGE_DIR" = "$TEST_IMPLICIT_STAGE"
- test -z "${CX_STAGE_PARENT_OWNER_OK+x}"
- test -z "${NVSHMEM_DISABLE_IB+x}"
- test -z "${CX_MORI_KERNEL_TYPE+x}${MORI_ENABLE_SDMA+x}"
-
- export COLLECTIVEX_OPERATOR_CONFIG_LOADED=$$
- export CX_SHARD_SKU=mi355x CX_NODES=1 CX_GPUS_PER_NODE=8
- export CX_LOCK_DIR=/validated/amd-locks CX_STAGE_DIR=/validated/amd-stage
- export CX_AUDIT_SALT="$(printf 'a%.0s' {1..64})"
- cx_lock_canonical_gha_env mi355x
- test "$CX_LOCK_DIR" = /validated/amd-locks
- test "$CX_STAGE_DIR" = /validated/amd-stage
- test "$MORI_COMMIT" = "$CX_MORI_COMMIT_MI355"
-
- export COLLECTIVEX_OPERATOR_CONFIG_LOADED=$$
- export CX_SHARD_SKU=mi355x CX_NODES=1 CX_GPUS_PER_NODE=8
- export CX_SQUASH_DIR=/validated/amd-shared
- export RUNNER_TEMP="$TEST_SHARED_PARENT/_work/_temp"
- export CX_AUDIT_SALT="$(printf 'a%.0s' {1..64})"
- unset CX_STAGE_DIR
- cx_lock_canonical_gha_env mi355x
- test "$CX_STAGE_DIR" = "$TEST_IMPLICIT_STAGE"
-
- unset COLLECTIVEX_CANONICAL_GHA
- unset COLLECTIVEX_OPERATOR_CONFIG_LOADED
- CX_IMAGE=manual CX_IMAGE_DIGEST=manual CX_NGPUS=3
- CX_MORI_KERNEL_TYPE=manual
- cx_lock_canonical_gha_env mi355x
- test "$CX_IMAGE:$CX_IMAGE_DIGEST:$CX_NGPUS:$CX_MORI_KERNEL_TYPE" = manual:manual:3:manual
- '''
- with tempfile.TemporaryDirectory(dir=Path.home()) as temporary:
- root = Path(temporary)
- home = root / "home"
- workspace = root / "workspace"
- implicit_stage = root / "implicit-stage"
- stage_target = root / "stage-target"
- stage_alias = root / "stage-alias"
- home.mkdir(mode=0o700)
- workspace.mkdir(mode=0o720)
- implicit_stage.mkdir(mode=0o700)
- stage_target.mkdir(mode=0o700)
- stage_alias.symlink_to(stage_target, target_is_directory=True)
- subprocess.run(
- ["bash", "-c", command, "_", str(common)],
- check=True,
- env={
- **os.environ,
- "HOME": str(home),
- "TEST_IMPLICIT_STAGE": str(implicit_stage),
- "TEST_STAGE_ALIAS": str(stage_alias),
- "TEST_STAGE_TARGET": str(stage_target),
- "TEST_SHARED_PARENT": str(root),
- "COLLECTIVEX_OPERATOR_CONFIG": "/dev/null",
- "GITHUB_WORKSPACE": str(workspace),
- },
- )
- self.assertEqual(list(workspace.iterdir()), [])
-
- def test_slurm_rendezvous_uses_relative_node_zero_and_validated_interface(self) -> None:
- common = ROOT / "runtime" / "common.sh"
- command = r'''
- set -euo pipefail
- source "$1"
- cx_host_exports() { printf '%s' HOME,PATH,USER; }
- squeue() { return 91; }
- scontrol() { return 92; }
- srun() {
- case "$*" in
- *" bash -s -- eth0") printf '%s\n' 192.0.2.7 ;;
- *" hostname -s") printf '%s\n' rank-zero-node ;;
- *) return 93 ;;
- esac
- }
- CX_SOCKET_IFNAME=eth0
- cx_resolve_slurm_rendezvous 123
- test "$MASTER_ADDR:$MASTER_PORT" = 192.0.2.7:29551
- unset CX_SOCKET_IFNAME
- CX_MASTER_PORT=30444
- cx_resolve_slurm_rendezvous 123
- test "$MASTER_ADDR:$MASTER_PORT" = rank-zero-node:30444
- '''
- subprocess.run(
- ["bash", "-c", command, "_", str(common)],
- check=True,
- )
-
- def test_implicit_stage_base_is_private_and_non_symlinked(self) -> None:
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary).resolve()
- safe_home = root / "safe-home"
- unsafe_home = root / "unsafe-home"
- linked_home = root / "linked-home"
- linked_child_home = root / "linked-child-home"
- safe_home.mkdir(mode=0o700)
- unsafe_home.mkdir(mode=0o770)
- unsafe_home.chmod(0o770)
- linked_home.symlink_to(safe_home, target_is_directory=True)
- linked_child_home.mkdir(mode=0o700)
- (linked_child_home / ".inferencex-collectivex-stage").symlink_to(
- safe_home, target_is_directory=True
- )
- command = r'''
- set -euo pipefail
- source "$1"
- base="$(cx_prepare_implicit_stage_base "$2")"
- test "$base" = "$2/.inferencex-collectivex-stage"
- test "$(stat -c '%a' "$base" 2>/dev/null || stat -f '%Lp' "$base")" = 700
- ! cx_prepare_implicit_stage_base "$3"
- test "$(cx_prepare_implicit_stage_base "$4")" = "$2/.inferencex-collectivex-stage"
- isolated="$(cx_prepare_implicit_stage_base "$2" runner-a)"
- test "$isolated" != "$2/.inferencex-collectivex-stage"
- case "$isolated" in "$2"/.inferencex-collectivex-stage-*) ;; *) exit 1 ;; esac
- ! cx_prepare_implicit_stage_base "$5"
- '''
- subprocess.run(
- [
- "bash", "-c", command, "_",
- str(ROOT / "runtime" / "common.sh"), str(safe_home),
- str(unsafe_home), str(linked_home), str(linked_child_home),
- ],
- check=True,
- env={**os.environ, "COLLECTIVEX_OPERATOR_CONFIG": "/dev/null"},
- )
-
- def test_canonical_amd_stage_uses_config_not_world_writable_workspace(self) -> None:
- common = ROOT / "runtime" / "common.sh"
- command = r'''
- source "$1"
- export COLLECTIVEX_CANONICAL_GHA=1 GITHUB_ACTIONS=true
- export GITHUB_RUN_ID=123 GITHUB_RUN_ATTEMPT=1
- export COLLECTIVEX_SOURCE_SHA=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
- export CX_SHARD_FILE=.shards/test.json CX_SHARD_SKU=mi325x
- export CX_NODES=1 CX_GPUS_PER_NODE=8 CX_SQUASH_DIR=/shared/containers
- export COLLECTIVEX_OPERATOR_CONFIG_LOADED=$$ CX_STAGE_DIR=/shared/amd-stage
- export CX_AUDIT_SALT="$(printf 'a%.0s' {1..64})"
- cx_lock_canonical_gha_env mi325x
- printf '%s' "$CX_STAGE_DIR"
- '''
- with tempfile.TemporaryDirectory(dir=Path.home()) as workspace:
- Path(workspace).chmod(0o702)
- result = subprocess.run(
- ["bash", "-c", command, "_", str(common)],
- text=True,
- capture_output=True,
- env={
- **os.environ,
- "COLLECTIVEX_OPERATOR_CONFIG": "/dev/null",
- "GITHUB_WORKSPACE": workspace,
- },
- )
- self.assertEqual(result.returncode, 0, result.stderr)
- self.assertEqual(result.stdout, "/shared/amd-stage")
- self.assertNotIn(workspace, result.stderr)
-
- def test_canonical_amd_stage_uses_config_not_symlinked_workspace(self) -> None:
- common = ROOT / "runtime" / "common.sh"
- command = r'''
- source "$1"
- export COLLECTIVEX_CANONICAL_GHA=1 GITHUB_ACTIONS=true
- export GITHUB_RUN_ID=123 GITHUB_RUN_ATTEMPT=1
- export COLLECTIVEX_SOURCE_SHA=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
- export CX_SHARD_FILE=.shards/test.json CX_SHARD_SKU=mi325x
- export CX_NODES=1 CX_GPUS_PER_NODE=8 CX_SQUASH_DIR=/shared/containers
- export COLLECTIVEX_OPERATOR_CONFIG_LOADED=$$ CX_STAGE_DIR=/shared/amd-stage
- export CX_AUDIT_SALT="$(printf 'a%.0s' {1..64})"
- cx_lock_canonical_gha_env mi325x
- printf '%s' "$CX_STAGE_DIR"
- '''
- with tempfile.TemporaryDirectory(dir=Path.home()) as temporary:
- root = Path(temporary)
- real = root / "real"
- real.mkdir()
- link = root / "workspace"
- link.symlink_to(real, target_is_directory=True)
- result = subprocess.run(
- ["bash", "-c", command, "_", str(common)],
- text=True,
- capture_output=True,
- env={
- **os.environ,
- "COLLECTIVEX_OPERATOR_CONFIG": "/dev/null",
- "GITHUB_WORKSPACE": str(link),
- },
- )
- self.assertEqual(result.returncode, 0, result.stderr)
- self.assertEqual(result.stdout, "/shared/amd-stage")
- self.assertNotIn(str(root), result.stderr)
-
- def test_image_selection_and_registry_verification_are_fail_closed(self) -> None:
- common = ROOT / "runtime" / "common.sh"
- command = r'''
- source "$1"
- test "$(cx_default_image mi325x)" = "$CX_IMAGE_AMD_MORI_MI325"
- test "$(cx_default_image mi355x)" = "$CX_IMAGE_AMD_MORI"
- pinned="sha256:$(printf 'a%.0s' {1..64})"
- curl() {
- case "$*" in
- *auth.docker.io*) printf '{"token":"test"}' ;;
- *) printf 'Docker-Content-Digest: %s\r\n' "$pinned" ;;
- esac
- }
- test "$(cx_resolve_registry_digest ubuntu:latest)" = "$pinned"
- test "$(cx_resolve_registry_digest docker.io/library/ubuntu:latest)" = "$pinned"
- ! (cx_resolve_registry_digest "ubuntu@$pinned")
- ! (cx_resolve_registry_digest ghcr.io/example/image:tag)
- ! (cx_resolve_registry_digest 'ubuntu@sha256:bad')
- curl() {
- case "$*" in *auth.docker.io*) printf '{"token":"test"}';; esac
- }
- ! (cx_resolve_registry_digest ubuntu:latest)
- cx_resolve_registry_digest() { printf '%s' "$CX_IMAGE_MULTIARCH_DIGEST"; }
- cx_verify_registry_image "$CX_IMAGE_MULTIARCH"
- test "$COLLECTIVEX_IMAGE_DIGEST_VERIFIED" = 1
- test "$COLLECTIVEX_IMAGE_DIGEST" = "$CX_IMAGE_MULTIARCH_DIGEST"
- cx_reverify_registry_image "$CX_IMAGE_MULTIARCH"
- cx_resolve_registry_digest() { printf 'sha256:%064d' 0; }
- ! (cx_reverify_registry_image "$CX_IMAGE_MULTIARCH")
- ! (cx_verify_registry_image "$CX_IMAGE_MULTIARCH")
- '''
- subprocess.run(
- ["bash", "-c", command, "_", str(common)],
- check=True,
- env={**os.environ, "COLLECTIVEX_OPERATOR_CONFIG": "/dev/null"},
- )
-
- def test_canonical_gha_requires_compute_visible_staging(self) -> None:
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary).resolve()
- repo = root / "repo"
- squash = root / "squash"
- source = repo / "experimental" / "CollectiveX"
- source.mkdir(parents=True)
- squash.mkdir(mode=0o700)
- (source / "public.py").write_text("public\n")
- (source / "private-infra.md").write_text("private\n")
- command = r'''
- set -euo pipefail
- source "$1"
- unset CX_SHARD_FILE CX_STAGE_DIR
- staged="$(COLLECTIVEX_CANONICAL_GHA=1; cx_stage_path "$2" "")"
- cx_stage_repo "$2" "$staged"
- test "$staged" != "$2"
- test -f "$staged/experimental/CollectiveX/public.py"
- test ! -e "$staged/experimental/CollectiveX/private-infra.md"
- cx_cleanup_stage "$staged" "$2"
- test ! -e "$staged"
- '''
- subprocess.run(
- ["bash", "-c", command, "_", str(ROOT / "runtime" / "common.sh"),
- str(repo)],
- check=True,
- env={
- **os.environ,
- "COLLECTIVEX_OPERATOR_CONFIG": "/dev/null",
- "CX_SQUASH_DIR": str(squash),
- },
- )
- self.assertEqual(list(squash.iterdir()), [])
-
- def test_manual_stage_does_not_write_to_checkout_parent(self) -> None:
- with tempfile.TemporaryDirectory() as temporary:
- parent = Path(temporary).resolve() / "readonly-parent"
- repo = parent / "repo"
- squash = parent / "squash"
- source = repo / "experimental" / "CollectiveX"
- source.mkdir(parents=True)
- squash.mkdir(mode=0o700)
- (source / "public.py").write_text("public\n")
- original_mode = parent.stat().st_mode & 0o777
- parent.chmod(0o555)
- try:
- command = r'''
- set -euo pipefail
- source "$1"
- unset CX_STAGE_DIR
- staged="$(cx_stage_path "$2" "")"
- cx_stage_repo "$2" "$staged"
- case "$staged" in "$3"/.collectivex-stage-*) ;; *) exit 1 ;; esac
- test -f "$staged/experimental/CollectiveX/public.py"
- test ! -e "$4/.collectivex-stage"
- cx_cleanup_stage "$staged" "$2"
- test ! -e "$staged"
- '''
- subprocess.run(
- [
- "bash", "-c", command, "_",
- str(ROOT / "runtime" / "common.sh"), str(repo),
- str(squash), str(parent),
- ],
- check=True,
- env={
- **os.environ,
- "COLLECTIVEX_OPERATOR_CONFIG": "/dev/null",
- "CX_SQUASH_DIR": str(squash),
- },
- )
- finally:
- parent.chmod(original_mode)
- self.assertEqual(
- sorted(path.name for path in parent.iterdir()),
- ["repo", "squash"],
- )
- self.assertEqual(list(squash.iterdir()), [])
-
- def test_stage_refuses_to_reuse_an_execution_child(self) -> None:
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary).resolve()
- repo = root / "repo"
- source = repo / "experimental" / "CollectiveX"
- source.mkdir(parents=True)
- (source / "public.py").write_text("public\n")
- base = root / "stage"
- child = base / "job_collision"
- child.mkdir(parents=True, mode=0o700)
- sentinel = child / "keep"
- sentinel.write_text("keep")
- command = r'''
- source "$1"
- ! (cx_stage_repo "$2" "$3/job_collision")
- '''
- subprocess.run(
- [
- "bash", "-c", command, "_",
- str(ROOT / "runtime" / "common.sh"), str(repo), str(base),
- ],
- check=True,
- env={
- **os.environ,
- "COLLECTIVEX_OPERATOR_CONFIG": "/dev/null",
- "COLLECTIVEX_CANONICAL_GHA": "1",
- "COLLECTIVEX_EXECUTION_ID": "collision",
- "CX_STAGE_DIR": str(base),
- },
- )
- self.assertEqual(sentinel.read_text(), "keep")
-
- def test_stage_removes_its_execution_child_when_copy_fails(self) -> None:
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary).resolve()
- repo = root / "repo"
- source = repo / "experimental" / "CollectiveX"
- source.mkdir(parents=True)
- (source / "public.py").write_text("public\n")
- base = root / "stage"
- base.mkdir(mode=0o700)
- sentinel = root / "python-called"
- command = r'''
- source "$1"
- python3() {
- if [[ "${2:-}" == */experimental/CollectiveX ]]; then
- : > "$PYTHON_CALLED"
- return 1
- fi
- command python3 "$@"
- }
- staged="$(cx_stage_path "$2" "$3")"
- ! cx_stage_repo "$2" "$staged"
- '''
- subprocess.run(
- [
- "bash", "-c", command, "_",
- str(ROOT / "runtime" / "common.sh"), str(repo), str(base),
- ],
- check=True,
- env={
- **os.environ,
- "COLLECTIVEX_OPERATOR_CONFIG": "/dev/null",
- "COLLECTIVEX_CANONICAL_GHA": "1",
- "COLLECTIVEX_EXECUTION_ID": f"test-{root.name}",
- "CX_STAGE_DIR": str(base),
- "PYTHON_CALLED": str(sentinel),
- },
- )
- self.assertTrue(sentinel.is_file())
- self.assertEqual(list(base.iterdir()), [])
-
- def test_interrupted_stage_is_cleanup_capable_before_copy(self) -> None:
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary).resolve()
- repo = root / "repo"
- source = repo / "experimental" / "CollectiveX"
- source.mkdir(parents=True)
- (source / "public.py").write_text("public\n")
- base = root / "stage"
- base.mkdir(mode=0o700)
- sibling = base / "keep"
- sibling.write_text("keep\n")
- command = r'''
- set -euo pipefail
- source "$1"
- export REPO_ROOT="$2" CX_BENCH=nccl-ep
- MOUNT_SRC="$(cx_stage_path "$REPO_ROOT" "$3")"
- cx_install_launcher_fail_safe
- rsync() { kill -TERM $$; return 143; }
- cx_stage_repo "$REPO_ROOT" "$MOUNT_SRC"
- '''
- result = subprocess.run(
- [
- "bash", "-c", command, "_",
- str(ROOT / "runtime" / "common.sh"), str(repo), str(base),
- ],
- text=True,
- capture_output=True,
- env={
- **os.environ,
- "COLLECTIVEX_OPERATOR_CONFIG": "/dev/null",
- "COLLECTIVEX_CANONICAL_GHA": "1",
- "COLLECTIVEX_EXECUTION_ID": "interrupted",
- "CX_STAGE_DIR": str(base),
- },
- )
- self.assertNotEqual(result.returncode, 0)
- self.assertFalse((base / "job_interrupted").exists())
- self.assertEqual(sibling.read_text(), "keep\n")
-
- def test_stage_base_and_early_cleanup_are_isolated(self) -> None:
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary).resolve()
- repo = root / "repo"
- source = repo / "experimental" / "CollectiveX"
- source.mkdir(parents=True)
- (source / "public.py").write_text("public\n")
- nested = repo / "stage"
- nested.mkdir(mode=0o700)
- group_writable = root / "group-stage"
- group_writable.mkdir(mode=0o770)
- group_writable.chmod(0o770)
- setgid = root / "setgid-stage"
- setgid.mkdir(mode=0o750)
- setgid.chmod(0o2750)
- command = r'''
- set -euo pipefail
- source "$1"
- ! (CX_STAGE_DIR="$3"; cx_stage_path "$2" "$3")
- ! (CX_STAGE_DIR="$4"; cx_stage_path "$2" "$4")
- export CX_STAGE_DIR="$5" COLLECTIVEX_EXECUTION_ID="setgid-$$"
- trap 'cx_cleanup_private_logs 0' EXIT
- staged="$(cx_stage_path "$2" "$CX_STAGE_DIR")"
- cx_stage_repo "$2" "$staged"
- chmod 2700 "$staged"
- cx_cleanup_stage "$staged" "$2"
- test ! -e "$staged"
- '''
- subprocess.run(
- [
- "bash", "-c", command, "_",
- str(ROOT / "runtime" / "common.sh"), str(repo), str(nested),
- str(group_writable), str(setgid),
- ],
- check=True,
- env={**os.environ, "COLLECTIVEX_OPERATOR_CONFIG": "/dev/null"},
- )
-
- early = r'''
- set -euo pipefail
- source "$1"
- export REPO_ROOT="$2" CX_STAGE_DIR="$3" CX_BENCH=nccl-ep
- export COLLECTIVEX_EXECUTION_ID="pre-marker-$$"
- MOUNT_SRC="$(cx_stage_path "$REPO_ROOT" "$CX_STAGE_DIR")"
- cx_install_launcher_fail_safe
- mkdir -m 700 "$MOUNT_SRC"
- exit 17
- '''
- result = subprocess.run(
- [
- "bash", "-c", early, "_",
- str(ROOT / "runtime" / "common.sh"), str(repo), str(setgid),
- ],
- text=True,
- capture_output=True,
- env={**os.environ, "COLLECTIVEX_OPERATOR_CONFIG": "/dev/null"},
- )
- self.assertEqual(result.returncode, 17, result.stderr)
- self.assertEqual(list(setgid.iterdir()), [])
-
- def test_backend_cache_reuses_v3_and_falls_back_once_without_repair(self) -> None:
- common = ROOT / "runtime" / "common.sh"
- with tempfile.TemporaryDirectory() as temporary:
- parent = Path(temporary) / "stage"
- parent.mkdir(mode=0o700)
- concurrent = Path(temporary) / "concurrent"
- concurrent.mkdir(mode=0o700)
- command = r'''
- set -euo pipefail
- source "$1"
- for worker in 1 2 3; do
- (
- cx_prepare_backend_cache "$2"
- printf '%s %s\n' "$CX_BACKEND_CACHE_SENTINEL_SHA256" \
- "$CX_PREPARED_BACKEND_CACHE" > "$3/$worker"
- ) &
- done
- wait
- cmp "$3/1" "$3/2"
- cmp "$3/1" "$3/3"
- cx_prepare_backend_cache "$2"
- first="$CX_PREPARED_BACKEND_CACHE"
- first_digest="$CX_BACKEND_CACHE_SENTINEL_SHA256"
- chmod 2700 "$first"
- cx_prepare_backend_cache "$2"
- second="$CX_PREPARED_BACKEND_CACHE"
- test "$first" = "$second"
- test "$first_digest" = "$CX_BACKEND_CACHE_SENTINEL_SHA256"
- test "$first" = "$(cd "$2" && pwd -P)/.collectivex-backend-cache-v3-$(id -u)"
- export CX_BACKEND_CACHE_ROOT="$first"
- cx_verify_backend_cache_mount
- export CX_BACKEND_CACHE_SENTINEL_SHA256="$(printf '0%.0s' {1..64})"
- ! cx_verify_backend_cache_mount
- '''
- subprocess.run(
- [
- "bash", "-c", command, "_", str(common), str(parent),
- str(concurrent),
- ],
- check=True,
- )
- cache = parent / f".collectivex-backend-cache-v3-{os.getuid()}"
- self.assertTrue(cache.is_dir())
- self.assertEqual(cache.stat().st_mode & 0o777, 0o700)
- self.assertEqual(
- list(cache.glob(".collectivex-mount-sentinel-v1.tmp.*")), []
- )
- alias = Path(temporary) / "stage-alias"
- alias.symlink_to(parent, target_is_directory=True)
- canonical = subprocess.run(
- [
- "bash", "-c",
- 'source "$1"; cx_prepare_backend_cache "$2"; '
- 'printf "%s\\n%s\\n" "$CX_PREPARED_BACKEND_CACHE" '
- '"$CX_BACKEND_CACHE_SENTINEL_SHA256"',
- "_", str(common), str(alias),
- ],
- text=True,
- capture_output=True,
- check=True,
- )
- cache_path, digest = canonical.stdout.splitlines()
- self.assertEqual(cache_path, str(cache.resolve()))
- self.assertRegex(digest, r"^[0-9a-f]{64}$")
- saved = parent / "saved-cache"
- cache.rename(saved)
- cache.mkdir(mode=0o700)
- replacement = cache / ".collectivex-mount-sentinel-v1"
- replacement.write_bytes(b"replacement".ljust(32, b"!"))
- replacement.chmod(0o600)
- replaced = subprocess.run(
- [
- "bash", "-c",
- 'source "$1"; export CX_BACKEND_CACHE_ROOT="$2" '
- 'CX_BACKEND_CACHE_SENTINEL_SHA256="$3"; '
- 'cx_verify_backend_cache_mount',
- "_", str(common), str(cache), digest,
- ]
- )
- self.assertNotEqual(replaced.returncode, 0)
- replacement.unlink()
- cache.rmdir()
- saved.rename(cache)
- (cache / ".collectivex-mount-sentinel-v1").unlink()
- cache.rmdir()
- target = Path(temporary) / "target"
- target.mkdir(mode=0o700)
- cache.symlink_to(target, target_is_directory=True)
- fallback = subprocess.run(
- [
- "bash", "-c",
- 'source "$1"; cx_prepare_backend_cache "$2"; '
- 'printf "%s\\n" "$CX_PREPARED_BACKEND_CACHE"',
- "_", str(common), str(parent),
- ],
- text=True,
- capture_output=True,
- check=True,
- )
- v4 = parent / f".collectivex-backend-cache-v4-{os.getuid()}"
- self.assertEqual(fallback.stdout.strip(), str(v4.resolve()))
- self.assertTrue(cache.is_symlink())
- self.assertTrue(v4.is_dir())
- (v4 / ".collectivex-mount-sentinel-v1").unlink()
- v4.rmdir()
- v4.symlink_to(target, target_is_directory=True)
- result = subprocess.run(
- [
- "bash", "-c", 'source "$1"; cx_prepare_backend_cache "$2"',
- "_", str(common), str(parent),
- ],
- text=True,
- capture_output=True,
- )
- self.assertNotEqual(result.returncode, 0)
- self.assertNotIn(str(parent), result.stderr)
- self.assertTrue(cache.is_symlink())
- self.assertTrue(v4.is_symlink())
-
- source = common.read_text().split("cx_prepare_backend_cache() {", 1)[1]
- program = source.split("<<'PY'\n", 1)[1].split("\nPY\n", 1)[0]
- with tempfile.TemporaryDirectory() as temporary:
- parent = Path(temporary) / "stage"
- parent.mkdir(mode=0o700)
- fake_os = types.ModuleType("os")
- fake_os.__dict__.update(os.__dict__)
- fake_os.fsync = mock.Mock(side_effect=OSError("forced fsync failure"))
- with (
- mock.patch.dict(sys.modules, {"os": fake_os}),
- mock.patch.object(sys, "argv", ["-", str(parent)]),
- mock.patch.object(sys, "stdout", io.StringIO()),
- self.assertRaises(SystemExit) as failure,
- ):
- exec(compile(program, "", "exec"), {})
- self.assertEqual(failure.exception.code, 1)
- self.assertEqual(
- list(parent.rglob(".collectivex-mount-sentinel-v1.tmp.*")), []
- )
-
- def test_nvidia_namespace_package_roots_come_from_distribution_files(self) -> None:
- runtime = ROOT / "runtime" / "run_in_container.sh"
- with tempfile.TemporaryDirectory() as temporary:
- site = Path(temporary) / "site"
- package = site / "nvidia" / "nccl"
- (package / "include").mkdir(parents=True)
- (package / "lib").mkdir()
- (package / "include" / "nccl.h").write_text("header\n")
- (package / "lib" / "libnccl.so.2").write_text("library\n")
- info = site / "nvidia_nccl_cu13-2.30.4.dist-info"
- info.mkdir()
- (info / "METADATA").write_text(
- "Metadata-Version: 2.1\nName: nvidia-nccl-cu13\nVersion: 2.30.4\n"
- )
- (info / "RECORD").write_text(
- "nvidia/nccl/include/nccl.h,,\n"
- "nvidia/nccl/lib/libnccl.so.2,,\n"
- "nvidia_nccl_cu13-2.30.4.dist-info/METADATA,,\n"
- "nvidia_nccl_cu13-2.30.4.dist-info/RECORD,,\n"
- )
- command = r'''
- set -euo pipefail
- eval "$(sed -n '/^cx_nvidia_package_root()/,/^}/p' "$1")"
- root="$(cx_nvidia_package_root nvidia-nccl-cu13 nccl)"
- test "$root" = "$2/nvidia/nccl"
- ! cx_nvidia_package_root nvidia-nccl-cu13 nvshmem
- '''
- subprocess.run(
- ["bash", "-c", command, "_", str(runtime), str(site.resolve())],
- check=True,
- env={**os.environ, "PYTHONPATH": str(site)},
- )
-
- def test_cuda_cccl_exports_the_resolved_jit_toolchain_root(self) -> None:
- runtime = ROOT / "runtime" / "run_in_container.sh"
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary)
- toolkit = root / "cuda-13.0"
- (toolkit / "bin").mkdir(parents=True)
- (toolkit / "include").mkdir()
- (toolkit / "lib64").mkdir()
- cccl = toolkit / "targets" / "x86_64-linux" / "include" / "cccl"
- cccl.mkdir(parents=True)
- nvcc = toolkit / "bin" / "nvcc"
- nvcc.write_text("#!/bin/sh\nexit 0\n")
- nvcc.chmod(0o755)
- alias = root / "cuda"
- alias.symlink_to(toolkit, target_is_directory=True)
- command = r'''
- set -euo pipefail
- eval "$(sed -n '/^cx_prepare_cuda_cccl()/,/^}/p' "$1")"
- cx_prepare_cuda_cccl
- test "$CUDA_HOME" = "$2"
- test "$CX_CUDA_CCCL" = "$2/targets/x86_64-linux/include/cccl"
- test "$CPATH" = "$2/targets/x86_64-linux/include/cccl:"
- test "$NVCC_PREPEND_FLAGS" = "-I$2/targets/x86_64-linux/include/cccl "
- '''
- subprocess.run(
- ["bash", "-c", command, "_", str(runtime), str(toolkit.resolve())],
- check=True,
- env={
- **os.environ,
- "PATH": f"{alias / 'bin'}:{os.environ['PATH']}",
- "CPATH": "",
- "NVCC_PREPEND_FLAGS": "",
- },
- )
-
- def test_deepep_v2_toolchain_rejects_overlay_lock_failure(self) -> None:
- runtime = ROOT / "runtime" / "run_in_container.sh"
- with tempfile.TemporaryDirectory() as temporary:
- command = r'''
- set -euo pipefail
- eval "$(sed -n '/^cx_prepare_deepep_toolchain()/,/^}/p' "$1")"
- cache_root="$2"
- cx_nvidia_package_root() { printf '%s' /unused; }
- cx_deepep_v2_root() { printf '%s' "$cache_root"; }
- cx_log() { :; }
- flock() { return 1; }
- ! cx_prepare_deepep_toolchain
- '''
- subprocess.run(
- ["bash", "-c", command, "_", str(runtime), temporary],
- check=True,
- )
-
- def test_pinned_source_fetch_retries_transient_failures(self) -> None:
- common = ROOT / "runtime" / "common.sh"
- with tempfile.TemporaryDirectory() as temporary:
- command = r'''
- set -euo pipefail
- eval "$(sed -n '/^cx_git()/,/^}/p' "$1")"
- eval "$(sed -n '/^cx_git_in_tree()/,/^}/p' "$1")"
- eval "$(sed -n '/^cx_fetch_revision()/,/^}/p' "$1")"
- attempts=0
- expected_directory="$(cd -P -- "$3" && pwd -P)"
- sleep() { :; }
- git() {
- local argument has_directory=0 has_trust=0
- if [ "$1" = '-c' ] && [ "$3" = init ]; then
- mkdir -p "${@: -1}"
- return 0
- fi
- for argument in "$@"; do
- [ "$argument" != '-C' ] || has_directory=1
- [ "$argument" != "safe.directory=$expected_directory" ] || has_trust=1
- [ "$argument" != 'safe.directory=*' ] || return 1
- done
- [ "$has_directory" = 0 ] || [ "$has_trust" = 1 ] || return 1
- case " $* " in
- *' fetch '*)
- attempts=$((attempts + 1))
- [ "$attempts" = 3 ]
- ;;
- *' rev-parse HEAD '*) printf '%s\n' "$revision" ;;
- *) return 0 ;;
- esac
- }
- cx_fetch_revision https://example.invalid/repo "$2" "$3"
- test "$attempts" = 3
- '''
- revision = "a" * 40
- subprocess.run(
- ["bash", "-c", command, "_", str(common), revision, temporary],
- check=True,
- )
-
- def test_git_tree_trust_is_exact_and_command_scoped(self) -> None:
- common = ROOT / "runtime" / "common.sh"
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary)
- repository = root / "repo"
- repository.mkdir()
- alias = root / "alias"
- alias.symlink_to(repository, target_is_directory=True)
- wildcard = root / "*"
- wildcard.mkdir()
- arguments = root / "arguments"
- command = r'''
- set -euo pipefail
- eval "$(sed -n '/^cx_git()/,/^}/p' "$1")"
- eval "$(sed -n '/^cx_git_in_tree()/,/^}/p' "$1")"
- arguments="$4"
- git() { printf '%s\n' "$@" > "$arguments"; }
- cx_git_in_tree "$2" status --porcelain
- ! cx_git_in_tree relative status
- ! cx_git_in_tree "$3" status
- ! cx_git_in_tree "$5" status
- '''
- subprocess.run(
- [
- "bash",
- "-c",
- command,
- "_",
- str(common),
- str(repository),
- str(alias),
- str(arguments),
- str(wildcard),
- ],
- check=True,
- )
- self.assertEqual(
- arguments.read_text().splitlines(),
- [
- "-c",
- "credential.helper=",
- "-c",
- f"safe.directory={repository.resolve()}",
- "-C",
- str(repository.resolve()),
- "status",
- "--porcelain",
- ],
- )
- self.assertNotIn("safe.directory=*", arguments.read_text())
-
- def test_runtime_materializes_the_verified_host_source_without_network(self) -> None:
- common = ROOT / "runtime" / "common.sh"
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary)
- seed = root / "seed"
- seed.mkdir()
- (seed / "pinned").write_text("source\n")
- destination = root / "build"
- fetched = root / "network-fetch"
- command = r'''
- set -euo pipefail
- source "$1"
- export CX_BACKEND_SOURCE_ROOT="$2/source"
- SEED="$3" FETCHED="$5"
- copy_mode=
- cx_backend_source_path() { printf '%s' "$SEED"; }
- cx_backend_source_is_valid() { test -f "$2/pinned"; }
- cx_fetch_revision() { : > "$FETCHED"; return 1; }
- cp() {
- test "$1" = -R
- copy_mode=recursive
- command cp "$@"
- }
- cx_materialize_backend_source deepep-hybrid "$4"
- test -f "$4/pinned"
- test "$copy_mode" = recursive
- python3 - "$4" <<'PY'
-import os
-import stat
-import sys
-assert stat.S_IMODE(os.stat(sys.argv[1]).st_mode) == 0o700
-PY
- test ! -e "$FETCHED"
- '''
- subprocess.run(
- [
- "bash", "-c", command, "_", str(common), str(root),
- str(seed), str(destination), str(fetched),
- ],
- check=True,
- )
-
- def test_backend_source_validation_rejects_status_errors_and_ignored_files(self) -> None:
- common = ROOT / "runtime" / "common.sh"
- with tempfile.TemporaryDirectory() as temporary:
- command = r'''
- set -euo pipefail
- source "$1"
- cx_backend_source_pin() { printf '%s|%s|' revision tree; }
- git() {
- case " $* " in
- *' rev-parse HEAD '*) printf '%s\n' revision ;;
- *' rev-parse HEAD^{tree} '*) printf '%s\n' tree ;;
- *' status --porcelain '*)
- [ "${GIT_OPTIONAL_LOCKS:-}" = 0 ] && [ "$mode" != status-error ]
- ;;
- *' ls-files --others --ignored '*)
- [ "$mode" != ignored ] || printf '%s\n' ignored.bin
- ;;
- *) return 1 ;;
- esac
- }
- mode=status-error
- ! cx_backend_source_is_valid backend "$2"
- mode=ignored
- ! cx_backend_source_is_valid backend "$2"
- mode=clean
- cx_backend_source_is_valid backend "$2"
- '''
- subprocess.run(
- ["bash", "-c", command, "_", str(common), temporary],
- check=True,
- )
-
- def test_deepep_v2_applies_only_the_pinned_upstream_nccl_check_fix(self) -> None:
- common = ROOT / "runtime" / "common.sh"
- with tempfile.TemporaryDirectory() as temporary:
- source = Path(temporary)
- init = source / "deep_ep" / "__init__.py"
- init.parent.mkdir()
- init.write_text(
- "for so in [line.strip().split(' ')[-1] for line in f if 'nccl' in line]:\n"
- )
- command = r'''
- set -euo pipefail
- source "$1"
- expected_file="$2/expected"
- sed "s/if 'nccl' in line/if 'libnccl' in line/" \
- "$2/deep_ep/__init__.py" > "$expected_file"
- CX_DEEPEP_V2_INIT_SHA256="$(sha256sum "$expected_file" | awk '{print $1}')"
- rm -f "$expected_file"
- cx_apply_deepep_v2_nccl_check_fix "$2"
- grep -Fq "if 'libnccl' in line" "$2/deep_ep/__init__.py"
- ! cx_apply_deepep_v2_nccl_check_fix "$2"
- '''
- subprocess.run(
- ["bash", "-c", command, "_", str(common), temporary],
- check=True,
- )
-
- def test_backend_source_root_normalizes_inherited_special_mode(self) -> None:
- common = ROOT / "runtime" / "common.sh"
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary)
- source_root = root / "experimental" / "CollectiveX" / ".cx_sources"
- source = source_root / "backend-revision"
- source.mkdir(parents=True)
- command = r'''
- set -euo pipefail
- source "$1"
- export COLLECTIVEX_EXECUTION_ID="source-mode-$$"
- trap 'cx_cleanup_private_logs 0' EXIT
- expected_mount="$2"
- expected_source="$3"
- expected_root="${expected_source%/*}"
- observed_mode=2700
- mock_stage_owner=4200
- mock_root_owner=4200
- chmod_calls=0
- chmod() {
- test "$1" = 700 && test "$2" = "$expected_root"
- chmod_calls=$((chmod_calls + 1))
- [ "$chmod_calls" = 2 ] || return 1
- observed_mode=700
- }
- stat() {
- case "$2" in
- %u)
- case "$3" in
- "$expected_mount") printf '%s\n' "$mock_stage_owner" ;;
- "$expected_root") printf '%s\n' "$mock_root_owner" ;;
- *) return 1 ;;
- esac
- ;;
- %a)
- case "$3" in
- "$expected_mount") printf '2700\n' ;;
- "$expected_root") printf '%s\n' "$observed_mode" ;;
- *) return 1 ;;
- esac
- ;;
- *) return 1 ;;
- esac
- }
- cx_backend_source_path() { printf '%s' "$expected_source"; }
- cx_backend_source_is_valid() {
- test "$1" = backend && test "$2" = "$expected_source"
- }
- cx_prepare_backend_source "$2" backend
- test "$observed_mode" = 2700
- test "$chmod_calls" = 0
- observed_mode=2750
- ! _cx_prepare_backend_source "$2" backend
- test "$chmod_calls" = 1
- _cx_prepare_backend_source "$2" backend
- test "$observed_mode" = 700
- mock_root_owner=4300
- ! _cx_prepare_backend_source "$2" backend
- '''
- subprocess.run(
- ["bash", "-c", command, "_", str(common), str(root), str(source)],
- check=True,
- )
-
- def test_canonical_backend_sources_use_verified_seed_without_network(self) -> None:
- common = ROOT / "runtime" / "common.sh"
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary)
- mount = root / "mount"
- source_root = mount / "experimental" / "CollectiveX" / ".cx_sources"
- seed_root = root / "seed"
- seeds = [
- seed_root / f"{backend}-revision"
- for backend in ("backend-one", "backend-two")
- ]
- mount.mkdir(mode=0o700)
- source_root.parent.mkdir(parents=True, mode=0o700)
- for seed in seeds:
- seed.mkdir(parents=True, mode=0o700)
- (seed / "pinned").write_text("source\n")
- network = root / "network"
- command = r'''
- set -euo pipefail
- source "$1"
- export COLLECTIVEX_CANONICAL_GHA=1
- export CX_BACKEND_SOURCE_SEED_ROOT="$4"
- export COLLECTIVEX_EXECUTION_ID="source-seed-$$"
- trap 'cx_cleanup_private_logs 0' EXIT
- NETWORK="$5"
- stat() {
- case "$2" in
- %u) printf '4200\n' ;;
- %a) printf '700\n' ;;
- *) return 1 ;;
- esac
- }
- cx_backend_source_path() { printf '%s/%s-revision' "$1" "$2"; }
- cx_backend_source_is_valid() { test -f "$2/pinned"; }
- cx_fetch_revision() { : > "$NETWORK"; return 1; }
- cx_prepare_backend_source "$2" backend-one
- cx_prepare_backend_source "$2" backend-two
- test -f "$3/backend-one-revision/pinned"
- test -f "$3/backend-two-revision/pinned"
- test ! -e "$NETWORK"
- rm -rf -- "$3/backend-one-revision" "$3/backend-two-revision"
- unset CX_BACKEND_SOURCE_SEED_ROOT
- ! _cx_prepare_backend_source "$2" backend-one
- test ! -e "$NETWORK"
- '''
- subprocess.run(
- [
- "bash", "-c", command, "_", str(common), str(mount),
- str(source_root), str(seed_root), str(network),
- ],
- check=True,
- )
-
- def test_deepep_hybrid_cache_reuse_revalidates_extensions(self) -> None:
- common = ROOT / "runtime" / "common.sh"
- runtime = ROOT / "runtime" / "run_in_container.sh"
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary)
- (root / "deep_ep_cpp.so").write_bytes(b"deep")
- (root / "hybrid_ep_cpp.so").write_bytes(b"hybrid")
- command = r'''
- set -euo pipefail
- chmod 700 "$3"
- source "$1"
- eval "$(sed -n '/^cx_deepep_hybrid_marker_content_sha256()/,/^}/p' "$2")"
- eval "$(sed -n '/^cx_deepep_hybrid_cache_is_valid()/,/^}/p' "$2")"
- revision=revision tree=tree
- cx_git() {
- case " $* " in
- *' rev-parse HEAD '*) printf '%s\n' "$revision" ;;
- *' rev-parse HEAD^{tree} '*) printf '%s\n' "$tree" ;;
- *' status --porcelain '*|*' ls-files --others '*) return 0 ;;
- *) return 1 ;;
- esac
- }
- cx_git_in_tree() { shift; cx_git "$@"; }
- marker="$3/.collectivex-complete"
- digest="$(cx_extension_pair_sha256 "$3" 'deep_ep_cpp*.so' 'hybrid_ep_cpp*.so')"
- (umask 077; printf '%s\n%s\n%s\n' "$revision" "$tree" "$digest" > "$marker")
- cx_deepep_hybrid_cache_is_valid "$3" "$marker" "$revision" "$tree"
- printf changed > "$3/hybrid_ep_cpp.so"
- ! cx_deepep_hybrid_cache_is_valid "$3" "$marker" "$revision" "$tree"
- printf hybrid > "$3/hybrid_ep_cpp.so"
- cp "$3/deep_ep_cpp.so" "$3/deep_ep_cpp-extra.so"
- ! cx_deepep_hybrid_cache_is_valid "$3" "$marker" "$revision" "$tree"
- '''
- subprocess.run(
- ["bash", "-c", command, "_", str(common), str(runtime), temporary],
- check=True,
- )
-
- def test_rack_backend_environment_is_shared_per_node_and_required(self) -> None:
- runtime = ROOT / "runtime" / "run_in_container.sh"
- launcher = (ROOT / "launchers" / "launch_gb-nv.sh").read_text()
- assignment = next(
- line for line in launcher.splitlines()
- if line.startswith("SOURCE_BACKEND_ENV=")
- )
- self.assertNotIn("/tmp/.cx_backend_env", launcher)
- self.assertIn('[ -f "$env_file" ] && [ -r "$env_file" ]', launcher)
- self.assertIn('[ ! -L "$env_file" ]', launcher)
- self.assertIn('$(stat -c "%u" "$env_root"):600', launcher)
- self.assertIn('case "$(stat -c "%a" "$env_root")" in 700|[1-7]700)', launcher)
- self.assertIn("node-${SLURM_NODEID}.sh", launcher)
- self.assertIn("HybridEPBuffer", launcher)
- self.assertIn('. "$env_file" || exit 66', launcher)
- with tempfile.TemporaryDirectory() as temporary:
- consumer = r'''
- eval "$1"
- env_root="$2/env"
- SOURCE_BACKEND_ENV="${SOURCE_BACKEND_ENV//\/ix\/experimental\/CollectiveX\/.cx_backend\/env/$env_root}"
- mkdir -p "$env_root"
- env_file="$env_root/node-1.sh"
- printf 'printf sourced > "$CX_SENTINEL"\n' > "$env_file"
- chmod 600 "$env_file"
- export CX_SENTINEL="$2/sentinel"
- stat() {
- [ "${STAT_FAIL:-0}" = 0 ] || return 1
- case "$2" in
- %a) printf '%s\n' "$ROOT_MODE" ;;
- %u) printf '1000\n' ;;
- %u:%a) printf '%s\n' "$FILE_OWNER_MODE" ;;
- *) return 2 ;;
- esac
- }
- run_case() {
- rm -f "$CX_SENTINEL"
- ROOT_MODE="$1" FILE_OWNER_MODE="$2" STAT_FAIL="$3" SLURM_NODEID="$4"
- ( eval "$SOURCE_BACKEND_ENV" )
- rc=$?
- [ "$rc" = "$5" ] || return 1
- if [ "$5" = 0 ]; then
- [ -f "$CX_SENTINEL" ]
- else
- [ ! -e "$CX_SENTINEL" ]
- fi
- }
- run_case 700 1000:600 0 1 0
- run_case 2700 1000:600 0 1 0
- run_case 755 1000:600 0 1 66
- run_case 700 1000:600 1 1 66
- run_case 700 2000:600 0 1 66
- mv "$env_file" "$env_file.real"
- ln -s "$env_file.real" "$env_file"
- run_case 700 1000:600 0 1 66
- rm "$env_file"
- mv "$env_file.real" "$env_file"
- run_case 700 1000:600 0 invalid 66
- '''
- subprocess.run(
- ["bash", "-c", consumer, "_", assignment, temporary],
- check=True,
- )
- command = r'''
- set -euo pipefail
- cd "$2"
- eval "$(sed -n '/^cx_persist_backend_env()/,/^}/p' "$1")"
- export SLURM_NODEID=1 PYTHONPATH=/ix/pinned DEEPEP_COMMIT=abc
- cx_persist_backend_env
- env_file="$PWD/.cx_backend/env/node-1.sh"
- test -f "$env_file"
- test "$(stat -f %Lp "$env_file" 2>/dev/null || stat -c %a "$env_file")" = 600
- unset PYTHONPATH DEEPEP_COMMIT
- . "$env_file"
- test "$PYTHONPATH" = /ix/pinned
- test "$DEEPEP_COMMIT" = abc
- SLURM_NODEID=invalid && ! cx_persist_backend_env
- '''
- subprocess.run(
- ["bash", "-c", command, "_", str(runtime), temporary],
- check=True,
- )
-
- def test_stage_cleanup_failure_fails_job_but_marks_allocation_safe(self) -> None:
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary)
- (root / "repo").mkdir()
- (root / "stage").mkdir()
- command = r'''
- source "$1"
- cx_write_cleanup_guard() {
- rm -f -- "$CX_JOB_ROOT/cleanup-safe" "$CX_JOB_ROOT/cleanup-unsafe"
- : > "$CX_JOB_ROOT/cleanup-$1"
- }
- cx_cleanup_stage() { return 1; }
- cx_cleanup_private_logs() { : > "$CX_JOB_ROOT/logs-deleted"; }
- export CX_JOB_ROOT="$2" REPO_ROOT="$2/repo" MOUNT_SRC="$2/stage"
- export COLLECTIVEX_CANONICAL_GHA=1 CX_ALLOCATION_UNCERTAIN=0
- unset CX_BENCH JOB_ID
- cx_launcher_cleanup 0
- '''
- result = subprocess.run(
- ["bash", "-c", command, "_", str(ROOT / "runtime" / "common.sh"),
- str(root)],
- text=True,
- capture_output=True,
- env={**os.environ, "COLLECTIVEX_OPERATOR_CONFIG": "/dev/null"},
- )
- self.assertEqual(result.returncode, 1, result.stderr)
- self.assertTrue((root / "cleanup-safe").is_file())
- self.assertFalse((root / "cleanup-unsafe").exists())
- self.assertFalse((root / "logs-deleted").exists())
-
- def test_generated_stage_cleanup_never_removes_configured_base(self) -> None:
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary)
- base = root / "stage"
- repo = root / "repo"
- generated = base / "job_execution"
- generated.mkdir(parents=True, mode=0o700)
- repo.mkdir()
- marker = generated / ".collectivex-stage-v1"
- marker.write_text("collectivex-stage-v1\nexecution\n")
- marker.chmod(0o600)
- (generated / "payload").write_text("temporary")
- subprocess.run(
- [
- "bash", "-c",
- 'source "$1"; cx_cleanup_stage "$2" "$3"; '
- '! cx_cleanup_stage "$4" "$3"',
- "_", str(ROOT / "runtime" / "common.sh"), str(generated),
- str(repo), str(base),
- ],
- check=True,
- env={
- **os.environ,
- "COLLECTIVEX_OPERATOR_CONFIG": "/dev/null",
- "COLLECTIVEX_EXECUTION_ID": "execution",
- "CX_STAGE_DIR": str(base),
- },
- )
- self.assertFalse(generated.exists())
- self.assertTrue(base.is_dir())
- self.assertTrue(repo.is_dir())
-
- def test_adapters_do_not_retain_dead_expected_methods(self) -> None:
- for path in HERE.glob("ep_*.py"):
- tree = ast.parse(path.read_text(), str(path))
- methods = {
- node.name for node in ast.walk(tree)
- if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))
- }
- self.assertNotIn("expected", methods, path.name)
-
- def test_artifact_safety_rejects_sensitive_material(self) -> None:
- private_address = ".".join(str(octet) for octet in (10, 0, 0, 1))
- secret = "github_pat_" + "A" * 24
- sensitive = {
- "ipv4": ({"note": private_address}, private_address),
- "ipv6": ({"note": "[2001:db8::1]:29500"}, "2001:db8::1"),
- "user-at-host": ({"note": "ssh admin@private-host"}, "admin@private-host"),
- "hostname": ({"note": "host=compute-17"}, "compute-17"),
- "private-dns": ({"note": "worker-7.cluster.local"}, "worker-7.cluster.local"),
- "suffixed-host": ({"worker_hostname": "relative"}, "worker_hostname"),
- "suffixed-address": ({"control_address": "relative"}, "control_address"),
- "suffixed-path": ({"scheduler_path": "relative"}, "scheduler_path"),
- "exact-address": ({"address": "relative"}, "address"),
- "exact-ip": ({"ip": "relative"}, "ip"),
- "camel-host": ({"workerHost": "relative"}, "workerHost"),
- "camel-path": ({"schedulerPath": "relative"}, "schedulerPath"),
- "acronym-gpu-uuid": ({"gpuUUID": "relative"}, "gpuUUID"),
- "acronym-device-uuid": ({"deviceUUID": "relative"}, "deviceUUID"),
- "acronym-pci-bus": ({"pciBusID": "relative"}, "pciBusID"),
- "mac-address": ({"note": "00:11:22:33:44:55"}, "00:11:22:33:44:55"),
- "ib-guid": ({"note": "00:11:22:33:44:55:66:77"}, "00:11:22:33:44:55:66:77"),
- "dgx-host": ({"note": "dgx-b300-001"}, "dgx-b300-001"),
- "cloud-host": ({"note": "ip-10-20-30-40"}, "ip-10-20-30-40"),
- "credential-field": ({"service_token": "short"}, "service_token"),
- "prefixed-token": ({"note": secret}, secret),
- "hf-token": ({"note": "hf_" + "A" * 24}, "hf_" + "A" * 24),
- "payment-token": ({"note": "sk_live_" + "A" * 24}, "sk_live_" + "A" * 24),
- "generic-secret": ({"note": "password=not-a-real-secret"}, "not-a-real-secret"),
- }
- for root in ("data", "it-share", "lustre", "raid", "nvme_home", "scratch", "gpfs", "fsx"):
- value = f"/{root}/collectivex/run"
- sensitive[f"private-root-{root}"] = ({"note": value}, value)
- for name, (document, offending) in sensitive.items():
- with self.subTest(name=name), self.assertRaises(
- artifact_safety.ArtifactSafetyError
- ) as caught:
- artifact_safety.assert_publication_safe([document])
- self.assertNotIn(offending, str(caught.exception))
-
- artifact_safety.assert_publication_safe([{
- "runner": "b300",
- "redaction": "sanitized-v1",
- "path": "datasets/" + "a" * 64 + "/dataset.json",
- "timing": "8:64:32",
- "image_digest": "sha256:" + "b" * 64,
- "source": "github.com",
- }])
- for ref in ("release@candidate", "worker1-feature", "sk-refactor-long-component-name"):
- artifact_safety.assert_publication_safe([{"ref": ref}])
-
- def test_artifact_safety_cli_does_not_echo_sensitive_values(self) -> None:
- private_value = ".".join(str(octet) for octet in (10, 24, 68, 12))
- with tempfile.TemporaryDirectory() as temporary:
- path = Path(temporary) / "artifact.json"
- path.write_text(json.dumps({"note": private_value}))
- result = subprocess.run(
- [sys.executable, str(ROOT / "artifact_safety.py"), str(path)],
- text=True,
- capture_output=True,
- )
- self.assertNotEqual(result.returncode, 0)
- self.assertIn("forbidden ipv4-address value", result.stderr)
- self.assertNotIn(private_value, result.stderr)
-
- def test_artifact_safety_rejects_linked_and_special_inputs(self) -> None:
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary)
- source = root / "source.json"
- source.write_text("{}")
- linked = root / "linked.json"
- linked.symlink_to(source)
- fifo = root / "fifo.json"
- os.mkfifo(fifo)
- for path in (linked, fifo):
- with self.subTest(path=path.name), self.assertRaises(
- artifact_safety.ArtifactSafetyError
- ):
- artifact_safety.load_documents([str(path)])
-
-
-if __name__ == "__main__":
- unittest.main()
From b09a270d5333609c4c7ca73a02287c33c1570ec5 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Wed, 8 Jul 2026 12:08:27 +0800
Subject: [PATCH 190/210] refactor(collectivex): strip publication subsystem to
neutral sweep artifacts
Reset scope to the MVP: schedule benchmarks, execute, validate emissions, and
upload neutral artifacts. Remove promotion, ranking, recommendation, channels,
bundles, and the qualification-repeat machinery. The frontend owns all display
and coverage decisions from the neutral matrix/result/summary artifacts. This
deliberately breaks the current frontend publication reader; that migration is a
separate effort.
Schemas: delete the channel, private-bundle, and public-dataset schemas. Strip
the precision, qualification-index, and allocation-stratum dimensions from
raw-case and terminal-outcome. Keep raw-case, samples, and terminal-outcome as
the neutral contract.
Contracts: validate document shape with jsonschema and keep only the cross-file
Python checks (scheduled-case-ID equality, detached-sample path/hash, one
terminal per scheduled case, complete shard delivery). Fold the artifact-safety
privacy boundary into validate_delivery and terminal emission, then delete
artifact_safety.py.
Matrix/capability/identity: drop the precision dimension and min/max_nodes;
schedule EP8/EP16 via ep_degrees plus an ep-sizes filter. Relocate the FP8
precision catalog from identity to ep_precision, where it stays callable but
dormant (default sweep is BF16 only). Remove required_publication from harness
emission so its case identity matches the matrix.
Tests trimmed to the neutral contract (four files, 731 LOC). CPU battery green;
matrix gen -> extract -> validate-control round-trips clean.
---
experimental/CollectiveX/artifact_safety.py | 212 --
experimental/CollectiveX/capability.py | 360 +--
experimental/CollectiveX/configs/suites.yaml | 32 -
experimental/CollectiveX/contracts.py | 2289 +++--------------
experimental/CollectiveX/identity.py | 203 +-
.../schemas/channel-v1.schema.json | 23 -
.../schemas/private-bundle-v1.schema.json | 163 --
.../schemas/public-dataset-v1.schema.json | 849 ------
.../schemas/raw-case-v1.schema.json | 149 +-
.../schemas/samples-v1.schema.json | 3 +-
.../schemas/terminal-outcome-v1.schema.json | 53 +-
experimental/CollectiveX/summarize.py | 18 +-
experimental/CollectiveX/sweep_matrix.py | 458 +---
experimental/CollectiveX/tests/ep_harness.py | 21 +-
.../CollectiveX/tests/ep_precision.py | 203 +-
.../tests/test_qualification_planning.py | 253 +-
.../tests/test_schema_v1_contract.py | 414 +--
17 files changed, 688 insertions(+), 5015 deletions(-)
delete mode 100644 experimental/CollectiveX/artifact_safety.py
delete mode 100644 experimental/CollectiveX/schemas/channel-v1.schema.json
delete mode 100644 experimental/CollectiveX/schemas/private-bundle-v1.schema.json
delete mode 100644 experimental/CollectiveX/schemas/public-dataset-v1.schema.json
diff --git a/experimental/CollectiveX/artifact_safety.py b/experimental/CollectiveX/artifact_safety.py
deleted file mode 100644
index 83d522fba8..0000000000
--- a/experimental/CollectiveX/artifact_safety.py
+++ /dev/null
@@ -1,212 +0,0 @@
-#!/usr/bin/env python3
-"""Fail-closed privacy check for CollectiveX public result documents."""
-from __future__ import annotations
-
-import argparse
-import ipaddress
-import json
-import os
-import re
-import stat
-
-
-SENSITIVE_FIELDS = frozenset({
- "environment", "env", "host", "hostname", "uuid", "gpu_uuid", "device_uuid",
- "pci_bus_id", "ip_address", "ip_addresses", "master_addr", "ssh", "ssh_target",
- "nodelist", "node_list", "nic_guid", "ib_guid", "topology_matrix", "rdma_devices",
- "user", "username", "password", "passwd", "secret", "token", "access_token",
- "api_token", "auth_token", "api_key", "private_key", "credential", "credentials",
- "address", "addresses", "ip", "ips",
-})
-SENSITIVE_FIELDS_COMPACT = frozenset(item.replace("_", "") for item in SENSITIVE_FIELDS)
-SENSITIVE_FIELD_SUFFIXES = (
- "_host", "_hostname", "_address", "_addresses", "_path", "_paths", "_ip", "_ips",
- "_password", "_passwd", "_secret", "_token", "_credential", "_credentials",
- "_uuid", "_guid", "_bus_id",
-)
-SENSITIVE_VALUE_PATTERNS = (
- ("private-path", re.compile(
- r"(? str:
- normalized = re.sub(r"([A-Z]+)([A-Z][a-z])", r"\1_\2", str(value).strip())
- normalized = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", normalized)
- return normalized.lower().replace("-", "_")
-
-
-def _sensitive_value_rule(value: str, *, contextual: bool = True) -> str | None:
- matched = next(
- (
- name for name, pattern in SENSITIVE_VALUE_PATTERNS
- if (contextual or name not in CONTEXTUAL_VALUE_RULES) and pattern.search(value)
- ),
- None,
- )
- if matched:
- return matched
- for candidate in IPV6_CANDIDATE.findall(value):
- try:
- address = candidate.split("%", 1)[0]
- if ipaddress.ip_address(address).version == 6:
- return "ipv6-address"
- except ValueError:
- continue
- return None
-
-
-def assert_publication_safe(docs: list[dict]) -> None:
- """Reject private infrastructure fields and value shapes."""
- def walk(value, doc_index: int, parent_field: str | None = None) -> None:
- if isinstance(value, dict):
- for key, child in value.items():
- field = _normalized_field(key)
- compact = field.replace("_", "")
- if (
- field in SENSITIVE_FIELDS
- or compact in SENSITIVE_FIELDS_COMPACT
- or field.endswith(SENSITIVE_FIELD_SUFFIXES)
- ):
- raise ArtifactSafetyError(
- f"artifact safety: doc[{doc_index}] contains forbidden private field"
- )
- key_rule = _sensitive_value_rule(str(key))
- if key_rule:
- raise ArtifactSafetyError(
- f"artifact safety: doc[{doc_index}] contains forbidden {key_rule} key"
- )
- walk(child, doc_index, field)
- elif isinstance(value, list):
- for child in value:
- walk(child, doc_index, parent_field)
- elif isinstance(value, str):
- rule = _sensitive_value_rule(value, contextual=parent_field != "ref")
- if rule:
- raise ArtifactSafetyError(
- f"artifact safety: doc[{doc_index}] contains forbidden {rule} value"
- )
-
- for index, doc in enumerate(docs):
- if not isinstance(doc, dict):
- raise ArtifactSafetyError(f"artifact safety: doc[{index}] is not a JSON object")
- walk(doc, index)
-
-
-def load_documents(paths: list[str]) -> list[dict]:
- docs: list[dict] = []
- for path in paths:
- try:
- metadata = os.lstat(path)
- except OSError as exc:
- raise ArtifactSafetyError("artifact safety: result file is unavailable") from exc
- if (
- not stat.S_ISREG(metadata.st_mode)
- or metadata.st_uid != os.getuid()
- or metadata.st_size <= 0
- or metadata.st_size > MAX_INPUT_BYTES
- ):
- raise ArtifactSafetyError("artifact safety: result file is unavailable")
- descriptor = -1
- try:
- descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
- opened = os.fstat(descriptor)
- if (
- not stat.S_ISREG(opened.st_mode)
- or (opened.st_dev, opened.st_ino, opened.st_size)
- != (metadata.st_dev, metadata.st_ino, metadata.st_size)
- ):
- raise ArtifactSafetyError("artifact safety: result file changed during open")
- with os.fdopen(descriptor, encoding="utf-8") as fh:
- descriptor = -1
- if path.endswith(".ndjson"):
- for line_number, line in enumerate(fh, 1):
- if not line.strip():
- continue
- try:
- docs.append(json.loads(line))
- except json.JSONDecodeError as exc:
- raise ArtifactSafetyError(
- f"artifact safety: malformed NDJSON at input line {line_number}"
- ) from exc
- else:
- docs.append(json.load(fh))
- except json.JSONDecodeError as exc:
- raise ArtifactSafetyError("artifact safety: malformed JSON input") from exc
- except (OSError, UnicodeError) as exc:
- raise ArtifactSafetyError("artifact safety: result file is unreadable") from exc
- finally:
- if descriptor >= 0:
- os.close(descriptor)
- if not docs:
- raise ArtifactSafetyError("artifact safety: no public result documents found")
- return docs
-
-
-def main() -> int:
- parser = argparse.ArgumentParser(description="Check CollectiveX result artifacts for private data")
- parser.add_argument("paths", nargs="+")
- args = parser.parse_args()
- try:
- docs = load_documents(args.paths)
- assert_publication_safe(docs)
- except ArtifactSafetyError as exc:
- parser.error(str(exc))
- print(f"artifact safety: {len(docs)} public document(s) passed")
- return 0
-
-
-if __name__ == "__main__":
- raise SystemExit(main())
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
index e19410dceb..59f0794716 100644
--- a/experimental/CollectiveX/capability.py
+++ b/experimental/CollectiveX/capability.py
@@ -204,211 +204,7 @@ def _platform(
),
}
-PRECISION_DISPOSITIONS = {
- "supported", "unsupported", "not-applicable", "provisional",
-}
-_NVIDIA_SKUS = (
- "h100-dgxc", "h200-dgxc", "b200-dgxc", "b300", "gb200", "gb300",
-)
-_DEEPEP_V2_PRECISION_SKUS = (
- "h200-dgxc", "b200-dgxc", "b300", "gb200", "gb300",
-)
-_HOPPER_UCCL_SKUS = ("h100-dgxc", "h200-dgxc")
-
-
-def _precision_rule(
- *,
- backend: str,
- skus: tuple[str, ...],
- ep_degrees: tuple[int, ...],
- mode: str,
- basis: str,
- disposition: str = "provisional",
-) -> dict[str, Any]:
- return {
- "backend": backend,
- "skus": skus,
- "ep_degrees": ep_degrees,
- "mode": mode,
- "disposition": disposition,
- "basis": basis,
- }
-
-
-_NORMAL_E4M3FN_PROFILE = "d-fp8-e4m3fn-b128-f32-prequantized.c-bf16"
-_NORMAL_E4M3FNUZ_PROFILE = "d-fp8-e4m3fnuz-b128-f32-prequantized.c-bf16"
-_LL_FP8_PROFILE = "d-fp8-e4m3fn-b128-f32-fused.c-bf16"
-_MORI_E4M3FN_DIRECT_PROFILE = "d-bf16.c-fp8-e4m3fn-direct-cast-noscale"
-_MORI_E4M3FN_BOTH_PROFILE = (
- "d-fp8-e4m3fn-b128-f32-prequantized.c-fp8-e4m3fn-direct-cast-noscale"
-)
-_MORI_E4M3FNUZ_DIRECT_PROFILE = "d-bf16.c-fp8-e4m3fnuz-direct-cast-noscale"
-_MORI_E4M3FNUZ_BOTH_PROFILE = (
- "d-fp8-e4m3fnuz-b128-f32-prequantized.c-fp8-e4m3fnuz-direct-cast-noscale"
-)
-
-# These are native-path candidates, not executable claims. A cell must be changed
-# from provisional to supported or unsupported after its pinned runtime probe.
-PRECISION_CAPABILITIES: dict[str, tuple[dict[str, Any], ...]] = {
- _NORMAL_E4M3FN_PROFILE: (
- _precision_rule(
- backend="deepep", skus=_NVIDIA_SKUS, ep_degrees=(8, 16), mode="normal",
- basis="deepep-v1-normal-prequantized-e4m3fn-block128-f32-scale",
- ),
- _precision_rule(
- backend="deepep-v2", skus=_DEEPEP_V2_PRECISION_SKUS,
- ep_degrees=(8, 16), mode="normal",
- basis="deepep-v2-normal-prequantized-e4m3fn-block128-f32-scale",
- ),
- _precision_rule(
- backend="deepep-hybrid", skus=_NVIDIA_SKUS,
- ep_degrees=(8, 16), mode="normal",
- basis="deepep-hybrid-normal-uint8-e4m3fn-block128-f32-scale",
- ),
- _precision_rule(
- backend="uccl", skus=_HOPPER_UCCL_SKUS, ep_degrees=(8, 16), mode="normal",
- basis="uccl-deepep-api-normal-prequantized-e4m3fn-block128-f32-scale",
- ),
- _precision_rule(
- backend="mori", skus=("mi355x",), ep_degrees=(8, 16), mode="normal",
- basis="mori-gfx950-normal-prequantized-ocp-e4m3fn-block128-f32-scale",
- ),
- ),
- _LL_FP8_PROFILE: (
- _precision_rule(
- backend="deepep", skus=_NVIDIA_SKUS, ep_degrees=(8, 16),
- mode="low-latency",
- basis="deepep-v1-low-latency-fused-e4m3fn-block128-f32-scale",
- ),
- _precision_rule(
- backend="uccl", skus=_HOPPER_UCCL_SKUS, ep_degrees=(8, 16),
- mode="low-latency",
- basis="uccl-deepep-api-low-latency-fused-e4m3fn-block128-f32-scale",
- ),
- ),
- _MORI_E4M3FN_DIRECT_PROFILE: (
- _precision_rule(
- backend="mori", skus=("mi355x",), ep_degrees=(8,), mode="normal",
- basis="mori-gfx950-ep8-intranode-e4m3fn-direct-cast-combine",
- ),
- ),
- _MORI_E4M3FN_BOTH_PROFILE: (
- _precision_rule(
- backend="mori", skus=("mi355x",), ep_degrees=(8,), mode="normal",
- basis="mori-gfx950-ep8-intranode-e4m3fn-dispatch-and-direct-cast-combine",
- ),
- ),
- _NORMAL_E4M3FNUZ_PROFILE: (
- _precision_rule(
- backend="mori", skus=("mi300x",), ep_degrees=(8, 16), mode="normal",
- basis="mori-gfx942-normal-prequantized-ocp-e4m3fnuz-block128-f32-scale",
- ),
- ),
- _MORI_E4M3FNUZ_DIRECT_PROFILE: (
- _precision_rule(
- backend="mori", skus=("mi300x",), ep_degrees=(8,), mode="normal",
- basis="mori-gfx942-ep8-asyncll-e4m3fnuz-direct-cast-combine",
- ),
- ),
- _MORI_E4M3FNUZ_BOTH_PROFILE: (
- _precision_rule(
- backend="mori", skus=("mi300x",), ep_degrees=(8,), mode="normal",
- basis="mori-gfx942-ep8-asyncll-e4m3fnuz-dispatch-and-direct-cast-combine",
- ),
- ),
-}
-
-PRECISION_CELL_OVERRIDES: dict[tuple[str, str, str, int, str], dict[str, str]] = {}
-
-# B300 EP16 runs over the RoCE GPU-fabric: nccl-ep and the DeepEP Hybrid control
-# path (native DOCA transport) execute there. DeepEP V1/V2 cannot -- their internode
-# path is NVSHMEM-IBGDA, which needs GDRCopy /dev/gdrdrv, and that char device is
-# unprovisioned on B300 hosts (module loaded, no device node; gdr_open returns NULL).
-# V1/V2 are additionally walled at the topology layer above; these precision-cell
-# overrides keep their declared native targets consistent (base_ok gates them anyway).
-for _profile, _backend, _mode in (
- (_NORMAL_E4M3FN_PROFILE, "deepep", "normal"),
- (_NORMAL_E4M3FN_PROFILE, "deepep-v2", "normal"),
- (_LL_FP8_PROFILE, "deepep", "low-latency"),
-):
- PRECISION_CELL_OVERRIDES[(_profile, _backend, "b300", 16, _mode)] = {
- "disposition": "unsupported",
- "basis": "deepep-ibgda-gdrcopy-unavailable-b300",
- }
-# DeepEP Hybrid's control path passes on B300 EP16, but its native FP8 precision
-# path is not yet validated on this fabric; keep the precision cell unsupported.
-PRECISION_CELL_OVERRIDES[(_NORMAL_E4M3FN_PROFILE, "deepep-hybrid", "b300", 16, "normal")] = {
- "disposition": "unsupported",
- "basis": "hybrid-native-precision-unvalidated-b300-ep16",
-}
-_VALIDATED_NATIVE_PROBE_CELLS = (
- # run, SKU, EP, backend, mode, profile, disposition, result
- ("28737315879", "b200-dgxc", 8, "deepep", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed"),
- ("28737315879", "b200-dgxc", 8, "deepep", "low-latency", _LL_FP8_PROFILE, "supported", "native-probe-passed"),
- ("28737315879", "b200-dgxc", 8, "deepep-hybrid", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "native-operation-failed"),
- ("28737315879", "b200-dgxc", 8, "deepep-v2", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "backend-construction-failed"),
- ("28745114766", "b200-dgxc", 16, "deepep", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "native-operation-timeout"),
- ("28747290376", "b200-dgxc", 16, "deepep", "low-latency", _LL_FP8_PROFILE, "unsupported", "native-operation-timeout"),
- ("28746910633", "b200-dgxc", 16, "deepep-v2", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "construction-consensus-accelerator-memory"),
- ("28748550531", "b200-dgxc", 16, "deepep-hybrid", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "backend-construction-failed"),
- ("28737422303", "h200-dgxc", 8, "deepep", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed"),
- ("28737422303", "h200-dgxc", 8, "deepep", "low-latency", _LL_FP8_PROFILE, "supported", "native-probe-passed"),
- ("28737422303", "h200-dgxc", 8, "deepep-hybrid", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "native-operation-failed"),
- ("28737422303", "h200-dgxc", 8, "deepep-v2", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "backend-construction-failed"),
- *(("28737422303", "h200-dgxc", 8, "uccl", mode, profile, "supported", "native-probe-passed")
- for profile, mode in ((_NORMAL_E4M3FN_PROFILE, "normal"), (_LL_FP8_PROFILE, "low-latency"))),
- *(("28737422902", "gb200", ep, "deepep", mode, profile, "supported", "native-probe-passed")
- for ep in (8, 16)
- for profile, mode in ((_NORMAL_E4M3FN_PROFILE, "normal"), (_LL_FP8_PROFILE, "low-latency"))),
- *(("28737422902", "gb200", ep, "deepep-v2", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed")
- for ep in (8, 16)),
- *(("28737422902", "gb200", ep, "deepep-hybrid", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "native-operation-failed")
- for ep in (8, 16)),
- ("28738113606", "h100-dgxc", 8, "deepep", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed"),
- ("28738113606", "h100-dgxc", 8, "deepep", "low-latency", _LL_FP8_PROFILE, "supported", "native-probe-passed"),
- ("28738113606", "h100-dgxc", 8, "deepep-hybrid", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "native-operation-failed"),
- ("28738113606", "h100-dgxc", 8, "uccl", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed"),
- ("28738113606", "h100-dgxc", 8, "uccl", "low-latency", _LL_FP8_PROFILE, "unsupported", "native-operation-timeout"),
- ("28745208954", "h100-dgxc", 16, "deepep", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed"),
- ("28745423523", "h100-dgxc", 16, "deepep", "low-latency", _LL_FP8_PROFILE, "supported", "native-probe-passed"),
- ("28745423523", "h100-dgxc", 16, "deepep-hybrid", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "native-operation-failed"),
- ("28745423523", "h100-dgxc", 16, "uccl", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed"),
- ("28745423523", "h100-dgxc", 16, "uccl", "low-latency", _LL_FP8_PROFILE, "supported", "native-probe-passed"),
- *(("28738445591", "gb300", ep, "deepep", mode, profile, "supported", "native-probe-passed")
- for ep in (8, 16)
- for profile, mode in ((_NORMAL_E4M3FN_PROFILE, "normal"), (_LL_FP8_PROFILE, "low-latency"))),
- *(("28738445591", "gb300", ep, "deepep-v2", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed")
- for ep in (8, 16)),
- *(("28738445591", "gb300", ep, "deepep-hybrid", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "native-operation-failed")
- for ep in (8, 16)),
- ("28738738793", "b300", 8, "deepep", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed"),
- ("28738738793", "b300", 8, "deepep-hybrid", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "native-operation-failed"),
- ("28739555164", "h200-dgxc", 16, "deepep", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed"),
- ("28739555164", "h200-dgxc", 16, "deepep-hybrid", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "native-operation-failed"),
- ("28740154697", "h200-dgxc", 16, "deepep", "low-latency", _LL_FP8_PROFILE, "supported", "native-probe-passed"),
- ("28740074613", "b300", 8, "deepep-v2", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed"),
- ("28740154697", "h200-dgxc", 16, "uccl", "low-latency", _LL_FP8_PROFILE, "supported", "native-probe-passed"),
- ("28740154697", "h200-dgxc", 16, "uccl", "normal", _NORMAL_E4M3FN_PROFILE, "supported", "native-probe-passed"),
- ("28750823474", "mi355x", 8, "mori", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "backend-construction-failed"),
- ("28750825814", "mi355x", 16, "mori", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "backend-construction-failed"),
- ("28740154697", "h200-dgxc", 16, "deepep-v2", "normal", _NORMAL_E4M3FN_PROFILE, "unsupported", "backend-setup-timeout"),
- ("28740533382", "mi355x", 8, "mori", "normal", _MORI_E4M3FN_BOTH_PROFILE, "unsupported", "backend-construction-failed"),
- ("28740533382", "mi355x", 8, "mori", "normal", _MORI_E4M3FN_DIRECT_PROFILE, "unsupported", "backend-construction-failed"),
- ("28782309414", "mi300x", 8, "mori", "normal", _NORMAL_E4M3FNUZ_PROFILE, "supported", "native-probe-passed"),
- ("28782309414", "mi300x", 8, "mori", "normal", _MORI_E4M3FNUZ_DIRECT_PROFILE, "unsupported", "backend-construction-failed"),
- ("28782309414", "mi300x", 8, "mori", "normal", _MORI_E4M3FNUZ_BOTH_PROFILE, "unsupported", "backend-construction-failed"),
- ("28788599713", "mi300x", 16, "mori", "normal", _NORMAL_E4M3FNUZ_PROFILE, "unsupported", "distributed-init-timeout"),
- ("28743235213", "b300", 8, "deepep", "low-latency", _LL_FP8_PROFILE, "supported", "native-probe-passed"),
-)
-PRECISION_CELL_OVERRIDES.update({
- (profile, backend, sku, ep, mode): {
- "basis": f"native-probe-v1-run-{run_id}-{result}",
- "disposition": disposition,
- }
- for run_id, sku, ep, backend, mode, profile, disposition, result
- in _VALIDATED_NATIVE_PROBE_CELLS
-})
def runtime_identity_issues(
sku: str, *, vendor: str, arch: str, machine: str, device_name: str,
device_count: int, world_size: int,
@@ -451,7 +247,7 @@ def _resolve_base(
eplb: bool = False,
mode: str = "normal",
) -> tuple[bool, str]:
- """Resolve the existing BF16 capability without a precision candidate."""
+ """Resolve the base BF16 capability for one SKU/backend/EP cell."""
platform, implementation = PLATFORMS.get(sku), BACKENDS.get(backend)
if platform is None:
return False, f"unknown GHA runner label {sku!r}"
@@ -497,66 +293,6 @@ def _resolve_base(
return True, "ok"
-def precision_targets(
- profile_names: tuple[str, ...] | list[str] | None = None,
-) -> list[dict[str, Any]]:
- """Expand exact native precision candidates into deterministic target cells."""
- names = list(PRECISION_CAPABILITIES) if profile_names is None else list(profile_names)
- unknown = sorted(set(names) - set(PRECISION_CAPABILITIES))
- if unknown:
- raise ValueError(f"unknown precision capability profiles {unknown}")
- targets: list[dict[str, Any]] = []
- seen: set[tuple[str, str, str, int, str]] = set()
- for profile_name in names:
- for rule in PRECISION_CAPABILITIES[profile_name]:
- for sku in rule["skus"]:
- for ep in rule["ep_degrees"]:
- key = (profile_name, rule["backend"], sku, ep, rule["mode"])
- if key in seen:
- raise RuntimeError(f"duplicate precision capability target {key}")
- seen.add(key)
- override = PRECISION_CELL_OVERRIDES.get(key, rule)
- targets.append({
- "precision_profile": profile_name,
- "backend": rule["backend"],
- "sku": sku,
- "ep": ep,
- "mode": rule["mode"],
- "disposition": override["disposition"],
- "basis": override["basis"],
- })
- return targets
-
-
-def provisional_precision_targets(
- profile_names: tuple[str, ...] | list[str] | None = None,
-) -> list[dict[str, Any]]:
- """Return probe-gated targets that must be eliminated before scheduling."""
- return [
- target for target in precision_targets(profile_names)
- if target["disposition"] == "provisional"
- ]
-
-
-def precision_target_declared(
- precision_profile: str,
- *,
- sku: str,
- backend: str,
- ep: int,
- mode: str,
-) -> bool:
- """Return whether a profile has an exact native candidate for this cell."""
- return any(
- target["precision_profile"] == precision_profile
- and target["sku"] == sku
- and target["backend"] == backend
- and target["ep"] == ep
- and target["mode"] == mode
- for target in precision_targets([precision_profile])
- )
-
-
def resolve_disposition(
sku: str,
backend: str,
@@ -566,9 +302,8 @@ def resolve_disposition(
routing: str = "uniform",
eplb: bool = False,
mode: str = "normal",
- precision_profile: str | None = None,
) -> tuple[str, str]:
- """Resolve a baseline or exact precision cell to its capability disposition."""
+ """Resolve a BF16 cell to its capability disposition."""
base_ok, base_detail = _resolve_base(
sku,
backend,
@@ -578,46 +313,7 @@ def resolve_disposition(
eplb=eplb,
mode=mode,
)
- if precision_profile is None or precision_profile == identity.V1_CONTROL_PRECISION_PROFILE:
- return ("supported", "ok") if base_ok else ("unsupported", base_detail)
- if precision_profile not in identity.V1_PRECISION_PROFILES:
- return "unsupported", f"unknown precision profile {precision_profile!r}"
- profile = identity.V1_PRECISION_PROFILES[precision_profile]
- if mode not in profile["modes"]:
- return (
- "not-applicable",
- f"precision profile {precision_profile} is not defined for {mode} mode",
- )
- if ep is None:
- platform = PLATFORMS.get(sku)
- if platform is None:
- return "unsupported", base_detail
- if nodes is None:
- ep = platform["ep_degrees"][0]
- else:
- matches = [
- degree for degree, topology in platform["topologies"].items()
- if topology["nodes"] == nodes
- ]
- if len(matches) != 1:
- return "unsupported", base_detail
- ep = matches[0]
- matches = [
- target for target in precision_targets([precision_profile])
- if target["sku"] == sku
- and target["backend"] == backend
- and target["ep"] == ep
- and target["mode"] == mode
- ]
- if not matches:
- return (
- "not-applicable",
- f"{precision_profile} has no native {backend} target on {sku} EP{ep}",
- )
- if not base_ok:
- return "unsupported", base_detail
- target = matches[0]
- return target["disposition"], target["basis"]
+ return ("supported", "ok") if base_ok else ("unsupported", base_detail)
def resolve(
@@ -629,7 +325,6 @@ def resolve(
routing: str = "uniform",
eplb: bool = False,
mode: str = "normal",
- precision_profile: str | None = None,
) -> tuple[bool, str]:
"""Return whether one fixed-v1 case can run on a public GHA runner label."""
disposition, detail = resolve_disposition(
@@ -640,54 +335,5 @@ def resolve(
routing=routing,
eplb=eplb,
mode=mode,
- precision_profile=precision_profile,
)
return disposition == "supported", detail
-
-
-def _validate_precision_capabilities() -> None:
- expected = set(identity.V1_PRECISION_PROFILES) - {
- identity.V1_CONTROL_PRECISION_PROFILE
- }
- if set(PRECISION_CAPABILITIES) != expected:
- raise RuntimeError("precision capability profiles differ from the identity registry")
- empty = sorted(
- profile for profile, rules in PRECISION_CAPABILITIES.items() if not rules
- )
- if empty:
- raise RuntimeError(f"precision profiles have no native targets: {empty}")
- declared_keys = {
- (
- target["precision_profile"], target["backend"], target["sku"],
- target["ep"], target["mode"],
- )
- for target in precision_targets()
- }
- if not set(PRECISION_CELL_OVERRIDES) <= declared_keys:
- raise RuntimeError("precision cell override has no declared native target")
- for target in precision_targets():
- if target["backend"] not in BACKENDS or target["sku"] not in PLATFORMS:
- raise RuntimeError(f"unknown precision target: {target}")
- if target["ep"] not in PLATFORMS[target["sku"]]["ep_degrees"]:
- raise RuntimeError(f"invalid precision target EP degree: {target}")
- if target["disposition"] not in PRECISION_DISPOSITIONS - {"not-applicable"}:
- raise RuntimeError(f"invalid declared precision disposition: {target}")
- if target["mode"] not in identity.V1_PRECISION_PROFILES[
- target["precision_profile"]
- ]["modes"]:
- raise RuntimeError(f"precision target mode differs from its profile: {target}")
- topology = topology_for(target["sku"], target["ep"])
- base_ok, base_detail = _resolve_base(
- target["sku"],
- target["backend"],
- ep=target["ep"],
- nodes=topology["nodes"] if topology is not None else None,
- mode=target["mode"],
- )
- if target["disposition"] in {"supported", "provisional"} and not base_ok:
- raise RuntimeError(
- f"precision target exceeds its backend capability: {target}: {base_detail}"
- )
-
-
-_validate_precision_capabilities()
diff --git a/experimental/CollectiveX/configs/suites.yaml b/experimental/CollectiveX/configs/suites.yaml
index 35a50bceaf..411d0914a1 100644
--- a/experimental/CollectiveX/configs/suites.yaml
+++ b/experimental/CollectiveX/configs/suites.yaml
@@ -10,7 +10,6 @@ suites:
routings: [uniform]
phases: [decode, prefill]
token_points_prefill: [256, 512]
- required_publication: official
ep-low-latency-v1:
mode: low-latency
@@ -21,34 +20,3 @@ suites:
routings: [uniform]
phases: [decode]
token_points_decode: [1, 2, 4, 8, 16, 32, 64, 128]
- required_publication: official
-
- ep-precision-normal-v1:
- mode: normal
- backends: [deepep, deepep-v2, uccl, deepep-hybrid, mori]
- workloads: [deepseek-v3-v1]
- platforms: [h100-dgxc, h200-dgxc, b300, b200-dgxc, gb300, gb200, mi300x, mi355x]
- ep_degrees: [8, 16]
- routings: [uniform]
- phases: [decode, prefill]
- token_points_decode: [128]
- token_points_prefill: [512]
- precision_profiles:
- - d-fp8-e4m3fn-b128-f32-prequantized.c-bf16
- - d-fp8-e4m3fnuz-b128-f32-prequantized.c-fp8-e4m3fnuz-direct-cast-noscale
- provisional: false
- required_publication: comparable-experimental
-
- ep-precision-low-latency-v1:
- mode: low-latency
- backends: [deepep, uccl]
- workloads: [deepseek-v3-v1]
- platforms: [h100-dgxc, h200-dgxc, b300, b200-dgxc, gb300, gb200, mi300x, mi355x]
- ep_degrees: [8, 16]
- routings: [uniform]
- phases: [decode]
- token_points_decode: [128]
- precision_profiles:
- - d-fp8-e4m3fn-b128-f32-fused.c-bf16
- provisional: false
- required_publication: comparable-experimental
diff --git a/experimental/CollectiveX/contracts.py b/experimental/CollectiveX/contracts.py
index 6a911faa05..449e83e28d 100644
--- a/experimental/CollectiveX/contracts.py
+++ b/experimental/CollectiveX/contracts.py
@@ -1,11 +1,26 @@
#!/usr/bin/env python3
-"""Strict native attempt contracts and metric validation for CollectiveX v1."""
+"""Result contracts for CollectiveX.
+
+JSON Schema (``schemas/*.schema.json``, Draft 2020-12) owns document shape and
+field constraints. The Python here covers only the facts JSON Schema cannot
+express: typed-identity recomputation, detached-sample path/hash joins, one
+terminal result per scheduled case, complete shard delivery, and the artifact
+privacy boundary. The public surface is intentionally small:
+
+ strict_json_load() - duplicate/non-finite-rejecting JSON reader
+ validate_result() - validate one emitted result file (raw+samples or terminal)
+ make_terminal_result() - build and self-validate one non-success outcome
+ validate_delivery() - reconcile a shard/matrix with its complete attempt set
+
+The backend-provenance and resource-projection helpers stay here so the bench
+emitter can call them; they move to the bench layer with the executable modules.
+"""
from __future__ import annotations
import argparse
import datetime as dt
-from functools import lru_cache
import hashlib
+import ipaddress
import json
import math
import os
@@ -14,32 +29,26 @@
import sys
from typing import Any, Iterable
-import artifact_safety
-import capability
-import identity
+from jsonschema import Draft202012Validator
-TESTS = Path(__file__).resolve().parent / "tests"
-sys.path.insert(0, str(TESTS))
-import eplb as eplb_contract # noqa: E402
-import workload as workload_contract # noqa: E402
+import identity
RAW_FORMAT = "collectivex.ep.v1"
SAMPLES_FORMAT = "collectivex.samples.v1"
TERMINAL_FORMAT = "collectivex.terminal.v1"
TERMINAL_CASE_FIELDS = {
"backend", "canonical", "eplb", "ep", "experts", "gpus_per_node", "hidden",
- "ladder", "mode", "nodes", "phase", "required_publication", "routing",
+ "ladder", "mode", "nodes", "phase", "routing",
"samples_per_point", "scale_out_transport", "scale_up_domain", "scale_up_transport",
"scope", "suite", "timing", "topk", "topology_class", "transport",
"warmup_semantics", "workload",
}
ALLOCATION_FACTOR_FIELDS = {
"artifact", "execution_id", "job", "repo", "run_attempt", "run_id", "runner",
- "source_sha", "qualification_index",
+ "source_sha",
}
GIT_RUN_FIELDS = {
- "artifact", "job", "qualification_index", "ref", "repo", "run_attempt", "run_id",
- "source_sha",
+ "artifact", "job", "ref", "repo", "run_attempt", "run_id", "source_sha",
}
PRE_EXECUTION_FAILURE_REASONS = {
"setup": "launcher-setup-failed",
@@ -66,14 +75,12 @@
CAPABILITY_FAILURE_REASONS = frozenset({
"backend-platform-unsupported",
"backend-token-capacity",
- "precision-profile-unsupported",
})
RETURN_CODE_FAILURE_MODES = {
5: "runtime-identity",
124: "timeout",
137: "timeout",
}
-PERCENTILES = ("p50", "p90", "p95", "p99")
V1_CONDITIONING_LADDERS = {
"decode": (1, 2, 4, 8, 16, 32, 64, 128),
"prefill": (1, 2, 4, 8, 16, 32, 64, 128, 256, 512),
@@ -107,7 +114,6 @@
"sortedcontainers": "2.4.0",
}
SCHEMA_DIR = Path(__file__).resolve().parent / "schemas"
-_SCHEMA_CACHE: dict[str, dict[str, Any]] = {}
REQUIRED_BACKEND_PROVENANCE = {
"deepep": (
"deepep_version", "deepep_commit", "backend_lineage", "allow_mnnvl",
@@ -132,43 +138,152 @@
"mori": ("mori_commit",),
"nccl-ep": ("nccl_version", "collective_library", "backend_lineage"),
}
-PROVENANCE_KEYS = {
- "allocated_qps", "allow_hybrid_mode", "allow_mnnvl", "allow_multiple_reduction",
- "api", "api_signature_sha256", "backend", "backend_lineage", "block_num",
- "block_num_floored", "block_num_target", "branch", "collective_library",
- "combine_dtype", "combine_warps", "communication_backend", "cuda_version",
- "deepep_commit", "deepep_distribution_version", "deepep_fix_pr",
- "deepep_nccl_check_commit", "deepep_nccl_check_fix_pr", "deepep_pr", "deepep_tree",
- "deepep_version", "deterministic", "device_cus",
- "device_sms", "dispatch_dtype", "dispatch_warps", "enable_sdma", "fmt_commit",
- "gin_enabled",
- "gpus_per_node", "heap_size",
- "impl", "jit_cache_key", "jit_cubins", "jit_kernel_keys", "jit_random_seed",
- "jit_shared_objects", "kernel_type",
- "loaded_libraries", "local_experts",
- "logical_scaleout_ranks",
- "logical_scaleup_ranks", "mapping_variant", "max_num_inp_token_per_rank",
- "max_num_tokens", "max_total_recv_tokens", "mnnvl_comm", "mode", "mori_commit",
- "nccl_communicator", "nccl_package_version", "nccl_version", "num_experts",
- "nvshmem_ibgda_nic_handler", "nvshmem_package_version",
- "num_max_tokens_per_rank", "num_nvl_bytes", "num_qps", "num_qps_per_rank",
- "num_rdma_bytes", "num_sms", "path",
- "physical_nvlink_ranks", "physical_rdma_ranks", "prefer_overlap_with_compute",
- "rdma_block_num",
- "realized_config", "reference_semantics", "requested_num_sms", "resource_mode", "routing_factor",
- "routing_metadata", "sm_fraction", "top_k",
- "torch_git_version", "torch_version", "transport", "trtllm", "tuned_source",
- "tuning_num_experts",
- "uccl_commit", "uccl_dependency_versions", "uccl_version", "uccl_wrapper_commit",
- "use_external_inp_buf",
- "workspace",
-}
class ContractError(ValueError):
- """A document differs from the native v1 contract."""
+ """A document differs from the CollectiveX result contract."""
+
+
+# ---------------------------------------------------------------------------
+# Artifact privacy boundary (kept narrow; enforced in delivery + terminal).
+# ---------------------------------------------------------------------------
+SENSITIVE_FIELDS = frozenset({
+ "environment", "env", "host", "hostname", "uuid", "gpu_uuid", "device_uuid",
+ "pci_bus_id", "ip_address", "ip_addresses", "master_addr", "ssh", "ssh_target",
+ "nodelist", "node_list", "nic_guid", "ib_guid", "topology_matrix", "rdma_devices",
+ "user", "username", "password", "passwd", "secret", "token", "access_token",
+ "api_token", "auth_token", "api_key", "private_key", "credential", "credentials",
+ "address", "addresses", "ip", "ips",
+})
+SENSITIVE_FIELDS_COMPACT = frozenset(item.replace("_", "") for item in SENSITIVE_FIELDS)
+SENSITIVE_FIELD_SUFFIXES = (
+ "_host", "_hostname", "_address", "_addresses", "_path", "_paths", "_ip", "_ips",
+ "_password", "_passwd", "_secret", "_token", "_credential", "_credentials",
+ "_uuid", "_guid", "_bus_id",
+)
+SENSITIVE_VALUE_PATTERNS = (
+ ("private-path", re.compile(
+ r"(? str:
+ normalized = re.sub(r"([A-Z]+)([A-Z][a-z])", r"\1_\2", str(value).strip())
+ normalized = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", normalized)
+ return normalized.lower().replace("-", "_")
+
+
+def _sensitive_value_rule(value: str, *, contextual: bool = True) -> str | None:
+ matched = next(
+ (
+ name for name, pattern in SENSITIVE_VALUE_PATTERNS
+ if (contextual or name not in CONTEXTUAL_VALUE_RULES) and pattern.search(value)
+ ),
+ None,
+ )
+ if matched:
+ return matched
+ for candidate in IPV6_CANDIDATE.findall(value):
+ try:
+ address = candidate.split("%", 1)[0]
+ if ipaddress.ip_address(address).version == 6:
+ return "ipv6-address"
+ except ValueError:
+ continue
+ return None
+def assert_publication_safe(docs: list[dict]) -> None:
+ """Reject private infrastructure fields and value shapes at the artifact boundary."""
+ def walk(value, doc_index: int, parent_field: str | None = None) -> None:
+ if isinstance(value, dict):
+ for key, child in value.items():
+ field = _normalized_field(key)
+ compact = field.replace("_", "")
+ if (
+ field in SENSITIVE_FIELDS
+ or compact in SENSITIVE_FIELDS_COMPACT
+ or field.endswith(SENSITIVE_FIELD_SUFFIXES)
+ ):
+ raise ArtifactSafetyError(
+ f"artifact safety: doc[{doc_index}] contains forbidden private field"
+ )
+ key_rule = _sensitive_value_rule(str(key))
+ if key_rule:
+ raise ArtifactSafetyError(
+ f"artifact safety: doc[{doc_index}] contains forbidden {key_rule} key"
+ )
+ walk(child, doc_index, field)
+ elif isinstance(value, list):
+ for child in value:
+ walk(child, doc_index, parent_field)
+ elif isinstance(value, str):
+ rule = _sensitive_value_rule(value, contextual=parent_field != "ref")
+ if rule:
+ raise ArtifactSafetyError(
+ f"artifact safety: doc[{doc_index}] contains forbidden {rule} value"
+ )
+
+ for index, doc in enumerate(docs):
+ if not isinstance(doc, dict):
+ raise ArtifactSafetyError(f"artifact safety: doc[{index}] is not a JSON object")
+ walk(doc, index)
+
+
+# ---------------------------------------------------------------------------
+# Backend emission helpers (called by the bench emitter, not by validation).
+# ---------------------------------------------------------------------------
def scheduled_case_profile(case: dict[str, Any], path: str = "case") -> dict[str, Any]:
"""Resolve an explicit scheduled mode to its immutable measurement profile."""
try:
@@ -177,14 +292,6 @@ def scheduled_case_profile(case: dict[str, Any], path: str = "case") -> dict[str
raise ContractError(f"{path}: {exc}") from exc
-def _scheduled_case(value: Any, path: str) -> dict[str, Any]:
- """Validate baseline or explicit-precision scheduled case fields."""
- fields = set(TERMINAL_CASE_FIELDS)
- if isinstance(value, dict) and "precision_profile" in value:
- fields.add("precision_profile")
- return _keys(value, fields, path)
-
-
def resolve_deepep_mnnvl(
*, requested: bool, signature_parameters: Iterable[str], deepep_commit: str | None
) -> tuple[dict[str, bool], str]:
@@ -263,92 +370,6 @@ def backend_version(provenance: dict[str, Any]) -> str | None:
return None
-def public_series_config(
- *, kernel_generation: Any, provenance: dict[str, Any],
- resource_profile: dict[str, Any], resource_mode: Any, device_product: Any,
-) -> dict[str, Any]:
- """Project raw implementation facts into the exact public configuration fields."""
- generation = None if kernel_generation == "n-a" else kernel_generation
- profile = "profile-" + _sha256_json(resource_profile)[:16]
- return {
- "backend": {
- "generation": generation,
- "version": backend_version(provenance),
- },
- "resource": {
- "mode": resource_mode,
- "profile": profile,
- "comm_units_kind": resource_profile.get("comm_units_kind"),
- "configured_units": resource_profile.get("configured_units"),
- },
- "system": {"label": str(device_product)[:160]},
- }
-
-
-def public_series_config_sha256(config: dict[str, Any]) -> str:
- """Commit the canonical public configuration projection into series identity."""
- return _sha256_json(config)
-
-
-SOURCE_BUILT_LIBRARY_ROLES = frozenset({
- "deepep-extension", "deepep-hybrid-extension",
-})
-
-
-def series_provenance(provenance: dict[str, Any]) -> dict[str, Any]:
- """Project stable semantic build identity while retaining raw binaries in private evidence."""
- projected = {
- key: value for key, value in provenance.items()
- if key not in {"jit_cache_key", "jit_shared_objects", "path", "sm_fraction"}
- }
- libraries = provenance.get("loaded_libraries")
- if isinstance(libraries, list):
- projected["loaded_libraries"] = [
- {
- "name": item.get("name"),
- "role": item.get("role"),
- "source_tree": provenance.get("deepep_tree"),
- }
- if isinstance(item, dict) and item.get("role") in SOURCE_BUILT_LIBRARY_ROLES
- else item
- for item in libraries
- ]
- jit_cubins = provenance.get("jit_cubins")
- if isinstance(jit_cubins, list):
- projected["jit_cubins"] = [
- {
- "cache_key": item.get("cache_key"),
- "sass_sha256": item.get("sass_sha256"),
- "source_sha256": item.get("source_sha256"),
- }
- if isinstance(item, dict)
- else item
- for item in jit_cubins
- ]
- return projected
-
-
-def routing_implementation_control_sha256(implementation: dict[str, Any]) -> str:
- """Bind routing cohorts to the same static build/generator and non-treatment configuration."""
- provenance = implementation.get("provenance")
- if not isinstance(provenance, dict):
- raise ContractError("implementation provenance is unavailable")
- semantic = series_provenance(provenance)
- treatment_fields = {
- "jit_cache_key", "jit_cubins", "jit_kernel_keys", "jit_shared_objects",
- "local_experts", "num_experts", "path", "realized_config", "sm_fraction",
- }
- return _sha256_json({
- "kernel_generation": implementation.get("kernel_generation"),
- "name": implementation.get("name"),
- "provenance": {
- key: value for key, value in semantic.items()
- if key not in treatment_fields
- },
- "resource_profile": implementation.get("resource_profile"),
- })
-
-
def _resolved_provenance_value(field: str, value: Any) -> bool:
if value is None or isinstance(value, (dict, list, tuple, set)) and not value:
return False
@@ -677,15 +698,13 @@ def backend_provenance_issues(backend: str, provenance: dict[str, Any]) -> list[
def provenance_complete(
provenance: dict[str, Any], backend: str, git_run: dict[str, Any] | None,
- *, allocation_stratum_sha256: Any, image_digest: Any, image_verified: Any,
- squash_sha256: Any,
+ *, image_digest: Any, image_verified: Any, squash_sha256: Any,
) -> bool:
+ """Return whether backend provenance and run identity are fully resolved."""
image = str(image_digest or "")
squash = str(squash_sha256 or "")
- allocation_stratum = str(allocation_stratum_sha256 or "")
return (
not backend_provenance_issues(backend, provenance)
- and bool(re.fullmatch(r"[0-9a-f]{64}", allocation_stratum))
and image_verified is True
and bool(re.fullmatch(r"sha256:[0-9a-f]{64}", image))
and bool(re.fullmatch(r"[0-9a-f]{64}", squash))
@@ -694,7 +713,10 @@ def provenance_complete(
)
-def strict_load(path: str | os.PathLike[str]) -> Any:
+# ---------------------------------------------------------------------------
+# JSON IO helpers.
+# ---------------------------------------------------------------------------
+def strict_json_load(path: str | os.PathLike[str]) -> Any:
"""Load JSON while rejecting duplicate keys and non-finite constants."""
def pairs(items):
result = {}
@@ -714,6 +736,21 @@ def constant(value):
raise ContractError(f"invalid JSON {path}: {exc}") from exc
+# Compatibility alias for existing runtime/summary callers.
+strict_load = strict_json_load
+
+
+def _finite_tree(value: Any, path: str = "$") -> None:
+ if isinstance(value, float) and not math.isfinite(value):
+ raise ContractError(f"{path} contains a non-finite number")
+ if isinstance(value, list):
+ for index, item in enumerate(value):
+ _finite_tree(item, f"{path}[{index}]")
+ elif isinstance(value, dict):
+ for key, item in value.items():
+ _finite_tree(item, f"{path}.{key}")
+
+
def canonical_json_bytes(value: Any) -> bytes:
"""Canonical finite JSON bytes for checksums and immutable artifacts."""
_finite_tree(value)
@@ -796,1685 +833,128 @@ def _integer(value: Any, path: str, *, minimum: int = 0) -> int:
return value
-def validate_conditioning_contract(value: Any, phase: str) -> dict[str, Any]:
- """Validate the exact phase-specific v1 conditioning schedule."""
- if phase not in V1_CONDITIONING_LADDERS:
- raise ContractError("raw conditioning phase is invalid")
- conditioning = _keys(
- value, {"contract", "ladder", "roundtrips_per_shape"},
- "raw.measurement.conditioning",
- )
- ladder = conditioning["ladder"]
- if (
- conditioning["contract"] != identity.V1_CASE_PROFILE["conditioning_contract"]
- or type(ladder) is not list
- or any(type(point) is not int for point in ladder)
- or ladder != list(V1_CONDITIONING_LADDERS[phase])
- or _integer(
- conditioning["roundtrips_per_shape"],
- "raw.measurement.conditioning.roundtrips_per_shape",
- minimum=1,
- ) != V1_CONDITIONING_ROUNDS_PER_SHAPE
- ):
- raise ContractError(f"raw {phase} conditioning contract differs")
- return conditioning
-
-
-def _number(value: Any, path: str, *, minimum: float | None = None) -> float:
- if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value):
- raise ContractError(f"{path} must be finite")
- result = float(value)
- if minimum is not None and result < minimum:
- raise ContractError(f"{path} must be >= {minimum}")
- return result
-
-
-def _finite_tree(value: Any, path: str = "$") -> None:
- if isinstance(value, float) and not math.isfinite(value):
- raise ContractError(f"{path} contains a non-finite number")
- if isinstance(value, list):
- for index, item in enumerate(value):
- _finite_tree(item, f"{path}[{index}]")
- elif isinstance(value, dict):
- for key, item in value.items():
- _finite_tree(item, f"{path}.{key}")
-
-
-def _typed(value: Any, kind: str, path: str) -> str:
- if not identity.is_typed_id(value, kind):
- raise ContractError(f"{path} is not a {kind} ID")
- return value
-
-
-def _sha256_json(value: Any) -> str:
- payload = json.dumps(
- value, allow_nan=False, ensure_ascii=False, sort_keys=True, separators=(",", ":")
- ).encode()
- return hashlib.sha256(payload).hexdigest()
-
-
-def _precision_byte_provenance(
- axis: dict[str, Any], logical_copies: int, hidden: int
-) -> dict[str, Any]:
- bits_per_value = {
- "bf16": 16,
- "fp8-e4m3fn": 8,
- "fp8-e4m3fnuz": 8,
- }.get(axis["communication_format"])
- if bits_per_value is None:
- raise ContractError("unknown communication precision format")
- scale_size = {None: 0, "f32": 4}.get(axis["scale_dtype"])
- if scale_size is None:
- raise ContractError("unknown communication scale dtype")
- group_size = axis["scale_group_size"]
- groups = math.ceil(hidden / group_size) if group_size is not None else 0
- activation = logical_copies * math.ceil(hidden * bits_per_value / 8)
- scales = logical_copies * groups * scale_size
- return {
- "accounting_contract": "activation-data-plus-scales-v1",
- "activation_data_bytes": activation,
- "scale_bytes": scales,
- "total_logical_bytes": activation + scales,
- }
-
-
-@lru_cache(maxsize=None)
-def _expected_eplb_calibration(
- routing: str,
- hidden: int,
- topk: int,
- logical_experts: int,
- physical_experts: int,
- ep_size: int,
- seed: int,
- reference_tokens_per_rank: int,
-) -> tuple[dict[str, Any], dict[str, Any]]:
- member, checksums, indices, _ = workload_contract.canonical_eplb_calibration_member(
- routing,
- hidden,
- topk,
- logical_experts,
- ep_size,
- reference_tokens_per_rank,
- seed,
- )
- load = [0] * logical_experts
- for row in indices:
- for expert in row:
- load[expert] += 1
- plan = eplb_contract.build_plan(load, physical_experts, ep_size)
- descriptor = {
- "calibration_token_offset": workload_contract.EPLB_CALIBRATION_TOKEN_OFFSET,
- "calibration_trace_sha256": checksums["trace"],
- "calibration_window": workload_contract.EPLB_CALIBRATION_WINDOW,
- "calibration_workload_id": member,
- }
- return plan, descriptor
-
-
-@lru_cache(maxsize=None)
-def _expected_eplb_plan(
- routing: str,
- topk: int,
- logical_experts: int,
- physical_experts: int,
- ep_size: int,
- seed: int,
- reference_tokens_per_rank: int,
- hidden: int = 7168,
-) -> dict[str, Any]:
- """Compatibility wrapper returning the disjoint calibration plan."""
- plan, _ = _expected_eplb_calibration(
- routing,
- hidden,
- topk,
- logical_experts,
- physical_experts,
- ep_size,
- seed,
- reference_tokens_per_rank,
- )
- return plan
-
-
-@lru_cache(maxsize=None)
-def _expected_canonical_trace(
- routing: str,
- hidden: int,
- topk: int,
- logical_experts: int,
- physical_experts: int,
- ep_size: int,
- tokens_per_rank: int,
- seed: int,
- eplb_enabled: bool,
- reference_tokens_per_rank: int,
-) -> tuple[str, dict[str, str], str, list[list[int]], list[list[float]]]:
- member, checksums, indices, weights = workload_contract.canonical_member(
- routing,
- hidden,
- topk,
- logical_experts,
- ep_size,
- tokens_per_rank,
- seed,
- )
- if eplb_enabled:
- plan = _expected_eplb_plan(
- routing,
- topk,
- logical_experts,
- physical_experts,
- ep_size,
- seed,
- reference_tokens_per_rank,
- hidden,
- )
- indices = eplb_contract.remap_rows(indices, plan)
- routing_hash = workload_contract.trace_checksums(indices, weights)["trace"]
- return member, checksums, routing_hash, indices, weights
-
-
-def _coefficient_of_variation(values: list[int]) -> float:
- mean = sum(values) / len(values)
- if mean == 0:
- return 0.0
- variance = sum((value - mean) ** 2 for value in values) / len(values)
- return variance**0.5 / mean
-
+# ---------------------------------------------------------------------------
+# Schema validation core (JSON Schema owns document shape).
+# ---------------------------------------------------------------------------
+_VALIDATOR_CACHE: dict[str, Draft202012Validator] = {}
-def _expected_routing_summary(
- indices: list[list[int]],
- weights: list[list[float]],
- *,
- physical_experts: int,
- ep_size: int,
- tokens_per_rank: int,
- gpus_per_node: int,
- scale_up_domain: int,
-) -> dict[str, Any]:
- """Recompute every published routing/load statistic without torch."""
- experts_per_rank = physical_experts // ep_size
- expert_load = [0] * physical_experts
- assignment_load = [0] * ep_size
- payload_load = [0] * ep_size
- fanouts: list[int] = []
- local = same_node = same_domain = copies = 0
- for token, row in enumerate(indices):
- destinations = {expert // experts_per_rank for expert in row}
- source = token // tokens_per_rank
- fanouts.append(len(destinations))
- for expert in row:
- expert_load[expert] += 1
- assignment_load[expert // experts_per_rank] += 1
- for destination in destinations:
- payload_load[destination] += 1
- copies += 1
- local += destination == source
- same_node += destination // gpus_per_node == source // gpus_per_node
- same_domain += destination // scale_up_domain == source // scale_up_domain
- fanout_histogram = [fanouts.count(value) for value in range(1, ep_size + 1)]
- expert_mean = sum(expert_load) / len(expert_load)
- return {
- "empty_expert_count": expert_load.count(0),
- "empty_rank_count": payload_load.count(0),
- "expert_assignment_rank_cv": _coefficient_of_variation(assignment_load),
- "expert_assignments_per_rank": assignment_load,
- "expert_load_cv": _coefficient_of_variation(expert_load),
- "expert_load_max": max(expert_load),
- "expert_load_mean": expert_mean,
- "expert_load_min": min(expert_load),
- "fanout_histogram": fanout_histogram,
- "fanout_max": max(fanouts),
- "fanout_mean": sum(fanouts) / len(fanouts),
- "fanout_min": min(fanouts),
- "hash": workload_contract.trace_checksums(indices, weights)["trace"],
- "hotspot_ratio": max(expert_load) / expert_mean if expert_mean else 0.0,
- "locality": {
- "placement": "packed",
- "local_rank_fraction": local / copies,
- "same_node_fraction": same_node / copies,
- "same_scaleup_domain_fraction": same_domain / copies,
- "cross_node_fraction": 1 - same_node / copies,
- "cross_domain_fraction": 1 - same_domain / copies,
- "gpus_per_node": gpus_per_node,
- "scale_up_domain": scale_up_domain,
- "copies": copies,
- },
- "payload_copies_per_rank": payload_load,
- "payload_rank_cv": _coefficient_of_variation(payload_load),
- "routed_copies": copies,
- "source_token_stats": {
- "min": tokens_per_rank,
- "mean": float(tokens_per_rank),
- "max": tokens_per_rank,
- "cv": 0.0,
- "empty_ranks": 0,
- "total": tokens_per_rank * ep_size,
- "ranks": ep_size,
- },
- }
-
-def _expected_histogram(samples: list[float], bins: int = 40) -> dict[str, Any]:
- low, high = min(samples), max(samples)
- if high <= low:
- return {"n": len(samples), "min": low, "max": high, "bins": bins, "counts": [len(samples)]}
- counts = [0] * bins
- span = high - low
- for sample in samples:
- index = min(bins - 1, int((sample - low) / span * bins))
- counts[index] += 1
- return {
- "n": len(samples),
- "min": round(low, 3),
- "max": round(high, 3),
- "bins": bins,
- "counts": counts,
- }
-
-
-def _expected_anomalies(
- tokens: int, components: dict[str, Any]
-) -> list[dict[str, Any]]:
- dispatch = components["dispatch"]["percentiles_us"]
- stage = components["stage"]["percentiles_us"]
- combine = components["combine"]["percentiles_us"]
- roundtrip = components["roundtrip"]["percentiles_us"]
- isolated = components["isolated_sum"]["percentiles_us"]
- anomalies: list[dict[str, Any]] = []
- if isolated is not None and roundtrip["p99"] > 3.0 * isolated["p99"]:
- anomalies.append({
- "type": "roundtrip_gt_isolated_sum",
- "T": tokens,
- "roundtrip_p99": round(roundtrip["p99"], 2),
- "isolated_sum_p99": round(isolated["p99"], 2),
- "ratio": round(roundtrip["p99"] / isolated["p99"], 2),
- "threshold": 3.0,
- })
- floor = (
- max(dispatch["p50"], combine["p50"], stage["p50"] if stage is not None else 0.0)
- if dispatch and combine else None
+def _validator(schema_file: str) -> Draft202012Validator:
+ validator = _VALIDATOR_CACHE.get(schema_file)
+ if validator is None:
+ try:
+ schema = json.loads((SCHEMA_DIR / schema_file).read_text())
+ except (OSError, json.JSONDecodeError) as exc:
+ raise ContractError(f"schema {schema_file} is unavailable: {exc}") from exc
+ Draft202012Validator.check_schema(schema)
+ validator = Draft202012Validator(schema)
+ _VALIDATOR_CACHE[schema_file] = validator
+ return validator
+
+
+def _schema_validate(document: Any, schema_file: str, label: str) -> None:
+ errors = sorted(
+ _validator(schema_file).iter_errors(document),
+ key=lambda error: list(error.absolute_path),
)
- if floor and roundtrip["p50"] < 0.95 * floor:
- anomalies.append({
- "type": "roundtrip_lt_component_floor",
- "T": tokens,
- "roundtrip_p50": round(roundtrip["p50"], 2),
- "component_floor_p50": round(floor, 2),
- })
- return anomalies
-
-
-def _validate_canonical_workload(
- workload: dict[str, Any],
- scheduled_case: dict[str, Any],
- rows: list[dict[str, Any]],
- eplb: dict[str, Any],
-) -> None:
- """Bind every canonical member and measured routing hash to its scheduled token row."""
- profile = identity.profile_for_case(scheduled_case)
- if eplb["enabled"]:
- plan = _expected_eplb_plan(
- scheduled_case["routing"],
- scheduled_case["topk"],
- scheduled_case["experts"],
- eplb["num_physical_experts"],
- scheduled_case["ep"],
- profile["seed"],
- profile["eplb_reference_tokens_per_rank"],
- scheduled_case["hidden"],
- )
- if eplb["mapping_hash"] != eplb_contract.mapping_hash(plan):
- raise ContractError("raw EPLB mapping differs from the frozen canonical plan")
-
- expected: dict[str, dict[str, str]] = {}
- for index, row in enumerate(rows):
- member, checksums, routing_hash, _, _ = _expected_canonical_trace(
- scheduled_case["routing"],
- scheduled_case["hidden"],
- scheduled_case["topk"],
- scheduled_case["experts"],
- eplb["num_physical_experts"],
- scheduled_case["ep"],
- row["tokens_per_rank"],
- profile["seed"],
- eplb["enabled"],
- profile["eplb_reference_tokens_per_rank"],
- )
- if row["routing"]["hash"] != routing_hash:
- raise ContractError(
- f"raw.measurement.rows[{index}].routing.hash differs from its canonical member"
- )
- expected[member] = checksums
- if (
- len(expected) != len(rows)
- or workload["members"] != sorted(expected)
- or workload["manifest_checksums"] != expected
- ):
- raise ContractError("raw canonical member set/checksums differ from scheduled rows")
- expected_workload_id = identity.workload_id({
- "members": [
- {"checksums": expected[member], "workload_id": member}
- for member in sorted(expected)
- ]
- })
- if workload["workload_id"] != expected_workload_id:
- raise ContractError("raw composite workload identity differs from scheduled rows")
-
-
-def _nearest_rank(samples: list[float], q: int) -> float:
- ordered = sorted(samples)
- return ordered[max(0, min(len(ordered) - 1, math.ceil(q / 100 * len(ordered)) - 1))]
-
-
-def _close(observed: Any, expected: float, path: str, tolerance: float = 1e-6) -> None:
- value = _number(observed, path)
- if not math.isclose(value, expected, rel_tol=tolerance, abs_tol=tolerance):
- raise ContractError(f"{path}={value} differs from recomputed {expected}")
-
-
-def _equivalent(
- observed: Any, expected: Any, path: str, *, tolerance: float = 1e-6
-) -> None:
- """Compare a recomputed JSON subtree while allowing only float roundoff."""
- if isinstance(expected, dict):
- value = _keys(observed, set(expected), path)
- for key, child in expected.items():
- _equivalent(value[key], child, f"{path}.{key}", tolerance=tolerance)
- return
- if isinstance(expected, list):
- if not isinstance(observed, list) or len(observed) != len(expected):
- raise ContractError(f"{path} differs from recomputed evidence")
- for index, child in enumerate(expected):
- _equivalent(observed[index], child, f"{path}[{index}]", tolerance=tolerance)
- return
- if isinstance(expected, float):
- _close(observed, expected, path, tolerance)
- return
- if type(observed) is not type(expected) or observed != expected:
- raise ContractError(f"{path} differs from recomputed evidence")
-
-
-def _schema_equal(left: Any, right: Any) -> bool:
- """JSON Schema equality: booleans are distinct from numbers."""
- if isinstance(left, bool) or isinstance(right, bool):
- return type(left) is type(right) and left == right
- if isinstance(left, dict) and isinstance(right, dict):
- return set(left) == set(right) and all(
- _schema_equal(left[key], right[key]) for key in left
- )
- if isinstance(left, list) and isinstance(right, list):
- return len(left) == len(right) and all(
- _schema_equal(a, b) for a, b in zip(left, right, strict=True)
- )
- return left == right
-
-
-def _schema_ref(root: dict[str, Any], reference: str) -> dict[str, Any]:
- if not reference.startswith("#/"):
- raise ContractError("native artifact schema contains a non-local reference")
- value: Any = root
- for part in reference[2:].split("/"):
- part = part.replace("~1", "/").replace("~0", "~")
- if not isinstance(value, dict) or part not in value:
- raise ContractError("native artifact schema contains a broken reference")
- value = value[part]
- if not isinstance(value, dict):
- raise ContractError("native artifact schema reference is not an object")
- return value
-
-
-def _schema_type_matches(value: Any, expected: str) -> bool:
- if expected == "null":
- return value is None
- if expected == "boolean":
- return type(value) is bool
- if expected == "object":
- return isinstance(value, dict)
- if expected == "array":
- return isinstance(value, list)
- if expected == "string":
- return isinstance(value, str)
- if expected == "number":
- return (
- not isinstance(value, bool)
- and isinstance(value, (int, float))
- and math.isfinite(value)
- )
- if expected == "integer":
- return (
- not isinstance(value, bool)
- and isinstance(value, (int, float))
- and math.isfinite(value)
- and float(value).is_integer()
- )
- raise ContractError(f"native artifact schema uses unsupported type {expected!r}")
-
-
-def _validate_schema_value(
- value: Any, schema: dict[str, Any], root: dict[str, Any], path: str
-) -> None:
- """Validate the bounded JSON Schema subset used by native artifact contracts."""
- if "$ref" in schema:
- _validate_schema_value(value, _schema_ref(root, schema["$ref"]), root, path)
- return
- if "oneOf" in schema:
- matches = 0
- for candidate in schema["oneOf"]:
- try:
- _validate_schema_value(value, candidate, root, path)
- except ContractError:
- continue
- matches += 1
- if matches != 1:
- raise ContractError(f"{path} must match exactly one native schema alternative")
- return
- expected_type = schema.get("type")
- if expected_type is not None and not _schema_type_matches(value, expected_type):
- raise ContractError(f"{path} is not a schema {expected_type}")
- if "const" in schema and not _schema_equal(value, schema["const"]):
- raise ContractError(f"{path} differs from its schema constant")
- if "enum" in schema and not any(_schema_equal(value, item) for item in schema["enum"]):
- raise ContractError(f"{path} is outside its schema enum")
-
- if isinstance(value, dict):
- required = set(schema.get("required", ()))
- properties = schema.get("properties", {})
- missing = required - set(value)
- if missing:
- raise ContractError(f"{path} lacks schema fields {sorted(missing)}")
- additional = schema.get("additionalProperties", True)
- extra = set(value) - set(properties)
- if additional is False and extra:
- raise ContractError(f"{path} has extra schema fields {sorted(extra)}")
- for key, item in value.items():
- if key in properties:
- _validate_schema_value(item, properties[key], root, f"{path}.{key}")
- elif isinstance(additional, dict):
- _validate_schema_value(item, additional, root, f"{path}.{key}")
- property_names = schema.get("propertyNames")
- if property_names is not None:
- for key in value:
- _validate_schema_value(key, property_names, root, f"{path}.")
-
- if isinstance(value, list):
- if len(value) < schema.get("minItems", 0):
- raise ContractError(f"{path} has too few schema items")
- maximum = schema.get("maxItems")
- if maximum is not None and len(value) > maximum:
- raise ContractError(f"{path} has too many schema items")
- if schema.get("uniqueItems") and any(
- _schema_equal(item, prior)
- for index, item in enumerate(value)
- for prior in value[:index]
- ):
- raise ContractError(f"{path} schema items are not unique")
- if "items" in schema:
- for index, item in enumerate(value):
- _validate_schema_value(item, schema["items"], root, f"{path}[{index}]")
-
- if isinstance(value, str):
- if len(value) < schema.get("minLength", 0):
- raise ContractError(f"{path} is shorter than its schema minimum")
- maximum = schema.get("maxLength")
- if maximum is not None and len(value) > maximum:
- raise ContractError(f"{path} is longer than its schema maximum")
- if "pattern" in schema and re.search(schema["pattern"], value) is None:
- raise ContractError(f"{path} does not match its schema pattern")
- if schema.get("format") == "date-time":
- try:
- parsed = dt.datetime.fromisoformat(value.replace("Z", "+00:00"))
- except ValueError as exc:
- raise ContractError(f"{path} is not a schema date-time") from exc
- if parsed.tzinfo is None:
- raise ContractError(f"{path} schema date-time lacks a timezone")
-
- if (
- not isinstance(value, bool)
- and isinstance(value, (int, float))
- and math.isfinite(value)
- ):
- if "minimum" in schema and value < schema["minimum"]:
- raise ContractError(f"{path} is below its schema minimum")
- if "maximum" in schema and value > schema["maximum"]:
- raise ContractError(f"{path} is above its schema maximum")
-
-
-def _validate_native_schema(name: str, value: Any) -> None:
- schema = _SCHEMA_CACHE.get(name)
- if schema is None:
- loaded = strict_load(SCHEMA_DIR / name)
- if not isinstance(loaded, dict):
- raise ContractError(f"native artifact schema {name} is not an object")
- schema = loaded
- _SCHEMA_CACHE[name] = schema
- _validate_schema_value(value, schema, schema, "$")
+ if errors:
+ error = errors[0]
+ location = "/".join(str(part) for part in error.absolute_path) or ""
+ raise ContractError(f"{label} schema error at {location}: {error.message}")
def validate_samples_document(document: Any) -> dict[str, Any]:
- _validate_native_schema("samples-v1.schema.json", document)
- doc = _keys(
- document,
- {"allocation_id", "attempt_id", "case_id", "format", "points",
- "qualification_index", "sampling", "schema_version", "series_id"},
- "samples",
- )
- if doc["format"] != SAMPLES_FORMAT or doc["schema_version"] != 1:
- raise ContractError("samples format/schema differs from v1")
- for field, kind in (
- ("allocation_id", "allocation"), ("attempt_id", "attempt"),
- ("case_id", "case"), ("series_id", "series"),
- ):
- _typed(doc[field], kind, f"samples.{field}")
- qualification_index = _integer(
- doc["qualification_index"], "samples.qualification_index", minimum=1
- )
- if qualification_index > 3:
- raise ContractError("samples.qualification_index must be in 1..3")
- sampling = _keys(
- doc["sampling"], {"iterations_per_trial", "reduction", "trials"}, "samples.sampling"
+ """Validate one detached exact-samples document against its schema."""
+ if not isinstance(document, dict):
+ raise ContractError("samples document must be an object")
+ _schema_validate(document, "samples-v1.schema.json", "samples")
+ return document
+
+
+def validate_raw_document(document: Any, samples: Any) -> dict[str, Any]:
+ """Validate a raw measurement document and its paired detached samples.
+
+ JSON Schema owns both document shapes. The cross-file checks here recompute
+ the typed identities from their factors and join the detached samples to the
+ measurement rows by point identity and sample digest.
+ """
+ if not isinstance(document, dict):
+ raise ContractError("raw document must be an object")
+ _schema_validate(document, "raw-case-v1.schema.json", "raw")
+ _schema_validate(samples, "samples-v1.schema.json", "samples")
+ ids = document["identity"]
+ case_id = identity.digest("case", ids["case_factors"])
+ allocation_id = identity.allocation_id(ids["allocation_factors"])
+ attempt_id = identity.attempt_id(
+ allocation=allocation_id, case=case_id, ordinal=ids["attempt_ordinal"]
)
- if (
- _integer(sampling["iterations_per_trial"], "samples.sampling.iterations_per_trial", minimum=1) != 8
- or _integer(sampling["trials"], "samples.sampling.trials", minimum=1) != 64
- or sampling["reduction"] != identity.V1_CASE_PROFILE["rank_reduction"]
+ if (case_id, allocation_id, attempt_id) != (
+ ids["case_id"], ids["allocation_id"], ids["attempt_id"]
):
- raise ContractError("samples must use the fixed 8x64 cross-rank-max contract")
- points = doc["points"]
- if not isinstance(points, list) or not points:
- raise ContractError("samples.points must be non-empty")
- seen = set()
- for index, point_value in enumerate(points):
- path = f"samples.points[{index}]"
- point = _keys(
- point_value,
- {"components", "evidence_id", "point_id", "sample_sha256", "tokens_per_rank"},
- path,
+ raise ContractError("raw typed identities do not match their factors")
+ series_id = ids["series_id"]
+ for field in ("case_id", "series_id", "allocation_id", "attempt_id"):
+ if samples.get(field) != ids[field]:
+ raise ContractError(f"detached samples {field} differs from raw identity")
+ rows = document["measurement"]["rows"]
+ sample_points = {point["point_id"]: point for point in samples["points"]}
+ if {row["point_id"] for row in rows} != set(sample_points):
+ raise ContractError("raw rows and detached samples cover different points")
+ success = document["outcome"]["status"] == "success"
+ for row in rows:
+ expected_point = identity.point_id(
+ series=series_id, tokens_per_rank=row["tokens_per_rank"]
)
- tokens = _integer(point["tokens_per_rank"], f"{path}.tokens_per_rank", minimum=1)
- if tokens in seen:
- raise ContractError(f"duplicate sample token point {tokens}")
- seen.add(tokens)
- _typed(point["point_id"], "point", f"{path}.point_id")
- _typed(point["evidence_id"], "evidence", f"{path}.evidence_id")
- components = _keys(
- point["components"], {"combine", "dispatch", "roundtrip", "stage"},
- f"{path}.components",
+ expected_evidence = identity.evidence_id(
+ point=expected_point, allocation=allocation_id, attempt=attempt_id,
+ sample_sha256=row["sample_sha256"],
)
- for name, component_value in components.items():
- component = _keys(
- component_value, {"availability", "sample_count", "trials"},
- f"{path}.components.{name}",
- )
- availability = component["availability"]
- count = _integer(component["sample_count"], f"{path}.components.{name}.sample_count")
- trials = component["trials"]
- if availability == "unavailable":
- if count != 0 or trials is not None or name == "roundtrip":
- raise ContractError(f"{path}.components.{name} has invalid unavailability")
- continue
- if availability != "measured" or not isinstance(trials, list) or len(trials) != 64:
- raise ContractError(f"{path}.components.{name} must contain 64 measured trials")
- if any(not isinstance(trial, list) or len(trial) != 8 for trial in trials):
- raise ContractError(f"{path}.components.{name} trials must each contain 8 samples")
- flattened = [
- _number(sample, f"{path}.components.{name}.trials", minimum=0.0)
- for trial in trials for sample in trial
- ]
- if count != 512 or len(flattened) != 512:
- raise ContractError(f"{path}.components.{name} must contain 512 samples")
- sample_base = {"components": components, "tokens_per_rank": tokens}
- if point["sample_sha256"] != _sha256_json(sample_base):
- raise ContractError(f"{path}.sample_sha256 differs")
- return doc
-
-
-def _validate_component(
- component_value: Any,
- sample_component: dict[str, Any] | None,
- path: str,
- *,
- derived: bool = False,
-) -> None:
- component = _keys(
- component_value, {"availability", "origin", "percentiles_us", "sample_count"}, path
- )
- availability = component["availability"]
- if availability == "unavailable":
- if component != {
- "availability": "unavailable", "origin": None,
- "percentiles_us": None, "sample_count": 0,
- }:
- raise ContractError(f"{path} has invalid unavailable representation")
- if sample_component and sample_component["availability"] != "unavailable":
- raise ContractError(f"{path} disagrees with samples")
- return
- expected_availability = "derived" if derived else "measured"
- expected_origin = "derived-percentile-sum" if derived else "measured"
- if availability != expected_availability or component["origin"] != expected_origin:
- raise ContractError(f"{path} has invalid availability/origin")
- percentiles = _keys(component["percentiles_us"], set(PERCENTILES), f"{path}.percentiles_us")
- if derived:
- if component["sample_count"] != 0:
- raise ContractError(f"{path}.sample_count must be zero for a derived value")
- return
- if sample_component is None or sample_component["availability"] != "measured":
- raise ContractError(f"{path} lacks measured sample evidence")
- flattened = [sample for trial in sample_component["trials"] for sample in trial]
- if component["sample_count"] != len(flattened):
- raise ContractError(f"{path}.sample_count differs from exact samples")
- for name, percentile in zip(PERCENTILES, (50, 90, 95, 99), strict=True):
- _close(percentiles[name], _nearest_rank(flattened, percentile), f"{path}.{name}")
-
-
-def _validate_oracle(
- value: Any,
- path: str,
- profile: dict[str, Any] | None = None,
- communication_precision: dict[str, Any] | None = None,
-) -> dict[str, Any]:
- profile = profile or identity.V1_NORMAL_CASE_PROFILE
- communication_precision = communication_precision or identity.precision_profile(
- identity.V1_CONTROL_PRECISION_PROFILE
- )
- tolerance = identity.combine_oracle_tolerances(communication_precision)
- oracle = _keys(
- value,
- {"atol", "checks", "combine_weight_semantics", "contract", "dispatch_sha256",
- "max_absolute_error", "max_elementwise_relative_error", "max_relative_error",
- "max_weight_error", "order_sha256", "ordering_contract", "passed", "receive_count",
- "rtol"},
- path,
- )
- if oracle["contract"] != profile["oracle_contract"]:
- raise ContractError(f"{path}.contract differs")
- checks = _keys(
- oracle["checks"],
- {"combine_values", "counts", "metadata", "multiplicity", "payload", "source_set",
- "weights"},
- f"{path}.checks",
- )
- if any(type(value) is not bool for value in checks.values()):
- raise ContractError(f"{path}.checks must be boolean")
- if type(oracle["passed"]) is not bool:
- raise ContractError(f"{path}.passed must be boolean")
- _integer(oracle["receive_count"], f"{path}.receive_count")
- _text(oracle["ordering_contract"], f"{path}.ordering_contract")
- expected_weight_semantics = (
- "gate-weighted-sum"
- if profile["combine_semantics"] == "gate-weighted"
- else "unweighted-rank-sum"
- )
- if oracle["combine_weight_semantics"] != expected_weight_semantics:
- raise ContractError(f"{path}.combine_weight_semantics differs from v1")
- _close(oracle["rtol"], tolerance["rtol"], f"{path}.rtol")
- _close(oracle["atol"], tolerance["atol"], f"{path}.atol")
- for field in ("dispatch_sha256", "order_sha256"):
- digest = oracle[field]
- if digest is not None and (
- not isinstance(digest, str) or len(digest) != 64
- or any(character not in "0123456789abcdef" for character in digest)
+ if row["point_id"] != expected_point or row["evidence_id"] != expected_evidence:
+ raise ContractError("raw point/evidence identity does not match its factors")
+ point = sample_points[row["point_id"]]
+ if (point["sample_sha256"], point["evidence_id"], point["tokens_per_rank"]) != (
+ row["sample_sha256"], row["evidence_id"], row["tokens_per_rank"]
):
- raise ContractError(f"{path}.{field} is not a SHA-256 digest")
- for field in (
- "max_absolute_error", "max_elementwise_relative_error", "max_relative_error",
- "max_weight_error",
- ):
- if oracle[field] is not None:
- _number(oracle[field], f"{path}.{field}", minimum=0.0)
- expected_pass = (
- all(checks.values())
- and oracle["max_relative_error"] is not None
- and oracle["max_relative_error"] < tolerance["rtol"]
- )
- if oracle["passed"] != expected_pass:
- raise ContractError(f"{path}.passed differs from its evidence")
- return oracle
+ raise ContractError("detached sample point differs from its raw row")
+ if success and row["correctness"]["passed"] is not True:
+ raise ContractError("successful raw case has a failed correctness check")
+ return document
-def _validate_precision_evidence(
- value: Any, profile_id: str, communication_precision: dict[str, Any], path: str
-) -> dict[str, Any]:
- precision = _keys(value, {"combine", "dispatch", "passed", "profile_id"}, path)
- if precision["profile_id"] != profile_id or type(precision["passed"]) is not bool:
- raise ContractError(f"{path} profile/outcome differs")
- for direction in ("dispatch", "combine"):
- axis_path = f"{path}.{direction}"
- axis = _keys(
- precision[direction],
- {"dequantized_semantics", "encoded_payload_valid", "max_abs_error",
- "max_rel_error", "passed", "saturation_count", "saturation_rate",
- "scales_finite", "scales_positive"},
- axis_path,
- )
- for field in ("dequantized_semantics", "encoded_payload_valid", "passed"):
- if type(axis[field]) is not bool:
- raise ContractError(f"{axis_path}.{field} must be boolean")
- scale_dtype = communication_precision[direction]["scale_dtype"]
- expects_scales = scale_dtype is not None
- for field in ("scales_finite", "scales_positive"):
- if expects_scales:
- if type(axis[field]) is not bool:
- raise ContractError(f"{axis_path}.{field} must be boolean")
- elif axis[field] is not None:
- raise ContractError(f"{axis_path}.{field} must be null without scales")
- saturation_count = _integer(
- axis["saturation_count"], f"{axis_path}.saturation_count"
- )
- saturation_rate = _number(
- axis["saturation_rate"], f"{axis_path}.saturation_rate", minimum=0.0
- )
- if saturation_rate > 1.0:
- raise ContractError(f"{axis_path}.saturation_rate must be <= 1")
- _number(axis["max_abs_error"], f"{axis_path}.max_abs_error", minimum=0.0)
- _number(axis["max_rel_error"], f"{axis_path}.max_rel_error", minimum=0.0)
- expected_pass = (
- axis["encoded_payload_valid"]
- and axis["dequantized_semantics"]
- and (not expects_scales or (axis["scales_finite"] and axis["scales_positive"]))
- and saturation_count == 0
- )
- if axis["passed"] != bool(expected_pass):
- raise ContractError(f"{axis_path}.passed differs from its evidence")
- expected_pass = precision["dispatch"]["passed"] and precision["combine"]["passed"]
- if precision["passed"] != expected_pass:
- raise ContractError(f"{path}.passed differs from direction evidence")
- return precision
-
-
-def validate_raw_document(document: Any, samples_document: Any) -> dict[str, Any]:
- """Validate identities, exact samples, formulas, privacy, and the native raw shape."""
- _validate_native_schema("raw-case-v1.schema.json", document)
- doc = _keys(
- document,
- {"case", "format", "generated_at", "identity", "implementation", "measurement",
- "outcome", "provenance", "record_type", "runtime_fingerprint", "sample_artifact",
- "schema_version", "topology", "workload"},
- "raw",
- )
- _finite_tree(doc)
- if doc["format"] != RAW_FORMAT or doc["schema_version"] != 1 or doc["record_type"] != "case-attempt":
- raise ContractError("raw format/schema/record type differs from v1")
- _text(doc["generated_at"], "raw.generated_at")
- identifiers = _keys(
- doc["identity"],
- {"allocation_factors", "allocation_id", "attempt_id", "attempt_ordinal", "case_factors",
- "case_id", "series_factors", "series_id"},
- "raw.identity",
- )
- for field, kind in (
- ("allocation_id", "allocation"), ("attempt_id", "attempt"),
- ("case_id", "case"), ("series_id", "series"),
- ):
- _typed(identifiers[field], kind, f"raw.identity.{field}")
- ordinal = _integer(identifiers["attempt_ordinal"], "raw.identity.attempt_ordinal", minimum=1)
- allocation_factors = _keys(
- identifiers["allocation_factors"], ALLOCATION_FACTOR_FIELDS,
- "raw.identity.allocation_factors",
- )
- qualification_index = _integer(
- allocation_factors["qualification_index"],
- "raw.identity.allocation_factors.qualification_index",
- minimum=1,
- )
- if qualification_index > 3:
- raise ContractError("raw qualification index must be in 1..3")
- case_factors = _keys(
- identifiers["case_factors"], {"case", "profile", "sku"},
- "raw.identity.case_factors",
- )
- scheduled_case = _scheduled_case(
- case_factors["case"], "raw.identity.case_factors.case"
- )
- profile = scheduled_case_profile(scheduled_case, "raw.identity.case_factors.case")
- if case_factors["profile"] != profile:
- raise ContractError("raw case profile differs from CollectiveX v1")
- _text(case_factors["sku"], "raw.identity.case_factors.sku")
- series_factors = _keys(
- identifiers["series_factors"],
- {"backend", "case_id", "image_digest", "implementation_contract_sha256",
- "public_config_sha256", "routing_control_sha256",
- "runtime_fingerprint_sha256", "source_sha", "squash_sha256", "workload_id"},
- "raw.identity.series_factors",
- )
- if identity.allocation_id(identifiers["allocation_factors"]) != identifiers["allocation_id"]:
- raise ContractError("allocation identity differs")
- if identity.digest("case", identifiers["case_factors"]) != identifiers["case_id"]:
- raise ContractError("case identity differs")
- if identity.series_id(identifiers["series_factors"]) != identifiers["series_id"]:
- raise ContractError("series identity differs")
- if identity.attempt_id(
- allocation=identifiers["allocation_id"], case=identifiers["case_id"], ordinal=ordinal
- ) != identifiers["attempt_id"]:
- raise ContractError("attempt identity differs")
-
- samples = validate_samples_document(samples_document)
- for field in ("allocation_id", "attempt_id", "case_id", "series_id"):
- if samples[field] != identifiers[field]:
- raise ContractError(f"samples.{field} differs from raw identity")
- if samples["qualification_index"] != qualification_index:
- raise ContractError("samples qualification index differs from raw allocation")
- sample_by_token = {point["tokens_per_rank"]: point for point in samples["points"]}
-
- case = _keys(
- doc["case"],
- {"attempt_ordinal", "backend", "eplb", "ep_size", "mode", "phase",
- "required_publication", "resource_mode", "runner", "shape", "suite", "workload_name"},
- "raw.case",
- )
- ep_size = _integer(case["ep_size"], "raw.case.ep_size", minimum=1)
- if case["attempt_ordinal"] != ordinal:
- raise ContractError("case attempt ordinal differs")
- for field in ("backend", "mode", "phase", "required_publication", "resource_mode", "runner",
- "suite", "workload_name"):
- _text(case[field], f"raw.case.{field}")
- shape = _keys(
- case["shape"],
- {"activation_profile", "combine_precision", "dispatch_precision", "eplb", "experts",
- "experts_per_rank", "hidden", "kernel_gen", "num_logical_experts",
- "precision_profile", "routing", "topk"},
- "raw.case.shape",
- )
- hidden = _integer(shape["hidden"], "raw.case.shape.hidden", minimum=1)
- topk = _integer(shape["topk"], "raw.case.shape.topk", minimum=1)
- physical_experts = _integer(
- shape["experts"], "raw.case.shape.experts", minimum=1
- )
- logical_experts = _integer(
- shape["num_logical_experts"],
- "raw.case.shape.num_logical_experts",
- minimum=1,
- )
- experts_per_rank = _integer(
- shape["experts_per_rank"], "raw.case.shape.experts_per_rank", minimum=1
- )
- precision_profile_id = scheduled_case.get(
- "precision_profile", identity.V1_CONTROL_PRECISION_PROFILE
- )
- communication_precision = identity.precision_profile(precision_profile_id)
- if (
- shape["precision_profile"] != precision_profile_id
- or shape["dispatch_precision"] != communication_precision["dispatch"]
- or shape["combine_precision"] != communication_precision["combine"]
- ):
- raise ContractError("raw communication precision differs from scheduled case")
- eplb = _keys(
- case["eplb"],
- {"calibration_token_offset", "calibration_trace_sha256", "calibration_window",
- "calibration_workload_id", "enabled", "imbalance_after", "imbalance_before",
- "mapping_hash", "max_replicas", "num_logical_experts", "num_physical_experts",
- "num_redundant", "planner", "reference_tokens_per_rank", "replicated_experts"},
- "raw.case.eplb",
- )
- if not isinstance(eplb["enabled"], bool):
- raise ContractError("raw.case.eplb.enabled must be boolean")
- expected_redundant = (
- profile["eplb_redundant_experts"] if eplb["enabled"] else 0
- )
- expected_physical = eplb_contract.physical_count(
- scheduled_case["experts"], expected_redundant, ep_size
- )
- if (
- shape["eplb"] != eplb["enabled"]
- or logical_experts != scheduled_case["experts"]
- or physical_experts != expected_physical
- or experts_per_rank * ep_size != physical_experts
- or eplb["num_logical_experts"] != logical_experts
- or eplb["num_physical_experts"] != physical_experts
- or eplb["num_redundant"] != expected_redundant
- ):
- raise ContractError("raw EPLB/shape dimensions differ from the frozen profile")
- if eplb["enabled"]:
- expected_plan, calibration_descriptor = _expected_eplb_calibration(
- scheduled_case["routing"],
- hidden,
- topk,
- logical_experts,
- physical_experts,
- ep_size,
- profile["seed"],
- profile["eplb_reference_tokens_per_rank"],
- )
- expected_eplb = {
- **calibration_descriptor,
- "enabled": True,
- "imbalance_after": expected_plan["imbalance_after"],
- "imbalance_before": expected_plan["imbalance_before"],
- "mapping_hash": eplb_contract.mapping_hash(expected_plan),
- "max_replicas": expected_plan["max_replicas"],
- "num_logical_experts": logical_experts,
- "num_physical_experts": physical_experts,
- "num_redundant": expected_redundant,
- "planner": profile["eplb_planner"],
- "reference_tokens_per_rank": profile[
- "eplb_reference_tokens_per_rank"
- ],
- "replicated_experts": expected_plan["replicated_experts"],
- }
- else:
- expected_eplb = {
- "calibration_token_offset": None,
- "calibration_trace_sha256": None,
- "calibration_window": None,
- "calibration_workload_id": None,
- "enabled": False,
- "imbalance_after": None,
- "imbalance_before": None,
- "mapping_hash": None,
- "max_replicas": None,
- "num_logical_experts": logical_experts,
- "num_physical_experts": physical_experts,
- "num_redundant": 0,
- "planner": None,
- "reference_tokens_per_rank": None,
- "replicated_experts": 0,
- }
- _equivalent(eplb, expected_eplb, "raw.case.eplb", tolerance=1e-9)
- if case_factors["sku"] != case["runner"]:
- raise ContractError("raw case runner differs from case identity")
-
- workload = _keys(
- doc["workload"],
- {"activation_generator", "activation_identity", "activation_profile",
- "cross_rank_consistent", "manifest_checksums", "members", "routing_generator", "source",
- "trace_hashes", "trace_signature", "workload_id"},
- "raw.workload",
- )
- if workload["source"] not in {"canonical-serialized", "seeded-runtime"}:
- raise ContractError("raw workload source is invalid")
- if workload["source"] == "canonical-serialized":
- _typed(workload["workload_id"], "workload", "raw.workload.workload_id")
- members = workload["members"]
- checksums = workload["manifest_checksums"]
- if (
- not isinstance(members, list)
- or not members
- or members != sorted(set(members))
- or not all(identity.is_typed_id(member, "workload") for member in members)
- or not isinstance(checksums, dict)
- or set(checksums) != set(members)
- ):
- raise ContractError("raw canonical workload members/checksums are invalid")
- for member, values in checksums.items():
- if (
- not isinstance(values, dict)
- or set(values) != {"topk_idx", "topk_weights", "trace"}
- or any(not re.fullmatch(r"[0-9a-f]{64}", str(value)) for value in values.values())
- ):
- raise ContractError(f"raw canonical workload checksums differ for {member}")
- expected_workload_id = identity.workload_id({
- "members": [
- {"checksums": checksums[member], "workload_id": member}
- for member in members
- ]
- })
- if workload["workload_id"] != expected_workload_id:
- raise ContractError("raw composite workload identity differs from its members")
- elif any(workload[field] is not None for field in ("members", "manifest_checksums", "workload_id")):
- raise ContractError("raw seeded workload cannot claim serialized members")
- if workload["cross_rank_consistent"] is not True:
- raise ContractError("raw workload is not consistent across ranks")
-
- measurement = _keys(
- doc["measurement"],
- {"component_order_contract", "conditioning", "contract", "execution_order_sha256",
- "qualification_index", "rows", "sampling", "source_allocation"},
- "raw.measurement",
- )
- if measurement["qualification_index"] != qualification_index:
- raise ContractError("raw measurement qualification index differs from allocation")
- if not isinstance(measurement["execution_order_sha256"], str) or not re.fullmatch(
- r"[0-9a-f]{64}", measurement["execution_order_sha256"]
- ):
- raise ContractError("raw measurement execution order digest is invalid")
- validate_conditioning_contract(measurement["conditioning"], case["phase"])
- sampling = _keys(
- measurement["sampling"],
- {"contract", "iterations_per_trial", "percentile_method", "reduction",
- "samples_per_component", "trials", "warmup_iterations", "warmup_semantics"},
- "raw.measurement.sampling",
- )
- expected_sampling = {
- "contract": profile["sampling_contract"], "iterations_per_trial": 8,
- "percentile_method": profile["percentile_method"],
- "reduction": profile["rank_reduction"],
- "samples_per_component": 512, "trials": 64, "warmup_iterations": 32,
- "warmup_semantics": "full-roundtrip-before-each-component-trial-point-v1",
- }
- if sampling != expected_sampling:
- raise ContractError("raw sampling contract differs from fixed-512-v1")
+def load_raw_attempt(path: str | os.PathLike[str]) -> dict[str, Any]:
+ """Load a raw attempt, verify its detached sample bytes, and validate both."""
+ document = strict_json_load(path)
+ artifact = _obj(_obj(document, "raw").get("sample_artifact"), "raw.sample_artifact")
+ sample_path = Path(path).with_name(_text(artifact.get("path"), "raw.sample_artifact.path"))
+ payload = sample_path.read_bytes()
if (
- case["mode"] != profile["mode"]
- or case["resource_mode"] != profile["resource_mode"]
- or measurement["contract"] != profile["contract"]
- or measurement["component_order_contract"] != profile["component_order_contract"]
- or measurement["source_allocation"] != "even"
- or shape["activation_profile"] != profile["activation_profile"]
- or workload["activation_generator"] != profile["activation_generator"]
- or workload["activation_profile"] != profile["activation_profile"]
- or workload["routing_generator"] != profile["routing_generator"]
+ len(payload) != artifact.get("bytes")
+ or hashlib.sha256(payload).hexdigest() != artifact.get("sha256")
):
- raise ContractError("raw case differs from the frozen v1 profile")
- expected_activation = hashlib.sha256(
- (
- f"counter|seed={profile['seed']}|hidden={hidden}|"
- f"gen={profile['activation_generator']}"
- ).encode()
- ).hexdigest()
- if workload["activation_identity"] != expected_activation:
- raise ContractError("raw activation identity differs from the frozen seed/profile")
- rows = measurement["rows"]
- if not isinstance(rows, list) or not rows:
- raise ContractError("raw.measurement.rows must be non-empty")
- seen_points = set()
- row_tokens = []
- recomputed_anomalies = 0
- for index, row_value in enumerate(rows):
- path = f"raw.measurement.rows[{index}]"
- row = _keys(
- row_value,
- {"anomalies", "byte_provenance", "components", "correctness", "evidence_id",
- "global_tokens", "point_id", "receive", "routing",
- "sample_histograms", "sample_sha256", "token_rate_at_latency_percentile",
- "tokens_per_rank"},
- path,
- )
- tokens = _integer(row["tokens_per_rank"], f"{path}.tokens_per_rank", minimum=1)
- row_tokens.append(tokens)
- if tokens in seen_points or tokens not in sample_by_token:
- raise ContractError(f"{path} token point is duplicate or missing samples")
- seen_points.add(tokens)
- if row["global_tokens"] != tokens * ep_size:
- raise ContractError(f"{path}.global_tokens formula differs")
- sample_point = sample_by_token[tokens]
- expected_point = identity.point_id(series=identifiers["series_id"], tokens_per_rank=tokens)
- if row["point_id"] != expected_point or sample_point["point_id"] != expected_point:
- raise ContractError(f"{path}.point_id differs")
- expected_evidence = identity.evidence_id(
- point=expected_point, allocation=identifiers["allocation_id"],
- attempt=identifiers["attempt_id"], sample_sha256=sample_point["sample_sha256"],
- )
- if row["evidence_id"] != expected_evidence or sample_point["evidence_id"] != expected_evidence:
- raise ContractError(f"{path}.evidence_id differs")
- if row["sample_sha256"] != sample_point["sample_sha256"]:
- raise ContractError(f"{path}.sample_sha256 differs")
- components = _keys(
- row["components"], {"combine", "dispatch", "isolated_sum", "roundtrip", "stage"},
- f"{path}.components",
- )
- for name in ("combine", "dispatch", "roundtrip", "stage"):
- _validate_component(
- components[name], sample_point["components"][name], f"{path}.components.{name}"
- )
- _validate_component(
- components["isolated_sum"], None, f"{path}.components.isolated_sum", derived=True
- )
- expected_stage_availability = (
- "measured"
- if communication_precision["dispatch"]["communication_format"] != "bf16"
- or (case["backend"] == "mori" and shape["kernel_gen"] == "intranode")
- else "unavailable"
- )
- if components["stage"]["availability"] != expected_stage_availability:
- raise ContractError(f"{path}.components.stage differs from adapter device work")
- _, _, _, expected_indices, expected_weights = _expected_canonical_trace(
- scheduled_case["routing"],
- hidden,
- topk,
- logical_experts,
- physical_experts,
- ep_size,
- tokens,
- profile["seed"],
- eplb["enabled"],
- profile["eplb_reference_tokens_per_rank"],
- )
- expected_routing = _expected_routing_summary(
- expected_indices,
- expected_weights,
- physical_experts=physical_experts,
- ep_size=ep_size,
- tokens_per_rank=tokens,
- gpus_per_node=scheduled_case["gpus_per_node"],
- scale_up_domain=scheduled_case["scale_up_domain"],
- )
- _equivalent(
- row["routing"], expected_routing, f"{path}.routing", tolerance=1e-5
- )
- expected_payload_counts = (
- expected_routing["expert_assignments_per_rank"]
- if profile["payload_unit"] == "token-expert"
- else expected_routing["payload_copies_per_rank"]
- )
- throughput = _keys(
- row["token_rate_at_latency_percentile"], set(PERCENTILES),
- f"{path}.token_rate_at_latency_percentile",
- )
- for percentile in PERCENTILES:
- latency = components["roundtrip"]["percentiles_us"][percentile]
- if latency <= 0:
- raise ContractError(f"{path} roundtrip latency must be positive")
- _close(
- throughput[percentile], row["global_tokens"] / (latency * 1e-6),
- f"{path}.token_rate_at_latency_percentile.{percentile}", 1e-9,
- )
- correctness = _keys(
- row["correctness"],
- {"contract", "max_relative_error", "passed", "precision", "rank_evidence", "scope"},
- f"{path}.correctness",
- )
- if (
- correctness["contract"] != profile["oracle_contract"]
- or correctness["scope"] != profile["correctness_scope"]
- or type(correctness["passed"]) is not bool
- ):
- raise ContractError(f"{path}.correctness contract differs")
- precision_evidence = _validate_precision_evidence(
- correctness["precision"], precision_profile_id, communication_precision,
- f"{path}.correctness.precision",
- )
- _number(
- correctness["max_relative_error"],
- f"{path}.correctness.max_relative_error",
- minimum=0.0,
- )
- rank_evidence = correctness["rank_evidence"]
- if not isinstance(rank_evidence, list) or len(rank_evidence) != ep_size:
- raise ContractError(f"{path}.correctness.rank_evidence must cover every rank")
- ranks = set()
- observed_max_error = 0.0
- evidence_passed = True
- for evidence_index, evidence_value in enumerate(rank_evidence):
- evidence_path = f"{path}.correctness.rank_evidence[{evidence_index}]"
- evidence = _keys(
- evidence_value,
- {"input_unchanged", "order_stable", "post_timing", "pre_timing", "rank"},
- evidence_path,
- )
- evidence_rank = _integer(evidence["rank"], f"{evidence_path}.rank")
- if evidence_rank >= ep_size:
- raise ContractError(f"{evidence_path}.rank is outside the EP group")
- ranks.add(evidence_rank)
- if type(evidence["input_unchanged"]) is not bool or type(evidence["order_stable"]) is not bool:
- raise ContractError(f"{evidence_path} stability fields must be boolean")
- pre = _validate_oracle(
- evidence["pre_timing"], f"{evidence_path}.pre_timing", profile,
- communication_precision,
- )
- post = _validate_oracle(
- evidence["post_timing"], f"{evidence_path}.post_timing", profile,
- communication_precision,
- )
- if (
- pre["receive_count"] != expected_payload_counts[evidence_rank]
- or post["receive_count"] != expected_payload_counts[evidence_rank]
- ):
- raise ContractError(
- f"{evidence_path}.receive_count differs from canonical routing"
- )
- expected_stability = all(
- pre[field] == post[field]
- for field in ("ordering_contract", "order_sha256", "dispatch_sha256")
- )
- if evidence["order_stable"] != expected_stability:
- raise ContractError(f"{evidence_path}.order_stable differs from the evidence")
- errors = [
- oracle["max_relative_error"]
- for oracle in (pre, post)
- if oracle["max_relative_error"] is not None
- ]
- observed_max_error = max([observed_max_error, *errors])
- evidence_passed = evidence_passed and all(
- (evidence["input_unchanged"], evidence["order_stable"], pre["passed"], post["passed"])
- )
- evidence_passed = evidence_passed and precision_evidence["passed"]
- if ranks != set(range(ep_size)) or correctness["passed"] != evidence_passed:
- raise ContractError(f"{path}.correctness rank coverage or outcome differs")
- _close(
- correctness["max_relative_error"], observed_max_error,
- f"{path}.correctness.max_relative_error",
- )
- if components["dispatch"]["availability"] == "measured":
- for percentile in PERCENTILES:
- expected = (
- components["dispatch"]["percentiles_us"][percentile]
- + (
- components["stage"]["percentiles_us"][percentile]
- if components["stage"]["availability"] == "measured"
- else 0.0
- )
- + components["combine"]["percentiles_us"][percentile]
- )
- _close(
- components["isolated_sum"]["percentiles_us"][percentile], expected,
- f"{path}.components.isolated_sum.{percentile}",
- )
- logical_copies = (
- sum(expected_routing["expert_assignments_per_rank"])
- if profile["payload_unit"] == "token-expert"
- else expected_routing["routed_copies"]
- )
- dispatch_bytes = _precision_byte_provenance(
- communication_precision["dispatch"], logical_copies, hidden
- )
- combine_bytes = _precision_byte_provenance(
- communication_precision["combine"], logical_copies, hidden
- )
- stage_bytes = {
- "accounting_contract": "activation-data-plus-scales-v1",
- "activation_data_bytes": 0,
- "scale_bytes": 0,
- "total_logical_bytes": 0,
- }
- roundtrip_bytes = {
- "accounting_contract": "activation-data-plus-scales-v1",
- **{
- field: dispatch_bytes[field] + combine_bytes[field]
- for field in (
- "activation_data_bytes", "scale_bytes", "total_logical_bytes"
- )
- },
- }
- expected_byte_provenance = {
- "combine": combine_bytes,
- "dispatch": dispatch_bytes,
- "roundtrip": roundtrip_bytes,
- "stage": stage_bytes,
- }
- _equivalent(
- row["byte_provenance"], expected_byte_provenance, f"{path}.byte_provenance"
- )
+ raise ContractError("sample artifact bytes or digest differ")
+ samples = strict_json_load(sample_path)
+ return validate_raw_document(document, samples)
- max_receive = max(expected_payload_counts)
- expected_receive = {
- "max": max_receive,
- "mean": sum(expected_payload_counts) / ep_size,
- "min": min(expected_payload_counts),
- "total": sum(expected_payload_counts),
- }
- _equivalent(row["receive"], expected_receive, f"{path}.receive")
- expected_histograms = {
- name: (
- _expected_histogram([
- sample
- for trial in sample_point["components"][name]["trials"]
- for sample in trial
- ])
- if sample_point["components"][name]["availability"] == "measured"
- else None
- )
- for name in ("dispatch", "stage", "combine", "roundtrip")
- }
- _equivalent(
- row["sample_histograms"], expected_histograms, f"{path}.sample_histograms"
- )
- expected_anomalies = _expected_anomalies(tokens, components)
- _equivalent(row["anomalies"], expected_anomalies, f"{path}.anomalies")
- recomputed_anomalies += len(expected_anomalies)
- if seen_points != set(sample_by_token):
- raise ContractError("raw rows and sample points differ")
- if row_tokens != sorted(row_tokens):
- raise ContractError("raw rows must follow the scheduled token ladder")
- expected_trace_hashes = sorted(row["routing"]["hash"] for row in rows)
- if workload["trace_hashes"] != expected_trace_hashes:
- raise ContractError("raw workload trace hashes differ from measured rows")
- expected_trace_signature = hashlib.sha256(
- "|".join(expected_trace_hashes).encode()
- ).hexdigest()
- if workload["trace_signature"] != expected_trace_signature:
- raise ContractError("raw workload trace signature differs from measured rows")
- implementation = _keys(
- doc["implementation"], {"kernel_generation", "name", "provenance", "resource_profile"},
- "raw.implementation",
- )
- if (
- implementation["name"] != case["backend"]
- or implementation["kernel_generation"] != shape["kernel_gen"]
- ):
- raise ContractError("raw implementation identity differs from the case")
- provenance_fields = _obj(implementation["provenance"], "raw.implementation.provenance")
- unknown = set(provenance_fields) - PROVENANCE_KEYS
- if unknown:
- raise ContractError(f"raw implementation provenance has unknown fields {sorted(unknown)}")
- if (
- implementation["name"] == "deepep-v2"
- and provenance_fields.get("deterministic") is not False
- ):
- raise ContractError("DeepEP V2 deterministic mode differs from the v1 kernel contract")
- if implementation["name"] == "deepep-v2" and (
- _integer(
- provenance_fields.get("tuning_num_experts"),
- "raw.implementation.provenance.tuning_num_experts",
- minimum=1,
- ) != logical_experts
- or _integer(
- provenance_fields.get("num_experts"),
- "raw.implementation.provenance.num_experts",
- minimum=1,
- ) != physical_experts
- ):
- raise ContractError("DeepEP V2 expert-count provenance differs from the case")
- if implementation["name"] == "deepep-hybrid":
- realized_config = provenance_fields.get("realized_config")
- jit_kernel_keys = provenance_fields.get("jit_kernel_keys")
- jit_shared_objects = provenance_fields.get("jit_shared_objects")
- domain_ranks, communication_domains = hybrid_communication_domains(
- ep_size, scheduled_case["scale_up_domain"]
- )
- if (
- not _hybrid_realized_config_is_valid(realized_config)
- or not _hybrid_jit_evidence_is_valid(jit_shared_objects, jit_kernel_keys)
- or realized_config["hidden_dim"] != shape["hidden"]
- or realized_config["num_of_experts_per_rank"] * ep_size != physical_experts
- or realized_config["num_of_ranks_per_node"] != domain_ranks
- or realized_config["num_of_nodes"] != communication_domains
- or realized_config["token_data_type"] != "UINT16"
- or any(
- len(artifact["rank_artifacts"]) != ep_size
- for artifact in jit_shared_objects
- )
- ):
- raise ContractError("DeepEP Hybrid realized config/JIT evidence differs from the case")
- if implementation["name"] == "nccl-ep" and implementation["kernel_generation"] != (
- collective_kernel_generation(provenance_fields.get("collective_library"))
- ):
- raise ContractError("NCCL/RCCL kernel generation differs from collective lineage")
- resource_profile = _obj(
- implementation["resource_profile"], "raw.implementation.resource_profile"
- )
- expected_resource_profile = project_resource_profile(provenance_fields)
- if resource_profile != expected_resource_profile:
- raise ContractError("raw resource profile differs from implementation provenance")
- topology = _keys(
- doc["topology"],
- {"device_count", "device_product", "gpus_per_node", "nodes", "placement",
- "realized_placement", "scale_out_transport", "scale_up_domain",
- "scale_up_transport", "scope", "topology_class", "transport", "world_size"},
- "raw.topology",
- )
- for field in ("device_count", "gpus_per_node", "nodes", "scale_up_domain", "world_size"):
- _integer(topology[field], f"raw.topology.{field}", minimum=1)
- for field in ("scale_up_transport", "scope", "topology_class", "transport"):
- _text(topology[field], f"raw.topology.{field}")
- if topology["scale_out_transport"] is not None:
- _text(topology["scale_out_transport"], "raw.topology.scale_out_transport")
- realized = _keys(
- topology["realized_placement"],
- {"gpus_per_node", "nodes", "ranks_per_node", "unique_local_ranks", "valid"},
- "raw.topology.realized_placement",
- )
- if realized != {
- "gpus_per_node": topology["gpus_per_node"],
- "nodes": topology["nodes"],
- "ranks_per_node": topology["gpus_per_node"],
- "unique_local_ranks": True,
- "valid": True,
- }:
- raise ContractError("raw realized placement differs from requested topology")
- if (
- topology["world_size"] != ep_size
- or topology["nodes"] * topology["gpus_per_node"] != ep_size
- or topology["device_count"] != topology["gpus_per_node"]
- or topology["placement"] != profile["placement"]
- or (
- topology["scope"] == "scale-up"
- and (
- ep_size > topology["scale_up_domain"]
- or topology["scale_out_transport"] is not None
- )
- )
- or (
- topology["scope"] == "scale-out"
- and (
- ep_size <= topology["scale_up_domain"]
- or ep_size % topology["scale_up_domain"] != 0
- or topology["scale_out_transport"] is None
- )
- )
- or topology["scope"] not in {"scale-up", "scale-out"}
- ):
- raise ContractError("raw topology dimensions differ from the case")
- if implementation["name"] == "deepep-v2":
- scale_out = scheduled_case["scope"] == "scale-out"
- expected_policy = (
- (True, True, "nccl-gin")
- if scale_out
- else (False, False, "nccl-device-lsa")
- )
- if (
- provenance_fields.get("allow_hybrid_mode"),
- provenance_fields.get("gin_enabled"),
- provenance_fields.get("communication_backend"),
- ) != expected_policy:
- raise ContractError("DeepEP V2 communication policy differs from the v1 contract")
- lsa_topology = tuple(
- _integer(
- provenance_fields.get(field),
- f"raw.implementation.provenance.{field}",
- minimum=1,
- )
- for field in (
- "physical_rdma_ranks", "physical_nvlink_ranks",
- "logical_scaleout_ranks", "logical_scaleup_ranks",
- )
- )
- domains = ep_size // scheduled_case["scale_up_domain"]
- expected_v2_topology = (
- (
- domains,
- scheduled_case["scale_up_domain"],
- domains,
- scheduled_case["scale_up_domain"],
- )
- if scale_out
- else (1, ep_size, 1, ep_size)
- )
- if lsa_topology != expected_v2_topology:
- raise ContractError("DeepEP V2 realized communication domains differ from topology")
- runtime = _keys(
- doc["runtime_fingerprint"],
- {"accelerator_runtime", "collective_library", "device", "driver_version", "framework",
- "machine", "python_version", "vendor"},
- "raw.runtime_fingerprint",
- )
- for field in ("machine", "python_version", "vendor"):
- _text(runtime[field], f"raw.runtime_fingerprint.{field}")
- runtime_device = _keys(
- runtime["device"], {"arch", "compute_units", "memory_bytes", "product", "warp_size"},
- "raw.runtime_fingerprint.device",
- )
- if topology["device_product"] != runtime_device["product"]:
- raise ContractError("raw topology and runtime device products differ")
- platform = capability.PLATFORMS.get(case["runner"])
- if platform is not None:
- identity_issues = capability.runtime_identity_issues(
- case["runner"], vendor=runtime["vendor"], arch=runtime_device["arch"],
- machine=runtime["machine"], device_name=runtime_device["product"],
- device_count=topology["device_count"], world_size=topology["world_size"],
- )
- registered_topology = capability.topology_for(case["runner"], ep_size)
- if identity_issues or (
- registered_topology is None
- or topology["gpus_per_node"] != platform["gpus_per_node"]
- or topology["scale_up_domain"] != platform["scale_up_domain"]
- or any(
- topology[field] != registered_topology[field]
- for field in (
- "nodes", "scope", "scale_up_transport", "scale_out_transport",
- "topology_class", "transport",
- )
- )
- ):
- raise ContractError(
- "raw runtime/topology differs from the scheduled SKU: "
- + "; ".join(identity_issues)
- )
- raw_provenance = _keys(
- doc["provenance"],
- {"allocation_stratum_sha256", "command", "distributed_launcher", "git_run",
- "image", "redaction"},
- "raw.provenance",
- )
- allocation_stratum = raw_provenance["allocation_stratum_sha256"]
- if workload["source"] == "canonical-serialized" and not (
- isinstance(allocation_stratum, str)
- and re.fullmatch(r"[0-9a-f]{64}", allocation_stratum)
- ):
- raise ContractError("canonical raw evidence is missing its private allocation stratum")
- image = _keys(
- raw_provenance["image"],
- {"arch", "digest", "digest_verified", "reference", "squash_sha256"},
- "raw.provenance.image",
- )
- if (
- image["digest_verified"] is not True
- or not isinstance(image["digest"], str)
- or not re.fullmatch(r"sha256:[0-9a-f]{64}", image["digest"])
- ):
- raise ContractError("raw image digest was not registry-verified")
- if raw_provenance["redaction"] != "sanitized-v1":
- raise ContractError("raw provenance redaction contract differs")
- git_run = raw_provenance["git_run"]
- if git_run is not None:
- git_run = _keys(git_run, GIT_RUN_FIELDS, "raw.provenance.git_run")
- if git_run["qualification_index"] != qualification_index:
- raise ContractError("raw git run qualification index differs from allocation")
- expected_provenance_complete = provenance_complete(
- provenance_fields,
- case["backend"],
- git_run,
- allocation_stratum_sha256=allocation_stratum,
- image_digest=image["digest"],
- image_verified=image["digest_verified"],
- squash_sha256=image["squash_sha256"],
- )
+def load_attempt(path: str | os.PathLike[str]) -> dict[str, Any]:
+ """Fully validate and return one raw or terminal attempt."""
+ document = strict_json_load(path)
+ if isinstance(document, dict) and document.get("format") == RAW_FORMAT:
+ return load_raw_attempt(path)
+ if isinstance(document, dict) and document.get("format") == TERMINAL_FORMAT:
+ return validate_terminal_document(document)
+ raise ContractError("unknown result attempt format")
- actual_scheduled_case = {
- "backend": case["backend"],
- "canonical": workload["source"] == "canonical-serialized",
- "eplb": eplb["enabled"],
- "ep": ep_size,
- "experts": shape["num_logical_experts"],
- "gpus_per_node": topology["gpus_per_node"],
- "hidden": hidden,
- "ladder": " ".join(map(str, row_tokens)),
- "mode": case["mode"],
- "nodes": topology["nodes"],
- "phase": case["phase"],
- "required_publication": case["required_publication"],
- "routing": shape["routing"],
- "samples_per_point": sampling["samples_per_component"],
- "scale_out_transport": topology["scale_out_transport"],
- "scale_up_domain": topology["scale_up_domain"],
- "scale_up_transport": topology["scale_up_transport"],
- "scope": topology["scope"],
- "suite": case["suite"],
- "timing": (
- f"{sampling['iterations_per_trial']}:{sampling['trials']}:"
- f"{sampling['warmup_iterations']}"
- ),
- "topk": shape["topk"],
- "topology_class": topology["topology_class"],
- "transport": topology["transport"],
- "warmup_semantics": sampling["warmup_semantics"],
- "workload": case["workload_name"],
- }
- if "precision_profile" in scheduled_case:
- actual_scheduled_case["precision_profile"] = shape["precision_profile"]
- if scheduled_case != actual_scheduled_case:
- mismatches = sorted(
- field for field in scheduled_case
- if scheduled_case[field] != actual_scheduled_case[field]
- )
- raise ContractError(f"raw data differs from scheduled case fields {mismatches}")
-
- if workload["source"] == "canonical-serialized":
- _validate_canonical_workload(workload, scheduled_case, rows, eplb)
-
- expected_series = {
- "backend": case["backend"],
- "case_id": identifiers["case_id"],
- "image_digest": image["digest"],
- "implementation_contract_sha256": _sha256_json({
- "kernel_generation": implementation["kernel_generation"],
- "name": implementation["name"],
- "provenance": series_provenance(provenance_fields),
- "resource_profile": resource_profile,
- }),
- "public_config_sha256": public_series_config_sha256(public_series_config(
- kernel_generation=implementation["kernel_generation"],
- provenance=provenance_fields,
- resource_profile=resource_profile,
- resource_mode=case["resource_mode"],
- device_product=topology["device_product"],
- )),
- "routing_control_sha256": routing_implementation_control_sha256(implementation),
- "runtime_fingerprint_sha256": _sha256_json(runtime),
- "source_sha": git_run["source_sha"] if git_run is not None else None,
- "squash_sha256": image["squash_sha256"],
- "workload_id": workload["workload_id"] or workload["trace_signature"],
- }
- if series_factors != expected_series:
- raise ContractError("raw series factors differ from measured implementation/runtime")
- expected_allocation = {
- "artifact": git_run["artifact"] if git_run is not None else None,
- "execution_id": allocation_factors["execution_id"],
- "job": git_run["job"] if git_run is not None else None,
- "qualification_index": qualification_index,
- "repo": git_run["repo"] if git_run is not None else None,
- "run_attempt": git_run["run_attempt"] if git_run is not None else None,
- "run_id": git_run["run_id"] if git_run is not None else None,
- "runner": case["runner"],
- "source_sha": git_run["source_sha"] if git_run is not None else None,
- }
- if allocation_factors != expected_allocation:
- raise ContractError("raw allocation factors differ from provenance")
- artifact = _keys(doc["sample_artifact"], {"bytes", "format", "path", "sha256"}, "raw.sample_artifact")
- if artifact["format"] != SAMPLES_FORMAT or Path(artifact["path"]).name != artifact["path"]:
- raise ContractError("raw.sample_artifact format/path is invalid")
- if not isinstance(artifact["sha256"], str) or len(artifact["sha256"]) != 64:
- raise ContractError("raw.sample_artifact.sha256 is invalid")
- _integer(artifact["bytes"], "raw.sample_artifact.bytes", minimum=1)
- outcome = _keys(doc["outcome"], {"publication_status", "reasons", "status", "validity"}, "raw.outcome")
- if outcome["status"] not in {"success", "invalid"} or outcome["publication_status"] not in {"diagnostic", "invalid"}:
- raise ContractError("raw outcome status is invalid")
- if not isinstance(outcome["reasons"], list) or not all(isinstance(x, str) for x in outcome["reasons"]):
- raise ContractError("raw outcome reasons must be strings")
- validity = _keys(
- outcome["validity"],
- {"anomaly_free", "execution_status", "measurement_conformance", "provenance_complete",
- "resource_conformance", "sampling_conformance", "semantic_correctness",
- "workload_identity", "workload_source"},
- "raw.outcome.validity",
- )
- correctness_passed = all(row["correctness"]["passed"] for row in rows)
- workload_consistent = workload["cross_rank_consistent"] is True
- expected_status = "success" if correctness_passed and workload_consistent else "invalid"
- expected_publication = "diagnostic" if expected_status == "success" else "invalid"
- if (
- outcome["status"] != expected_status
- or outcome["publication_status"] != expected_publication
- or bool(outcome["reasons"]) == (expected_status == "success")
- or validity["execution_status"] != "complete"
- or validity["semantic_correctness"] != ("pass" if correctness_passed else "fail")
- or validity["workload_identity"] != (
- "consistent-across-ranks" if workload_consistent else "inconsistent"
- )
- or validity["workload_source"] != workload["source"]
- or validity["measurement_conformance"] != "conformant"
- or validity["sampling_conformance"] != "conformant"
- or validity["resource_conformance"] != resource_profile["conformance_class"]
- or validity["anomaly_free"] != (recomputed_anomalies == 0)
- or validity["provenance_complete"] is not expected_provenance_complete
- ):
- raise ContractError("raw outcome differs from its measurement evidence")
- artifact_safety.assert_publication_safe([doc])
- return doc
+def validate_result(path: str | os.PathLike[str]) -> dict[str, Any]:
+ """Validate one emitted result file (raw+samples or terminal) and return it."""
+ return load_attempt(path)
-def make_terminal_document(
+
+# ---------------------------------------------------------------------------
+# Terminal outcome construction and validation.
+# ---------------------------------------------------------------------------
+def make_terminal_result(
*,
allocation_factors: dict[str, Any],
attempt_ordinal: int,
@@ -2530,41 +1010,24 @@ def make_terminal_document(
return validate_terminal_document(document)
+# Compatibility alias for existing matrix/runtime callers.
+make_terminal_document = make_terminal_result
+
+
def validate_terminal_document(document: Any) -> dict[str, Any]:
- _validate_native_schema("terminal-outcome-v1.schema.json", document)
- doc = _keys(
- document,
- {"case", "format", "generated_at", "identity", "outcome", "provenance", "record_type",
- "schema_version"},
- "terminal",
- )
- if doc["format"] != TERMINAL_FORMAT or doc["schema_version"] != 1 or doc["record_type"] != "terminal-outcome":
- raise ContractError("terminal format/schema/record type differs from v1")
- ids = _keys(doc["identity"], {
- "allocation_factors", "allocation_id", "attempt_id", "attempt_ordinal",
- "case_factors", "case_id",
- }, "terminal.identity")
- for field, kind in (("allocation_id", "allocation"), ("attempt_id", "attempt"), ("case_id", "case")):
- _typed(ids[field], kind, f"terminal.identity.{field}")
- ordinal = _integer(ids["attempt_ordinal"], "terminal.identity.attempt_ordinal", minimum=1)
- case = _scheduled_case(doc["case"], "terminal.case")
- factors = _keys(ids["case_factors"], {"case", "profile", "sku"}, "terminal.identity.case_factors")
+ """Validate a terminal outcome against its schema and cross-field semantics."""
+ if not isinstance(document, dict):
+ raise ContractError("terminal document must be an object")
+ _schema_validate(document, "terminal-outcome-v1.schema.json", "terminal")
+ ids = document["identity"]
+ case = document["case"]
+ factors = ids["case_factors"]
if factors["case"] != case or factors["profile"] != scheduled_case_profile(
case, "terminal.case"
):
raise ContractError("terminal case factors differ from the scheduled case/profile")
- _text(factors["sku"], "terminal.identity.case_factors.sku")
- allocation = _keys(
- ids["allocation_factors"], ALLOCATION_FACTOR_FIELDS,
- "terminal.identity.allocation_factors",
- )
- qualification_index = _integer(
- allocation["qualification_index"],
- "terminal.identity.allocation_factors.qualification_index",
- minimum=1,
- )
- if qualification_index > 3:
- raise ContractError("terminal qualification index must be in 1..3")
+ allocation = ids["allocation_factors"]
+ ordinal = ids["attempt_ordinal"]
expected_case = identity.digest("case", factors)
expected_allocation = identity.allocation_id(allocation)
expected_attempt = identity.attempt_id(
@@ -2574,90 +1037,48 @@ def validate_terminal_document(document: Any) -> dict[str, Any]:
expected_case, expected_allocation, expected_attempt
):
raise ContractError("terminal typed identities do not match their factors")
- provenance = _keys(
- doc["provenance"], {"git_run", "control_sha256", "redaction", "source"},
- "terminal.provenance",
- )
+ provenance = document["provenance"]
git_run = provenance["git_run"]
- if git_run is not None:
- git_run = _keys(git_run, GIT_RUN_FIELDS, "terminal.provenance.git_run")
- if git_run["qualification_index"] != qualification_index:
- raise ContractError(
- "terminal git run qualification index differs from allocation"
- )
- control = provenance["control_sha256"]
- if control is not None and (
- not isinstance(control, str) or len(control) != 64
- or any(char not in "0123456789abcdef" for char in control)
- ):
- raise ContractError("terminal control_sha256 is invalid")
- if provenance["redaction"] != "sanitized-v1":
- raise ContractError("terminal redaction contract differs")
- source = _text(provenance["source"], "terminal.provenance.source")
- outcome = _keys(
- doc["outcome"], {"failure_mode", "reason", "return_code", "status"}, "terminal.outcome"
- )
- if outcome["status"] not in {"failed", "invalid", "unsupported"}:
- raise ContractError("terminal outcome status is invalid")
- failure_mode = _text(outcome["failure_mode"], "terminal.outcome.failure_mode")
- reason = _text(outcome["reason"], "terminal.outcome.reason")
- _integer(outcome["return_code"], "terminal.outcome.return_code")
+ outcome = document["outcome"]
+ failure_mode = outcome["failure_mode"]
+ source = provenance["source"]
if source == "runtime-emitter":
expected_runner = factors["sku"]
- expected_reason = RUNTIME_FAILURE_REASONS.get(failure_mode)
- valid_outcome = outcome["status"] == "failed" and reason == expected_reason
+ valid_outcome = (
+ outcome["status"] == "failed"
+ and outcome["reason"] == RUNTIME_FAILURE_REASONS.get(failure_mode)
+ )
elif source == "post-emit-command":
expected_runner = factors["sku"]
- expected_reason = POST_EMIT_FAILURE_REASONS.get(failure_mode)
- valid_outcome = outcome["status"] == "failed" and reason == expected_reason
+ valid_outcome = (
+ outcome["status"] == "failed"
+ and outcome["reason"] == POST_EMIT_FAILURE_REASONS.get(failure_mode)
+ )
elif source == "matrix-capability-resolver":
expected_runner = "capability-resolver"
valid_outcome = (
outcome["status"] == "unsupported"
and failure_mode == "capability"
- and reason in CAPABILITY_FAILURE_REASONS
+ and outcome["reason"] in CAPABILITY_FAILURE_REASONS
)
else:
raise ContractError("terminal provenance source is not registered")
if not valid_outcome:
raise ContractError("terminal source and outcome are not registered")
- expected_allocation = {
+ expected_allocation_factors = {
"artifact": git_run["artifact"] if git_run is not None else None,
"execution_id": allocation["execution_id"],
"job": git_run["job"] if git_run is not None else None,
- "qualification_index": qualification_index,
"repo": git_run["repo"] if git_run is not None else None,
"run_attempt": git_run["run_attempt"] if git_run is not None else None,
"run_id": git_run["run_id"] if git_run is not None else None,
"runner": expected_runner,
"source_sha": git_run["source_sha"] if git_run is not None else None,
}
- if allocation != expected_allocation:
+ if allocation != expected_allocation_factors:
raise ContractError("terminal allocation factors differ from provenance or source")
- artifact_safety.assert_publication_safe([doc])
- return doc
-
-
-def load_raw_attempt(path: str | os.PathLike[str]) -> dict[str, Any]:
- document = strict_load(path)
- artifact = _obj(document, "raw").get("sample_artifact")
- artifact = _obj(artifact, "raw.sample_artifact")
- sample_path = Path(path).with_name(_text(artifact.get("path"), "raw.sample_artifact.path"))
- payload = sample_path.read_bytes()
- if len(payload) != artifact.get("bytes") or hashlib.sha256(payload).hexdigest() != artifact.get("sha256"):
- raise ContractError("sample artifact bytes or digest differ")
- samples = strict_load(sample_path)
- return validate_raw_document(document, samples)
-
-
-def load_attempt(path: str | os.PathLike[str]) -> dict[str, Any]:
- """Fully validate and return one native raw or terminal attempt."""
- document = strict_load(path)
- if isinstance(document, dict) and document.get("format") == RAW_FORMAT:
- return load_raw_attempt(path)
- if isinstance(document, dict) and document.get("format") == TERMINAL_FORMAT:
- return validate_terminal_document(document)
- raise ContractError("unknown native attempt format")
+ assert_publication_safe([document])
+ return document
def quarantine_invalid_attempt(path: str | os.PathLike[str]) -> bool:
@@ -2683,37 +1104,9 @@ def quarantine_invalid_attempt(path: str | os.PathLike[str]) -> bool:
return True
-def normalize_attempt(document: dict[str, Any]) -> dict[str, Any]:
- """Return the publisher-facing projection after native validation."""
- if document.get("format") == RAW_FORMAT:
- ids = document["identity"]
- return {
- "allocation_id": ids["allocation_id"],
- "attempt_id": ids["attempt_id"],
- "case": document["case"],
- "case_id": ids["case_id"],
- "generated_at": document["generated_at"],
- "outcome": document["outcome"],
- "points": document["measurement"]["rows"],
- "runtime_fingerprint": document["runtime_fingerprint"],
- "series_id": ids["series_id"],
- }
- if document.get("format") == TERMINAL_FORMAT:
- ids = document["identity"]
- return {
- "allocation_id": ids["allocation_id"],
- "attempt_id": ids["attempt_id"],
- "case": document["case"],
- "case_id": ids["case_id"],
- "generated_at": document["generated_at"],
- "outcome": document["outcome"],
- "points": [],
- "runtime_fingerprint": None,
- "series_id": None,
- }
- raise ContractError("unknown attempt format")
-
-
+# ---------------------------------------------------------------------------
+# Terminal construction from the runtime environment.
+# ---------------------------------------------------------------------------
def _env_integer(name: str, default: int) -> int:
try:
return int(os.environ.get(name, str(default)))
@@ -2733,10 +1126,9 @@ def _terminal_case_from_environment(backend: str, phase: str) -> dict[str, Any]:
if phase == "decode"
else "128 256 512 1024 2048 4096"
)
- case = {
+ return {
"suite": os.environ.get("CX_SUITE") or "manual",
"workload": os.environ.get("CX_WORKLOAD_NAME") or "manual",
- "required_publication": os.environ.get("CX_REQUIRED_PUBLICATION") or "diagnostic",
"backend": backend,
"mode": os.environ.get("CX_MODE", "normal"),
"routing": os.environ.get("CX_ROUTING", "uniform"),
@@ -2766,10 +1158,6 @@ def _terminal_case_from_environment(backend: str, phase: str) -> dict[str, Any]:
"scale_up_transport": os.environ.get("CX_SCALE_UP_TRANSPORT", "unknown"),
"scale_out_transport": os.environ.get("CX_SCALE_OUT_TRANSPORT") or None,
}
- precision_profile = os.environ.get("CX_PRECISION_PROFILE") or None
- if precision_profile is not None:
- case["precision_profile"] = precision_profile
- return case
def _git_run_from_environment() -> dict[str, Any] | None:
@@ -2787,7 +1175,6 @@ def value(name: str) -> str | None:
}
if not any(item is not None for item in git_run.values()):
return None
- git_run["qualification_index"] = _env_integer("CX_QUALIFICATION_INDEX", 1)
return git_run
@@ -2798,7 +1185,6 @@ def _allocation_factors_from_environment(
"artifact": git_run["artifact"] if git_run is not None else None,
"execution_id": os.environ.get("COLLECTIVEX_EXECUTION_ID") or None,
"job": git_run["job"] if git_run is not None else None,
- "qualification_index": _env_integer("CX_QUALIFICATION_INDEX", 1),
"repo": git_run["repo"] if git_run is not None else None,
"run_attempt": git_run["run_attempt"] if git_run is not None else None,
"run_id": git_run["run_id"] if git_run is not None else None,
@@ -2824,7 +1210,7 @@ def make_terminal_from_environment(
}
git_run = _git_run_from_environment()
control = os.environ.get("COLLECTIVEX_CONTROL_SHA256") or None
- return make_terminal_document(
+ return make_terminal_result(
allocation_factors=_allocation_factors_from_environment(runner, git_run),
attempt_ordinal=_env_integer("CX_ATTEMPT_ID", 1),
case=case,
@@ -2852,9 +1238,9 @@ def _write_document(path: str | os.PathLike[str], document: dict[str, Any]) -> N
def demote_raw_attempt(path: str | os.PathLike[str], return_code: int) -> dict[str, Any]:
"""Replace a rank-zero raw result when the distributed command later fails."""
destination = Path(path)
- raw = strict_load(destination)
+ raw = strict_json_load(destination)
if not isinstance(raw, dict) or raw.get("format") != RAW_FORMAT:
- raise ContractError("only a native raw attempt can be demoted")
+ raise ContractError("only a raw attempt can be demoted")
ids = _obj(raw.get("identity"), "raw.identity")
required = {
"allocation_factors", "allocation_id", "attempt_id", "attempt_ordinal",
@@ -2866,7 +1252,7 @@ def demote_raw_attempt(path: str | os.PathLike[str], return_code: int) -> dict[s
git_run = _obj(raw.get("provenance"), "raw.provenance").get("git_run")
if git_run is not None:
git_run = _keys(git_run, GIT_RUN_FIELDS, "raw.provenance.git_run")
- terminal = make_terminal_document(
+ terminal = make_terminal_result(
allocation_factors=ids["allocation_factors"],
attempt_ordinal=ids["attempt_ordinal"],
case=ids["case_factors"]["case"],
@@ -2889,16 +1275,19 @@ def demote_raw_attempt(path: str | os.PathLike[str], return_code: int) -> dict[s
return terminal
-def validate_attempt_paths(paths: list[str]) -> int:
+# ---------------------------------------------------------------------------
+# Shard / matrix delivery reconciliation.
+# ---------------------------------------------------------------------------
+def _validate_attempt_paths(paths: list[str]) -> int:
"""Fully validate a result directory's attempts and paired sample artifacts."""
if not paths or len(paths) != len(set(paths)):
- raise ContractError("validate-many requires unique result paths")
+ raise ContractError("delivery requires unique result paths")
sample_paths: set[Path] = set()
referenced_samples: set[Path] = set()
attempt_count = 0
for raw_path in paths:
path = Path(raw_path).resolve()
- document = strict_load(path)
+ document = strict_json_load(path)
if isinstance(document, dict) and document.get("format") == RAW_FORMAT:
document = load_raw_attempt(path)
referenced_samples.add(path.with_name(document["sample_artifact"]["path"]))
@@ -2914,16 +1303,16 @@ def validate_attempt_paths(paths: list[str]) -> int:
if sample_paths != referenced_samples:
raise ContractError("sample artifacts are missing, orphaned, or outside the validated set")
if attempt_count == 0:
- raise ContractError("result set contains no native attempts")
+ raise ContractError("result set contains no attempts")
return attempt_count
def validate_delivery(
paths: list[str], source_path: str, *, disposition: str | None = None
) -> int:
- """Reconcile a shard or matrix disposition with its complete native attempt set."""
+ """Reconcile a shard or matrix disposition with its complete attempt set."""
source_file = Path(source_path).resolve()
- source = strict_load(source_file)
+ source = strict_json_load(source_file)
if isinstance(source, dict) and source.get("format") == "collectivex.matrix.v1":
if disposition is None:
raise ContractError("matrix delivery validation requires a disposition")
@@ -2949,12 +1338,13 @@ def validate_delivery(
if not expected or len(expected) != expected_count:
raise ContractError("delivery source has empty or duplicate case coverage")
- validate_attempt_paths(paths)
+ _validate_attempt_paths(paths)
attempts = []
for raw_path in paths:
- document = strict_load(raw_path)
+ document = strict_json_load(raw_path)
if isinstance(document, dict) and document.get("format") in {RAW_FORMAT, TERMINAL_FORMAT}:
attempts.append(load_attempt(raw_path))
+ assert_publication_safe(attempts)
by_case: dict[str, list[dict[str, Any]]] = {}
attempt_ids = set()
allocation_ids = set()
@@ -3012,7 +1402,7 @@ def validate_delivery(
def main() -> int:
- parser = argparse.ArgumentParser(description="CollectiveX native attempt contracts")
+ parser = argparse.ArgumentParser(description="CollectiveX result contracts")
subparsers = parser.add_subparsers(dest="command", required=True)
probe = subparsers.add_parser("probe")
probe.add_argument("path")
@@ -3026,8 +1416,6 @@ def main() -> int:
demote = subparsers.add_parser("demote")
demote.add_argument("path")
demote.add_argument("--return-code", required=True, type=int)
- validate_many = subparsers.add_parser("validate-many")
- validate_many.add_argument("paths", nargs="+")
quarantine = subparsers.add_parser("quarantine-invalid")
quarantine.add_argument("path")
delivery = subparsers.add_parser("validate-delivery")
@@ -3061,9 +1449,6 @@ def main() -> int:
_write_document(args.out, document)
print(f"preserved terminal outcome ({document['outcome']['failure_mode']})")
return 0
- if args.command == "validate-many":
- print(f"validated {validate_attempt_paths(args.paths)} native attempts")
- return 0
if args.command == "quarantine-invalid":
quarantine_invalid_attempt(args.path)
return 0
diff --git a/experimental/CollectiveX/identity.py b/experimental/CollectiveX/identity.py
index d574a1189e..7b691fb411 100644
--- a/experimental/CollectiveX/identity.py
+++ b/experimental/CollectiveX/identity.py
@@ -5,7 +5,6 @@
import hashlib
import json
import re
-from copy import deepcopy
from typing import Any
IDENTITY_VERSION = 1
@@ -66,180 +65,6 @@
"low-latency": V1_LOW_LATENCY_CASE_PROFILE,
}
-V1_CONTROL_PRECISION_PROFILE = "d-bf16.c-bf16"
-V1_NORMAL_PRECISION_PROFILE_IDS = (
- "d-fp8-e4m3fn-b128-f32-prequantized.c-bf16",
- "d-fp8-e4m3fnuz-b128-f32-prequantized.c-fp8-e4m3fnuz-direct-cast-noscale",
-)
-V1_LOW_LATENCY_PRECISION_PROFILE_IDS = (
- "d-fp8-e4m3fn-b128-f32-fused.c-bf16",
-)
-
-
-def _communication_axis(
- *,
- api_input_dtype: str,
- api_output_dtype: str,
- communication_format: str,
- scale_dtype: str | None,
- scale_layout: str,
- scale_group_size: int | None,
- padding_contract: str,
- alignment_contract: str,
- quantization_origin: str,
- conversion_boundary: str,
-) -> dict[str, Any]:
- return {
- "api_input_dtype": api_input_dtype,
- "api_output_dtype": api_output_dtype,
- "communication_format": communication_format,
- "scale_dtype": scale_dtype,
- "scale_layout": scale_layout,
- "scale_group_size": scale_group_size,
- "padding_contract": padding_contract,
- "alignment_contract": alignment_contract,
- "quantization_origin": quantization_origin,
- "conversion_boundary": conversion_boundary,
- }
-
-
-_BF16_AXIS = _communication_axis(
- api_input_dtype="bf16",
- api_output_dtype="bf16",
- communication_format="bf16",
- scale_dtype=None,
- scale_layout="none",
- scale_group_size=None,
- padding_contract="none",
- alignment_contract="native-bf16-vector-alignment",
- quantization_origin="none",
- conversion_boundary="none",
-)
-_FP8_E4M3FN_PREQUANTIZED_DISPATCH = _communication_axis(
- api_input_dtype="fp8-e4m3fn-with-f32-scale",
- api_output_dtype="fp8-e4m3fn-with-f32-scale",
- communication_format="fp8-e4m3fn",
- scale_dtype="f32",
- scale_layout="per-token-hidden-block",
- scale_group_size=128,
- padding_contract="right-zero-pad-hidden-to-128",
- alignment_contract="hidden-block-128",
- quantization_origin="caller-prequantized",
- conversion_boundary="before-dispatch-timing",
-)
-_FP8_E4M3FNUZ_PREQUANTIZED_DISPATCH = _communication_axis(
- api_input_dtype="fp8-e4m3fnuz-with-f32-scale",
- api_output_dtype="fp8-e4m3fnuz-with-f32-scale",
- communication_format="fp8-e4m3fnuz",
- scale_dtype="f32",
- scale_layout="per-token-hidden-block",
- scale_group_size=128,
- padding_contract="right-zero-pad-hidden-to-128",
- alignment_contract="hidden-block-128",
- quantization_origin="caller-prequantized",
- conversion_boundary="before-dispatch-timing",
-)
-_FP8_E4M3FN_FUSED_DISPATCH = _communication_axis(
- api_input_dtype="bf16",
- api_output_dtype="fp8-e4m3fn-with-f32-scale",
- communication_format="fp8-e4m3fn",
- scale_dtype="f32",
- scale_layout="per-token-hidden-block",
- scale_group_size=128,
- padding_contract="right-zero-pad-hidden-to-128",
- alignment_contract="hidden-block-128",
- quantization_origin="backend-fused",
- conversion_boundary="inside-dispatch-timing",
-)
-_FP8_E4M3FN_DIRECT_CAST_COMBINE = _communication_axis(
- api_input_dtype="bf16",
- api_output_dtype="bf16",
- communication_format="fp8-e4m3fn",
- scale_dtype=None,
- scale_layout="none",
- scale_group_size=None,
- padding_contract="none",
- alignment_contract="native-fp8-vector-alignment",
- quantization_origin="backend-internal-direct-cast",
- conversion_boundary="inside-combine-timing",
-)
-_FP8_E4M3FNUZ_DIRECT_CAST_COMBINE = _communication_axis(
- api_input_dtype="bf16",
- api_output_dtype="bf16",
- communication_format="fp8-e4m3fnuz",
- scale_dtype=None,
- scale_layout="none",
- scale_group_size=None,
- padding_contract="none",
- alignment_contract="native-fp8-vector-alignment",
- quantization_origin="backend-internal-direct-cast",
- conversion_boundary="inside-combine-timing",
-)
-
-V1_PRECISION_PROFILES: dict[str, dict[str, Any]] = {
- V1_CONTROL_PRECISION_PROFILE: {
- "modes": ["normal", "low-latency"],
- "dispatch": _BF16_AXIS,
- "combine": _BF16_AXIS,
- },
- "d-fp8-e4m3fn-b128-f32-prequantized.c-bf16": {
- "modes": ["normal"],
- "dispatch": _FP8_E4M3FN_PREQUANTIZED_DISPATCH,
- "combine": _BF16_AXIS,
- },
- "d-fp8-e4m3fn-b128-f32-fused.c-bf16": {
- "modes": ["low-latency"],
- "dispatch": _FP8_E4M3FN_FUSED_DISPATCH,
- "combine": _BF16_AXIS,
- },
- "d-bf16.c-fp8-e4m3fn-direct-cast-noscale": {
- "modes": ["normal"],
- "dispatch": _BF16_AXIS,
- "combine": _FP8_E4M3FN_DIRECT_CAST_COMBINE,
- },
- "d-fp8-e4m3fn-b128-f32-prequantized.c-fp8-e4m3fn-direct-cast-noscale": {
- "modes": ["normal"],
- "dispatch": _FP8_E4M3FN_PREQUANTIZED_DISPATCH,
- "combine": _FP8_E4M3FN_DIRECT_CAST_COMBINE,
- },
- "d-fp8-e4m3fnuz-b128-f32-prequantized.c-bf16": {
- "modes": ["normal"],
- "dispatch": _FP8_E4M3FNUZ_PREQUANTIZED_DISPATCH,
- "combine": _BF16_AXIS,
- },
- "d-bf16.c-fp8-e4m3fnuz-direct-cast-noscale": {
- "modes": ["normal"],
- "dispatch": _BF16_AXIS,
- "combine": _FP8_E4M3FNUZ_DIRECT_CAST_COMBINE,
- },
- "d-fp8-e4m3fnuz-b128-f32-prequantized.c-fp8-e4m3fnuz-direct-cast-noscale": {
- "modes": ["normal"],
- "dispatch": _FP8_E4M3FNUZ_PREQUANTIZED_DISPATCH,
- "combine": _FP8_E4M3FNUZ_DIRECT_CAST_COMBINE,
- },
-}
-
-V1_COMBINE_ORACLE_TOLERANCES = {
- "bf16": {"atol": 2e-2, "rtol": 5e-2},
- "fp8-direct-cast": {"atol": 4e-2, "rtol": 8e-2},
-}
-
-
-def combine_oracle_tolerances(communication_precision: dict[str, Any]) -> dict[str, float]:
- """Return the frozen combine-oracle gate for one exact native codec."""
- combine = communication_precision["combine"]
- communication_format = combine["communication_format"]
- if communication_format == "bf16":
- key = "bf16"
- elif (
- communication_format in {"fp8-e4m3fn", "fp8-e4m3fnuz"}
- and combine["quantization_origin"] == "backend-internal-direct-cast"
- ):
- key = "fp8-direct-cast"
- else:
- raise ValueError("precision profile has no frozen combine-oracle tolerance")
- return dict(V1_COMBINE_ORACLE_TOLERANCES[key])
-
def case_profile(mode: str) -> dict[str, Any]:
"""Return the immutable measurement profile for one scheduled mode."""
@@ -249,32 +74,16 @@ def case_profile(mode: str) -> dict[str, Any]:
raise IdentityError(f"unknown CollectiveX case mode {mode!r}") from exc
-def precision_profile(name: str) -> dict[str, Any]:
- """Return one exact dispatch/combine communication-format profile."""
- try:
- profile = V1_PRECISION_PROFILES[name]
- except KeyError as exc:
- raise IdentityError(f"unknown CollectiveX precision profile {name!r}") from exc
- return {"profile_id": name, **deepcopy(profile)}
-
-
def profile_for_case(case: dict[str, Any]) -> dict[str, Any]:
- """Resolve a scheduled case's explicit mode to its identity profile."""
+ """Resolve a scheduled case's explicit mode to its identity profile.
+
+ Dispatch and combine are fixed BF16 benchmark facts, so a case's identity
+ profile is fully determined by its measurement mode.
+ """
mode = case.get("mode")
if not isinstance(mode, str):
raise IdentityError("scheduled case mode is missing")
- base = case_profile(mode)
- precision_name = case.get("precision_profile")
- if precision_name is None:
- return base
- if not isinstance(precision_name, str):
- raise IdentityError("scheduled case precision_profile must be a string")
- precision = precision_profile(precision_name)
- if mode not in precision["modes"]:
- raise IdentityError(
- f"precision profile {precision_name!r} is not valid in mode {mode!r}"
- )
- return {**base, "communication_precision": precision}
+ return case_profile(mode)
class IdentityError(ValueError):
diff --git a/experimental/CollectiveX/schemas/channel-v1.schema.json b/experimental/CollectiveX/schemas/channel-v1.schema.json
deleted file mode 100644
index 87ffa86b03..0000000000
--- a/experimental/CollectiveX/schemas/channel-v1.schema.json
+++ /dev/null
@@ -1,23 +0,0 @@
-{
- "$schema": "https://json-schema.org/draft/2020-12/schema",
- "$id": "https://inferencex.com/schemas/collectivex/channel-v1.schema.json",
- "title": "CollectiveX public channel v1",
- "type": "object",
- "additionalProperties": false,
- "required": ["format","channel","dataset","generated_at"],
- "properties": {
- "format": {"const": "collectivex.channel.v1"},
- "channel": {"const": "dev-latest"},
- "dataset": {
- "type": "object",
- "additionalProperties": false,
- "required": ["path","sha256","bytes"],
- "properties": {
- "path": {"type": "string","pattern": "^datasets/[0-9a-f]{64}/dataset\\.json$"},
- "sha256": {"type": "string","pattern": "^[0-9a-f]{64}$"},
- "bytes": {"type": "integer","minimum": 1,"maximum": 33554432}
- }
- },
- "generated_at": {"type": "string","format": "date-time"}
- }
-}
diff --git a/experimental/CollectiveX/schemas/private-bundle-v1.schema.json b/experimental/CollectiveX/schemas/private-bundle-v1.schema.json
deleted file mode 100644
index 7891196927..0000000000
--- a/experimental/CollectiveX/schemas/private-bundle-v1.schema.json
+++ /dev/null
@@ -1,163 +0,0 @@
-{
- "$schema": "https://json-schema.org/draft/2020-12/schema",
- "$id": "https://inferencex.com/schemas/collectivex/private-bundle-v1.schema.json",
- "title": "CollectiveX private attempt bundle v1",
- "type": "object",
- "additionalProperties": false,
- "required": [
- "format",
- "schema_version",
- "created_at",
- "ingest_id",
- "run",
- "matrix",
- "sources",
- "attempts",
- "coverage",
- "runtime_fingerprints",
- "checksums",
- "validation"
- ],
- "properties": {
- "format": {"const": "collectivex.private.bundle.v1"},
- "schema_version": {"const": 1},
- "created_at": {"type": "string","format": "date-time"},
- "ingest_id": {"$ref": "#/$defs/sha256"},
- "run": {
- "type": "object",
- "additionalProperties": false,
- "required": ["repository","run_id","run_attempt","qualification_index","source_sha"],
- "properties": {
- "repository": {"type": "string","pattern": "^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$"},
- "run_id": {"type": "string","pattern": "^[1-9][0-9]*$"},
- "run_attempt": {"type": "integer","minimum": 1},
- "qualification_index": {"type":"integer","minimum":1,"maximum":3},
- "source_sha": {"type": "string","pattern": "^[0-9a-f]{40}$"}
- }
- },
- "matrix": {"$ref": "#/$defs/file"},
- "sources": {"type": "array","minItems": 1,"uniqueItems": true,"items": {"$ref": "#/$defs/source"}},
- "attempts": {
- "type": "array",
- "minItems": 1,
- "items": {
- "type": "object",
- "additionalProperties": false,
- "required": [
- "attempt_id",
- "allocation_id",
- "case_id",
- "outcome",
- "reason",
- "selected",
- "document",
- "samples",
- "runtime_fingerprint_sha256",
- "series_ids",
- "evidence_ids"
- ],
- "properties": {
- "attempt_id": {"$ref": "#/$defs/attemptId"},
- "allocation_id": {"$ref": "#/$defs/allocationId"},
- "case_id": {"$ref": "#/$defs/caseId"},
- "outcome": {"$ref": "#/$defs/outcome"},
- "reason": {"$ref": "#/$defs/reason"},
- "selected": {"type": "boolean"},
- "document": {"$ref": "#/$defs/file"},
- "samples": {"oneOf": [{"type": "null"},{"$ref": "#/$defs/file"}]},
- "runtime_fingerprint_sha256": {"oneOf": [{"type": "null"},{"$ref": "#/$defs/sha256"}]},
- "series_ids": {"type": "array","uniqueItems": true,"items": {"$ref": "#/$defs/seriesId"}},
- "evidence_ids": {"type": "array","uniqueItems": true,"items": {"$ref": "#/$defs/evidenceId"}}
- }
- }
- },
- "coverage": {
- "type": "object",
- "additionalProperties": false,
- "required": ["expected_cases","terminal_cases","complete","outcome_counts","selections"],
- "properties": {
- "expected_cases": {"type": "integer","minimum": 1},
- "terminal_cases": {"type": "integer","minimum": 0},
- "complete": {"type": "boolean"},
- "outcome_counts": {"$ref": "#/$defs/outcomeCounts"},
- "selections": {
- "type": "array",
- "minItems": 1,
- "items": {
- "type": "object",
- "additionalProperties": false,
- "required": ["case_id","selected_attempt_id","outcome"],
- "properties": {
- "case_id": {"$ref": "#/$defs/caseId"},
- "selected_attempt_id": {"$ref": "#/$defs/attemptId"},
- "outcome": {"$ref": "#/$defs/outcome"}
- }
- }
- }
- }
- },
- "runtime_fingerprints": {"type": "array","uniqueItems": true,"items": {"$ref": "#/$defs/sha256"}},
- "checksums": {"$ref": "#/$defs/file"},
- "validation": {
- "type": "object",
- "additionalProperties": false,
- "required": ["policy","passed","checks"],
- "properties": {
- "policy": {"const": "collectivex-publisher-v1"},
- "passed": {"const": true},
- "checks": {
- "type": "array",
- "minItems": 1,
- "uniqueItems": true,
- "items": {"type": "string","pattern": "^[a-z0-9][a-z0-9.-]*$"}
- }
- }
- }
- },
- "$defs": {
- "sha256": {"type": "string","pattern": "^[0-9a-f]{64}$"},
- "caseId": {"type": "string","pattern": "^cxcase-v1-[0-9a-f]{64}$"},
- "seriesId": {"type": "string","pattern": "^cxseries-v1-[0-9a-f]{64}$"},
- "evidenceId": {"type": "string","pattern": "^cxevidence-v1-[0-9a-f]{64}$"},
- "allocationId": {"type": "string","pattern": "^cxallocation-v1-[0-9a-f]{64}$"},
- "attemptId": {"type": "string","pattern": "^cxattempt-v1-[0-9a-f]{64}$"},
- "reason": {"oneOf": [{"type": "null"},{"type": "string","pattern": "^[a-z0-9][a-z0-9.-]*$","maxLength": 96}]},
- "outcome": {"enum": ["success","unsupported","failed","invalid","diagnostic"]},
- "outcomeCounts": {
- "type": "object",
- "additionalProperties": false,
- "required": ["success","unsupported","failed","invalid","diagnostic"],
- "properties": {
- "success": {"type": "integer","minimum": 0},
- "unsupported": {"type": "integer","minimum": 0},
- "failed": {"type": "integer","minimum": 0},
- "invalid": {"type": "integer","minimum": 0},
- "diagnostic": {"type": "integer","minimum": 0}
- }
- },
- "file": {
- "type": "object",
- "additionalProperties": false,
- "required": ["path","sha256","bytes"],
- "properties": {
- "path": {"type": "string","pattern": "^[A-Za-z0-9_.-]+(?:/[A-Za-z0-9_.-]+)*$"},
- "sha256": {"$ref": "#/$defs/sha256"},
- "bytes": {"type": "integer","minimum": 1}
- }
- },
- "source": {
- "type": "object",
- "additionalProperties": false,
- "required": ["path","sha256","bytes","artifact_name"],
- "properties": {
- "path": {"type": "string","pattern": "^[A-Za-z0-9_.-]+(?:/[A-Za-z0-9_.-]+)*$"},
- "sha256": {"$ref": "#/$defs/sha256"},
- "bytes": {"type": "integer","minimum": 1},
- "artifact_name": {
- "type": "string",
- "pattern": "^cx(?:unsupported|shard-[a-z0-9][a-z0-9_.-]{0,127})-[1-9][0-9]*-[1-9][0-9]*$"
- }
- }
- }
- }
-}
diff --git a/experimental/CollectiveX/schemas/public-dataset-v1.schema.json b/experimental/CollectiveX/schemas/public-dataset-v1.schema.json
deleted file mode 100644
index 867fbfe1bf..0000000000
--- a/experimental/CollectiveX/schemas/public-dataset-v1.schema.json
+++ /dev/null
@@ -1,849 +0,0 @@
-{
- "$schema": "https://json-schema.org/draft/2020-12/schema",
- "$id": "https://inferencex.com/schemas/collectivex/public-dataset-v1.schema.json",
- "title": "CollectiveX sanitized public dataset v1",
- "type": "object",
- "additionalProperties": false,
- "required": [
- "format",
- "schema_version",
- "generated_at",
- "source_bundle_ids",
- "promotion",
- "coverage",
- "attempts",
- "series",
- "cohorts",
- "rankings",
- "recommendations",
- "sensitivities"
- ],
- "properties": {
- "format": {"const": "collectivex.public.v1"},
- "schema_version": {"const": 1},
- "generated_at": {"type": "string","format": "date-time"},
- "source_bundle_ids": {"type": "array","uniqueItems": true,"items": {"$ref": "#/$defs/sha256"}},
- "promotion": {
- "type": "object",
- "additionalProperties": false,
- "required": [
- "status",
- "reason",
- "matrix_id",
- "allocation_ids",
- "required_allocations",
- "qualification_indices",
- "requested_cases",
- "terminal_cases",
- "measured_cases",
- "unsupported_cases",
- "requested_points",
- "terminal_points",
- "measured_points",
- "unsupported_points",
- "policy"
- ],
- "properties": {
- "status": {"enum": ["promoted","diagnostic","quarantined"]},
- "reason": {"$ref": "#/$defs/reason"},
- "matrix_id": {"oneOf": [{"type": "null"},{"$ref": "#/$defs/sha256"}]},
- "allocation_ids": {"type": "array","uniqueItems": true,"items": {"$ref": "#/$defs/allocationId"}},
- "required_allocations": {"const": 1},
- "qualification_indices": {"type":"array","minItems":0,"maxItems":1,"uniqueItems":true,"items":{"enum":[1]}},
- "coverage_scope": {"enum": ["full","partial"]},
- "covered_skus": {"type": "array","uniqueItems": true,"items": {"$ref": "#/$defs/safeId"}},
- "requested_cases": {"type": "integer","minimum": 0},
- "terminal_cases": {"type": "integer","minimum": 0},
- "measured_cases": {"type": "integer","minimum": 0},
- "unsupported_cases": {"type": "integer","minimum": 0},
- "requested_points": {"type": "integer","minimum": 0},
- "terminal_points": {"type": "integer","minimum": 0},
- "measured_points": {"type": "integer","minimum": 0},
- "unsupported_points": {"type": "integer","minimum": 0},
- "policy": {"const": "collectivex-decision-grade-v1"}
- }
- },
- "coverage": {"type": "array","uniqueItems":true,"items": {"$ref": "#/$defs/coverage"}},
- "attempts": {"type": "array","items": {"$ref": "#/$defs/attempt"}},
- "series": {"type": "array","items": {"$ref": "#/$defs/series"}},
- "cohorts": {"type": "array","items": {"$ref": "#/$defs/cohort"}},
- "rankings": {"type": "array","items": {"$ref": "#/$defs/ranking"}},
- "recommendations": {"type": "array","items": {"$ref": "#/$defs/recommendation"}},
- "sensitivities": {"type": "array","items": {"$ref": "#/$defs/sensitivity"}}
- },
- "$defs": {
- "sha256": {"type": "string","pattern": "^[0-9a-f]{64}$"},
- "caseId": {"type": "string","pattern": "^cxcase-v1-[0-9a-f]{64}$"},
- "workloadId": {"type": "string","pattern": "^cxwork-v1-[0-9a-f]{64}$"},
- "seriesId": {"type": "string","pattern": "^cxseries-v1-[0-9a-f]{64}$"},
- "pointId": {"type": "string","pattern": "^cxpoint-v1-[0-9a-f]{64}$"},
- "evidenceId": {"type": "string","pattern": "^cxevidence-v1-[0-9a-f]{64}$"},
- "allocationId": {"type": "string","pattern": "^cxallocation-v1-[0-9a-f]{64}$"},
- "attemptId": {"type": "string","pattern": "^cxattempt-v1-[0-9a-f]{64}$"},
- "safeId": {"type": "string","pattern": "^[a-z0-9][a-z0-9_.-]*$","maxLength": 128},
- "communicationAxis": {
- "type":"object",
- "additionalProperties":false,
- "required": [
- "alignment_contract","api_input_dtype","api_output_dtype","communication_format",
- "conversion_boundary","padding_contract","quantization_origin","scale_dtype",
- "scale_group_size","scale_layout"
- ],
- "properties": {
- "alignment_contract":{"enum":["native-bf16-vector-alignment","hidden-block-128","native-fp8-vector-alignment","value-block-64"]},
- "api_input_dtype":{"enum":["bf16","fp8-e4m3fn-with-f32-scale","fp8-e4m3fnuz-with-f32-scale"]},
- "api_output_dtype":{"enum":["bf16","fp8-e4m3fn-with-f32-scale","fp8-e4m3fnuz-with-f32-scale"]},
- "communication_format":{"enum":["bf16","fp8-e4m3fn","fp8-e4m3fnuz"]},
- "conversion_boundary":{"enum":["none","before-dispatch-timing","inside-dispatch-timing","inside-combine-timing"]},
- "padding_contract":{"enum":["none","right-zero-pad-hidden-to-128","right-zero-pad-values-to-64"]},
- "quantization_origin":{"enum":["none","caller-prequantized","backend-fused","backend-internal","backend-internal-direct-cast"]},
- "scale_dtype":{"oneOf":[{"type":"null"},{"enum":["f32"]}]},
- "scale_group_size":{"oneOf":[{"type":"null"},{"enum":[64,128]}]},
- "scale_layout":{"enum":["none","per-token-hidden-block","dynamic-per-64-values"]}
- }
- },
- "precisionProfile": {
- "enum": [
- "d-bf16.c-bf16",
- "d-fp8-e4m3fn-b128-f32-prequantized.c-bf16",
- "d-fp8-e4m3fn-b128-f32-fused.c-bf16",
- "d-bf16.c-fp8-e4m3fn-direct-cast-noscale",
- "d-fp8-e4m3fn-b128-f32-prequantized.c-fp8-e4m3fn-direct-cast-noscale",
- "d-fp8-e4m3fnuz-b128-f32-prequantized.c-bf16",
- "d-bf16.c-fp8-e4m3fnuz-direct-cast-noscale",
- "d-fp8-e4m3fnuz-b128-f32-prequantized.c-fp8-e4m3fnuz-direct-cast-noscale"
- ]
- },
- "byteAccounting": {
- "type":"object",
- "additionalProperties":false,
- "required":["accounting_contract","activation_data_bytes","scale_bytes","total_logical_bytes"],
- "properties": {
- "accounting_contract":{"const":"activation-data-plus-scales-v1"},
- "activation_data_bytes":{"type":"integer","minimum":0},
- "scale_bytes":{"type":"integer","minimum":0},
- "total_logical_bytes":{"type":"integer","minimum":0}
- }
- },
- "publicationTier": {"enum": ["official","comparable-experimental"]},
- "label": {"type": "string","minLength": 1,"maxLength": 160},
- "nullableLabel": {"oneOf": [{"type": "null"},{"$ref": "#/$defs/label"}]},
- "reason": {"oneOf": [{"type": "null"},{"type": "string","pattern": "^[a-z0-9][a-z0-9.-]*$","maxLength": 96}]},
- "reasonId": {"type":"string","pattern":"^[a-z0-9][a-z0-9.-]*$","maxLength":96},
- "outcome": {"enum": ["success","unsupported","failed","invalid","diagnostic"]},
- "precisionAxisEvidence": {
- "type":"object",
- "additionalProperties":false,
- "required":[
- "dequantized_semantics","encoded_payload_valid","max_abs_error","max_rel_error",
- "passed","saturation_count","saturation_rate","scales_finite","scales_positive"
- ],
- "properties": {
- "dequantized_semantics":{"type":"boolean"},
- "encoded_payload_valid":{"type":"boolean"},
- "max_abs_error":{"type":"number","minimum":0},
- "max_rel_error":{"type":"number","minimum":0},
- "passed":{"type":"boolean"},
- "saturation_count":{"type":"integer","minimum":0},
- "saturation_rate":{"type":"number","minimum":0,"maximum":1},
- "scales_finite":{"oneOf":[{"type":"null"},{"type":"boolean"}]},
- "scales_positive":{"oneOf":[{"type":"null"},{"type":"boolean"}]}
- }
- },
- "precisionEvidence": {
- "type":"object",
- "additionalProperties":false,
- "required":["combine","dispatch","passed","profile_id"],
- "properties": {
- "combine":{"$ref":"#/$defs/precisionAxisEvidence"},
- "dispatch":{"$ref":"#/$defs/precisionAxisEvidence"},
- "passed":{"type":"boolean"},
- "profile_id":{"$ref":"#/$defs/precisionProfile"}
- }
- },
- "pointCorrectness": {
- "type":"object",
- "additionalProperties":false,
- "required":["semantic_pass","precision"],
- "properties": {
- "semantic_pass":{"type":"boolean"},
- "precision":{"$ref":"#/$defs/precisionEvidence"}
- }
- },
- "coverageTopology": {
- "type": "object",
- "additionalProperties": false,
- "required": [
- "ep_size",
- "nodes",
- "gpus_per_node",
- "scale_up_domain",
- "scope",
- "scale_up_transport",
- "scale_out_transport",
- "transport",
- "topology_class"
- ],
- "properties": {
- "ep_size": {"type": "integer","minimum": 1},
- "nodes": {"type": "integer","minimum": 1},
- "gpus_per_node": {"type": "integer","minimum": 1},
- "scale_up_domain": {"type": "integer","minimum": 1},
- "scope": {"enum": ["scale-up","scale-out"]},
- "scale_up_transport": {"$ref": "#/$defs/safeId"},
- "scale_out_transport": {"oneOf": [{"type": "null"},{"$ref": "#/$defs/safeId"}]},
- "transport": {"$ref": "#/$defs/safeId"},
- "topology_class": {"$ref": "#/$defs/safeId"}
- }
- },
- "coverageResource": {
- "type":"object",
- "additionalProperties":false,
- "required":["mode","profile","comm_units_kind","configured_units"],
- "properties": {
- "mode":{"const":"fixed-profile"},
- "profile":{"oneOf":[{"type":"null"},{"$ref":"#/$defs/safeId"}]},
- "comm_units_kind":{"$ref":"#/$defs/nullableLabel"},
- "configured_units":{"oneOf":[{"type":"null"},{"type":"integer","minimum":1}]}
- }
- },
- "coveragePoint": {
- "type":"object",
- "additionalProperties":false,
- "allOf":[
- {
- "if":{"properties":{"terminal_status":{"const":"measured"}},"required":["terminal_status"]},
- "then":{"properties":{"point_id":{"$ref":"#/$defs/pointId"},"series_id":{"$ref":"#/$defs/seriesId"},"reason":{"type":"null"}}}
- },
- {
- "if":{"properties":{"terminal_status":{"const":"unsupported"}},"required":["terminal_status"]},
- "then":{"properties":{"point_id":{"type":"null"},"series_id":{"type":"null"},"reason":{"$ref":"#/$defs/reasonId"}}}
- },
- {
- "if":{"properties":{"terminal_status":{"enum":["failed","invalid"]}},"required":["terminal_status"]},
- "then":{"properties":{"reason":{"$ref":"#/$defs/reasonId"}}}
- }
- ],
- "required":["point_id","series_id","tokens_per_rank","global_tokens","terminal_status","reason"],
- "properties": {
- "point_id":{"oneOf":[{"type":"null"},{"$ref":"#/$defs/pointId"}]},
- "series_id":{"oneOf":[{"type":"null"},{"$ref":"#/$defs/seriesId"}]},
- "tokens_per_rank":{"type":"integer","minimum":1},
- "global_tokens":{"type":"integer","minimum":1},
- "terminal_status":{"enum":["measured","unsupported","failed","invalid","diagnostic"]},
- "reason":{"$ref":"#/$defs/reason"}
- }
- },
- "coverage": {
- "type": "object",
- "additionalProperties": false,
- "required": [
- "case_id",
- "label",
- "required",
- "sku",
- "suite",
- "workload",
- "publication_tier",
- "backend",
- "backend_generation",
- "mode",
- "phase",
- "routing",
- "eplb",
- "precision_profile",
- "dispatch_precision",
- "combine_precision",
- "resource",
- "topology",
- "points",
- "disposition",
- "selected_attempt_id",
- "outcome",
- "failure_mode",
- "reason",
- "attempt_ids"
- ],
- "properties": {
- "case_id": {"$ref": "#/$defs/caseId"},
- "label": {"$ref": "#/$defs/label"},
- "required": {"type": "boolean"},
- "sku": {"$ref": "#/$defs/safeId"},
- "suite": {"$ref":"#/$defs/safeId"},
- "workload": {"$ref":"#/$defs/safeId"},
- "publication_tier": {"$ref":"#/$defs/publicationTier"},
- "backend": {"$ref": "#/$defs/safeId"},
- "backend_generation": {"$ref":"#/$defs/nullableLabel"},
- "mode": {"enum": ["normal","low-latency"]},
- "phase": {"enum": ["decode","prefill"]},
- "routing": {"enum":["uniform"]},
- "eplb": {"type":"boolean"},
- "precision_profile": {"$ref":"#/$defs/precisionProfile"},
- "dispatch_precision": {"$ref":"#/$defs/communicationAxis"},
- "combine_precision": {"$ref":"#/$defs/communicationAxis"},
- "resource": {"$ref":"#/$defs/coverageResource"},
- "topology": {"$ref": "#/$defs/coverageTopology"},
- "points": {"type":"array","minItems":1,"uniqueItems":true,"items":{"$ref":"#/$defs/coveragePoint"}},
- "disposition": {"enum": ["runnable","unsupported"]},
- "selected_attempt_id": {"oneOf": [{"type": "null"},{"$ref": "#/$defs/attemptId"}]},
- "outcome": {"$ref": "#/$defs/outcome"},
- "failure_mode": {"$ref": "#/$defs/reason"},
- "reason": {"$ref": "#/$defs/reason"},
- "attempt_ids": {"type": "array","uniqueItems": true,"items": {"$ref": "#/$defs/attemptId"}}
- }
- },
- "attempt": {
- "type": "object",
- "additionalProperties": false,
- "required": [
- "attempt_id",
- "evidence",
- "case_id",
- "allocation_id",
- "run_id",
- "run_attempt",
- "qualification_index",
- "attempt_index",
- "selected",
- "outcome",
- "failure_mode",
- "reason",
- "series_id",
- "completed_at"
- ],
- "properties": {
- "attempt_id": {"$ref": "#/$defs/attemptId"},
- "evidence": {
- "type": "array",
- "uniqueItems": true,
- "items": {
- "type": "object",
- "additionalProperties": false,
- "required": ["evidence_id","point_id"],
- "properties": {"evidence_id": {"$ref": "#/$defs/evidenceId"},"point_id": {"$ref": "#/$defs/pointId"}}
- }
- },
- "case_id": {"$ref": "#/$defs/caseId"},
- "allocation_id": {"$ref": "#/$defs/allocationId"},
- "run_id": {"type": "string","pattern": "^[1-9][0-9]*$"},
- "run_attempt": {"type": "integer","minimum": 1},
- "qualification_index": {"type":"integer","minimum":1,"maximum":3},
- "attempt_index": {"type": "integer","minimum": 1},
- "selected": {"type": "boolean"},
- "outcome": {"$ref": "#/$defs/outcome"},
- "failure_mode": {"$ref": "#/$defs/reason"},
- "reason": {"$ref": "#/$defs/reason"},
- "series_id": {"oneOf": [{"type": "null"},{"$ref": "#/$defs/seriesId"}]},
- "completed_at": {"oneOf": [{"type": "null"},{"type": "string","format": "date-time"}]}
- }
- },
- "eligibility": {
- "type": "object",
- "additionalProperties": false,
- "allOf": [{
- "if": {"properties": {"decision_grade": {"const": true}},"required": ["decision_grade"]},
- "then": {"properties": {"reasons": {"maxItems": 0}}},
- "else": {"properties": {"reasons": {"minItems": 1}}}
- }],
- "required": [
- "decision_grade",
- "allocation_ids",
- "complete",
- "correct",
- "measured_roundtrip_p99",
- "stable_ordering",
- "reasons"
- ],
- "properties": {
- "decision_grade": {"type": "boolean"},
- "allocation_ids": {"type": "array","uniqueItems": true,"items": {"$ref": "#/$defs/allocationId"}},
- "complete": {"type": "boolean"},
- "correct": {"type": "boolean"},
- "measured_roundtrip_p99": {"type": "boolean"},
- "stable_ordering": {"type": "boolean"},
- "reasons": {
- "type": "array",
- "uniqueItems": true,
- "items": {"type": "string","pattern": "^[a-z0-9][a-z0-9.-]*$","maxLength": 96}
- }
- }
- },
- "percentiles": {
- "type": "object",
- "additionalProperties": false,
- "required": ["p50","p90","p95","p99"],
- "properties": {
- "p50": {"type": "number","exclusiveMinimum": 0},
- "p90": {"type": "number","exclusiveMinimum": 0},
- "p95": {"type": "number","exclusiveMinimum": 0},
- "p99": {"type": "number","exclusiveMinimum": 0}
- }
- },
- "ratePercentiles": {
- "type": "object",
- "additionalProperties": false,
- "required": ["p50","p90","p95","p99"],
- "properties": {
- "p50": {"type": "number","minimum": 0},
- "p90": {"type": "number","minimum": 0},
- "p95": {"type": "number","minimum": 0},
- "p99": {"type": "number","minimum": 0}
- }
- },
- "component": {
- "type": "object",
- "additionalProperties": false,
- "allOf":[
- {
- "if":{"properties":{"origin":{"const":"measured"}},"required":["origin"]},
- "then":{"properties":{"sample_count":{"const":512}}}
- },
- {
- "if":{"properties":{"origin":{"const":"derived"}},"required":["origin"]},
- "then":{"properties":{"sample_count":{"type":"null"}}}
- }
- ],
- "required": ["origin","latency_us","byte_provenance","activation_data_rate_gbps_at_latency_percentile","total_logical_data_rate_gbps_at_latency_percentile","sample_count"],
- "properties": {
- "origin": {"enum": ["measured","derived"]},
- "latency_us": {"$ref": "#/$defs/percentiles"},
- "byte_provenance": {"$ref":"#/$defs/byteAccounting"},
- "activation_data_rate_gbps_at_latency_percentile": {"oneOf":[{"type":"null"},{"$ref":"#/$defs/ratePercentiles"}]},
- "total_logical_data_rate_gbps_at_latency_percentile": {"oneOf":[{"type":"null"},{"$ref":"#/$defs/ratePercentiles"}]},
- "sample_count": {"oneOf": [{"type": "null"},{"type": "integer","minimum": 1}]}
- }
- },
- "nullableComponent": {"oneOf": [{"type": "null"},{"$ref": "#/$defs/component"}]},
- "trialDiagnosticComponent": {
- "type":"object",
- "additionalProperties":false,
- "required":["drift_flagged","first_last_median_ratio","outlier_flagged","robust_outlier_fraction","trial_count"],
- "properties": {
- "drift_flagged":{"type":"boolean"},
- "first_last_median_ratio":{"type":"number","minimum":1},
- "outlier_flagged":{"type":"boolean"},
- "robust_outlier_fraction":{"type":"number","minimum":0,"maximum":1},
- "trial_count":{"const":64}
- }
- },
- "nullableTrialDiagnosticComponent": {"oneOf":[{"type":"null"},{"$ref":"#/$defs/trialDiagnosticComponent"}]},
- "trialDiagnostics": {
- "type":"object",
- "additionalProperties":false,
- "required":["components","flagged","reasons"],
- "properties": {
- "components": {
- "type":"object",
- "additionalProperties":false,
- "required":["dispatch","stage","combine","roundtrip"],
- "properties": {
- "dispatch":{"$ref":"#/$defs/nullableTrialDiagnosticComponent"},
- "stage":{"$ref":"#/$defs/nullableTrialDiagnosticComponent"},
- "combine":{"$ref":"#/$defs/nullableTrialDiagnosticComponent"},
- "roundtrip":{"$ref":"#/$defs/nullableTrialDiagnosticComponent"}
- }
- },
- "flagged":{"type":"boolean"},
- "reasons":{"type":"array","maxItems":2,"uniqueItems":true,"items":{"enum":["trial-drift","trial-outliers"]}}
- }
- },
- "point": {
- "type": "object",
- "additionalProperties": false,
- "required": [
- "point_id",
- "tokens_per_rank",
- "global_tokens",
- "anomalies",
- "correctness",
- "trial_diagnostics",
- "routing",
- "components",
- "roundtrip_token_rate_at_latency_percentile",
- "evidence_ids"
- ],
- "properties": {
- "point_id": {"$ref": "#/$defs/pointId"},
- "tokens_per_rank": {"type": "integer","minimum": 1},
- "global_tokens": {"type": "integer","minimum": 1},
- "anomalies": {"type":"array","maxItems":16,"uniqueItems":true,"items":{"$ref":"#/$defs/reasonId"}},
- "correctness": {"$ref":"#/$defs/pointCorrectness"},
- "trial_diagnostics":{"$ref":"#/$defs/trialDiagnostics"},
- "routing": {
- "type": "object",
- "additionalProperties": false,
- "required": [
- "fanout_mean",
- "recv_tokens_max",
- "expert_load_cv",
- "payload_rank_cv",
- "hotspot_ratio",
- "empty_expert_count",
- "empty_rank_count",
- "routed_copies"
- ],
- "properties": {
- "fanout_mean": {"type": "number","minimum": 0},
- "recv_tokens_max": {"type": "integer","minimum": 0},
- "expert_load_cv": {"type": "number","minimum": 0},
- "payload_rank_cv": {"type": "number","minimum": 0},
- "hotspot_ratio": {"type": "number","minimum": 0},
- "empty_expert_count": {"type": "integer","minimum": 0},
- "empty_rank_count": {"type": "integer","minimum": 0},
- "routed_copies": {"type": "integer","minimum": 1}
- }
- },
- "components": {
- "type": "object",
- "additionalProperties": false,
- "required": ["dispatch","stage","combine","roundtrip","isolated_sum"],
- "properties": {
- "dispatch": {"$ref": "#/$defs/nullableComponent"},
- "stage": {"$ref":"#/$defs/nullableComponent"},
- "combine": {"$ref": "#/$defs/nullableComponent"},
- "roundtrip": {"$ref": "#/$defs/nullableComponent"},
- "isolated_sum": {"$ref": "#/$defs/nullableComponent"}
- }
- },
- "roundtrip_token_rate_at_latency_percentile": {"$ref": "#/$defs/percentiles"},
- "evidence_ids": {"type": "array","minItems":1,"maxItems":3,"uniqueItems": true,"items": {"$ref": "#/$defs/evidenceId"}}
- }
- },
- "series": {
- "type": "object",
- "additionalProperties": false,
- "required": [
- "series_id",
- "label",
- "status",
- "case_ids",
- "allocation_ids",
- "model",
- "suite",
- "mode",
- "phase",
- "publication_tier",
- "backend",
- "build",
- "system",
- "workload",
- "eplb",
- "resource",
- "measurement",
- "points",
- "eligibility"
- ],
- "properties": {
- "series_id": {"$ref": "#/$defs/seriesId"},
- "label": {"$ref": "#/$defs/label"},
- "status": {"enum": ["decision-grade","diagnostic"]},
- "case_ids": {"type": "array","minItems": 1,"uniqueItems": true,"items": {"$ref": "#/$defs/caseId"}},
- "allocation_ids": {"type": "array","minItems": 1,"uniqueItems": true,"items": {"$ref": "#/$defs/allocationId"}},
- "model": {"$ref": "#/$defs/safeId"},
- "suite": {"$ref": "#/$defs/safeId"},
- "mode": {"enum": ["normal","low-latency"]},
- "phase": {"enum": ["decode","prefill"]},
- "publication_tier": {"$ref": "#/$defs/publicationTier"},
- "backend": {
- "type": "object",
- "additionalProperties": false,
- "required": ["id","label","role","generation","version"],
- "properties": {
- "id": {"$ref": "#/$defs/safeId"},
- "label": {"$ref": "#/$defs/label"},
- "role": {"enum": ["library","reference"]},
- "generation": {"$ref": "#/$defs/nullableLabel"},
- "version": {"$ref": "#/$defs/nullableLabel"}
- }
- },
- "build": {
- "type": "object",
- "additionalProperties": false,
- "required": ["implementation_contract_sha256","public_config_sha256","routing_control_sha256","runtime_fingerprint_sha256","image_digest","source_sha","squash_sha256"],
- "properties": {
- "implementation_contract_sha256": {"$ref": "#/$defs/sha256"},
- "public_config_sha256": {"$ref": "#/$defs/sha256"},
- "routing_control_sha256": {"$ref": "#/$defs/sha256"},
- "runtime_fingerprint_sha256": {"$ref": "#/$defs/sha256"},
- "image_digest": {"type": "string","pattern": "^sha256:[0-9a-f]{64}$"},
- "source_sha": {"type": "string","pattern": "^[0-9a-f]{40,64}$"},
- "squash_sha256": {"$ref": "#/$defs/sha256"}
- }
- },
- "system": {
- "type": "object",
- "additionalProperties": false,
- "required": ["sku","label","vendor","topology_class","transport","scale_up_transport","scale_out_transport","scope","nodes","gpus_per_node","scale_up_domain","world_size","ep_size","placement"],
- "properties": {
- "sku": {"$ref": "#/$defs/safeId"},
- "label": {"$ref": "#/$defs/label"},
- "vendor": {"enum": ["nvidia","amd"]},
- "topology_class": {"$ref": "#/$defs/safeId"},
- "transport": {"$ref": "#/$defs/safeId"},
- "scale_up_transport": {"$ref": "#/$defs/safeId"},
- "scale_out_transport": {"oneOf": [{"type": "null"},{"$ref": "#/$defs/safeId"}]},
- "scope": {"enum": ["scale-up","scale-out"]},
- "nodes": {"type": "integer","minimum": 1},
- "gpus_per_node": {"type": "integer","minimum": 1},
- "scale_up_domain": {"type": "integer","minimum": 1},
- "world_size": {"type": "integer","minimum": 1},
- "ep_size": {"type": "integer","minimum": 1},
- "placement": {"enum": ["packed"]}
- }
- },
- "workload": {
- "type": "object",
- "additionalProperties": false,
- "required": [
- "workload_id",
- "hidden",
- "top_k",
- "experts",
- "routing",
- "eplb",
- "precision_profile",
- "dispatch_precision",
- "combine_precision",
- "activation_profile"
- ],
- "properties": {
- "workload_id": {"$ref": "#/$defs/workloadId"},
- "hidden": {"type": "integer","minimum": 1},
- "top_k": {"type": "integer","minimum": 1},
- "experts": {"type": "integer","minimum": 1},
- "routing": {"enum": ["uniform"]},
- "eplb": {"type": "boolean"},
- "precision_profile": {"$ref":"#/$defs/precisionProfile"},
- "dispatch_precision": {"$ref":"#/$defs/communicationAxis"},
- "combine_precision": {"$ref":"#/$defs/communicationAxis"},
- "activation_profile": {"const": "canonical-counter-source-v4"}
- }
- },
- "eplb": {
- "type": "object",
- "additionalProperties": false,
- "allOf":[
- {
- "if":{"properties":{"enabled":{"const":true}},"required":["enabled"]},
- "then":{"properties":{
- "calibration_workload_id":{"$ref":"#/$defs/workloadId"},
- "calibration_trace_sha256":{"$ref":"#/$defs/sha256"},
- "calibration_window":{"const":"collectivex-eplb-calibration-window-v1"},
- "calibration_token_offset":{"type":"integer","minimum":0}
- }},
- "else":{"properties":{
- "calibration_workload_id":{"type":"null"},
- "calibration_trace_sha256":{"type":"null"},
- "calibration_window":{"type":"null"},
- "calibration_token_offset":{"type":"null"}
- }}
- }
- ],
- "required": [
- "enabled",
- "calibration_workload_id",
- "calibration_trace_sha256",
- "calibration_window",
- "calibration_token_offset",
- "planner",
- "mapping_sha256",
- "logical_experts",
- "physical_experts",
- "redundant_experts",
- "reference_tokens_per_rank",
- "replicated_experts",
- "max_replicas",
- "imbalance_before",
- "imbalance_after"
- ],
- "properties": {
- "enabled": {"type": "boolean"},
- "calibration_workload_id": {"oneOf":[{"type":"null"},{"$ref":"#/$defs/workloadId"}]},
- "calibration_trace_sha256": {"oneOf":[{"type":"null"},{"$ref":"#/$defs/sha256"}]},
- "calibration_window": {"oneOf":[{"type":"null"},{"const":"collectivex-eplb-calibration-window-v1"}]},
- "calibration_token_offset": {"oneOf":[{"type":"null"},{"type":"integer","minimum":0}]},
- "planner": {"$ref": "#/$defs/nullableLabel"},
- "mapping_sha256": {"oneOf": [{"type": "null"},{"$ref": "#/$defs/sha256"}]},
- "logical_experts": {"type": "integer","minimum": 1},
- "physical_experts": {"type": "integer","minimum": 1},
- "redundant_experts": {"type": "integer","minimum": 0},
- "reference_tokens_per_rank": {"oneOf": [{"type": "null"},{"type": "integer","minimum": 1}]},
- "replicated_experts": {"type": "integer","minimum": 0},
- "max_replicas": {"oneOf": [{"type": "null"},{"type": "integer","minimum": 0}]},
- "imbalance_before": {"oneOf": [{"type": "null"},{"type": "number","minimum": 0}]},
- "imbalance_after": {"oneOf": [{"type": "null"},{"type": "number","minimum": 0}]}
- }
- },
- "resource": {
- "type": "object",
- "additionalProperties": false,
- "required": ["mode","profile","comm_units_kind","configured_units"],
- "properties": {
- "mode": {"const": "fixed-profile"},
- "profile": {"$ref": "#/$defs/safeId"},
- "comm_units_kind": {"$ref": "#/$defs/nullableLabel"},
- "configured_units": {"oneOf": [{"type": "null"},{"type": "integer","minimum": 1}]}
- }
- },
- "measurement": {
- "type": "object",
- "additionalProperties": false,
- "required": [
- "contract",
- "component_order_contract",
- "combine_semantics",
- "payload_unit",
- "sampling_contract",
- "iters",
- "trials",
- "warmups",
- "samples_per_component",
- "qualification_indices",
- "headline_component",
- "headline_percentile"
- ],
- "properties": {
- "contract": {"enum": ["layout-and-dispatch-v1","expert-packed-weighted-combine-v1"]},
- "component_order_contract": {"const":"qualification-hash-rotated-components-v1"},
- "combine_semantics": {"enum": ["activation-only","gate-weighted"]},
- "payload_unit": {"enum": ["token-rank","token-expert"]},
- "sampling_contract": {"const": "fixed-512-v1"},
- "iters": {"const": 8},
- "trials": {"const": 64},
- "warmups": {"const": 32},
- "samples_per_component": {"const": 512},
- "qualification_indices": {"type":"array","minItems":1,"maxItems":3,"uniqueItems":true,"items":{"enum":[1,2,3]}},
- "headline_component": {"const": "roundtrip"},
- "headline_percentile": {"const": "p99"}
- }
- },
- "points": {"type": "array","minItems": 1,"items": {"$ref": "#/$defs/point"}},
- "eligibility": {"$ref": "#/$defs/eligibility"}
- }
- },
- "cohort": {
- "type": "object",
- "additionalProperties": false,
- "required": [
- "cohort_id",
- "kind",
- "label",
- "description",
- "series_ids",
- "controlled_factors",
- "varying_factors",
- "publication_tier",
- "eligibility"
- ],
- "properties": {
- "cohort_id": {"type": "string","pattern": "^cxcohort-v1-[0-9a-f]{64}$"},
- "kind": {"enum": ["library","chip","system","dispatch-precision","combine-precision","precision-pair"]},
- "label": {"$ref": "#/$defs/label"},
- "description": {"$ref": "#/$defs/label"},
- "series_ids": {"type": "array","minItems": 2,"uniqueItems": true,"items": {"$ref": "#/$defs/seriesId"}},
- "controlled_factors": {"type": "array","minItems": 1,"uniqueItems": true,"items": {"$ref": "#/$defs/safeId"}},
- "varying_factors": {"type": "array","minItems": 1,"uniqueItems": true,"items": {"$ref": "#/$defs/safeId"}},
- "publication_tier": {"$ref": "#/$defs/publicationTier"},
- "eligibility": {"$ref": "#/$defs/eligibility"}
- }
- },
- "metric": {
- "type": "object",
- "additionalProperties": false,
- "required": ["operation","statistic","measure","objective","tokens_per_rank","phase"],
- "properties": {
- "operation": {"const": "roundtrip"},
- "statistic": {"enum": ["p50","p99"]},
- "measure": {"enum": ["latency_us","activation_data_rate_gbps_at_latency_percentile","total_logical_data_rate_gbps_at_latency_percentile"]},
- "objective": {"enum": ["min","max"]},
- "tokens_per_rank": {"type": "integer","minimum": 1},
- "phase": {"enum": ["decode","prefill"]}
- }
- },
- "ranking": {
- "type": "object",
- "additionalProperties": false,
- "required": ["ranking_id","cohort_id","label","metric","entries","publication_tier","eligibility"],
- "properties": {
- "ranking_id": {"type": "string","pattern": "^cxranking-v1-[0-9a-f]{64}$"},
- "cohort_id": {"type": "string","pattern": "^cxcohort-v1-[0-9a-f]{64}$"},
- "label": {"$ref": "#/$defs/label"},
- "metric": {"$ref": "#/$defs/metric"},
- "entries": {
- "type": "array",
- "minItems": 2,
- "items": {
- "type": "object",
- "additionalProperties": false,
- "required": ["rank","series_id","point_id","value","unit"],
- "properties": {
- "rank": {"type": "integer","minimum": 1},
- "series_id": {"$ref": "#/$defs/seriesId"},
- "point_id": {"$ref": "#/$defs/pointId"},
- "value": {"type": "number","exclusiveMinimum": 0},
- "unit": {"enum": ["us","GB/s"]}
- }
- }
- },
- "publication_tier": {"$ref": "#/$defs/publicationTier"},
- "eligibility": {"$ref": "#/$defs/eligibility"}
- }
- },
- "recommendation": {
- "type": "object",
- "additionalProperties": false,
- "required": [
- "recommendation_id",
- "cohort_id",
- "label",
- "objective",
- "series_id",
- "point_id",
- "value",
- "unit",
- "rationale",
- "publication_tier",
- "eligibility"
- ],
- "properties": {
- "recommendation_id": {"type": "string","pattern": "^cxrecommendation-v1-[0-9a-f]{64}$"},
- "cohort_id": {"type": "string","pattern": "^cxcohort-v1-[0-9a-f]{64}$"},
- "label": {"$ref": "#/$defs/label"},
- "objective": {"enum": ["min-p50-latency","min-p99-latency","max-activation-data-rate-at-p50-latency","max-activation-data-rate-at-p99-latency","max-total-logical-data-rate-at-p50-latency","max-total-logical-data-rate-at-p99-latency"]},
- "series_id": {"$ref": "#/$defs/seriesId"},
- "point_id": {"$ref": "#/$defs/pointId"},
- "value": {"type": "number","exclusiveMinimum": 0},
- "unit": {"enum": ["us","GB/s"]},
- "rationale": {"$ref": "#/$defs/label"},
- "publication_tier": {"const": "official"},
- "eligibility": {"$ref": "#/$defs/eligibility"}
- }
- },
- "sensitivity": {
- "type": "object",
- "additionalProperties": false,
- "required": [
- "sensitivity_id",
- "cohort_id",
- "label",
- "baseline_series_id",
- "candidate_series_id",
- "metric",
- "signed_change_ratio",
- "publication_tier",
- "eligibility"
- ],
- "properties": {
- "sensitivity_id": {"type": "string","pattern": "^cxsensitivity-v1-[0-9a-f]{64}$"},
- "cohort_id": {"type": "string","pattern": "^cxcohort-v1-[0-9a-f]{64}$"},
- "label": {"$ref": "#/$defs/label"},
- "baseline_series_id": {"$ref": "#/$defs/seriesId"},
- "candidate_series_id": {"$ref": "#/$defs/seriesId"},
- "metric": {"$ref": "#/$defs/metric"},
- "signed_change_ratio": {"type": "number"},
- "publication_tier": {"$ref": "#/$defs/publicationTier"},
- "eligibility": {"$ref": "#/$defs/eligibility"}
- }
- }
- }
-}
diff --git a/experimental/CollectiveX/schemas/raw-case-v1.schema.json b/experimental/CollectiveX/schemas/raw-case-v1.schema.json
index 4975157404..5f6c83977f 100644
--- a/experimental/CollectiveX/schemas/raw-case-v1.schema.json
+++ b/experimental/CollectiveX/schemas/raw-case-v1.schema.json
@@ -77,55 +77,6 @@
"type": "object"
},
"nullable_sha256": {"oneOf":[{"type":"null"},{"pattern":"^[0-9a-f]{64}$","type":"string"}]},
- "precision_profile": {
- "enum": [
- "d-bf16.c-bf16",
- "d-fp8-e4m3fn-b128-f32-prequantized.c-bf16",
- "d-fp8-e4m3fn-b128-f32-fused.c-bf16",
- "d-bf16.c-fp8-e4m3fn-direct-cast-noscale",
- "d-fp8-e4m3fn-b128-f32-prequantized.c-fp8-e4m3fn-direct-cast-noscale",
- "d-fp8-e4m3fnuz-b128-f32-prequantized.c-bf16",
- "d-bf16.c-fp8-e4m3fnuz-direct-cast-noscale",
- "d-fp8-e4m3fnuz-b128-f32-prequantized.c-fp8-e4m3fnuz-direct-cast-noscale"
- ]
- },
- "communication_axis": {
- "additionalProperties": false,
- "properties": {
- "alignment_contract": {"enum":["native-bf16-vector-alignment","hidden-block-128","native-fp8-vector-alignment","value-block-64"]},
- "api_input_dtype": {"enum":["bf16","fp8-e4m3fn-with-f32-scale","fp8-e4m3fnuz-with-f32-scale"]},
- "api_output_dtype": {"enum":["bf16","fp8-e4m3fn-with-f32-scale","fp8-e4m3fnuz-with-f32-scale"]},
- "communication_format": {"enum":["bf16","fp8-e4m3fn","fp8-e4m3fnuz"]},
- "conversion_boundary": {"enum":["none","before-dispatch-timing","inside-dispatch-timing","inside-combine-timing"]},
- "padding_contract": {"enum":["none","right-zero-pad-hidden-to-128","right-zero-pad-values-to-64"]},
- "quantization_origin": {"enum":["none","caller-prequantized","backend-fused","backend-internal","backend-internal-direct-cast"]},
- "scale_dtype": {"oneOf":[{"type":"null"},{"enum":["f32"]}]},
- "scale_group_size": {"oneOf":[{"type":"null"},{"enum":[64,128]}]},
- "scale_layout": {"enum":["none","per-token-hidden-block","dynamic-per-64-values"]}
- },
- "required": [
- "alignment_contract","api_input_dtype","api_output_dtype","communication_format",
- "conversion_boundary","padding_contract","quantization_origin","scale_dtype",
- "scale_group_size","scale_layout"
- ],
- "type": "object"
- },
- "communication_precision": {
- "additionalProperties": false,
- "properties": {
- "combine": {"$ref":"#/$defs/communication_axis"},
- "dispatch": {"$ref":"#/$defs/communication_axis"},
- "modes": {
- "items":{"enum":["normal","low-latency"]},
- "minItems":1,
- "type":"array",
- "uniqueItems":true
- },
- "profile_id": {"$ref":"#/$defs/precision_profile"}
- },
- "required": ["combine","dispatch","modes","profile_id"],
- "type": "object"
- },
"byte_accounting": {
"additionalProperties": false,
"properties": {
@@ -139,36 +90,6 @@
],
"type": "object"
},
- "precision_axis_evidence": {
- "additionalProperties": false,
- "properties": {
- "dequantized_semantics": {"type":"boolean"},
- "encoded_payload_valid": {"type":"boolean"},
- "max_abs_error": {"minimum":0,"type":"number"},
- "max_rel_error": {"minimum":0,"type":"number"},
- "passed": {"type":"boolean"},
- "saturation_count": {"minimum":0,"type":"integer"},
- "saturation_rate": {"maximum":1,"minimum":0,"type":"number"},
- "scales_finite": {"oneOf":[{"type":"null"},{"type":"boolean"}]},
- "scales_positive": {"oneOf":[{"type":"null"},{"type":"boolean"}]}
- },
- "required": [
- "dequantized_semantics","encoded_payload_valid","max_abs_error","max_rel_error",
- "passed","saturation_count","saturation_rate","scales_finite","scales_positive"
- ],
- "type":"object"
- },
- "precision_evidence": {
- "additionalProperties": false,
- "properties": {
- "combine": {"$ref":"#/$defs/precision_axis_evidence"},
- "dispatch": {"$ref":"#/$defs/precision_axis_evidence"},
- "passed": {"type":"boolean"},
- "profile_id": {"$ref":"#/$defs/precision_profile"}
- },
- "required": ["combine","dispatch","passed","profile_id"],
- "type":"object"
- },
"case_profile": {
"additionalProperties": false,
"allOf": [
@@ -201,7 +122,6 @@
"combine_dtype": {"const":"bf16"},
"combine_quant_mode": {"const":"none"},
"combine_semantics": {"enum":["activation-only","gate-weighted"]},
- "communication_precision": {"$ref":"#/$defs/communication_precision"},
"component_order_contract": {"const":"qualification-hash-rotated-components-v1"},
"conditioning_contract": {"const":"fixed-phase-ramp-8-roundtrips-v1"},
"contract": {"enum":["layout-and-dispatch-v1","expert-packed-weighted-combine-v1"]},
@@ -351,8 +271,6 @@
"mode": {"enum":["normal","low-latency"]},
"nodes": {"minimum":1,"type":"integer"},
"phase": {"enum":["decode","prefill"]},
- "precision_profile": {"$ref":"#/$defs/precision_profile"},
- "required_publication": {"enum":["official","comparable-experimental"]},
"routing": {"enum":["uniform"]},
"samples_per_point": {"const":512},
"scale_out_transport": {"oneOf":[{"type":"null"},{"maxLength":128,"pattern":"^[a-z0-9][a-z0-9_.-]*$","type":"string"}]},
@@ -379,7 +297,6 @@
"mode",
"nodes",
"phase",
- "required_publication",
"routing",
"samples_per_point",
"scale_out_transport",
@@ -405,40 +322,13 @@
"repo": {"pattern":"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$","type":"string"},
"run_attempt": {"pattern":"^[1-9][0-9]*$","type":"string"},
"run_id": {"pattern":"^[1-9][0-9]*$","type":"string"},
- "qualification_index": {"maximum":3,"minimum":1,"type":"integer"},
"source_sha": {"pattern":"^[0-9a-f]{40}$","type":"string"}
},
- "required": ["artifact","job","qualification_index","ref","repo","run_attempt","run_id","source_sha"],
+ "required": ["artifact","job","ref","repo","run_attempt","run_id","source_sha"],
"type": "object"
}
},
"additionalProperties": false,
- "allOf": [
- {
- "if": {
- "properties": {
- "workload": {
- "properties": {"source": {"const":"canonical-serialized"}},
- "required": ["source"]
- }
- },
- "required": ["workload"]
- },
- "then": {
- "properties": {
- "provenance": {
- "properties": {
- "allocation_stratum_sha256": {
- "pattern":"^[0-9a-f]{64}$",
- "type":"string"
- }
- },
- "required": ["allocation_stratum_sha256"]
- }
- }
- }
- }
- ],
"properties": {
"case": {
"additionalProperties": false,
@@ -507,36 +397,29 @@
},
"mode": {"enum":["normal","low-latency"]},
"phase": {"enum":["decode","prefill"]},
- "required_publication": {"enum":["official","comparable-experimental"]},
"resource_mode": {"const":"fixed-profile"},
"runner": {"maxLength":128,"pattern":"^[a-z0-9][a-z0-9_.-]*$","type":"string"},
"shape": {
"additionalProperties": false,
"properties": {
"activation_profile": {"const":"canonical-counter-source-v4"},
- "combine_precision": {"$ref":"#/$defs/communication_axis"},
- "dispatch_precision": {"$ref":"#/$defs/communication_axis"},
"eplb": {"type":"boolean"},
"experts": {"minimum":1,"type":"integer"},
"experts_per_rank": {"minimum":1,"type":"integer"},
"hidden": {"minimum":1,"type":"integer"},
"kernel_gen": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]},
"num_logical_experts": {"minimum":1,"type":"integer"},
- "precision_profile": {"$ref":"#/$defs/precision_profile"},
"routing": {"enum":["uniform"]},
"topk": {"minimum":1,"type":"integer"}
},
"required": [
"activation_profile",
- "combine_precision",
- "dispatch_precision",
"eplb",
"experts",
"experts_per_rank",
"hidden",
"kernel_gen",
"num_logical_experts",
- "precision_profile",
"routing",
"topk"
],
@@ -552,7 +435,6 @@
"ep_size",
"mode",
"phase",
- "required_publication",
"resource_mode",
"runner",
"shape",
@@ -575,11 +457,10 @@
"repo": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]},
"run_attempt": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]},
"run_id": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]},
- "qualification_index": {"maximum":3,"minimum":1,"type":"integer"},
"runner": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]},
"source_sha": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]}
},
- "required": ["artifact","execution_id","job","qualification_index","repo","run_attempt","run_id","runner","source_sha"],
+ "required": ["artifact","execution_id","job","repo","run_attempt","run_id","runner","source_sha"],
"type": "object"
},
"allocation_id": {"pattern":"^cxallocation-v1-[0-9a-f]{64}$","type":"string"},
@@ -600,24 +481,16 @@
"additionalProperties": false,
"properties": {
"backend": {"maxLength":128,"pattern":"^[a-z0-9][a-z0-9_.-]*$","type":"string"},
- "implementation_contract_sha256": {"pattern":"^[0-9a-f]{64}$","type":"string"},
- "public_config_sha256": {"pattern":"^[0-9a-f]{64}$","type":"string"},
- "routing_control_sha256": {"pattern":"^[0-9a-f]{64}$","type":"string"},
"case_id": {"pattern":"^cxcase-v1-[0-9a-f]{64}$","type":"string"},
"image_digest": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]},
- "runtime_fingerprint_sha256": {"pattern":"^[0-9a-f]{64}$","type":"string"},
"source_sha": {"oneOf":[{"type":"null"},{"pattern":"^[0-9a-f]{40}$","type":"string"}]},
"squash_sha256": {"oneOf":[{"type":"null"},{"pattern":"^[0-9a-f]{64}$","type":"string"}]},
"workload_id": {"pattern":"^cxwork-v1-[0-9a-f]{64}$","type":"string"}
},
"required": [
"backend",
- "implementation_contract_sha256",
- "public_config_sha256",
- "routing_control_sha256",
"case_id",
"image_digest",
- "runtime_fingerprint_sha256",
"source_sha",
"squash_sha256",
"workload_id"
@@ -862,8 +735,6 @@
"type": "object"
},
"contract": {"enum":["layout-and-dispatch-v1","expert-packed-weighted-combine-v1"]},
- "execution_order_sha256": {"pattern":"^[0-9a-f]{64}$","type":"string"},
- "qualification_index": {"maximum":3,"minimum":1,"type":"integer"},
"rows": {
"items": {
"additionalProperties": false,
@@ -904,7 +775,6 @@
"contract": {"enum":["expert-specific-transform-v1","expert-assignment-transform-v1"]},
"max_relative_error": {"minimum":0,"type":"number"},
"passed": {"type":"boolean"},
- "precision": {"$ref":"#/$defs/precision_evidence"},
"rank_evidence": {
"items": {
"additionalProperties": false,
@@ -923,7 +793,7 @@
},
"scope": {"enum":["dispatch-metadata-and-transformed-combine","expert-assignment-and-weighted-combine"]}
},
- "required": ["contract","max_relative_error","passed","precision","rank_evidence","scope"],
+ "required": ["contract","max_relative_error","passed","rank_evidence","scope"],
"type": "object"
},
"evidence_id": {"pattern":"^cxevidence-v1-[0-9a-f]{64}$","type":"string"},
@@ -1110,8 +980,6 @@
"component_order_contract",
"conditioning",
"contract",
- "execution_order_sha256",
- "qualification_index",
"rows",
"sampling",
"source_allocation"
@@ -1121,7 +989,6 @@
"outcome": {
"additionalProperties": false,
"properties": {
- "publication_status": {"enum":["diagnostic","invalid"]},
"reasons": {"items":{"type":"string"},"type":"array"},
"status": {"enum":["success","invalid"]},
"validity": {
@@ -1151,18 +1018,12 @@
"type": "object"
}
},
- "required": ["publication_status","reasons","status","validity"],
+ "required": ["reasons","status","validity"],
"type": "object"
},
"provenance": {
"additionalProperties": false,
"properties": {
- "allocation_stratum_sha256": {
- "oneOf": [
- {"type":"null"},
- {"pattern":"^[0-9a-f]{64}$","type":"string"}
- ]
- },
"command": {"minLength":1,"type":"string"},
"distributed_launcher": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]},
"git_run": {"oneOf":[{"type":"null"},{"$ref":"#/$defs/git_run"}]},
@@ -1182,7 +1043,7 @@
},
"redaction": {"const":"sanitized-v1"}
},
- "required": ["allocation_stratum_sha256","command","distributed_launcher","git_run","image","redaction"],
+ "required": ["command","distributed_launcher","git_run","image","redaction"],
"type": "object"
},
"record_type": {"const":"case-attempt"},
diff --git a/experimental/CollectiveX/schemas/samples-v1.schema.json b/experimental/CollectiveX/schemas/samples-v1.schema.json
index f216860a80..e1c34fda51 100644
--- a/experimental/CollectiveX/schemas/samples-v1.schema.json
+++ b/experimental/CollectiveX/schemas/samples-v1.schema.json
@@ -4,7 +4,7 @@
"title": "CollectiveX exact private samples v1",
"type": "object",
"additionalProperties": false,
- "required": ["allocation_id","attempt_id","case_id","format","points","qualification_index","sampling","schema_version","series_id"],
+ "required": ["allocation_id","attempt_id","case_id","format","points","sampling","schema_version","series_id"],
"properties": {
"allocation_id": {"$ref": "#/$defs/allocationId"},
"attempt_id": {"$ref": "#/$defs/attemptId"},
@@ -36,7 +36,6 @@
}
}
},
- "qualification_index": {"type":"integer","minimum":1,"maximum":3},
"sampling": {
"type": "object",
"additionalProperties": false,
diff --git a/experimental/CollectiveX/schemas/terminal-outcome-v1.schema.json b/experimental/CollectiveX/schemas/terminal-outcome-v1.schema.json
index 64babf7e9d..1ffa88fcae 100644
--- a/experimental/CollectiveX/schemas/terminal-outcome-v1.schema.json
+++ b/experimental/CollectiveX/schemas/terminal-outcome-v1.schema.json
@@ -132,58 +132,17 @@
"reason": {
"enum": [
"backend-platform-unsupported",
- "backend-token-capacity",
- "precision-profile-unsupported"
+ "backend-token-capacity"
]
}
}
},
"sha256": {"type": "string","pattern": "^[0-9a-f]{64}$"},
"safeId": {"type": "string","pattern": "^[a-z0-9][a-z0-9_.-]*$","maxLength": 128},
- "precisionProfile": {
- "enum":[
- "d-bf16.c-bf16",
- "d-fp8-e4m3fn-b128-f32-prequantized.c-bf16",
- "d-fp8-e4m3fn-b128-f32-fused.c-bf16",
- "d-bf16.c-fp8-e4m3fn-direct-cast-noscale",
- "d-fp8-e4m3fn-b128-f32-prequantized.c-fp8-e4m3fn-direct-cast-noscale",
- "d-fp8-e4m3fnuz-b128-f32-prequantized.c-bf16",
- "d-bf16.c-fp8-e4m3fnuz-direct-cast-noscale",
- "d-fp8-e4m3fnuz-b128-f32-prequantized.c-fp8-e4m3fnuz-direct-cast-noscale"
- ]
- },
"caseId": {"type": "string","pattern": "^cxcase-v1-[0-9a-f]{64}$"},
"allocationId": {"type": "string","pattern": "^cxallocation-v1-[0-9a-f]{64}$"},
"attemptId": {"type": "string","pattern": "^cxattempt-v1-[0-9a-f]{64}$"},
"nullableText": {"oneOf": [{"type": "null"},{"type": "string","minLength": 1}]},
- "communicationAxis": {
- "type":"object",
- "additionalProperties":false,
- "required":["alignment_contract","api_input_dtype","api_output_dtype","communication_format","conversion_boundary","padding_contract","quantization_origin","scale_dtype","scale_group_size","scale_layout"],
- "properties": {
- "alignment_contract":{"enum":["native-bf16-vector-alignment","hidden-block-128","native-fp8-vector-alignment","value-block-64"]},
- "api_input_dtype":{"enum":["bf16","fp8-e4m3fn-with-f32-scale","fp8-e4m3fnuz-with-f32-scale"]},
- "api_output_dtype":{"enum":["bf16","fp8-e4m3fn-with-f32-scale","fp8-e4m3fnuz-with-f32-scale"]},
- "communication_format":{"enum":["bf16","fp8-e4m3fn","fp8-e4m3fnuz"]},
- "conversion_boundary":{"enum":["none","before-dispatch-timing","inside-dispatch-timing","inside-combine-timing"]},
- "padding_contract":{"enum":["none","right-zero-pad-hidden-to-128","right-zero-pad-values-to-64"]},
- "quantization_origin":{"enum":["none","caller-prequantized","backend-fused","backend-internal","backend-internal-direct-cast"]},
- "scale_dtype":{"oneOf":[{"type":"null"},{"enum":["f32"]}]},
- "scale_group_size":{"oneOf":[{"type":"null"},{"enum":[64,128]}]},
- "scale_layout":{"enum":["none","per-token-hidden-block","dynamic-per-64-values"]}
- }
- },
- "communicationPrecision": {
- "type":"object",
- "additionalProperties":false,
- "required":["combine","dispatch","modes","profile_id"],
- "properties": {
- "combine":{"$ref":"#/$defs/communicationAxis"},
- "dispatch":{"$ref":"#/$defs/communicationAxis"},
- "modes":{"type":"array","minItems":1,"uniqueItems":true,"items":{"enum":["normal","low-latency"]}},
- "profile_id":{"$ref":"#/$defs/precisionProfile"}
- }
- },
"caseProfile": {
"type":"object",
"additionalProperties":false,
@@ -218,7 +177,6 @@
"combine_dtype":{"const":"bf16"},
"combine_quant_mode":{"const":"none"},
"combine_semantics":{"enum":["activation-only","gate-weighted"]},
- "communication_precision":{"$ref":"#/$defs/communicationPrecision"},
"component_order_contract":{"const":"qualification-hash-rotated-components-v1"},
"conditioning_contract":{"const":"fixed-phase-ramp-8-roundtrips-v1"},
"contract":{"enum":["layout-and-dispatch-v1","expert-packed-weighted-combine-v1"]},
@@ -244,12 +202,11 @@
"allocationFactors": {
"type": "object",
"additionalProperties": false,
- "required": ["artifact","execution_id","job","qualification_index","repo","run_attempt","run_id","runner","source_sha"],
+ "required": ["artifact","execution_id","job","repo","run_attempt","run_id","runner","source_sha"],
"properties": {
"artifact": {"$ref": "#/$defs/nullableText"},
"execution_id": {"$ref": "#/$defs/nullableText"},
"job": {"$ref": "#/$defs/nullableText"},
- "qualification_index": {"type":"integer","minimum":1,"maximum":3},
"repo": {"$ref": "#/$defs/nullableText"},
"run_attempt": {"$ref": "#/$defs/nullableText"},
"run_id": {"$ref": "#/$defs/nullableText"},
@@ -260,11 +217,10 @@
"gitRun": {
"type": "object",
"additionalProperties": false,
- "required": ["artifact","job","qualification_index","ref","repo","run_attempt","run_id","source_sha"],
+ "required": ["artifact","job","ref","repo","run_attempt","run_id","source_sha"],
"properties": {
"artifact": {"type": "string","minLength": 1},
"job": {"type": "string","minLength": 1},
- "qualification_index": {"type":"integer","minimum":1,"maximum":3},
"ref": {"type": "string","minLength": 1},
"repo": {"type": "string","pattern": "^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$"},
"run_attempt": {"type": "string","pattern": "^[1-9][0-9]*$"},
@@ -287,7 +243,6 @@
"mode",
"nodes",
"phase",
- "required_publication",
"routing",
"samples_per_point",
"scale_out_transport",
@@ -314,8 +269,6 @@
"mode": {"enum": ["normal","low-latency"]},
"nodes": {"type": "integer","minimum": 1},
"phase": {"enum": ["decode","prefill"]},
- "precision_profile": {"$ref":"#/$defs/precisionProfile"},
- "required_publication": {"enum": ["official","comparable-experimental","diagnostic"]},
"routing": {"enum": ["uniform"]},
"samples_per_point": {"const": 512},
"scale_out_transport": {"oneOf": [{"type": "null"},{"$ref": "#/$defs/safeId"}]},
diff --git a/experimental/CollectiveX/summarize.py b/experimental/CollectiveX/summarize.py
index 3752db6b9d..df0e1eb53f 100644
--- a/experimental/CollectiveX/summarize.py
+++ b/experimental/CollectiveX/summarize.py
@@ -16,7 +16,7 @@ def load_results(directory: str, runner: str | None, timestamp: str | None) -> l
if timestamp and timestamp not in path.name:
continue
try:
- document = contracts.strict_load(path)
+ document = contracts.strict_json_load(path)
if document.get("format") == contracts.RAW_FORMAT:
documents.append(contracts.load_raw_attempt(path))
elif document.get("format") == contracts.TERMINAL_FORMAT:
@@ -26,7 +26,7 @@ def load_results(directory: str, runner: str | None, timestamp: str | None) -> l
return documents
-def _identity(document: dict) -> tuple[str, str, str, str, bool, str, int]:
+def _identity(document: dict) -> tuple[str, str, str, str, bool, int]:
case = document["case"]
if document["format"] == contracts.RAW_FORMAT:
routing = case["shape"]["routing"]
@@ -37,7 +37,7 @@ def _identity(document: dict) -> tuple[str, str, str, str, bool, str, int]:
sku = document["identity"]["case_factors"]["sku"]
return (
sku, case["suite"], routing, case["phase"], eplb,
- case["required_publication"], case.get("ep_size", case.get("ep", 0)),
+ case.get("ep_size", case.get("ep", 0)),
)
@@ -55,16 +55,16 @@ def render(documents: list[dict], markdown: bool) -> str:
if markdown:
lines = [
"## CollectiveX EP results", "",
- "| sku | backend | suite | phase | routing | tier | ep | outcome | T* | p50 us | p99 us |",
- "|---|---|---|---|---|---|--:|---|--:|--:|--:|",
+ "| sku | backend | suite | phase | routing | ep | outcome | T* | p50 us | p99 us |",
+ "|---|---|---|---|---|--:|---|--:|--:|--:|",
]
for document in documents:
- sku, suite, routing, phase, eplb, tier, ep = _identity(document)
+ sku, suite, routing, phase, eplb, ep = _identity(document)
backend = document["case"]["backend"]
token, p50, p99 = _headline(document)
lines.append(
f"| {sku} | `{backend}` | {suite} | {phase} | "
- f"{routing}{'+eplb' if eplb else ''} | {tier} | {ep} | "
+ f"{routing}{'+eplb' if eplb else ''} | {ep} | "
f"{document['outcome']['status']} | {token} | {p50} | {p99} |"
)
if not documents:
@@ -72,12 +72,12 @@ def render(documents: list[dict], markdown: bool) -> str:
return "\n".join(lines)
lines = ["CollectiveX EP results", "======================"]
for document in documents:
- sku, suite, routing, phase, eplb, tier, ep = _identity(document)
+ sku, suite, routing, phase, eplb, ep = _identity(document)
backend = document["case"]["backend"]
token, _, p99 = _headline(document)
lines.append(
f" {sku:<10} {backend:<16} {suite:<13} {phase:<7} "
- f"{routing}{'+eplb' if eplb else ''} {tier} ep{ep} "
+ f"{routing}{'+eplb' if eplb else ''} ep{ep} "
f"{document['outcome']['status']} T={token} roundtrip_p99_us={p99}"
)
return "\n".join(lines)
diff --git a/experimental/CollectiveX/sweep_matrix.py b/experimental/CollectiveX/sweep_matrix.py
index 2df801733a..5f0efdb82f 100644
--- a/experimental/CollectiveX/sweep_matrix.py
+++ b/experimental/CollectiveX/sweep_matrix.py
@@ -1,9 +1,9 @@
#!/usr/bin/env python3
-"""Resolve CollectiveX v1 suites and extract validated execution shards.
+"""Resolve CollectiveX suites and extract validated execution shards.
Mode changes measurement semantics and therefore participates in case identity.
-Precision sensitivity uses allowlisted communication profiles; provisional native
-paths remain outside the executable matrix until their probes are resolved.
+Dispatch and combine are fixed BF16 benchmark facts, so a case's coordinates are
+suite/workload/backend/topology only; the matrix schedules, it never ranks.
"""
from __future__ import annotations
@@ -41,7 +41,6 @@
V1_SUITE_CONTRACTS = {
"ep-core-v1": {
"mode": "normal",
- "publication": "official",
"coordinates": {
("normal", "decode", "uniform", False),
("normal", "prefill", "uniform", False),
@@ -53,51 +52,24 @@
},
"ep-low-latency-v1": {
"mode": "low-latency",
- "publication": "official",
"backends": {"deepep", "uccl"},
"coordinates": {("low-latency", "decode", "uniform", False)},
"ladders": {"decode": tuple(ep_harness.DECODE_LADDER)},
},
- "ep-precision-normal-v1": {
- "mode": "normal",
- "publication": "comparable-experimental",
- "backends": {"deepep", "deepep-v2", "uccl", "deepep-hybrid", "mori"},
- "precision_profiles": identity.V1_NORMAL_PRECISION_PROFILE_IDS,
- "coordinates": {
- ("normal", "decode", "uniform", False),
- ("normal", "prefill", "uniform", False),
- },
- "ladders": {"decode": (128,), "prefill": (512,)},
- },
- "ep-precision-low-latency-v1": {
- "mode": "low-latency",
- "publication": "comparable-experimental",
- "backends": {"deepep", "uccl"},
- "precision_profiles": identity.V1_LOW_LATENCY_PRECISION_PROFILE_IDS,
- "coordinates": {("low-latency", "decode", "uniform", False)},
- "ladders": {"decode": (128,)},
- },
}
IDENTIFIER = re.compile(r"[a-z0-9][a-z0-9.-]*")
SUITE_FIELDS = {
"backends", "ep_degrees", "eplb", "mode", "phases", "platforms",
- "precision_profiles", "provisional", "required_publication", "routings",
- "token_points", "token_points_decode", "token_points_prefill", "workloads",
+ "routings", "token_points", "token_points_decode", "token_points_prefill",
+ "workloads",
}
SUITE_REQUIRED = {
- "ep_degrees", "mode", "phases", "platforms", "required_publication", "routings",
- "workloads",
+ "ep_degrees", "mode", "phases", "platforms", "routings", "workloads",
}
TOPOLOGY_FIELDS = (
"nodes", "gpus_per_node", "scale_up_domain", "scope", "scale_up_transport",
"scale_out_transport", "transport", "topology_class",
)
-QUALIFICATION_INDICES = range(1, 4)
-CANONICAL_V1_EXECUTION_PLAN_SHA256 = {
- 1: "1636f01be0434f5e7e7a633f20b8ee52088d4eaa0528b333544a87f7a6e86950",
- 2: "8d98af40cb054f7b686c0b3ecfaa0f340fbab7e3bf34b9696f7256c647307992",
- 3: "fd6fafb06e2ba29374946e173876e7c27684391fb0f4a2c6590779ea804e62ee",
-}
class MatrixError(ValueError):
@@ -229,52 +201,6 @@ def validate_config_documents(
raise SystemExit(f"suite {name}.backends differs from the frozen v1 catalog")
if expected_backends is None and "backends" in suite:
raise SystemExit(f"suite {name}.backends must be omitted")
- expected_profiles = contract.get("precision_profiles")
- if expected_profiles is None:
- if "precision_profiles" in suite or "provisional" in suite:
- raise SystemExit(
- f"suite {name} cannot add precision fields to a baseline suite"
- )
- precision_profiles: list[str] = []
- else:
- precision_profiles = _list(
- suite.get("precision_profiles"),
- f"suite {name}.precision_profiles",
- str,
- set(identity.V1_PRECISION_PROFILES),
- )
- if tuple(precision_profiles) != expected_profiles:
- raise SystemExit(
- f"suite {name}.precision_profiles differs from the frozen v1 catalog"
- )
- if identity.V1_CONTROL_PRECISION_PROFILE in precision_profiles:
- raise SystemExit(
- f"suite {name} must reference existing BF16 evidence, not duplicate it"
- )
- if any(
- mode not in identity.V1_PRECISION_PROFILES[profile]["modes"]
- for profile in precision_profiles
- ):
- raise SystemExit(f"suite {name} contains a precision profile for another mode")
- if type(suite.get("provisional")) is not bool:
- raise SystemExit(f"suite {name}.provisional must be a boolean")
- unresolved = cap.provisional_precision_targets(precision_profiles)
- if suite["provisional"] != bool(unresolved):
- raise SystemExit(
- f"suite {name}.provisional must track unresolved capability targets"
- )
- candidates = cap.precision_targets(precision_profiles)
- covered_candidates = [
- target for target in candidates
- if target["backend"] in suite_backends
- and target["sku"] in suite["platforms"]
- and target["ep"] in suite["ep_degrees"]
- and target["mode"] == mode
- ]
- if covered_candidates != candidates:
- raise SystemExit(
- f"suite {name} does not cover every declared precision target"
- )
suite_workloads = _list(suite["workloads"], f"suite {name}.workloads", str)
unknown = sorted(set(suite_workloads) - set(registry))
if unknown:
@@ -288,16 +214,6 @@ def validate_config_documents(
eplb = _list(suite.get("eplb", [False]), f"suite {name}.eplb", bool)
if True in eplb:
raise SystemExit(f"suite {name}: EPLB is unavailable for v1 uniform routing")
- if suite["required_publication"] not in {"official", "comparable-experimental"}:
- raise SystemExit(f"suite {name}.required_publication is invalid")
- if suite["required_publication"] != contract["publication"]:
- raise SystemExit(
- f"suite {name}.required_publication differs from the frozen v1 catalog"
- )
- if suite["required_publication"] == "official":
- unverified = [item for item in suite_workloads if not registry[item].get("verified_against")]
- if unverified:
- raise SystemExit(f"suite {name}: official workloads need verified_against: {unverified}")
degrees = _list(suite["ep_degrees"], f"suite {name}.ep_degrees", int)
if degrees != [8, 16]:
raise SystemExit(f"suite {name}.ep_degrees must be exactly [8, 16]")
@@ -352,20 +268,15 @@ def _ladder(suite: dict[str, Any], phase: str) -> str:
def _v1_requested_ladder(case: dict[str, Any]) -> str:
"""Bind extracted controls to the frozen v1 suite and workload catalog."""
suite = V1_SUITE_CONTRACTS.get(case.get("suite"))
- expected_profiles = None if suite is None else suite.get("precision_profiles")
- precision_profile = case.get("precision_profile")
coordinate = (
case.get("mode"), case.get("phase"), case.get("routing"), case.get("eplb")
)
if (
suite is None
or coordinate not in suite["coordinates"]
- or case.get("required_publication") != suite["publication"]
or (
case.get("workload"), case.get("hidden"), case.get("topk"), case.get("experts")
) != V1_WORKLOAD
- or (expected_profiles is None and precision_profile is not None)
- or (expected_profiles is not None and precision_profile not in expected_profiles)
):
raise MatrixError("case differs from the frozen v1 suite/workload catalog")
return " ".join(map(str, suite["ladders"][case["phase"]]))
@@ -375,19 +286,9 @@ def _expected_disposition(
sku: str, case: dict[str, Any]
) -> tuple[str, str | None, str | None]:
requested_ladder = _v1_requested_ladder(case)
- precision_profile = case.get("precision_profile")
- if precision_profile is not None and not cap.precision_target_declared(
- precision_profile,
- sku=sku,
- backend=case["backend"],
- ep=case["ep"],
- mode=case["mode"],
- ):
- raise MatrixError("precision case is not an exact native capability target")
disposition, detail = cap.resolve_disposition(
sku, case["backend"], ep=case["ep"], nodes=case["nodes"],
routing=case["routing"], eplb=case["eplb"], mode=case["mode"],
- precision_profile=precision_profile,
)
if disposition == "supported":
if case["ladder"] != requested_ladder:
@@ -396,15 +297,8 @@ def _expected_disposition(
if case["ladder"] != requested_ladder:
raise MatrixError("unsupported case ladder differs from the frozen v1 suite catalog")
if disposition == "unsupported":
- reason = (
- "precision-profile-unsupported"
- if precision_profile is not None
- else "backend-platform-unsupported"
- )
- return "unsupported", reason, detail
- if disposition == "provisional":
- raise MatrixError("provisional precision target entered the executable matrix")
- raise MatrixError("not-applicable precision tuple entered the requested matrix")
+ return "unsupported", "backend-platform-unsupported", detail
+ raise MatrixError(f"invalid capability disposition {disposition!r}")
def _case_id(sku: str, case: dict[str, Any]) -> str:
@@ -416,7 +310,7 @@ def _case_id(sku: str, case: dict[str, Any]) -> str:
def _semantic_points(sku: str, case: dict[str, Any]) -> list[str]:
execution = {
key: value for key, value in case.items()
- if key not in {"canonical", "case_id", "ladder", "required_publication", "suite", "workload"}
+ if key not in {"canonical", "case_id", "ladder", "suite", "workload"}
}
return [
json.dumps(
@@ -452,22 +346,17 @@ def resolve_matrix(
backends: str = "",
only_sku: str = "",
exclude_skus: str = "",
- min_nodes: int = 0,
- max_nodes: int = 0,
ep_sizes: str = "",
max_cases: int = 128,
) -> dict[str, Any]:
"""Resolve suite configuration into allocation-sized workflow shards."""
if max_cases <= 0:
raise SystemExit("--max-cases must be positive")
- if min_nodes < 0 or max_nodes < 0 or (min_nodes and max_nodes and min_nodes > max_nodes):
- raise SystemExit("invalid node bounds")
- # --ep-sizes narrows the matrix to specific expert-parallel degrees, dispatch-time
- # like --max-nodes: "8" keeps every EP8 shard and drops EP16, so a comprehensive run
- # can co-schedule the 8-GPU SKUs' single-node EP8 with the GB SKUs' two-node EP8
- # without dispatching any EP16 leg. Blank keeps every degree. The resulting matrix is
- # a partial (still publishable) subset; the publisher binds every promoted case to the
- # canonical catalog, so a filtered matrix cannot invent or reclassify cases.
+ # --ep-sizes narrows the matrix to specific expert-parallel degrees at dispatch
+ # time: "8" keeps every EP8 shard and drops EP16, so a comprehensive run can
+ # co-schedule the 8-GPU SKUs' single-node EP8 with the GB SKUs' two-node EP8
+ # without dispatching any EP16 leg. Blank keeps every degree. The resulting
+ # matrix is a partial subset that only omits cases; it never reclassifies them.
selected_eps: set[int] = set()
for value in (part.strip() for part in ep_sizes.split(",")):
if not value:
@@ -477,11 +366,8 @@ def resolve_matrix(
selected_eps.add(int(value))
if only_sku and only_sku not in cap.PLATFORMS:
raise SystemExit(f"unknown --only-sku {only_sku!r}; have {sorted(cap.PLATFORMS)}")
- # --exclude-skus narrows the matrix to a partial (still publishable) subset by
- # dropping whole runner pools — e.g. exclude a SKU whose cluster is unavailable
- # so a versioned "tagged + success" run covers only the SKUs it can actually
- # exercise. The publisher binds every promoted case to the canonical catalog,
- # so a narrowed matrix cannot invent or reclassify cases; it only omits them.
+ # --exclude-skus narrows the matrix to a subset by dropping whole runner pools
+ # — e.g. exclude a SKU whose cluster is unavailable. It only omits cases.
excluded = {value.strip() for value in exclude_skus.split(",") if value.strip()}
unknown_excluded = sorted(excluded - set(cap.PLATFORMS))
if unknown_excluded:
@@ -497,7 +383,7 @@ def resolve_matrix(
registry = suites_document["suites"]
select_all = suites == "all"
names = (
- [name for name, suite in registry.items() if not suite.get("provisional", False)]
+ list(registry)
if select_all
else [value.strip() for value in suites.split(",") if value.strip()]
)
@@ -506,16 +392,6 @@ def resolve_matrix(
unknown = sorted(set(names) - set(registry))
if unknown:
raise SystemExit(f"unknown suites {unknown}; have {sorted(registry)}")
- blocked = [name for name in names if registry[name].get("provisional", False)]
- if blocked:
- unresolved = sum(
- len(cap.provisional_precision_targets(registry[name]["precision_profiles"]))
- for name in blocked
- )
- raise SystemExit(
- f"provisional precision suites cannot be scheduled: {blocked}; "
- f"resolve {unresolved} capability targets first"
- )
targets = _select_backends(backend, backends)
shards: dict[tuple[str, str, int], list[dict[str, Any]]] = {}
@@ -527,7 +403,6 @@ def resolve_matrix(
phases = suite["phases"]
routings = suite["routings"]
eplb_values = suite.get("eplb", [False])
- precision_profiles = suite.get("precision_profiles", [None])
suite_backends = set(suite.get("backends", cap.SWEEP_BACKENDS))
suite_targets = [target for target in targets if target in suite_backends]
if not suite_targets:
@@ -538,30 +413,18 @@ def resolve_matrix(
if platform_name in excluded:
continue
ep_degrees = suite["ep_degrees"]
- for workload, ep, phase, routing, eplb, target, precision_profile in itertools.product(
+ for workload, ep, phase, routing, eplb, target in itertools.product(
suite["workloads"], ep_degrees, phases, routings, eplb_values,
- suite_targets, precision_profiles,
+ suite_targets,
):
if selected_eps and ep not in selected_eps:
continue
- if precision_profile is not None and not cap.precision_target_declared(
- precision_profile,
- sku=platform_name,
- backend=target,
- ep=ep,
- mode=mode,
- ):
- continue
topology = cap.topology_for(platform_name, ep)
if topology is None:
raise SystemExit(
f"suite {suite_name}: {platform_name} EP{ep} is not registered"
)
nodes = int(topology["nodes"])
- if min_nodes and nodes < min_nodes:
- continue
- if max_nodes and nodes > max_nodes:
- continue
capability_disposition, capability_detail = cap.resolve_disposition(
platform_name,
target,
@@ -570,7 +433,6 @@ def resolve_matrix(
routing=routing,
eplb=bool(eplb),
mode=mode,
- precision_profile=precision_profile,
)
hidden, topk, experts = _dims(workloads, workload)
@@ -583,7 +445,6 @@ def add_case(
case: dict[str, Any] = {
"suite": suite_name,
"workload": workload,
- "required_publication": suite["required_publication"],
"backend": target,
"routing": routing,
"phase": phase,
@@ -600,8 +461,6 @@ def add_case(
"canonical": True,
**{field: topology[field] for field in TOPOLOGY_FIELDS},
}
- if precision_profile is not None:
- case["precision_profile"] = precision_profile
for signature in _semantic_points(platform_name, case):
if signature in scheduled:
raise SystemExit(
@@ -622,22 +481,11 @@ def add_case(
shards.setdefault((platform_name, target, nodes), []).append(case)
requested_ladder = _ladder(suite, phase)
- if capability_disposition == "not-applicable":
- continue
- if capability_disposition == "provisional":
- raise SystemExit(
- f"suite {suite_name}: provisional target escaped its suite gate: "
- f"{precision_profile} {target} {platform_name} EP{ep}"
- )
if capability_disposition == "unsupported":
add_case(
requested_ladder,
"unsupported",
- (
- "precision-profile-unsupported"
- if precision_profile is not None
- else "backend-platform-unsupported"
- ),
+ "backend-platform-unsupported",
capability_detail,
)
continue
@@ -714,118 +562,6 @@ def _positive_int(value: Any, field: str) -> int:
return value
-def _qualification_index(value: Any, field: str = "qualification_index") -> int:
- if type(value) is not int or value not in QUALIFICATION_INDICES:
- raise MatrixError(f"{field} must be an integer in 1..3")
- return value
-
-
-def _requested_qualification_index(value: int | None = None) -> int:
- if value is not None:
- return _qualification_index(value)
- raw = os.environ.get("CX_QUALIFICATION_INDEX", "1")
- if raw not in {"1", "2", "3"}:
- raise MatrixError("CX_QUALIFICATION_INDEX must be an integer in 1..3")
- return int(raw)
-
-
-def _case_precision_profile(case: dict[str, Any]) -> str:
- profile = case.get("precision_profile", identity.V1_CONTROL_PRECISION_PROFILE)
- if not isinstance(profile, str) or profile not in identity.V1_PRECISION_PROFILES:
- raise MatrixError("qualification case has an invalid precision profile")
- return profile
-
-
-def _qualification_digest(
- shard_id: str,
- case_id: str,
- profile_id: str,
- qualification_index: int,
-) -> bytes:
- return hashlib.sha256(
- "\0".join((shard_id, case_id, profile_id, str(qualification_index))).encode()
- ).digest()
-
-
-def _rotate(values: list[Any], offset: int) -> list[Any]:
- if not values:
- return []
- position = offset % len(values)
- return values[position:] + values[:position]
-
-
-def _seeded_qualification_order(
- shard_id: str,
- cases: list[dict[str, Any]],
- qualification_index: int,
-) -> list[dict[str, Any]]:
- by_profile: dict[str, list[dict[str, Any]]] = {}
- for case in cases:
- by_profile.setdefault(_case_precision_profile(case), []).append(case)
- profiles = sorted(
- by_profile,
- key=lambda profile: _qualification_digest(
- shard_id, "profile-order", profile, 1
- ),
- )
- profiles = _rotate(profiles, qualification_index - 1)
- groups: dict[str, list[dict[str, Any]]] = {}
- for profile in profiles:
- group = sorted(
- by_profile[profile],
- key=lambda case: _qualification_digest(
- shard_id, case["case_id"], profile, qualification_index
- ),
- )
- groups[profile] = _rotate(group, qualification_index - 1)
- interleaved = [
- groups[profile][position]
- for position in range(max(map(len, groups.values())))
- for profile in profiles
- if position < len(groups[profile])
- ]
- return interleaved
-
-
-def qualification_execution_order(
- shard_id: str,
- cases: list[dict[str, Any]],
- qualification_index: int,
-) -> list[dict[str, Any]]:
- """Return one deterministic, repeat-specific permutation of a shard's cases."""
- index = _qualification_index(qualification_index)
- if not isinstance(shard_id, str) or not IDENTIFIER.fullmatch(shard_id):
- raise MatrixError("qualification shard ID is invalid")
- if not isinstance(cases, list) or not cases:
- raise MatrixError("qualification planning requires at least one case")
- selected: list[dict[str, Any]] = []
- seen: set[tuple[str, ...]] = set()
- for current in range(1, index + 1):
- candidate = _seeded_qualification_order(shard_id, cases, current)
- signature = tuple(case["case_id"] for case in candidate)
- if len(cases) >= current and signature in seen:
- for offset in range(1, len(candidate)):
- rotated = _rotate(candidate, offset)
- rotated_signature = tuple(case["case_id"] for case in rotated)
- if rotated_signature not in seen:
- candidate = rotated
- signature = rotated_signature
- break
- seen.add(signature)
- selected = candidate
- return selected
-
-
-def execution_plan_sha256(cases: list[dict[str, Any]]) -> str:
- """Bind an execution plan to only its ordered case and precision-profile IDs."""
- plan = [
- [case["case_id"], _case_precision_profile(case)]
- for case in cases
- ]
- payload = json.dumps(plan, ensure_ascii=True, separators=(",", ":")).encode()
- return hashlib.sha256(payload).hexdigest()
-
-
def execution_weight(cases: list[dict[str, Any]]) -> int:
"""Return deterministic GPU-point work used to bound workflow parallelism."""
if not isinstance(cases, list) or not cases:
@@ -840,35 +576,6 @@ def execution_weight(cases: list[dict[str, Any]]) -> int:
return weight
-def qualification_execution_plan_sha256(
- matrix: dict[str, Any], qualification_index: int
-) -> str:
- """Bind one qualification repeat to every shard's ordered case plan."""
- index = _qualification_index(qualification_index)
- document = validate_matrix_document(matrix)
- requested = {
- item["case"]["case_id"]: item["case"]
- for item in document["requested_cases"]
- }
- plan = []
- for shard in sorted(document["include"], key=lambda item: item["id"]):
- ordered = qualification_execution_order(
- shard["id"],
- [requested[case_id] for case_id in shard["case_ids"]],
- index,
- )
- plan.append([
- shard["id"], shard["execution_weight"], execution_plan_sha256(ordered),
- ])
- payload = json.dumps(
- {"qualification_index": index, "shards": plan},
- ensure_ascii=True,
- sort_keys=True,
- separators=(",", ":"),
- ).encode()
- return hashlib.sha256(payload).hexdigest()
-
-
def validate_shard_control(
shard: dict[str, Any],
*,
@@ -876,7 +583,6 @@ def validate_shard_control(
backend: str,
nodes: int,
require_runnable: bool = True,
- qualification_index: int | None = None,
) -> None:
"""Validate one shard against the workflow cell that requested it."""
if not isinstance(shard, dict):
@@ -885,7 +591,7 @@ def validate_shard_control(
raise MatrixError("shard platform/backend is not registered")
top_fields = {
"schema_version", "id", "sku", "backend", "nodes", "n", "cases",
- "qualification_index", "execution_plan_sha256", "execution_weight",
+ "execution_weight",
}
if (
set(shard) != top_fields
@@ -895,19 +601,6 @@ def validate_shard_control(
raise MatrixError("shard fields or schema version differ from v1 contract")
if not isinstance(shard.get("id"), str) or not IDENTIFIER.fullmatch(shard["id"]):
raise MatrixError("shard has invalid id")
- observed_qualification = _qualification_index(
- shard.get("qualification_index"), "shard.qualification_index"
- )
- if (
- qualification_index is not None
- and observed_qualification != _qualification_index(qualification_index)
- ):
- raise MatrixError("shard qualification_index differs from the requested repeat")
- if (
- not isinstance(shard.get("execution_plan_sha256"), str)
- or re.fullmatch(r"[0-9a-f]{64}", shard["execution_plan_sha256"]) is None
- ):
- raise MatrixError("shard execution_plan_sha256 is invalid")
for field, expected in (("sku", sku), ("backend", backend)):
if shard.get(field) != expected:
raise MatrixError(
@@ -923,21 +616,14 @@ def validate_shard_control(
if _positive_int(shard.get("n"), "shard.n") != len(cases):
raise MatrixError("shard.n does not match the number of cases")
seen: set[str] = set()
- base_required = {
- "case_id", "suite", "workload", "required_publication", "backend", "routing",
- "mode", "phase", "ep", "eplb", "hidden", "topk", "experts",
- "samples_per_point",
- "warmup_semantics", "ladder", "timing", "canonical",
+ required = {
+ "case_id", "suite", "workload", "backend", "routing", "mode", "phase", "ep",
+ "eplb", "hidden", "topk", "experts", "samples_per_point", "warmup_semantics",
+ "ladder", "timing", "canonical",
} | set(TOPOLOGY_FIELDS)
for index, case in enumerate(cases):
if not isinstance(case, dict):
raise MatrixError(f"case {index} must be a JSON object")
- suite_contract = V1_SUITE_CONTRACTS.get(case.get("suite"))
- required = base_required | (
- {"precision_profile"}
- if suite_contract is not None and "precision_profiles" in suite_contract
- else set()
- )
fields = set(case)
if fields != required:
raise MatrixError(
@@ -951,24 +637,16 @@ def validate_shard_control(
raise MatrixError(f"duplicate case_id {case_id}")
seen.add(case_id)
string_fields = [
- "suite", "workload", "required_publication", "backend", "mode", "routing",
- "phase", "warmup_semantics", "ladder", "timing",
+ "suite", "workload", "backend", "mode", "routing", "phase",
+ "warmup_semantics", "ladder", "timing",
]
- if "precision_profile" in required:
- string_fields.append("precision_profile")
for field in string_fields:
if not isinstance(case[field], str) or not case[field]:
raise MatrixError(f"case {index}.{field} must be a non-empty string")
- identifier_fields = [
- "suite", "workload", "required_publication", "backend", "routing", "phase",
- ]
- if "precision_profile" in required:
- identifier_fields.append("precision_profile")
+ identifier_fields = ["suite", "workload", "backend", "routing", "phase"]
for field in identifier_fields:
if not IDENTIFIER.fullmatch(case[field]):
raise MatrixError(f"case {index}.{field} is not a safe identifier")
- if case["required_publication"] not in {"official", "comparable-experimental"}:
- raise MatrixError(f"case {index} has invalid publication requirement")
case_identity = {key: value for key, value in case.items() if key != "case_id"}
if case_id != _case_id(sku, case_identity):
raise MatrixError(f"case {index} case_id does not match its contents")
@@ -1032,15 +710,6 @@ def validate_shard_control(
shard.get("execution_weight"), "shard.execution_weight"
) != execution_weight(cases):
raise MatrixError("shard execution_weight differs from its cases")
- expected_order = qualification_execution_order(
- shard["id"], cases, observed_qualification
- )
- if [case["case_id"] for case in cases] != [
- case["case_id"] for case in expected_order
- ]:
- raise MatrixError("shard cases differ from the qualification execution order")
- if shard["execution_plan_sha256"] != execution_plan_sha256(cases):
- raise MatrixError("shard execution_plan_sha256 differs from its ordered cases")
def validate_matrix_document(document: Any) -> dict[str, Any]:
@@ -1105,10 +774,6 @@ def validate_matrix_document(document: Any) -> dict[str, Any]:
"nodes": nodes,
"n": 1,
"execution_weight": execution_weight(requested_case_plan),
- "qualification_index": 1,
- "execution_plan_sha256": execution_plan_sha256(
- requested_case_plan
- ),
"cases": requested_case_plan,
},
sku=sku,
@@ -1189,10 +854,8 @@ def extract_shard(
sku: str,
backend: str,
nodes: int,
- qualification_index: int | None = None,
) -> dict[str, Any]:
"""Extract one strictly matched shard control file, writing it atomically."""
- qualification = _requested_qualification_index(qualification_index)
document = validate_matrix_document(_strict_json_load(Path(matrix_path)))
include = document["include"]
matches = [item for item in include if isinstance(item, dict) and item.get("id") == shard_id]
@@ -1203,11 +866,7 @@ def extract_shard(
item["case"]["case_id"]: item
for item in document["requested_cases"]
}
- cases = qualification_execution_order(
- source["id"],
- [requested[case_id]["case"] for case_id in source["case_ids"]],
- qualification,
- )
+ cases = [requested[case_id]["case"] for case_id in source["case_ids"]]
control = {
"schema_version": 1,
"id": source.get("id"),
@@ -1216,17 +875,9 @@ def extract_shard(
"nodes": source.get("nodes"),
"n": source.get("n"),
"execution_weight": source.get("execution_weight"),
- "qualification_index": qualification,
- "execution_plan_sha256": execution_plan_sha256(cases),
"cases": cases,
}
- validate_shard_control(
- control,
- sku=sku,
- backend=backend,
- nodes=nodes,
- qualification_index=qualification,
- )
+ validate_shard_control(control, sku=sku, backend=backend, nodes=nodes)
output = Path(output_path)
output.parent.mkdir(parents=True, exist_ok=True)
temporary = output.with_name(f".{output.name}.tmp-{os.getpid()}")
@@ -1248,32 +899,26 @@ def emit_unsupported(
document = validate_matrix_document(_strict_json_load(source))
control_sha256 = hashlib.sha256(source.read_bytes()).hexdigest()
generated_at = dt.datetime.now(dt.timezone.utc).isoformat()
- try:
- qualification_index = int(os.environ.get("CX_QUALIFICATION_INDEX", "1"))
- except ValueError as exc:
- raise MatrixError("CX_QUALIFICATION_INDEX must be an integer in 1..3") from exc
- if qualification_index not in range(1, 4):
- raise MatrixError("CX_QUALIFICATION_INDEX must be in 1..3")
git_run = {
"run_id": os.environ.get("GITHUB_RUN_ID"),
"run_attempt": os.environ.get("GITHUB_RUN_ATTEMPT"),
- "qualification_index": qualification_index,
"ref": os.environ.get("GITHUB_REF_NAME") or os.environ.get("GITHUB_REF"),
"source_sha": os.environ.get("COLLECTIVEX_SOURCE_SHA") or os.environ.get("GITHUB_SHA"),
"repo": os.environ.get("GITHUB_REPOSITORY"),
"job": os.environ.get("GITHUB_JOB"),
"artifact": os.environ.get("COLLECTIVEX_ARTIFACT_NAME"),
}
+ if not any(value is not None for value in git_run.values()):
+ git_run = None
allocation_factors = {
- "artifact": git_run["artifact"],
+ "artifact": git_run["artifact"] if git_run is not None else None,
"execution_id": os.environ.get("COLLECTIVEX_EXECUTION_ID"),
- "job": git_run["job"],
- "qualification_index": qualification_index,
- "repo": git_run["repo"],
- "run_attempt": git_run["run_attempt"],
- "run_id": git_run["run_id"],
+ "job": git_run["job"] if git_run is not None else None,
+ "repo": git_run["repo"] if git_run is not None else None,
+ "run_attempt": git_run["run_attempt"] if git_run is not None else None,
+ "run_id": git_run["run_id"] if git_run is not None else None,
"runner": "capability-resolver",
- "source_sha": git_run["source_sha"],
+ "source_sha": git_run["source_sha"] if git_run is not None else None,
}
destination = Path(output_dir)
destination.mkdir(parents=True, exist_ok=True)
@@ -1291,10 +936,9 @@ def emit_unsupported(
case_id = identity.digest("case", case_factors)
if case_id != scheduled["case_id"]:
raise MatrixError(f"unsupported case identity differs for {scheduled['case_id']}")
- attempt_ordinal = 1
record = contracts.make_terminal_document(
allocation_factors=allocation_factors,
- attempt_ordinal=attempt_ordinal,
+ attempt_ordinal=1,
case=case,
case_factors=case_factors,
control_sha256=control_sha256,
@@ -1327,17 +971,8 @@ def frontend_catalog(matrix: dict[str, Any]) -> dict[str, Any]:
document = validate_matrix_document(matrix)
matrix_bytes = contracts.canonical_json_bytes(document) + b"\n"
cases = []
- precision_profiles: dict[str, dict[str, Any]] = {}
for wrapper in document["requested_cases"]:
case = wrapper["case"]
- precision_profile = case.get(
- "precision_profile", identity.V1_CONTROL_PRECISION_PROFILE
- )
- precision = identity.precision_profile(precision_profile)
- precision_profiles[precision_profile] = {
- "dispatch": precision["dispatch"],
- "combine": precision["combine"],
- }
cases.append({
"backend": case["backend"],
"backend_generation": None,
@@ -1350,8 +985,6 @@ def frontend_catalog(matrix: dict[str, Any]) -> dict[str, Any]:
),
"mode": case["mode"],
"phase": case["phase"],
- "precision_profile": precision_profile,
- "publication_tier": case["required_publication"],
"reason": wrapper["reason"],
"required": True,
"resource": {
@@ -1381,14 +1014,13 @@ def frontend_catalog(matrix: dict[str, Any]) -> dict[str, Any]:
"format": "collectivex.frontend-catalog.v1",
"matrix_sha256": hashlib.sha256(matrix_bytes).hexdigest(),
"point_count": sum(len(case["points"]) for case in cases),
- "precision_profiles": dict(sorted(precision_profiles.items())),
"schema_version": 1,
"cases": cases,
}
def main() -> int:
- parser = argparse.ArgumentParser(description="CollectiveX v1 matrix resolver")
+ parser = argparse.ArgumentParser(description="CollectiveX matrix resolver")
parser.add_argument("--suites", default="all", help="'all' or comma-list of suites")
parser.add_argument("--backend", default="", help="select one EP backend")
parser.add_argument("--backends", default="", help="'all' or comma-list of EP backends")
@@ -1398,8 +1030,6 @@ def main() -> int:
default="",
help="comma-list of runner pools to drop (partial matrix); disjoint from --only-sku",
)
- parser.add_argument("--min-nodes", type=int, default=0)
- parser.add_argument("--max-nodes", type=int, default=0)
parser.add_argument(
"--ep-sizes",
default="",
@@ -1415,7 +1045,6 @@ def main() -> int:
parser.add_argument("--expect-sku", default="")
parser.add_argument("--expect-backend", default="")
parser.add_argument("--expect-nodes", type=int, default=0)
- parser.add_argument("--qualification-index", type=int, default=None)
parser.add_argument("--out", default="")
args = parser.parse_args()
@@ -1436,15 +1065,11 @@ def main() -> int:
)
try:
control = _strict_json_load(Path(args.validate_control))
- qualification = _requested_qualification_index(
- args.qualification_index
- )
validate_shard_control(
control,
sku=args.expect_sku,
backend=args.expect_backend,
nodes=args.expect_nodes,
- qualification_index=qualification,
)
except MatrixError as exc:
parser.error(str(exc))
@@ -1465,7 +1090,6 @@ def main() -> int:
sku=args.expect_sku,
backend=args.expect_backend,
nodes=args.expect_nodes,
- qualification_index=args.qualification_index,
)
except MatrixError as exc:
parser.error(str(exc))
@@ -1479,8 +1103,6 @@ def main() -> int:
backends=args.backends,
only_sku=args.only_sku,
exclude_skus=args.exclude_skus,
- min_nodes=args.min_nodes,
- max_nodes=args.max_nodes,
ep_sizes=args.ep_sizes,
max_cases=args.max_cases,
)
diff --git a/experimental/CollectiveX/tests/ep_harness.py b/experimental/CollectiveX/tests/ep_harness.py
index fd1f3a29a7..6a94523ddd 100644
--- a/experimental/CollectiveX/tests/ep_harness.py
+++ b/experimental/CollectiveX/tests/ep_harness.py
@@ -46,6 +46,7 @@
import types
import contracts
+import ep_precision
import identity
import workload as workload_contract
@@ -91,7 +92,7 @@
def _oracle_tolerances(backend) -> tuple[float, float]:
- tolerance = identity.combine_oracle_tolerances(backend.communication_precision)
+ tolerance = ep_precision.combine_oracle_tolerances(backend.communication_precision)
return tolerance["atol"], tolerance["rtol"]
EPLB_REDUNDANT_EXPERTS = 32
@@ -165,7 +166,7 @@ def add_common_args(ap: argparse.ArgumentParser) -> None:
ap.add_argument(
"--precision-profile",
default="",
- choices=("", *identity.V1_PRECISION_PROFILES),
+ choices=("", *ep_precision.V1_PRECISION_PROFILES),
help="exact native dispatch/combine communication profile; blank selects BF16 control",
)
ap.add_argument("--phase", default="decode", choices=["decode", "prefill"],
@@ -184,7 +185,6 @@ def add_common_args(ap: argparse.ArgumentParser) -> None:
ap.add_argument("--case-id", default="")
ap.add_argument("--suite", default="")
ap.add_argument("--workload-name", default="")
- ap.add_argument("--required-publication", default="")
ap.add_argument("--seed", type=int, default=ROUTING_SEED)
ap.add_argument(
"--qualification-index",
@@ -687,7 +687,7 @@ def _precision_evidence(backend, problem, view, combined, expected_combined) ->
)
return evidence
profile_id = getattr(backend, "precision_profile_id", None)
- if profile_id != identity.V1_CONTROL_PRECISION_PROFILE:
+ if profile_id != ep_precision.V1_CONTROL_PRECISION_PROFILE:
failed = _baseline_precision_axis()
failed.update({"encoded_payload_valid": False, "dequantized_semantics": False,
"passed": False})
@@ -1213,14 +1213,11 @@ def run_sweep(args, backend, torch, dist, device, rank: int, world_size: int) ->
"""Drive the source-tokens-per-rank sweep for one fully-specified line."""
mode = getattr(args, "mode", "normal")
requested_precision = getattr(args, "precision_profile", "") or None
- resolved_precision_id = requested_precision or identity.V1_CONTROL_PRECISION_PROFILE
+ resolved_precision_id = requested_precision or ep_precision.V1_CONTROL_PRECISION_PROFILE
try:
- profile_case = {"mode": mode}
- if requested_precision is not None:
- profile_case["precision_profile"] = requested_precision
- case_profile = identity.profile_for_case(profile_case)
- communication_precision = identity.precision_profile(resolved_precision_id)
- except identity.IdentityError as exc:
+ case_profile = identity.profile_for_case({"mode": mode})
+ communication_precision = ep_precision.precision_profile(resolved_precision_id)
+ except (identity.IdentityError, ep_precision.PrecisionError) as exc:
if rank == 0:
print(f"ERROR: {exc}")
return 2
@@ -1737,7 +1734,6 @@ def component(percentiles, count, *, derived=False):
"mode": mode,
"nodes": nodes,
"phase": args.phase,
- "required_publication": args.required_publication or "diagnostic",
"routing": args.routing,
"samples_per_point": TIMED_SAMPLES_PER_POINT,
"scale_up_domain": args.scale_up_domain or (args.gpus_per_node or ep_size),
@@ -1939,7 +1935,6 @@ def component(percentiles, count, *, derived=False):
"ep_size": ep_size,
"mode": mode,
"phase": args.phase,
- "required_publication": args.required_publication or "diagnostic",
"resource_mode": "fixed-profile",
"runner": args.runner,
"shape": shape,
diff --git a/experimental/CollectiveX/tests/ep_precision.py b/experimental/CollectiveX/tests/ep_precision.py
index 01440b8f54..e01b5a0353 100644
--- a/experimental/CollectiveX/tests/ep_precision.py
+++ b/experimental/CollectiveX/tests/ep_precision.py
@@ -7,13 +7,205 @@
from dataclasses import dataclass
from typing import Any, Iterable
-import identity
-
class PrecisionError(RuntimeError):
"""A requested precision profile cannot be realized by the pinned API."""
+# ---------------------------------------------------------------------------
+# Native communication-precision catalog.
+#
+# Dispatch and combine are fixed BF16 benchmark facts for the default sweep, so
+# no scheduled case selects an FP8 profile. The FP8 axes below stay callable so
+# an adapter can still exercise its native codec directly; they are dormant, not
+# removed. This catalog lives with the codec (not in ``identity``) because it is
+# a property of the communication kernels, not of the neutral artifact identity.
+# ---------------------------------------------------------------------------
+
+
+def _communication_axis(
+ *,
+ api_input_dtype: str,
+ api_output_dtype: str,
+ communication_format: str,
+ scale_dtype: str | None,
+ scale_layout: str,
+ scale_group_size: int | None,
+ padding_contract: str,
+ alignment_contract: str,
+ quantization_origin: str,
+ conversion_boundary: str,
+) -> dict[str, Any]:
+ return {
+ "api_input_dtype": api_input_dtype,
+ "api_output_dtype": api_output_dtype,
+ "communication_format": communication_format,
+ "scale_dtype": scale_dtype,
+ "scale_layout": scale_layout,
+ "scale_group_size": scale_group_size,
+ "padding_contract": padding_contract,
+ "alignment_contract": alignment_contract,
+ "quantization_origin": quantization_origin,
+ "conversion_boundary": conversion_boundary,
+ }
+
+
+_BF16_AXIS = _communication_axis(
+ api_input_dtype="bf16",
+ api_output_dtype="bf16",
+ communication_format="bf16",
+ scale_dtype=None,
+ scale_layout="none",
+ scale_group_size=None,
+ padding_contract="none",
+ alignment_contract="native-bf16-vector-alignment",
+ quantization_origin="none",
+ conversion_boundary="none",
+)
+_FP8_E4M3FN_PREQUANTIZED_DISPATCH = _communication_axis(
+ api_input_dtype="fp8-e4m3fn-with-f32-scale",
+ api_output_dtype="fp8-e4m3fn-with-f32-scale",
+ communication_format="fp8-e4m3fn",
+ scale_dtype="f32",
+ scale_layout="per-token-hidden-block",
+ scale_group_size=128,
+ padding_contract="right-zero-pad-hidden-to-128",
+ alignment_contract="hidden-block-128",
+ quantization_origin="caller-prequantized",
+ conversion_boundary="before-dispatch-timing",
+)
+_FP8_E4M3FNUZ_PREQUANTIZED_DISPATCH = _communication_axis(
+ api_input_dtype="fp8-e4m3fnuz-with-f32-scale",
+ api_output_dtype="fp8-e4m3fnuz-with-f32-scale",
+ communication_format="fp8-e4m3fnuz",
+ scale_dtype="f32",
+ scale_layout="per-token-hidden-block",
+ scale_group_size=128,
+ padding_contract="right-zero-pad-hidden-to-128",
+ alignment_contract="hidden-block-128",
+ quantization_origin="caller-prequantized",
+ conversion_boundary="before-dispatch-timing",
+)
+_FP8_E4M3FN_FUSED_DISPATCH = _communication_axis(
+ api_input_dtype="bf16",
+ api_output_dtype="fp8-e4m3fn-with-f32-scale",
+ communication_format="fp8-e4m3fn",
+ scale_dtype="f32",
+ scale_layout="per-token-hidden-block",
+ scale_group_size=128,
+ padding_contract="right-zero-pad-hidden-to-128",
+ alignment_contract="hidden-block-128",
+ quantization_origin="backend-fused",
+ conversion_boundary="inside-dispatch-timing",
+)
+_FP8_E4M3FN_DIRECT_CAST_COMBINE = _communication_axis(
+ api_input_dtype="bf16",
+ api_output_dtype="bf16",
+ communication_format="fp8-e4m3fn",
+ scale_dtype=None,
+ scale_layout="none",
+ scale_group_size=None,
+ padding_contract="none",
+ alignment_contract="native-fp8-vector-alignment",
+ quantization_origin="backend-internal-direct-cast",
+ conversion_boundary="inside-combine-timing",
+)
+_FP8_E4M3FNUZ_DIRECT_CAST_COMBINE = _communication_axis(
+ api_input_dtype="bf16",
+ api_output_dtype="bf16",
+ communication_format="fp8-e4m3fnuz",
+ scale_dtype=None,
+ scale_layout="none",
+ scale_group_size=None,
+ padding_contract="none",
+ alignment_contract="native-fp8-vector-alignment",
+ quantization_origin="backend-internal-direct-cast",
+ conversion_boundary="inside-combine-timing",
+)
+
+V1_CONTROL_PRECISION_PROFILE = "d-bf16.c-bf16"
+V1_NORMAL_PRECISION_PROFILE_IDS = (
+ "d-fp8-e4m3fn-b128-f32-prequantized.c-bf16",
+ "d-fp8-e4m3fnuz-b128-f32-prequantized.c-fp8-e4m3fnuz-direct-cast-noscale",
+)
+V1_LOW_LATENCY_PRECISION_PROFILE_IDS = (
+ "d-fp8-e4m3fn-b128-f32-fused.c-bf16",
+)
+
+V1_PRECISION_PROFILES: dict[str, dict[str, Any]] = {
+ V1_CONTROL_PRECISION_PROFILE: {
+ "modes": ["normal", "low-latency"],
+ "dispatch": _BF16_AXIS,
+ "combine": _BF16_AXIS,
+ },
+ "d-fp8-e4m3fn-b128-f32-prequantized.c-bf16": {
+ "modes": ["normal"],
+ "dispatch": _FP8_E4M3FN_PREQUANTIZED_DISPATCH,
+ "combine": _BF16_AXIS,
+ },
+ "d-fp8-e4m3fn-b128-f32-fused.c-bf16": {
+ "modes": ["low-latency"],
+ "dispatch": _FP8_E4M3FN_FUSED_DISPATCH,
+ "combine": _BF16_AXIS,
+ },
+ "d-bf16.c-fp8-e4m3fn-direct-cast-noscale": {
+ "modes": ["normal"],
+ "dispatch": _BF16_AXIS,
+ "combine": _FP8_E4M3FN_DIRECT_CAST_COMBINE,
+ },
+ "d-fp8-e4m3fn-b128-f32-prequantized.c-fp8-e4m3fn-direct-cast-noscale": {
+ "modes": ["normal"],
+ "dispatch": _FP8_E4M3FN_PREQUANTIZED_DISPATCH,
+ "combine": _FP8_E4M3FN_DIRECT_CAST_COMBINE,
+ },
+ "d-fp8-e4m3fnuz-b128-f32-prequantized.c-bf16": {
+ "modes": ["normal"],
+ "dispatch": _FP8_E4M3FNUZ_PREQUANTIZED_DISPATCH,
+ "combine": _BF16_AXIS,
+ },
+ "d-bf16.c-fp8-e4m3fnuz-direct-cast-noscale": {
+ "modes": ["normal"],
+ "dispatch": _BF16_AXIS,
+ "combine": _FP8_E4M3FNUZ_DIRECT_CAST_COMBINE,
+ },
+ "d-fp8-e4m3fnuz-b128-f32-prequantized.c-fp8-e4m3fnuz-direct-cast-noscale": {
+ "modes": ["normal"],
+ "dispatch": _FP8_E4M3FNUZ_PREQUANTIZED_DISPATCH,
+ "combine": _FP8_E4M3FNUZ_DIRECT_CAST_COMBINE,
+ },
+}
+
+V1_COMBINE_ORACLE_TOLERANCES = {
+ "bf16": {"atol": 2e-2, "rtol": 5e-2},
+ "fp8-direct-cast": {"atol": 4e-2, "rtol": 8e-2},
+}
+
+
+def precision_profile(name: str) -> dict[str, Any]:
+ """Return one exact dispatch/combine communication-format profile."""
+ try:
+ profile = V1_PRECISION_PROFILES[name]
+ except KeyError as exc:
+ raise PrecisionError(f"unknown CollectiveX precision profile {name!r}") from exc
+ return {"profile_id": name, **deepcopy(profile)}
+
+
+def combine_oracle_tolerances(communication_precision: dict[str, Any]) -> dict[str, float]:
+ """Return the frozen combine-oracle gate for one exact native codec."""
+ combine = communication_precision["combine"]
+ fmt = combine["communication_format"]
+ if fmt == "bf16":
+ key = "bf16"
+ elif (
+ fmt in {"fp8-e4m3fn", "fp8-e4m3fnuz"}
+ and combine["quantization_origin"] == "backend-internal-direct-cast"
+ ):
+ key = "fp8-direct-cast"
+ else:
+ raise PrecisionError("precision profile has no frozen combine-oracle tolerance")
+ return dict(V1_COMBINE_ORACLE_TOLERANCES[key])
+
+
@dataclass(frozen=True)
class DispatchEncoding:
"""One dispatch input plus its post-codec semantic representation."""
@@ -35,12 +227,9 @@ def resolve_precision(
"""Resolve and validate the exact profile requested for one adapter."""
profile_id = (
getattr(args, "precision_profile", "")
- or identity.V1_CONTROL_PRECISION_PROFILE
+ or V1_CONTROL_PRECISION_PROFILE
)
- try:
- profile = identity.precision_profile(profile_id)
- except identity.IdentityError as exc:
- raise PrecisionError(str(exc)) from exc
+ profile = precision_profile(profile_id) # raises PrecisionError on unknown
if mode not in profile["modes"]:
raise PrecisionError(
f"precision profile {profile_id!r} is not valid in mode {mode!r}"
diff --git a/experimental/CollectiveX/tests/test_qualification_planning.py b/experimental/CollectiveX/tests/test_qualification_planning.py
index 0e1993475e..90b62c06af 100644
--- a/experimental/CollectiveX/tests/test_qualification_planning.py
+++ b/experimental/CollectiveX/tests/test_qualification_planning.py
@@ -1,9 +1,7 @@
#!/usr/bin/env python3
-"""CPU-only tests for qualification-specific shard execution planning."""
+"""CPU-only tests for matrix resolution, subset selection, and shard extraction."""
from __future__ import annotations
-import copy
-import hashlib
import json
import os
from pathlib import Path
@@ -17,9 +15,7 @@
ROOT = HERE.parent
sys.path[:0] = [str(ROOT), str(HERE)]
-import identity # noqa: E402
import sweep_matrix # noqa: E402
-import capability # noqa: E402
def _canonical(value: object) -> bytes:
@@ -28,7 +24,7 @@ def _canonical(value: object) -> bytes:
).encode()
-class QualificationPlanningTest(unittest.TestCase):
+class MatrixPlanningTest(unittest.TestCase):
@classmethod
def setUpClass(cls) -> None:
cls.matrix = sweep_matrix.validate_matrix_document(
@@ -38,166 +34,40 @@ def setUpClass(cls) -> None:
)
cls.shard = next(item for item in cls.matrix["include"] if item["n"] >= 3)
- def test_matrix_semantics_do_not_depend_on_qualification_index(self) -> None:
- expected = _canonical(self.matrix)
- for qualification_index in (1, 2, 3):
- with mock.patch.dict(
- os.environ,
- {"CX_QUALIFICATION_INDEX": str(qualification_index)},
- ):
- observed = sweep_matrix.validate_matrix_document(
- sweep_matrix.resolve_matrix(
- backend="deepep", only_sku="h100-dgxc", max_cases=128
- )
- )
- self.assertEqual(_canonical(observed), expected)
-
- def test_extract_shard_has_deterministic_distinct_exact_plans(self) -> None:
+ def test_extract_shard_is_deterministic_and_matrix_preserving(self) -> None:
matrix_bytes = _canonical(self.matrix) + b"\n"
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary)
matrix_path = root / "matrix.json"
matrix_path.write_bytes(matrix_bytes)
- original_digest = hashlib.sha256(matrix_path.read_bytes()).hexdigest()
- controls = []
- for qualification_index in (1, 2, 3):
- first = sweep_matrix.extract_shard(
- matrix_path,
- self.shard["id"],
- root / f"q{qualification_index}-first.json",
- sku=self.shard["sku"],
- backend=self.shard["backend"],
- nodes=self.shard["nodes"],
- qualification_index=qualification_index,
- )
- repeated = sweep_matrix.extract_shard(
- matrix_path,
- self.shard["id"],
- root / f"q{qualification_index}-repeated.json",
- sku=self.shard["sku"],
- backend=self.shard["backend"],
- nodes=self.shard["nodes"],
- qualification_index=qualification_index,
- )
- self.assertEqual(first, repeated)
- self.assertEqual(first["qualification_index"], qualification_index)
- self.assertEqual(
- {case["case_id"] for case in first["cases"]},
- set(self.shard["case_ids"]),
- )
- plan = [
- [
- case["case_id"],
- case.get(
- "precision_profile",
- identity.V1_CONTROL_PRECISION_PROFILE,
- ),
- ]
- for case in first["cases"]
- ]
- self.assertEqual(
- first["execution_plan_sha256"],
- hashlib.sha256(
- json.dumps(plan, separators=(",", ":")).encode()
- ).hexdigest(),
- )
- sweep_matrix.validate_shard_control(
- first,
- sku=self.shard["sku"],
- backend=self.shard["backend"],
- nodes=self.shard["nodes"],
- qualification_index=qualification_index,
- )
- controls.append(first)
- self.assertEqual(
- len({control["execution_plan_sha256"] for control in controls}), 3
+ first = sweep_matrix.extract_shard(
+ matrix_path,
+ self.shard["id"],
+ root / "first.json",
+ sku=self.shard["sku"],
+ backend=self.shard["backend"],
+ nodes=self.shard["nodes"],
)
- self.assertEqual(
- len({tuple(case["case_id"] for case in control["cases"])
- for control in controls}),
- 3,
+ repeated = sweep_matrix.extract_shard(
+ matrix_path,
+ self.shard["id"],
+ root / "repeated.json",
+ sku=self.shard["sku"],
+ backend=self.shard["backend"],
+ nodes=self.shard["nodes"],
)
+ self.assertEqual(first, repeated)
self.assertEqual(
- hashlib.sha256(matrix_path.read_bytes()).hexdigest(), original_digest
+ [case["case_id"] for case in first["cases"]], self.shard["case_ids"]
)
-
- def test_precision_profiles_and_cases_rotate_across_repeats(self) -> None:
- profiles = list(identity.V1_PRECISION_PROFILES)[:3]
- cases = [
- {
- "case_id": identity.digest("case", {"fixture": index}),
- "precision_profile": profiles[index % len(profiles)],
- }
- for index in range(9)
- ]
- plans = [
- sweep_matrix.qualification_execution_order(
- "qualification-fixture", cases, qualification_index
+ sweep_matrix.validate_shard_control(
+ first,
+ sku=self.shard["sku"],
+ backend=self.shard["backend"],
+ nodes=self.shard["nodes"],
)
- for qualification_index in (1, 2, 3)
- ]
- expected_ids = {case["case_id"] for case in cases}
- self.assertTrue(all(
- {case["case_id"] for case in plan} == expected_ids for plan in plans
- ))
- self.assertEqual(
- len({tuple(case["case_id"] for case in plan) for plan in plans}), 3
- )
- self.assertEqual(
- len({tuple(case["precision_profile"] for case in plan) for plan in plans}),
- 3,
- )
-
- def test_matrix_execution_plan_digest_is_repeat_specific_and_stable(self) -> None:
- digests = [
- sweep_matrix.qualification_execution_plan_sha256(self.matrix, index)
- for index in (1, 2, 3)
- ]
- self.assertEqual(len(set(digests)), 3)
- self.assertTrue(all(len(digest) == 64 for digest in digests))
- self.assertEqual(
- digests,
- [
- sweep_matrix.qualification_execution_plan_sha256(
- copy.deepcopy(self.matrix), index
- )
- for index in (1, 2, 3)
- ],
- )
- self.assertTrue(all(shard["execution_weight"] > 0 for shard in self.matrix["include"]))
- tampered = copy.deepcopy(self.matrix)
- tampered["include"][0]["execution_weight"] += 1
- with self.assertRaisesRegex(
- sweep_matrix.MatrixError, "execution_weight differs from its cases"
- ):
- sweep_matrix.qualification_execution_plan_sha256(tampered, 1)
-
- def test_full_v1_execution_plans_match_the_frozen_digests(self) -> None:
- matrix = sweep_matrix.validate_matrix_document(
- sweep_matrix.resolve_matrix(suites="all", backends="all", max_cases=128)
- )
- if capability.provisional_precision_targets():
- self.assertNotEqual(
- [
- sweep_matrix.qualification_execution_plan_sha256(matrix, index)
- for index in (1, 2, 3)
- ],
- [
- sweep_matrix.CANONICAL_V1_EXECUTION_PLAN_SHA256[index]
- for index in (1, 2, 3)
- ],
- )
- return
- self.assertEqual(
- [
- sweep_matrix.qualification_execution_plan_sha256(matrix, index)
- for index in (1, 2, 3)
- ],
- [
- sweep_matrix.CANONICAL_V1_EXECUTION_PLAN_SHA256[index]
- for index in (1, 2, 3)
- ],
- )
+ # Extraction is read-only: the matrix on disk is untouched.
+ self.assertEqual(matrix_path.read_bytes(), matrix_bytes)
def test_full_v1_matrix_emits_every_unsupported_terminal_outcome(self) -> None:
matrix = sweep_matrix.validate_matrix_document(
@@ -211,7 +81,6 @@ def test_full_v1_matrix_emits_every_unsupported_terminal_outcome(self) -> None:
"COLLECTIVEX_ARTIFACT_NAME": "cxunsupported-123-1",
"COLLECTIVEX_EXECUTION_ID": "123_1_unsupported",
"COLLECTIVEX_SOURCE_SHA": "a" * 40,
- "CX_QUALIFICATION_INDEX": "1",
"GITHUB_JOB": "setup",
"GITHUB_REF_NAME": "collectivex",
"GITHUB_REPOSITORY": "SemiAnalysisAI/InferenceX",
@@ -227,7 +96,7 @@ def test_full_v1_matrix_emits_every_unsupported_terminal_outcome(self) -> None:
documents = [json.loads(path.read_text()) for path in written]
self.assertEqual(len(documents), len(expected))
- self.assertEqual(len(documents), 155)
+ self.assertTrue(documents)
self.assertEqual(
{document["outcome"]["reason"] for document in documents},
{item["reason"] for item in expected},
@@ -237,7 +106,7 @@ def test_full_v1_matrix_emits_every_unsupported_terminal_outcome(self) -> None:
{item["case"]["case_id"] for item in expected},
)
- def test_exclude_skus_narrows_matrix_to_a_publishable_subset(self) -> None:
+ def test_exclude_skus_narrows_matrix_to_a_subset(self) -> None:
full = sweep_matrix.validate_matrix_document(
sweep_matrix.resolve_matrix(suites="all", backends="all", max_cases=128)
)
@@ -270,7 +139,7 @@ def test_exclude_skus_rejects_unknown_and_conflicting_pools(self) -> None:
with self.assertRaisesRegex(SystemExit, "disjoint pools"):
sweep_matrix.resolve_matrix(only_sku="b300", exclude_skus="b300")
- def test_ep_sizes_narrows_matrix_to_a_publishable_subset(self) -> None:
+ def test_ep_sizes_narrows_matrix_to_a_subset(self) -> None:
full = sweep_matrix.validate_matrix_document(
sweep_matrix.resolve_matrix(suites="all", backends="all", max_cases=128)
)
@@ -295,7 +164,6 @@ def ep8_by_id(matrix: dict[str, object]) -> dict[str, object]:
if item["case"]["ep"] == 8
}
self.assertEqual(ep8_by_id(partial), ep8_by_id(full))
- # Every include shard cell that survives is single-degree EP8 (no EP16 shard).
self.assertTrue(
all(item["case"]["ep"] == 8 for item in partial["requested_cases"])
)
@@ -321,77 +189,14 @@ def test_frontend_catalog_covers_every_requested_case_and_point(self) -> None:
{item["case_id"] for item in catalog["cases"]},
{item["case"]["case_id"] for item in self.matrix["requested_cases"]},
)
- self.assertEqual(
- set(catalog["precision_profiles"]),
- {item["precision_profile"] for item in catalog["cases"]},
- )
self.assertTrue(all(item["required"] for item in catalog["cases"]))
self.assertTrue(all(item["backend_generation"] is None for item in catalog["cases"]))
self.assertEqual(
{item["resource"]["mode"] for item in catalog["cases"]},
{"fixed-profile"},
)
- for item in catalog["cases"]:
- profile = identity.precision_profile(item["precision_profile"])
- projected = catalog["precision_profiles"][item["precision_profile"]]
- self.assertEqual(projected["dispatch"], profile["dispatch"])
- self.assertEqual(projected["combine"], profile["combine"])
self.assertLess(len(_canonical(catalog)) + 1, 1024 * 1024)
- def test_invalid_qualification_controls_are_rejected(self) -> None:
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary)
- matrix_path = root / "matrix.json"
- matrix_path.write_bytes(_canonical(self.matrix) + b"\n")
- arguments = {
- "sku": self.shard["sku"],
- "backend": self.shard["backend"],
- "nodes": self.shard["nodes"],
- }
- for invalid in (0, 4, True):
- with self.subTest(qualification_index=invalid), self.assertRaisesRegex(
- sweep_matrix.MatrixError, "integer in 1..3"
- ):
- sweep_matrix.extract_shard(
- matrix_path,
- self.shard["id"],
- root / "invalid.json",
- qualification_index=invalid,
- **arguments,
- )
- with mock.patch.dict(os.environ, {"CX_QUALIFICATION_INDEX": "invalid"}):
- with self.assertRaisesRegex(
- sweep_matrix.MatrixError, "integer in 1..3"
- ):
- sweep_matrix.extract_shard(
- matrix_path,
- self.shard["id"],
- root / "invalid-env.json",
- **arguments,
- )
-
- with mock.patch.dict(os.environ, {}, clear=True):
- control = sweep_matrix.extract_shard(
- matrix_path,
- self.shard["id"],
- root / "default.json",
- **arguments,
- )
- self.assertEqual(control["qualification_index"], 1)
-
- invalid_control = copy.deepcopy(control)
- invalid_control["qualification_index"] = 4
- with self.assertRaisesRegex(
- sweep_matrix.MatrixError, "integer in 1..3"
- ):
- sweep_matrix.validate_shard_control(invalid_control, **arguments)
- tampered = copy.deepcopy(control)
- tampered["execution_plan_sha256"] = "0" * 64
- with self.assertRaisesRegex(
- sweep_matrix.MatrixError, "differs from its ordered cases"
- ):
- sweep_matrix.validate_shard_control(tampered, **arguments)
-
if __name__ == "__main__":
unittest.main()
diff --git a/experimental/CollectiveX/tests/test_schema_v1_contract.py b/experimental/CollectiveX/tests/test_schema_v1_contract.py
index 9d1a1324d1..15b8e552e9 100644
--- a/experimental/CollectiveX/tests/test_schema_v1_contract.py
+++ b/experimental/CollectiveX/tests/test_schema_v1_contract.py
@@ -1,5 +1,5 @@
#!/usr/bin/env python3
-"""Focused structural tests for the fail-closed CollectiveX V1 schemas."""
+"""Focused structural tests for the fail-closed CollectiveX neutral schemas."""
from __future__ import annotations
import copy
@@ -14,8 +14,18 @@
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
-import identity # noqa: E402
-import contracts # noqa: E402
+
+# The scheduled case is the neutral contract every artifact agrees on. Both the
+# raw evidence schema and the terminal-outcome schema must enforce this exact set
+# of 24 fields with additionalProperties:false so a producer cannot invent,
+# rename, or drop a scheduling field on either side of a run.
+SCHEDULED_CASE_FIELDS = frozenset({
+ "backend", "canonical", "ep", "eplb", "experts", "gpus_per_node", "hidden",
+ "ladder", "mode", "nodes", "phase", "routing", "samples_per_point",
+ "scale_out_transport", "scale_up_domain", "scale_up_transport", "scope",
+ "suite", "timing", "topk", "topology_class", "transport", "warmup_semantics",
+ "workload",
+})
def _load(name: str) -> dict:
@@ -32,174 +42,51 @@ def _definition_validator(schema: dict, name: str) -> jsonschema.Validator:
)
-class CollectiveXV1SchemaContractTest(unittest.TestCase):
+class CollectiveXSchemaContractTest(unittest.TestCase):
@classmethod
def setUpClass(cls) -> None:
cls.raw = _load("raw-case-v1.schema.json")
cls.samples = _load("samples-v1.schema.json")
- cls.public = _load("public-dataset-v1.schema.json")
- cls.bundle = _load("private-bundle-v1.schema.json")
cls.terminal = _load("terminal-outcome-v1.schema.json")
def test_all_checked_in_schemas_are_draft_2020_12_valid(self) -> None:
- for path in sorted((ROOT / "schemas").glob("*.schema.json")):
+ schemas = sorted((ROOT / "schemas").glob("*.schema.json"))
+ # The publication schemas are gone; only the three neutral ones remain.
+ self.assertEqual(
+ {path.name for path in schemas},
+ {
+ "raw-case-v1.schema.json",
+ "samples-v1.schema.json",
+ "terminal-outcome-v1.schema.json",
+ },
+ )
+ for path in schemas:
with self.subTest(path=path.name):
schema = json.loads(path.read_text())
jsonschema.Draft202012Validator.check_schema(schema)
self.assertFalse(schema["additionalProperties"])
- def test_raw_backend_provenance_schema_matches_runtime_vocabulary(self) -> None:
- provenance = self.raw["properties"]["implementation"]["properties"][
- "provenance"
- ]
+ def test_scheduled_case_is_identical_across_raw_and_terminal(self) -> None:
+ # Format consts pin the record type of each neutral artifact.
self.assertEqual(
- set(provenance["propertyNames"]["enum"]),
- contracts.PROVENANCE_KEYS,
+ self.raw["properties"]["format"]["const"], "collectivex.ep.v1"
)
-
- def test_precision_catalog_and_axes_are_exact_and_strict(self) -> None:
- expected = set(identity.V1_PRECISION_PROFILES)
- self.assertEqual(set(self.raw["$defs"]["precision_profile"]["enum"]), expected)
- self.assertEqual(set(self.public["$defs"]["precisionProfile"]["enum"]), expected)
- self.assertEqual(set(self.terminal["$defs"]["precisionProfile"]["enum"]), expected)
-
- axis_keys = {
- "alignment_contract",
- "api_input_dtype",
- "api_output_dtype",
- "communication_format",
- "conversion_boundary",
- "padding_contract",
- "quantization_origin",
- "scale_dtype",
- "scale_group_size",
- "scale_layout",
- }
- for schema, name in (
- (self.raw, "communication_axis"),
- (self.public, "communicationAxis"),
- (self.terminal, "communicationAxis"),
- ):
- with self.subTest(schema=schema["$id"]):
- axis = schema["$defs"][name]
- self.assertFalse(axis["additionalProperties"])
- self.assertEqual(set(axis["required"]), axis_keys)
- self.assertEqual(set(axis["properties"]), axis_keys)
-
- axis_validator = _definition_validator(self.raw, "communication_axis")
- profile_validator = _definition_validator(self.raw, "communication_precision")
- for name in sorted(expected):
- profile = identity.precision_profile(name)
- with self.subTest(profile=name):
- axis_validator.validate(profile["dispatch"])
- axis_validator.validate(profile["combine"])
- profile_validator.validate(profile)
- raw_case_profile = _definition_validator(self.raw, "case_profile")
- terminal_case_profile = _definition_validator(self.terminal, "caseProfile")
- for case in (
- {"mode": "normal"},
- {
- "mode": "normal",
- "precision_profile": "d-fp8-e4m3fn-b128-f32-prequantized.c-bf16",
- },
- {
- "mode": "low-latency",
- "precision_profile": "d-fp8-e4m3fn-b128-f32-fused.c-bf16",
- },
- ):
- resolved = identity.profile_for_case(case)
- raw_case_profile.validate(resolved)
- terminal_case_profile.validate(resolved)
-
- shape = self.raw["properties"]["case"]["properties"]["shape"]
- self.assertIn("precision_profile", shape["required"])
- self.assertIn("dispatch_precision", shape["required"])
- self.assertIn("combine_precision", shape["required"])
- self.assertNotIn("dispatch_dtype", shape["properties"])
- self.assertNotIn("quant", shape["properties"])
-
- workload = self.public["$defs"]["series"]["properties"]["workload"]
- self.assertIn("precision_profile", workload["required"])
- self.assertIn("dispatch_precision", workload["required"])
- self.assertIn("combine_precision", workload["required"])
- self.assertNotIn("dispatch_dtype", workload["properties"])
- self.assertNotIn("combine_dtype", workload["properties"])
-
- oracle = self.raw["$defs"]["oracle"]["properties"]
- tolerances = {
- tuple(identity.combine_oracle_tolerances(profile).values())
- for profile in (
- identity.precision_profile(name) for name in sorted(expected)
- )
- }
- self.assertEqual(set(oracle["atol"]["enum"]), {value[0] for value in tolerances})
- self.assertEqual(set(oracle["rtol"]["enum"]), {value[1] for value in tolerances})
-
- profile = self.raw["$defs"]["case_profile"]
self.assertEqual(
- profile["properties"]["activation_generator"]["const"],
- identity.V1_NORMAL_CASE_PROFILE["activation_generator"],
+ self.terminal["properties"]["format"]["const"], "collectivex.terminal.v1"
)
self.assertEqual(
- profile["properties"]["activation_profile"]["const"],
- identity.V1_NORMAL_CASE_PROFILE["activation_profile"],
- )
- self.assertEqual(
- profile["properties"]["source_identity_contract"]["const"],
- identity.V1_NORMAL_CASE_PROFILE["source_identity_contract"],
- )
-
- def test_qualification_index_is_bound_across_private_and_public_records(self) -> None:
- paths = (
- self.raw["properties"]["measurement"]["properties"]["qualification_index"],
- self.raw["properties"]["identity"]["properties"]["allocation_factors"]["properties"]["qualification_index"],
- self.raw["$defs"]["git_run"]["properties"]["qualification_index"],
- self.samples["properties"]["qualification_index"],
- self.bundle["properties"]["run"]["properties"]["qualification_index"],
- self.terminal["$defs"]["allocationFactors"]["properties"]["qualification_index"],
- self.terminal["$defs"]["gitRun"]["properties"]["qualification_index"],
- self.public["$defs"]["attempt"]["properties"]["qualification_index"],
- )
- for value in paths:
- self.assertEqual((value["minimum"], value["maximum"]), (1, 3))
- promotion_indices = self.public["properties"]["promotion"]["properties"]["qualification_indices"]
- series_indices = self.public["$defs"]["series"]["properties"]["measurement"]["properties"]["qualification_indices"]
- for schema, valid_values, invalid_values in (
- (promotion_indices, ([], [1]), ([0], [2], [1, 2], [1, 2, 3])),
- (series_indices, ([1], [2, 3], [1, 2, 3]), ([0], [4], [1, 1], [1, 2, 3, 1])),
- ):
- validator = jsonschema.Draft202012Validator(schema)
- for value in valid_values:
- validator.validate(value)
- for value in invalid_values:
- with self.assertRaises(jsonschema.ValidationError):
- validator.validate(value)
- measurement = self.raw["properties"]["measurement"]
- self.assertIn("execution_order_sha256", measurement["required"])
- self.assertEqual(
- measurement["properties"]["execution_order_sha256"]["pattern"],
- "^[0-9a-f]{64}$",
+ self.samples["properties"]["format"]["const"], "collectivex.samples.v1"
)
- def test_private_allocation_stratum_is_required_only_in_raw_canonical_evidence(self) -> None:
- provenance = self.raw["properties"]["provenance"]
- self.assertIn("allocation_stratum_sha256", provenance["required"])
- stratum = provenance["properties"]["allocation_stratum_sha256"]
- jsonschema.Draft202012Validator(stratum).validate(None)
- jsonschema.Draft202012Validator(stratum).validate("a" * 64)
- with self.assertRaises(jsonschema.ValidationError):
- jsonschema.Draft202012Validator(stratum).validate("A" * 64)
- conditional = self.raw["allOf"][0]
- self.assertEqual(
- conditional["if"]["properties"]["workload"]["properties"]["source"],
- {"const": "canonical-serialized"},
- )
- canonical_stratum = conditional["then"]["properties"]["provenance"][
- "properties"
- ]["allocation_stratum_sha256"]
- with self.assertRaises(jsonschema.ValidationError):
- jsonschema.Draft202012Validator(canonical_stratum).validate(None)
- self.assertNotIn("allocation_stratum_sha256", json.dumps(self.public))
+ raw_case = self.raw["$defs"]["scheduled_case"]
+ terminal_case = self.terminal["$defs"]["case"]
+ for descriptor in (raw_case, terminal_case):
+ self.assertFalse(descriptor["additionalProperties"])
+ self.assertEqual(set(descriptor["properties"]), SCHEDULED_CASE_FIELDS)
+ self.assertEqual(set(descriptor["required"]), SCHEDULED_CASE_FIELDS)
+ # No precision dimension survives on the scheduled case.
+ self.assertNotIn("precision_profile", raw_case["properties"])
+ self.assertNotIn("precision_profile", terminal_case["properties"])
def test_stage_samples_are_absent_or_exactly_512(self) -> None:
validator = _definition_validator(self.samples, "component")
@@ -225,56 +112,22 @@ def test_stage_samples_are_absent_or_exactly_512(self) -> None:
with self.assertRaises(jsonschema.ValidationError):
validator.validate(broken)
- raw_components = self.raw["properties"]["measurement"]["properties"]["rows"]["items"]["properties"]["components"]
- public_components = self.public["$defs"]["point"]["properties"]["components"]
+ raw_components = self.raw["properties"]["measurement"]["properties"]["rows"][
+ "items"
+ ]["properties"]["components"]
self.assertIn("stage", raw_components["required"])
- self.assertIn("stage", public_components["required"])
- def test_byte_provenance_supports_precision_aware_rates(self) -> None:
+ def test_byte_accounting_is_exact_and_strict(self) -> None:
expected = {
"accounting_contract",
"activation_data_bytes",
"scale_bytes",
"total_logical_bytes",
}
- for schema, name in (
- (self.raw, "byte_accounting"),
- (self.public, "byteAccounting"),
- ):
- definition = schema["$defs"][name]
- self.assertFalse(definition["additionalProperties"])
- self.assertEqual(set(definition["required"]), expected)
- self.assertEqual(set(definition["properties"]), expected)
- component = self.public["$defs"]["component"]
- self.assertIn("byte_provenance", component["required"])
- self.assertIn("activation_data_rate_gbps_at_latency_percentile", component["required"])
- self.assertIn("total_logical_data_rate_gbps_at_latency_percentile", component["required"])
-
- def test_raw_correctness_carries_directional_precision_evidence(self) -> None:
- correctness = self.raw["properties"]["measurement"]["properties"]["rows"]["items"]["properties"]["correctness"]
- self.assertIn("precision", correctness["required"])
- evidence = self.raw["$defs"]["precision_evidence"]
- self.assertFalse(evidence["additionalProperties"])
- self.assertEqual(
- set(evidence["required"]),
- {"profile_id", "dispatch", "combine", "passed"},
- )
- axis = self.raw["$defs"]["precision_axis_evidence"]
- self.assertFalse(axis["additionalProperties"])
- self.assertEqual(
- set(axis["required"]),
- {
- "encoded_payload_valid",
- "scales_finite",
- "scales_positive",
- "dequantized_semantics",
- "saturation_count",
- "saturation_rate",
- "max_abs_error",
- "max_rel_error",
- "passed",
- },
- )
+ definition = self.raw["$defs"]["byte_accounting"]
+ self.assertFalse(definition["additionalProperties"])
+ self.assertEqual(set(definition["required"]), expected)
+ self.assertEqual(set(definition["properties"]), expected)
def test_eplb_calibration_provenance_is_explicit(self) -> None:
fields = {
@@ -283,175 +136,10 @@ def test_eplb_calibration_provenance_is_explicit(self) -> None:
"calibration_window",
"calibration_token_offset",
}
- raw = self.raw["properties"]["case"]["properties"]["eplb"]
- public = self.public["$defs"]["series"]["properties"]["eplb"]
- for descriptor in (raw, public):
- self.assertTrue(fields <= set(descriptor["required"]))
- self.assertTrue(fields <= set(descriptor["properties"]))
- self.assertFalse(descriptor["additionalProperties"])
-
- def test_public_coverage_is_a_full_case_and_point_inventory(self) -> None:
- coverage = self.public["$defs"]["coverage"]
- dimensions = {
- "sku",
- "suite",
- "workload",
- "publication_tier",
- "backend",
- "backend_generation",
- "resource",
- "topology",
- "phase",
- "mode",
- "routing",
- "eplb",
- "precision_profile",
- "dispatch_precision",
- "combine_precision",
- "points",
- }
- self.assertTrue(dimensions <= set(coverage["required"]))
- self.assertFalse(coverage["additionalProperties"])
- point = self.public["$defs"]["coveragePoint"]
- self.assertEqual(
- set(point["properties"]["terminal_status"]["enum"]),
- {"measured", "unsupported", "failed", "invalid", "diagnostic"},
- )
- self.assertIn("tokens_per_rank", point["required"])
- self.assertIn("global_tokens", point["required"])
- self.assertIn("reason", point["required"])
-
- promotion = self.public["properties"]["promotion"]
- counts = {
- "requested_cases",
- "terminal_cases",
- "measured_cases",
- "unsupported_cases",
- "requested_points",
- "terminal_points",
- "measured_points",
- "unsupported_points",
- }
- self.assertTrue(counts <= set(promotion["required"]))
- self.assertFalse(promotion["additionalProperties"])
-
- def test_public_coverage_point_reason_tracks_terminal_status(self) -> None:
- validator = _definition_validator(self.public, "coveragePoint")
- point_id = "cxpoint-v1-" + "1" * 64
- series_id = "cxseries-v1-" + "2" * 64
- measured = {
- "point_id": point_id,
- "series_id": series_id,
- "tokens_per_rank": 8,
- "global_tokens": 64,
- "terminal_status": "measured",
- "reason": None,
- }
- validator.validate(measured)
- unsupported = {
- **measured,
- "point_id": None,
- "series_id": None,
- "terminal_status": "unsupported",
- "reason": "backend-platform-unsupported",
- }
- validator.validate(unsupported)
- failed = {**unsupported, "terminal_status": "failed", "reason": "execution-failed"}
- validator.validate(failed)
-
- for broken in (
- {**measured, "reason": "unexpected-reason"},
- {**unsupported, "reason": None},
- {**failed, "reason": None},
- {**unsupported, "reason": "contains spaces"},
- ):
- with self.assertRaises(jsonschema.ValidationError):
- validator.validate(broken)
-
- def test_public_measured_point_has_bounded_detail(self) -> None:
- point = self.public["$defs"]["point"]
- self.assertFalse(point["additionalProperties"])
- self.assertTrue({"anomalies", "correctness"} <= set(point["required"]))
- self.assertNotIn("stability", point["properties"])
- self.assertNotIn("correct", point["properties"])
-
- axis = {
- "encoded_payload_valid": True,
- "scales_finite": None,
- "scales_positive": None,
- "dequantized_semantics": True,
- "saturation_count": 0,
- "saturation_rate": 0.0,
- "max_abs_error": 0.0,
- "max_rel_error": 0.0,
- "passed": True,
- }
- correctness = {
- "semantic_pass": True,
- "precision": {
- "profile_id": identity.V1_CONTROL_PRECISION_PROFILE,
- "dispatch": axis,
- "combine": copy.deepcopy(axis),
- "passed": True,
- },
- }
- correctness_validator = _definition_validator(self.public, "pointCorrectness")
- correctness_validator.validate(correctness)
- broken_correctness = copy.deepcopy(correctness)
- broken_correctness["precision"]["dispatch"]["unexpected"] = True
- with self.assertRaises(jsonschema.ValidationError):
- correctness_validator.validate(broken_correctness)
-
- anomalies = point["properties"]["anomalies"]
- self.assertEqual(anomalies["maxItems"], 16)
- self.assertTrue(anomalies["uniqueItems"])
- anomaly_validator = jsonschema.Draft202012Validator({
- "$schema": "https://json-schema.org/draft/2020-12/schema",
- "$defs": self.public["$defs"],
- "$ref": "#/$defs/point/properties/anomalies",
- })
- anomaly_validator.validate(["roundtrip-gt-isolated-sum"])
- with self.assertRaises(jsonschema.ValidationError):
- anomaly_validator.validate([f"anomaly-{index}" for index in range(17)])
-
- def test_zero_byte_component_rates_and_trial_count_are_bound(self) -> None:
- # A measured but zero-logical-byte component (e.g. UCCL host-staging's
- # ``stage``: real latency, zero bytes) reports a 0.0 GB/s rate. Rate
- # percentiles therefore allow 0 (``ratePercentiles``, minimum 0), while
- # latency percentiles stay strictly positive (``percentiles``,
- # exclusiveMinimum 0). This keeps a zero-byte measurement honest instead
- # of forcing it to null or dropping the component.
- defs = self.public["$defs"]
-
- def _ref_validator(pointer: str) -> jsonschema.Validator:
- return jsonschema.Draft202012Validator({
- "$schema": "https://json-schema.org/draft/2020-12/schema",
- "$defs": defs,
- "$ref": pointer,
- })
-
- for field in (
- "activation_data_rate_gbps_at_latency_percentile",
- "total_logical_data_rate_gbps_at_latency_percentile",
- ):
- rate = _ref_validator(f"#/$defs/component/properties/{field}")
- rate.validate({"p50": 0.0, "p90": 0.0, "p95": 0.0, "p99": 0.0})
- rate.validate(None)
- with self.assertRaises(jsonschema.ValidationError):
- rate.validate({"p50": -1.0, "p90": 0.0, "p95": 0.0, "p99": 0.0})
-
- latency = _ref_validator("#/$defs/component/properties/latency_us")
- latency.validate({"p50": 1.0, "p90": 1.0, "p95": 1.0, "p99": 1.0})
- with self.assertRaises(jsonschema.ValidationError):
- latency.validate({"p50": 0.0, "p90": 1.0, "p95": 1.0, "p99": 1.0})
-
- # A single qualification run yields exactly 64 trial medians per point.
- trial_count = _ref_validator(
- "#/$defs/trialDiagnosticComponent/properties/trial_count"
- )
- trial_count.validate(64)
- with self.assertRaises(jsonschema.ValidationError):
- trial_count.validate(192)
+ descriptor = self.raw["properties"]["case"]["properties"]["eplb"]
+ self.assertTrue(fields <= set(descriptor["required"]))
+ self.assertTrue(fields <= set(descriptor["properties"]))
+ self.assertFalse(descriptor["additionalProperties"])
if __name__ == "__main__":
From 6737d2df97779444447d64b2d86fd458c1892a27 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Wed, 8 Jul 2026 12:16:44 +0800
Subject: [PATCH 191/210] refactor(collectivex): reduce sweep workflow to
setup+sweep
Drop the probe-summary, autopublish, publish, and refresh jobs together
with the release-marker/digest logic and the 11 publication-oriented
inputs. The workflow now only resolves the shard matrix and executes each
cell, uploading neutral matrix and per-shard result artifacts. Display,
promotion, ranking, and dataset building move out of this repo.
---
.github/workflows/collectivex-sweep.yml | 721 ++----------------------
1 file changed, 43 insertions(+), 678 deletions(-)
diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml
index 08f4e10d3c..18e83e5b9a 100644
--- a/.github/workflows/collectivex-sweep.yml
+++ b/.github/workflows/collectivex-sweep.yml
@@ -1,8 +1,10 @@
# CollectiveX Sweep — one structured run instead of thousands of dispatches.
#
-# Shape: sweep/probe runs setup -> GPU cells; publish and refresh stay on disposable
-# GitHub-hosted storage and skip GPU jobs. This file is registered on the default
-# branch, so its collectivex branch revision can be dispatched with --ref.
+# Shape: setup resolves the suites into a shard matrix, then one GPU cell runs per
+# shard, executes the benchmark, validates the emitted neutral artifacts, and uploads
+# them. This repo only schedules, executes, validates, and uploads — it does not
+# promote, rank, recommend, or decide what a frontend displays. This file is registered
+# on the default branch, so its collectivex branch revision can be dispatched with --ref.
name: CollectiveX Sweep
permissions:
actions: read
@@ -10,11 +12,6 @@ permissions:
on:
workflow_dispatch:
inputs:
- operation:
- description: Operation to execute
- type: choice
- default: sweep
- options: [sweep, probe-precision, publish, refresh]
backend:
description: "EP library to sweep — 'all' runs every EP backend in one matrix"
type: choice
@@ -25,62 +22,24 @@ on:
type: string
default: all
only_sku:
- description: Restrict to one V1 GHA runner pool (h100-dgxc|h200-dgxc|b300|b200-dgxc|gb200|gb300|mi300x|mi355x); blank = all
+ description: Restrict to one GHA runner pool (h100-dgxc|h200-dgxc|b300|b200-dgxc|gb200|gb300|mi300x|mi355x); blank = all
type: string
default: ''
exclude_skus:
description: Comma-list of runner pools to drop from the matrix (partial run, e.g. b300); disjoint from only_sku; blank = none
type: string
default: ''
- probe_id:
- description: Optional exact provisional probe ID; valid only for probe-precision
- type: string
- default: ''
- min_nodes:
- description: Keep only shards with at least this node/tray count (2 keeps every EP16 and GB EP8; blank = all)
- type: string
- default: ''
- max_nodes:
- description: Keep only shards with at most this node/tray count (1 keeps non-GB EP8; blank = all)
- type: string
- default: ''
ep_sizes:
description: Keep only shards whose expert-parallel degree is in this comma-list (8 keeps EP8 only, dropping EP16 so GB SKUs co-schedule with 8-GPU SKUs; blank = all)
type: string
default: ''
- max_cases:
- description: Max cases per shard cell before chunking into another GHA job (128 = no chunking for current suites)
- type: string
- default: '128'
- release_tag:
- description: "Release version: 'unversioned' (diagnostic, cannot publish) or a positive integer N to publish as version N (incrementable: 1, 2, 3, ...)"
- type: string
- default: unversioned
- qualification_index:
- description: Deterministic execution-order index; V1 qualification uses 1, then 2, then 3
- type: choice
- default: '1'
- options: ['1', '2', '3']
- publish_run_id:
- description: For publish, one successful versioned sweep run ID (qualification index 1)
- type: string
- default: ''
- refresh_run_id:
- description: For refresh, source publication workflow run ID
- type: string
- default: ''
- refresh_digest:
- description: For refresh, exact publication dataset SHA-256
- type: string
- default: ''
concurrency:
- group: cx-${{ inputs.operation }}-${{ github.ref }}-${{ inputs.release_tag }}-${{ inputs.backend }}-${{ inputs.only_sku }}-${{ inputs.probe_id }}
+ group: cx-${{ github.ref }}-${{ inputs.backend }}-${{ inputs.only_sku }}
cancel-in-progress: false
jobs:
# ---- setup: resolve the suites into the shard matrix (the "pending jobs" node) ----
setup:
- if: ${{ inputs.operation == 'sweep' || inputs.operation == 'probe-precision' }}
runs-on: ubuntu-latest
outputs:
matrix: ${{ steps.gen.outputs.matrix }}
@@ -95,82 +54,32 @@ jobs:
working-directory: experimental/CollectiveX
env:
INPUT_BACKEND: ${{ inputs.backend }}
- INPUT_OPERATION: ${{ inputs.operation }}
INPUT_SUITES: ${{ inputs.suites }}
INPUT_ONLY_SKU: ${{ inputs.only_sku }}
INPUT_EXCLUDE_SKUS: ${{ inputs.exclude_skus }}
- INPUT_PROBE_ID: ${{ inputs.probe_id }}
- INPUT_MIN_NODES: ${{ inputs.min_nodes }}
- INPUT_MAX_NODES: ${{ inputs.max_nodes }}
INPUT_EP_SIZES: ${{ inputs.ep_sizes }}
- INPUT_MAX_CASES: ${{ inputs.max_cases }}
- INPUT_QUALIFICATION_INDEX: ${{ inputs.qualification_index }}
- CX_QUALIFICATION_INDEX: ${{ inputs.qualification_index }}
COLLECTIVEX_SOURCE_SHA: ${{ github.sha }}
COLLECTIVEX_ARTIFACT_NAME: cxunsupported-${{ github.run_id }}-${{ github.run_attempt }}
COLLECTIVEX_EXECUTION_ID: ${{ github.run_id }}_${{ github.run_attempt }}_unsupported
run: |
set -euo pipefail
- if [ "$INPUT_OPERATION" = probe-precision ] && [ "${{ inputs.release_tag }}" != unversioned ]; then
- echo 'precision probes cannot carry a release version' >&2
- exit 1
- fi
- if [ "$INPUT_OPERATION" = sweep ] && [ "${{ inputs.release_tag }}" != unversioned ]; then
- # The release version is a positive integer N (v1, v2, v3, ...); it is
- # incrementable, never a hardcoded tag. 'unversioned' stays diagnostic.
- [[ "${{ inputs.release_tag }}" =~ ^[1-9][0-9]*$ ]] || {
- echo 'release_tag must be "unversioned" or a positive integer version N' >&2
- exit 1
- }
- [[ "$INPUT_QUALIFICATION_INDEX" =~ ^[123]$ ]] || {
- echo 'versioned sweeps require qualification_index 1, 2, or 3' >&2
- exit 1
- }
- # A versioned sweep may be filtered to a subset of the canonical matrix
- # ("tagged + success" contract) — a full run and a partial run (e.g.
- # only_sku, or a single backend) are both publishable. The publisher
- # binds every promoted case to the canonical catalog, so a filtered
- # matrix cannot invent or reclassify cases; it only narrows coverage.
- python3 - <<'PY'
- import capability
-
- if capability.provisional_precision_targets():
- raise SystemExit("versioned sweeps require every precision capability cell to be resolved")
- PY
- fi
- if [ "$INPUT_OPERATION" = probe-precision ]; then
- args=(--workflow-plan --backend "$INPUT_BACKEND" --out matrix_full.json)
- [ -n "$INPUT_ONLY_SKU" ] && args+=(--only-sku "$INPUT_ONLY_SKU")
- [ -n "$INPUT_PROBE_ID" ] && args+=(--probe-id "$INPUT_PROBE_ID")
- [ -n "$INPUT_MIN_NODES" ] && args+=(--min-nodes "$INPUT_MIN_NODES")
- [ -n "$INPUT_MAX_NODES" ] && args+=(--max-nodes "$INPUT_MAX_NODES")
- python3 tests/probe_precision.py "${args[@]}"
- else
- [ -z "$INPUT_PROBE_ID" ] || { echo 'probe_id requires probe-precision' >&2; exit 1; }
- args=(--suites "$INPUT_SUITES" --max-cases "$INPUT_MAX_CASES")
- case "$INPUT_BACKEND" in
- all) args+=(--backends all) ;;
- *) args+=(--backend "$INPUT_BACKEND") ;;
- esac
- [ -n "$INPUT_ONLY_SKU" ] && args+=(--only-sku "$INPUT_ONLY_SKU")
- [ -n "$INPUT_EXCLUDE_SKUS" ] && args+=(--exclude-skus "$INPUT_EXCLUDE_SKUS")
- [ -n "$INPUT_MIN_NODES" ] && args+=(--min-nodes "$INPUT_MIN_NODES")
- [ -n "$INPUT_MAX_NODES" ] && args+=(--max-nodes "$INPUT_MAX_NODES")
- [ -n "$INPUT_EP_SIZES" ] && args+=(--ep-sizes "$INPUT_EP_SIZES")
- python3 sweep_matrix.py "${args[@]}" --out matrix_full.json >/dev/null
- fi
- python3 artifact_safety.py matrix_full.json
+ args=(--suites "$INPUT_SUITES")
+ case "$INPUT_BACKEND" in
+ all) args+=(--backends all) ;;
+ *) args+=(--backend "$INPUT_BACKEND") ;;
+ esac
+ [ -n "$INPUT_ONLY_SKU" ] && args+=(--only-sku "$INPUT_ONLY_SKU")
+ [ -n "$INPUT_EXCLUDE_SKUS" ] && args+=(--exclude-skus "$INPUT_EXCLUDE_SKUS")
+ [ -n "$INPUT_EP_SIZES" ] && args+=(--ep-sizes "$INPUT_EP_SIZES")
+ python3 sweep_matrix.py "${args[@]}" --out matrix_full.json >/dev/null
SLIM=$(python3 -c "import json;m=json.load(open('matrix_full.json'));print(json.dumps({'include':[{k:v for k,v in x.items() if k!='case_ids'} for x in m['include']]}))")
{
echo "matrix=$SLIM"
echo "n=$(python3 -c "import json;print(len(json.load(open('matrix_full.json'))['include']))")"
echo "source_backends=$(python3 -c "import json;m=json.load(open('matrix_full.json'));print(' '.join(sorted({x['backend'] for x in m['include']} & {'deepep-v2','deepep-hybrid'})))")"
- echo "max_parallel=$(python3 -c "import json;m=json.load(open('matrix_full.json'));w=max(x['execution_weight'] for x in m['include']);b=64 if '$INPUT_OPERATION' == 'probe-precision' else 4096;print(max(1,min(10,b//w)))")"
+ echo "max_parallel=$(python3 -c "import json;m=json.load(open('matrix_full.json'));w=max(x['execution_weight'] for x in m['include']);print(max(1,min(10,4096//w)))")"
} >> "$GITHUB_OUTPUT"
- unsupported_n=0
- if [ "$INPUT_OPERATION" = sweep ]; then
- unsupported_n=$(python3 -c "import json;m=json.load(open('matrix_full.json'));print(sum(x['disposition']=='unsupported' for x in m['requested_cases']))")
- fi
+ unsupported_n=$(python3 -c "import json;m=json.load(open('matrix_full.json'));print(sum(x['disposition']=='unsupported' for x in m['requested_cases']))")
echo "unsupported_n=$unsupported_n" >> "$GITHUB_OUTPUT"
if [ "$unsupported_n" -gt 0 ]; then
python3 sweep_matrix.py --emit-unsupported-from matrix_full.json \
@@ -222,90 +131,17 @@ jobs:
path: ${{ runner.temp }}/collectivex-backend-sources.tar
if-no-files-found: error
overwrite: true
- retention-days: ${{ inputs.release_tag != 'unversioned' && 90 || 3 }}
+ retention-days: 14
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: cxsweep-matrix-${{ github.run_id }}
path: experimental/CollectiveX/matrix_full.json
if-no-files-found: error
overwrite: true
- retention-days: ${{ inputs.release_tag != 'unversioned' && 90 || 3 }}
- - name: Create release marker
- if: ${{ inputs.operation == 'sweep' && inputs.release_tag != 'unversioned' }}
- env:
- EXPECTED_MATRIX_SHA256: 5894bab58d3deb2bcee51baa075ca5f5d324b4292ac1cef9f6bc08a07ab1d9a3
- RUN_ID: ${{ github.run_id }}
- RUN_ATTEMPT: ${{ github.run_attempt }}
- SOURCE_SHA: ${{ github.sha }}
- QUALIFICATION_INDEX: ${{ inputs.qualification_index }}
- RELEASE_VERSION: ${{ inputs.release_tag }}
- run: |
- set -euo pipefail
- destination="$RUNNER_TEMP/collectivex-release"
- install -d -m 700 "$destination"
- python3 - "$destination/release.json" <<'PY'
- import hashlib
- import json
- import os
- import pathlib
- import sys
-
- sys.path.insert(0, str(pathlib.Path("experimental/CollectiveX").resolve()))
- import sweep_matrix
-
- matrix = pathlib.Path("experimental/CollectiveX/matrix_full.json").read_bytes()
- matrix_sha256 = hashlib.sha256(matrix).hexdigest()
- # A versioned run may cover the full canonical matrix or a filtered subset.
- # A full run stays pinned to the locked matrix + execution plan (strict
- # integrity); a partial run records its own matrix + plan digests so a
- # "tagged + success" subset is still a verifiable, publishable marker.
- coverage_scope = (
- "full" if matrix_sha256 == os.environ["EXPECTED_MATRIX_SHA256"] else "partial"
- )
- version = int(os.environ["RELEASE_VERSION"])
- if version < 1:
- raise SystemExit("release version must be a positive integer")
- qualification_index = int(os.environ["QUALIFICATION_INDEX"])
- execution_plan_sha256 = sweep_matrix.qualification_execution_plan_sha256(
- json.loads(matrix), qualification_index
- )
- if coverage_scope == "full" and (
- execution_plan_sha256
- != sweep_matrix.CANONICAL_V1_EXECUTION_PLAN_SHA256.get(qualification_index)
- ):
- raise SystemExit("a full release version requires the locked execution plan")
- marker = {
- "coverage_scope": coverage_scope,
- "execution_plan_sha256": execution_plan_sha256,
- "format": "collectivex.release-tag.v1",
- "matrix_sha256": matrix_sha256,
- "qualification_index": qualification_index,
- "run_attempt": os.environ["RUN_ATTEMPT"],
- "run_id": os.environ["RUN_ID"],
- "source_sha": os.environ["SOURCE_SHA"],
- "version": version,
- }
- pathlib.Path(sys.argv[1]).write_text(
- json.dumps(marker, sort_keys=True, separators=(",", ":")) + "\n"
- )
- PY
- python3 experimental/CollectiveX/artifact_safety.py "$destination/release.json"
- - name: Upload release marker
- if: ${{ inputs.operation == 'sweep' && inputs.release_tag != 'unversioned' }}
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
- with:
- name: cxrelease-${{ inputs.release_tag }}-${{ github.run_id }}-${{ github.run_attempt }}
- path: ${{ runner.temp }}/collectivex-release/release.json
- if-no-files-found: error
- retention-days: 90
- - name: Validate unsupported artifact safety
- id: unsupported_safety
- if: ${{ always() && fromJSON(steps.gen.outputs.unsupported_n) > 0 }}
- run: |
- python3 experimental/CollectiveX/artifact_safety.py experimental/CollectiveX/unsupported/*.json
+ retention-days: 14
- name: Validate unsupported outcomes
id: unsupported_contracts
- if: ${{ always() && fromJSON(steps.gen.outputs.unsupported_n) > 0 && steps.unsupported_safety.outcome == 'success' }}
+ if: ${{ always() && fromJSON(steps.gen.outputs.unsupported_n) > 0 }}
env:
COLLECTIVEX_ARTIFACT_NAME: cxunsupported-${{ github.run_id }}-${{ github.run_attempt }}
COLLECTIVEX_EXECUTION_ID: ${{ github.run_id }}_${{ github.run_attempt }}_unsupported
@@ -315,17 +151,17 @@ jobs:
--disposition unsupported \
experimental/CollectiveX/unsupported/*.json
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
- if: ${{ always() && fromJSON(steps.gen.outputs.unsupported_n) > 0 && steps.unsupported_contracts.outcome == 'success' && steps.unsupported_safety.outcome == 'success' }}
+ if: ${{ always() && fromJSON(steps.gen.outputs.unsupported_n) > 0 && steps.unsupported_contracts.outcome == 'success' }}
with:
name: cxunsupported-${{ github.run_id }}-${{ github.run_attempt }}
path: experimental/CollectiveX/unsupported/*.json
if-no-files-found: error
- retention-days: ${{ inputs.release_tag != 'unversioned' && 90 || 3 }}
+ retention-days: 14
# ---- sweep: ONE matrix cell per shard (the parent job with child jobs) ----
sweep:
needs: setup
- if: ${{ (inputs.operation == 'sweep' || inputs.operation == 'probe-precision') && fromJSON(needs.setup.outputs.n) > 0 }}
+ if: ${{ fromJSON(needs.setup.outputs.n) > 0 }}
strategy:
fail-fast: false
max-parallel: ${{ fromJSON(needs.setup.outputs.max_parallel) }}
@@ -339,16 +175,14 @@ jobs:
CX_SCALE_UP_DOMAIN: ${{ matrix.scale_up_domain }}
CX_SHARD_FILE: .shards/${{ matrix.id }}.json
CX_SHARD_SKU: ${{ matrix.sku }}
- CX_PRECISION_PROBE: ${{ inputs.operation == 'probe-precision' && '1' || '0' }}
COLLECTIVEX_CANONICAL_GHA: '1'
COLLECTIVEX_SOURCE_SHA: ${{ github.sha }}
- COLLECTIVEX_ARTIFACT_NAME: ${{ inputs.operation == 'probe-precision' && format('cxprobe-{0}-{1}-{2}', matrix.id, github.run_id, github.run_attempt) || format('cxshard-{0}-{1}-{2}', matrix.id, github.run_id, github.run_attempt) }}
+ COLLECTIVEX_ARTIFACT_NAME: ${{ format('cxshard-{0}-{1}-{2}', matrix.id, github.run_id, github.run_attempt) }}
# Consolidated shards run one bounded build-group in one Slurm allocation.
# MI300X's compute partition has a 180-minute ceiling; the other production
# pools accept 300 minutes. Allocations release as soon as the shard finishes.
- CX_TIME: ${{ inputs.operation == 'probe-precision' && '90' || (matrix.sku == 'mi300x' && '180' || '300') }}
+ CX_TIME: ${{ matrix.sku == 'mi300x' && '180' || '300' }}
COLLECTIVEX_EXECUTION_ID: ${{ github.run_id }}_${{ github.run_attempt }}_${{ matrix.id }}
- CX_QUALIFICATION_INDEX: ${{ inputs.qualification_index }}
CX_JOB_PARENT: ${{ matrix.launcher == 'mi-amds' && format('/tmp/inferencex-collectivex-parent-{0}-{1}-{2}', github.run_id, github.run_attempt, matrix.id) || '/tmp' }}
CX_JOB_ROOT: ${{ matrix.launcher == 'mi-amds' && format('/tmp/inferencex-collectivex-parent-{0}-{1}-{2}/inferencex-collectivex-{0}-{1}-{2}', github.run_id, github.run_attempt, matrix.id) || format('/tmp/inferencex-collectivex-{0}-{1}-{2}', github.run_id, github.run_attempt, matrix.id) }}
CX_SOURCE_ROOT: ${{ matrix.launcher == 'mi-amds' && format('/tmp/inferencex-collectivex-parent-{0}-{1}-{2}/inferencex-collectivex-{0}-{1}-{2}/source', github.run_id, github.run_attempt, matrix.id) || format('/tmp/inferencex-collectivex-{0}-{1}-{2}/source', github.run_id, github.run_attempt, matrix.id) }}
@@ -495,24 +329,14 @@ jobs:
set -euo pipefail
cd "$CX_SOURCE_ROOT/experimental/CollectiveX" 2>/dev/null \
|| { echo "CollectiveX source is unavailable" >&2; exit 1; }
- if [ '${{ inputs.operation }}' = probe-precision ]; then
- python3 tests/probe_precision.py \
- --extract-from "$CX_JOB_ROOT/control/matrix_full.json" \
- --probe-id '${{ matrix.id }}' \
- --expect-sku '${{ matrix.sku }}' \
- --expect-backend '${{ matrix.backend }}' \
- --expect-nodes '${{ matrix.nodes }}' \
- --out '${{ env.CX_SHARD_FILE }}'
- else
- python3 sweep_matrix.py \
- --extract-from "$CX_JOB_ROOT/control/matrix_full.json" \
- --shard-id '${{ matrix.id }}' \
- --expect-sku '${{ matrix.sku }}' \
- --expect-backend '${{ matrix.backend }}' \
- --expect-nodes '${{ matrix.nodes }}' \
- --out '${{ env.CX_SHARD_FILE }}' >/dev/null
- fi
- - name: Execute ${{ inputs.operation }} cell ${{ matrix.id }}
+ python3 sweep_matrix.py \
+ --extract-from "$CX_JOB_ROOT/control/matrix_full.json" \
+ --shard-id '${{ matrix.id }}' \
+ --expect-sku '${{ matrix.sku }}' \
+ --expect-backend '${{ matrix.backend }}' \
+ --expect-nodes '${{ matrix.nodes }}' \
+ --out '${{ env.CX_SHARD_FILE }}' >/dev/null
+ - name: Execute sweep cell ${{ matrix.id }}
id: sweep_shard
env:
COLLECTIVEX_OPERATOR_CONFIG_CONTENT: ${{ secrets.COLLECTIVEX_OPERATOR_CONFIG_V1 }}
@@ -642,29 +466,17 @@ jobs:
set -euo pipefail
[ -f "$CX_JOB_ROOT/cleanup-safe" ] && [ ! -e "$CX_JOB_ROOT/cleanup-unsafe" ] \
|| { echo "CollectiveX allocation cleanup was not confirmed" >&2; exit 1; }
- - name: Validate shard artifact safety
- id: artifact_safety
- if: ${{ always() && steps.allocation_cleanup.outcome == 'success' && (inputs.operation != 'probe-precision' || steps.sweep_shard.outcome == 'success') }}
- run: |
- cd "$CX_SOURCE_ROOT" 2>/dev/null \
- || { echo "CollectiveX source is unavailable" >&2; exit 1; }
- python3 experimental/CollectiveX/artifact_safety.py experimental/CollectiveX/results/*.json
- name: Validate shard delivery completeness
id: delivery_contracts
- if: ${{ always() && steps.artifact_safety.outcome == 'success' }}
+ if: ${{ always() && steps.allocation_cleanup.outcome == 'success' }}
run: |
cd "$CX_SOURCE_ROOT" 2>/dev/null \
|| { echo "CollectiveX source is unavailable" >&2; exit 1; }
- if [ '${{ inputs.operation }}' = probe-precision ]; then
- python3 experimental/CollectiveX/tests/probe_precision.py \
- --validate-manifest experimental/CollectiveX/results/*.json
- else
- python3 experimental/CollectiveX/contracts.py validate-delivery \
- --source "experimental/CollectiveX/${CX_SHARD_FILE}" \
- experimental/CollectiveX/results/*.json
- fi
+ python3 experimental/CollectiveX/contracts.py validate-delivery \
+ --source "experimental/CollectiveX/${CX_SHARD_FILE}" \
+ experimental/CollectiveX/results/*.json
- name: Shard summary
- if: ${{ inputs.operation == 'sweep' && always() && steps.artifact_safety.outcome == 'success' && steps.delivery_contracts.outcome == 'success' }}
+ if: ${{ always() && steps.delivery_contracts.outcome == 'success' }}
run: |
cd "$CX_SOURCE_ROOT" 2>/dev/null \
|| { echo "CollectiveX source is unavailable" >&2; exit 1; }
@@ -672,7 +484,7 @@ jobs:
--results-dir experimental/CollectiveX/results --markdown >> "$GITHUB_STEP_SUMMARY" || true
- name: Stage shard artifact
id: stage_artifact
- if: ${{ always() && steps.delivery_contracts.outcome == 'success' && steps.artifact_safety.outcome == 'success' }}
+ if: ${{ always() && steps.delivery_contracts.outcome == 'success' }}
run: |
set -euo pipefail
cd "$CX_SOURCE_ROOT" 2>/dev/null \
@@ -680,14 +492,14 @@ jobs:
cp -- experimental/CollectiveX/results/*.json "$CX_JOB_ROOT/artifact/"
- name: Upload shard results
id: upload_artifact
- if: always() && steps.stage_artifact.outcome == 'success' && steps.delivery_contracts.outcome == 'success' && steps.artifact_safety.outcome == 'success'
+ if: always() && steps.stage_artifact.outcome == 'success' && steps.delivery_contracts.outcome == 'success'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
- name: ${{ inputs.operation == 'probe-precision' && format('cxprobe-{0}-{1}-{2}', matrix.id, github.run_id, github.run_attempt) || format('cxshard-{0}-{1}-{2}', matrix.id, github.run_id, github.run_attempt) }}
+ name: ${{ format('cxshard-{0}-{1}-{2}', matrix.id, github.run_id, github.run_attempt) }}
path: |
${{ env.CX_JOB_ROOT }}/artifact/*.json
if-no-files-found: error
- retention-days: ${{ inputs.release_tag != 'unversioned' && 90 || 3 }}
+ retention-days: 14
- name: Cleanup isolated workspace
if: ${{ always() && steps.source.outputs.prepared == 'true' }}
run: |
@@ -704,7 +516,6 @@ jobs:
|| { echo "CollectiveX allocation cleanup was not confirmed; retaining isolated files" >&2; exit 1; }
if [ '${{ steps.sweep_shard.outcome }}' = success ] \
&& [ '${{ steps.allocation_cleanup.outcome }}' = success ] \
- && [ '${{ steps.artifact_safety.outcome }}' = success ] \
&& [ '${{ steps.delivery_contracts.outcome }}' = success ] \
&& [ '${{ steps.stage_artifact.outcome }}' = success ] \
&& [ '${{ steps.upload_artifact.outcome }}' = success ] \
@@ -723,449 +534,3 @@ jobs:
rm -rf -- "$CX_JOB_ROOT"
[ "$CX_JOB_PARENT" = /tmp ] || rm -f -- "$CX_JOB_PARENT"
- probe-summary:
- needs: [setup, sweep]
- if: ${{ always() && inputs.operation == 'probe-precision' && needs.sweep.result == 'success' }}
- runs-on: ubuntu-latest
- timeout-minutes: 15
- steps:
- - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v5.0.0
- with: { clean: true, persist-credentials: false }
- - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
- with:
- name: cxsweep-matrix-${{ github.run_id }}
- path: ${{ runner.temp }}/collectivex-probes/control
- - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
- with:
- pattern: cxprobe-*-${{ github.run_id }}-${{ github.run_attempt }}
- merge-multiple: true
- path: ${{ runner.temp }}/collectivex-probes/results
- - name: Validate exact precision probe coverage
- run: |
- set -euo pipefail
- plan="$RUNNER_TEMP/collectivex-probes/control/matrix_full.json"
- shopt -s nullglob
- manifests=("$RUNNER_TEMP"/collectivex-probes/results/*.json)
- [ "${#manifests[@]}" -gt 0 ] || {
- echo 'precision probe artifacts are empty' >&2
- exit 1
- }
- python3 experimental/CollectiveX/artifact_safety.py "$plan" "${manifests[@]}"
- python3 experimental/CollectiveX/tests/probe_precision.py \
- --validate-bundle "$plan" --validate-manifest "${manifests[@]}"
- - name: Upload validated precision probe bundle
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
- with:
- name: cxprecision-probes-${{ github.run_id }}-${{ github.run_attempt }}
- path: ${{ runner.temp }}/collectivex-probes
- if-no-files-found: error
- retention-days: 30
-
- # ---- autopublish: a successful versioned sweep self-publishes its own run ----
- # A "tagged + success" sweep (full OR partial) emits its own promoted
- # publication artifact from the same run, so the frontend can discover many
- # independent runs of a version Just-In-Time without a separate manual publish step.
- autopublish:
- needs: [setup, sweep]
- if: ${{ inputs.operation == 'sweep' && inputs.release_tag != 'unversioned' && inputs.qualification_index == '1' && needs.sweep.result == 'success' }}
- runs-on: ubuntu-latest
- timeout-minutes: 30
- env:
- REPOSITORY: ${{ github.repository }}
- RUN_ID: ${{ github.run_id }}
- RUN_ATTEMPT: ${{ github.run_attempt }}
- SOURCE_SHA: ${{ github.sha }}
- QUALIFICATION_INDEX: ${{ inputs.qualification_index }}
- RELEASE_VERSION: ${{ inputs.release_tag }}
- steps:
- - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v5.0.0
- with:
- ref: ${{ github.sha }}
- clean: true
- persist-credentials: false
- - name: Install publisher dependencies
- run: python3 -m pip install --quiet -r experimental/CollectiveX/requirements.txt
- - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
- with:
- path: ${{ runner.temp }}/collectivex-downloads
- - name: Build promoted publication
- run: |
- set -euo pipefail
- store="$RUNNER_TEMP/collectivex-publisher"
- run_dir="$RUNNER_TEMP/collectivex-downloads"
- output="$RUNNER_TEMP/collectivex-publication"
- umask 027
- mkdir -m 750 "$store" "$output"
- matrix="$run_dir/cxsweep-matrix-$RUN_ID/matrix_full.json"
- [ -f "$matrix" ] || {
- echo 'run is missing its exact matrix artifact' >&2
- exit 1
- }
- mapfile -t artifacts < <(
- find "$run_dir" -mindepth 1 -maxdepth 1 -type d \
- \( -name "cxshard-*-$RUN_ID-$RUN_ATTEMPT" \
- -o -name "cxunsupported-$RUN_ID-$RUN_ATTEMPT" \) -print | sort
- )
- [ "${#artifacts[@]}" -gt 0 ] || {
- echo 'run has no result artifacts' >&2
- exit 1
- }
- artifact_args=()
- for artifact in "${artifacts[@]}"; do
- artifact_args+=(--artifact "$artifact")
- done
- result=$(
- python3 experimental/CollectiveX/publisher.py --store-root "$store" ingest \
- --matrix "$matrix" \
- "${artifact_args[@]}" \
- --repository "$REPOSITORY" \
- --run-id "$RUN_ID" \
- --run-attempt "$RUN_ATTEMPT" \
- --qualification-index "$QUALIFICATION_INDEX" \
- --source-sha "$SOURCE_SHA"
- )
- bundle_id=$(jq -er '.bundle_id' <<< "$result")
- result=$(
- python3 experimental/CollectiveX/publisher.py --store-root "$store" promote \
- --bundle "$bundle_id"
- )
- dataset_id=$(jq -er '.dataset_sha256' <<< "$result")
- dataset="$store/public/datasets/$dataset_id/dataset.json"
- [ -f "$dataset" ] || {
- echo 'publisher did not install the promoted dataset' >&2
- exit 1
- }
- publication="$output/collectivex_public_v1_$dataset_id.ndjson"
- cp -- "$dataset" "$publication"
- python3 experimental/CollectiveX/artifact_safety.py "$publication"
- python3 experimental/CollectiveX/publisher.py --store-root "$store" verify \
- --channel dev-latest --bundle "$bundle_id"
- [ "$(stat -c %s "$publication")" -le 33554432 ] || {
- echo 'publication exceeds the 32 MiB frontend limit' >&2
- exit 1
- }
- sha256sum "$publication"
- {
- echo "## CollectiveX publication (auto) — version $RELEASE_VERSION"
- echo
- echo "Dataset: \`$dataset_id\`"
- echo
- echo "Source run: \`$RUN_ID\` attempt \`$RUN_ATTEMPT\`"
- } >> "$GITHUB_STEP_SUMMARY"
- - name: Upload JIT publication artifact
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
- with:
- name: cxpublication-${{ inputs.release_tag }}-${{ github.run_id }}-${{ github.run_attempt }}
- path: ${{ runner.temp }}/collectivex-publication/*.ndjson
- if-no-files-found: error
- retention-days: 90
-
- publish:
- if: ${{ inputs.operation == 'publish' }}
- runs-on: ubuntu-latest
- timeout-minutes: 30
- env:
- GH_TOKEN: ${{ github.token }}
- RUN_ID: ${{ inputs.publish_run_id }}
- steps:
- - name: Verify source run
- id: runs
- env:
- REPOSITORY: ${{ github.repository }}
- run: |
- set -euo pipefail
- run_id="$RUN_ID"
- [[ "$run_id" =~ ^[1-9][0-9]*$ ]] || {
- echo 'publish_run_id must be a single decimal run ID' >&2
- exit 1
- }
-
- : > "$RUNNER_TEMP/collectivex-runs.tsv"
- metadata=$(gh api "repos/$REPOSITORY/actions/runs/$run_id")
- name=$(jq -r '.name' <<< "$metadata")
- path=$(jq -r '.path' <<< "$metadata")
- branch=$(jq -r '.head_branch' <<< "$metadata")
- status=$(jq -r '.status' <<< "$metadata")
- conclusion=$(jq -r '.conclusion' <<< "$metadata")
- sha=$(jq -r '.head_sha' <<< "$metadata")
- attempt=$(jq -r '.run_attempt' <<< "$metadata")
- [ "$name" = 'CollectiveX Sweep' ] \
- && [ "$path" = '.github/workflows/collectivex-sweep.yml' ] \
- && [ "$branch" = 'collectivex' ] \
- && [ "$status" = 'completed' ] \
- && [ "$conclusion" = 'success' ] \
- && [[ "$attempt" =~ ^[1-9][0-9]*$ ]] \
- && [[ "$sha" =~ ^[0-9a-f]{40}$ ]] || {
- echo "run $run_id is not an eligible V1 sweep" >&2
- exit 1
- }
- source_sha="$sha"
- printf '%s\t%s\t%s\n' "$run_id" "$attempt" "$sha" \
- >> "$RUNNER_TEMP/collectivex-runs.tsv"
- echo "source_sha=$source_sha" >> "$GITHUB_OUTPUT"
-
- - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v5.0.0
- with:
- ref: ${{ steps.runs.outputs.source_sha }}
- clean: true
- persist-credentials: false
-
- - name: Install publisher dependencies
- run: python3 -m pip install --quiet -r experimental/CollectiveX/requirements.txt
-
- - name: Build promoted publication
- id: build
- env:
- REPOSITORY: ${{ github.repository }}
- run: |
- set -euo pipefail
- store="$RUNNER_TEMP/collectivex-publisher"
- downloads="$RUNNER_TEMP/collectivex-downloads"
- output="$RUNNER_TEMP/collectivex-publication"
- umask 027
- mkdir -m 750 "$store" "$downloads" "$output"
- : > "$RUNNER_TEMP/collectivex-bundles.txt"
- : > "$RUNNER_TEMP/collectivex-qualification-indices.txt"
- : > "$RUNNER_TEMP/collectivex-versions.txt"
-
- while IFS=$'\t' read -r run_id attempt source_sha; do
- run_dir="$downloads/$run_id"
- mkdir -m 750 "$run_dir"
- gh run download "$run_id" --repo "$REPOSITORY" --dir "$run_dir"
- matrix="$run_dir/cxsweep-matrix-$run_id/matrix_full.json"
- # The release marker's version is embedded in its artifact name
- # (cxrelease---); discover it version-agnostically.
- marker=$(
- find "$run_dir" -mindepth 2 -maxdepth 2 -type f \
- -regextype posix-extended \
- -regex ".*/cxrelease-[1-9][0-9]*-$run_id-$attempt/release\.json" -print -quit
- )
- [ -f "$matrix" ] || {
- echo "run $run_id is missing its exact matrix artifact" >&2
- exit 1
- }
- [ -n "$marker" ] && [ -f "$marker" ] || {
- echo "run $run_id is not tagged for a release version" >&2
- exit 1
- }
- version=$(jq -r '.version' "$marker")
- [[ "$version" =~ ^[1-9][0-9]*$ ]] || {
- echo "run $run_id has a non-integer release version" >&2
- exit 1
- }
- matrix_sha=$(sha256sum "$matrix" | cut -d' ' -f1)
- qualification_index=$(jq -r '.qualification_index' "$marker")
- execution_plan_sha=$(
- PYTHONPATH=experimental/CollectiveX python3 -c \
- 'import json,sys,sweep_matrix as s; print(s.qualification_execution_plan_sha256(json.load(open(sys.argv[1], encoding="utf-8")), int(sys.argv[2])))' \
- "$matrix" "$qualification_index"
- )
- jq -e \
- --arg run_id "$run_id" \
- --arg attempt "$attempt" \
- --arg source_sha "$source_sha" \
- --arg matrix_sha "$matrix_sha" \
- --arg execution_plan_sha "$execution_plan_sha" \
- 'keys == ["coverage_scope","execution_plan_sha256","format","matrix_sha256","qualification_index","run_attempt","run_id","source_sha","version"]
- and .format == "collectivex.release-tag.v1"
- and (.version | type == "number" and . >= 1)
- and (.coverage_scope == "full" or .coverage_scope == "partial")
- and .run_id == $run_id
- and .run_attempt == $attempt
- and .source_sha == $source_sha
- and .qualification_index == 1
- and .matrix_sha256 == $matrix_sha
- and .execution_plan_sha256 == $execution_plan_sha' \
- "$marker" >/dev/null || {
- echo "run $run_id has an invalid release marker" >&2
- exit 1
- }
- printf '%s\n' "$version" >> "$RUNNER_TEMP/collectivex-versions.txt"
- jq -r '.qualification_index' "$marker" \
- >> "$RUNNER_TEMP/collectivex-qualification-indices.txt"
-
- mapfile -t artifacts < <(
- find "$run_dir" -mindepth 1 -maxdepth 1 -type d \
- \( -name "cxshard-*-$run_id-$attempt" \
- -o -name "cxunsupported-$run_id-$attempt" \) -print | sort
- )
- [ "${#artifacts[@]}" -gt 0 ] || {
- echo "run $run_id has no result artifacts" >&2
- exit 1
- }
- artifact_args=()
- for artifact in "${artifacts[@]}"; do
- artifact_args+=(--artifact "$artifact")
- done
- result=$(
- python3 experimental/CollectiveX/publisher.py --store-root "$store" ingest \
- --matrix "$matrix" \
- "${artifact_args[@]}" \
- --repository "$REPOSITORY" \
- --run-id "$run_id" \
- --run-attempt "$attempt" \
- --qualification-index "$qualification_index" \
- --source-sha "$source_sha"
- )
- bundle_id=$(jq -er '.bundle_id' <<< "$result")
- printf '%s\n' "$bundle_id" >> "$RUNNER_TEMP/collectivex-bundles.txt"
- done < "$RUNNER_TEMP/collectivex-runs.tsv"
-
- [ "$(sort -n "$RUNNER_TEMP/collectivex-qualification-indices.txt" | tr '\n' ' ')" = '1 ' ] || {
- echo 'source run must be qualification index 1 exactly once' >&2
- exit 1
- }
-
- release_version=$(sort -un "$RUNNER_TEMP/collectivex-versions.txt")
- [ "$(printf '%s\n' "$release_version" | wc -l)" -eq 1 ] || {
- echo 'publish source runs disagree on the release version' >&2
- exit 1
- }
- echo "version=$release_version" >> "$GITHUB_OUTPUT"
-
- mapfile -t bundle_ids < "$RUNNER_TEMP/collectivex-bundles.txt"
- promote_args=()
- for bundle_id in "${bundle_ids[@]}"; do
- promote_args+=(--bundle "$bundle_id")
- done
- result=$(
- python3 experimental/CollectiveX/publisher.py --store-root "$store" promote \
- "${promote_args[@]}"
- )
- dataset_id=$(jq -er '.dataset_sha256' <<< "$result")
- dataset="$store/public/datasets/$dataset_id/dataset.json"
- [ -f "$dataset" ] || {
- echo 'publisher did not install the promoted dataset' >&2
- exit 1
- }
- publication="$output/collectivex_public_v1_$dataset_id.ndjson"
- cp -- "$dataset" "$publication"
- python3 experimental/CollectiveX/artifact_safety.py "$publication"
- python3 experimental/CollectiveX/publisher.py --store-root "$store" verify \
- --channel dev-latest "${promote_args[@]}"
- [ "$(stat -c %s "$publication")" -le 33554432 ] || {
- echo 'publication exceeds the 32 MiB frontend limit' >&2
- exit 1
- }
- sha256sum "$publication"
- {
- echo "## CollectiveX publication — version $release_version"
- echo
- echo "Dataset: \`$dataset_id\`"
- echo
- echo 'Source run:'
- sed 's/^/- `/' "$RUNNER_TEMP/collectivex-runs.tsv" | sed 's/$/`/'
- } >> "$GITHUB_STEP_SUMMARY"
-
- - name: Upload JIT publication artifact
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
- with:
- name: cxpublication-${{ steps.build.outputs.version }}-${{ github.run_id }}-${{ github.run_attempt }}
- path: ${{ runner.temp }}/collectivex-publication/*.ndjson
- if-no-files-found: error
- retention-days: 90
-
- refresh:
- if: ${{ inputs.operation == 'refresh' }}
- runs-on: ubuntu-latest
- timeout-minutes: 10
- env:
- GH_TOKEN: ${{ github.token }}
- SOURCE_RUN_ID: ${{ inputs.refresh_run_id }}
- EXPECTED_DIGEST: ${{ inputs.refresh_digest }}
- steps:
- - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v5.0.0
- with: { clean: true, persist-credentials: false }
-
- - name: Revalidate promoted publication
- id: refresh
- env:
- REPOSITORY: ${{ github.repository }}
- run: |
- set -euo pipefail
- [[ "$SOURCE_RUN_ID" =~ ^[1-9][0-9]*$ ]] || {
- echo 'refresh_run_id must be a decimal workflow run ID' >&2
- exit 1
- }
- [[ "$EXPECTED_DIGEST" =~ ^[0-9a-f]{64}$ ]] || {
- echo 'refresh_digest must be a SHA-256 digest' >&2
- exit 1
- }
- metadata=$(gh api "repos/$REPOSITORY/actions/runs/$SOURCE_RUN_ID")
- path=$(jq -r '.path' <<< "$metadata")
- branch=$(jq -r '.head_branch' <<< "$metadata")
- status=$(jq -r '.status' <<< "$metadata")
- conclusion=$(jq -r '.conclusion' <<< "$metadata")
- attempt=$(jq -r '.run_attempt' <<< "$metadata")
- [ "$path" = '.github/workflows/collectivex-sweep.yml' ] \
- && [ "$branch" = 'collectivex' ] \
- && [ "$status" = completed ] \
- && [ "$conclusion" = success ] \
- && [[ "$attempt" =~ ^[1-9][0-9]*$ ]] || {
- echo 'refresh source is not an eligible CollectiveX workflow run' >&2
- exit 1
- }
-
- # The publication artifact carries its version in its name
- # (cxpublication---); discover it from the run.
- pub_name=$(
- gh api --paginate "repos/$REPOSITORY/actions/runs/$SOURCE_RUN_ID/artifacts" \
- --jq '.artifacts[].name' \
- | grep -E "^cxpublication-[1-9][0-9]*-$SOURCE_RUN_ID-$attempt$" \
- | sort -u
- )
- [ "$(printf '%s\n' "$pub_name" | grep -c .)" -eq 1 ] || {
- echo 'refresh source has no single versioned publication artifact' >&2
- exit 1
- }
- release_version=$(printf '%s\n' "$pub_name" | sed -E 's/^cxpublication-([1-9][0-9]*)-.*/\1/')
- echo "version=$release_version" >> "$GITHUB_OUTPUT"
-
- source="$RUNNER_TEMP/collectivex-refresh-source"
- output="$RUNNER_TEMP/collectivex-publication"
- install -d -m 750 "$source" "$output"
- gh run download "$SOURCE_RUN_ID" --repo "$REPOSITORY" \
- --name "$pub_name" --dir "$source"
- mapfile -t files < <(find "$source" -mindepth 1 -maxdepth 1 -type f -print)
- [ "${#files[@]}" -eq 1 ] || {
- echo 'refresh source must contain exactly one root file' >&2
- exit 1
- }
- expected_name="collectivex_public_v1_${EXPECTED_DIGEST}.ndjson"
- [ "$(basename "${files[0]}")" = "$expected_name" ] || {
- echo 'refresh source filename differs from its requested digest' >&2
- exit 1
- }
- [ "$(sha256sum "${files[0]}" | cut -d' ' -f1)" = "$EXPECTED_DIGEST" ] || {
- echo 'refresh source bytes differ from their requested digest' >&2
- exit 1
- }
- [ "$(stat -c %s "${files[0]}")" -le 33554432 ] || {
- echo 'refresh source exceeds the 32 MiB frontend limit' >&2
- exit 1
- }
- python3 - "${files[0]}" <<'PY'
- import json
- import pathlib
- import sys
-
- path = pathlib.Path(sys.argv[1])
- text = path.read_text(encoding="utf-8")
- if not text.endswith("\n") or text.count("\n") != 1:
- raise SystemExit("publication must contain exactly one NDJSON record")
- document = json.loads(text)
- if document.get("format") != "collectivex.public.v1":
- raise SystemExit("publication format is invalid")
- if document.get("promotion", {}).get("status") != "promoted":
- raise SystemExit("publication is not promoted")
- PY
- python3 experimental/CollectiveX/artifact_safety.py "${files[0]}"
- cp -- "${files[0]}" "$output/$expected_name"
-
- - name: Upload refreshed JIT publication artifact
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
- with:
- name: cxpublication-${{ steps.refresh.outputs.version }}-${{ github.run_id }}-${{ github.run_attempt }}
- path: ${{ runner.temp }}/collectivex-publication/*.ndjson
- if-no-files-found: error
- retention-days: 90
From 26c60185903a3e810511c38139c7466fea225788 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Wed, 8 Jul 2026 12:21:50 +0800
Subject: [PATCH 192/210] refactor(collectivex): move executable bench modules
to bench/
Relocate the 15 runnable EP modules (adapters, harness, routing,
workload, eplb, precision codec, run_ep/make_workloads entrypoints) from
tests/ into a dedicated bench/ directory so tests/ holds only unit tests.
Retarget sweep_matrix's importer, the two unit tests' path setup, the
runtime shell launch paths, and run_ep's stamped reproduction command.
Sibling and top-level imports resolve via file-relative sys.path, so the
move is mechanical; probe_precision paths stay pending removal in the
runtime prune.
---
.../CollectiveX/{tests => bench}/ep_backend.py | 0
experimental/CollectiveX/{tests => bench}/ep_deepep.py | 0
.../CollectiveX/{tests => bench}/ep_deepep_family.py | 0
.../CollectiveX/{tests => bench}/ep_deepep_hybrid.py | 0
.../CollectiveX/{tests => bench}/ep_deepep_v2.py | 0
.../CollectiveX/{tests => bench}/ep_harness.py | 0
experimental/CollectiveX/{tests => bench}/ep_mori.py | 0
experimental/CollectiveX/{tests => bench}/ep_nccl.py | 0
.../CollectiveX/{tests => bench}/ep_precision.py | 0
experimental/CollectiveX/{tests => bench}/ep_uccl.py | 0
experimental/CollectiveX/{tests => bench}/eplb.py | 0
.../CollectiveX/{tests => bench}/make_workloads.py | 6 +++---
experimental/CollectiveX/{tests => bench}/routing.py | 0
experimental/CollectiveX/{tests => bench}/run_ep.py | 4 ++--
experimental/CollectiveX/{tests => bench}/workload.py | 0
experimental/CollectiveX/runtime/common.sh | 4 ++--
experimental/CollectiveX/runtime/run_in_container.sh | 10 +++++-----
experimental/CollectiveX/sweep_matrix.py | 2 +-
experimental/CollectiveX/tests/test_ep_backend.py | 4 ++--
.../CollectiveX/tests/test_ep_precision_adapters.py | 4 ++--
20 files changed, 17 insertions(+), 17 deletions(-)
rename experimental/CollectiveX/{tests => bench}/ep_backend.py (100%)
rename experimental/CollectiveX/{tests => bench}/ep_deepep.py (100%)
rename experimental/CollectiveX/{tests => bench}/ep_deepep_family.py (100%)
rename experimental/CollectiveX/{tests => bench}/ep_deepep_hybrid.py (100%)
rename experimental/CollectiveX/{tests => bench}/ep_deepep_v2.py (100%)
rename experimental/CollectiveX/{tests => bench}/ep_harness.py (100%)
rename experimental/CollectiveX/{tests => bench}/ep_mori.py (100%)
rename experimental/CollectiveX/{tests => bench}/ep_nccl.py (100%)
rename experimental/CollectiveX/{tests => bench}/ep_precision.py (100%)
rename experimental/CollectiveX/{tests => bench}/ep_uccl.py (100%)
rename experimental/CollectiveX/{tests => bench}/eplb.py (100%)
rename experimental/CollectiveX/{tests => bench}/make_workloads.py (96%)
rename experimental/CollectiveX/{tests => bench}/routing.py (100%)
rename experimental/CollectiveX/{tests => bench}/run_ep.py (99%)
rename experimental/CollectiveX/{tests => bench}/workload.py (100%)
diff --git a/experimental/CollectiveX/tests/ep_backend.py b/experimental/CollectiveX/bench/ep_backend.py
similarity index 100%
rename from experimental/CollectiveX/tests/ep_backend.py
rename to experimental/CollectiveX/bench/ep_backend.py
diff --git a/experimental/CollectiveX/tests/ep_deepep.py b/experimental/CollectiveX/bench/ep_deepep.py
similarity index 100%
rename from experimental/CollectiveX/tests/ep_deepep.py
rename to experimental/CollectiveX/bench/ep_deepep.py
diff --git a/experimental/CollectiveX/tests/ep_deepep_family.py b/experimental/CollectiveX/bench/ep_deepep_family.py
similarity index 100%
rename from experimental/CollectiveX/tests/ep_deepep_family.py
rename to experimental/CollectiveX/bench/ep_deepep_family.py
diff --git a/experimental/CollectiveX/tests/ep_deepep_hybrid.py b/experimental/CollectiveX/bench/ep_deepep_hybrid.py
similarity index 100%
rename from experimental/CollectiveX/tests/ep_deepep_hybrid.py
rename to experimental/CollectiveX/bench/ep_deepep_hybrid.py
diff --git a/experimental/CollectiveX/tests/ep_deepep_v2.py b/experimental/CollectiveX/bench/ep_deepep_v2.py
similarity index 100%
rename from experimental/CollectiveX/tests/ep_deepep_v2.py
rename to experimental/CollectiveX/bench/ep_deepep_v2.py
diff --git a/experimental/CollectiveX/tests/ep_harness.py b/experimental/CollectiveX/bench/ep_harness.py
similarity index 100%
rename from experimental/CollectiveX/tests/ep_harness.py
rename to experimental/CollectiveX/bench/ep_harness.py
diff --git a/experimental/CollectiveX/tests/ep_mori.py b/experimental/CollectiveX/bench/ep_mori.py
similarity index 100%
rename from experimental/CollectiveX/tests/ep_mori.py
rename to experimental/CollectiveX/bench/ep_mori.py
diff --git a/experimental/CollectiveX/tests/ep_nccl.py b/experimental/CollectiveX/bench/ep_nccl.py
similarity index 100%
rename from experimental/CollectiveX/tests/ep_nccl.py
rename to experimental/CollectiveX/bench/ep_nccl.py
diff --git a/experimental/CollectiveX/tests/ep_precision.py b/experimental/CollectiveX/bench/ep_precision.py
similarity index 100%
rename from experimental/CollectiveX/tests/ep_precision.py
rename to experimental/CollectiveX/bench/ep_precision.py
diff --git a/experimental/CollectiveX/tests/ep_uccl.py b/experimental/CollectiveX/bench/ep_uccl.py
similarity index 100%
rename from experimental/CollectiveX/tests/ep_uccl.py
rename to experimental/CollectiveX/bench/ep_uccl.py
diff --git a/experimental/CollectiveX/tests/eplb.py b/experimental/CollectiveX/bench/eplb.py
similarity index 100%
rename from experimental/CollectiveX/tests/eplb.py
rename to experimental/CollectiveX/bench/eplb.py
diff --git a/experimental/CollectiveX/tests/make_workloads.py b/experimental/CollectiveX/bench/make_workloads.py
similarity index 96%
rename from experimental/CollectiveX/tests/make_workloads.py
rename to experimental/CollectiveX/bench/make_workloads.py
index 5ceaba70f0..65dd2064d9 100644
--- a/experimental/CollectiveX/tests/make_workloads.py
+++ b/experimental/CollectiveX/bench/make_workloads.py
@@ -4,17 +4,17 @@
dir that runs then consume via `run_ep.py --workload-dir`. One trace is emitted per global-token
count because global token count is part of workload identity.
- python3 tests/make_workloads.py --out-dir /path/to/cx_workloads \\
+ python3 bench/make_workloads.py --out-dir /path/to/cx_workloads \\
--routing uniform --ep 8 --hidden 7168 --topk 8 --experts 256 --seed 67 \\
--tokens-ladder "1 2 4 8 16 32 64 128 256 512"
Or by the named v1 workload in configs/workloads.yaml. Explicit dimension flags still override it:
- python3 tests/make_workloads.py --out-dir /path/to/cx_workloads --workload deepseek-v3-v1 --routing uniform --ep 8
+ python3 bench/make_workloads.py --out-dir /path/to/cx_workloads --workload deepseek-v3-v1 --routing uniform --ep 8
--id-only prints the content-bound workload_id per ladder point without torch/numpy:
- python3 tests/make_workloads.py --workload deepseek-v3-v1 --ep 8 --id-only
+ python3 bench/make_workloads.py --workload deepseek-v3-v1 --ep 8 --id-only
Generate every routing the suites need by running once per --routing. Idempotent (same id => same
file). The dir is the cross-hardware artifact: copy it to each cluster so all consume identical bytes.
diff --git a/experimental/CollectiveX/tests/routing.py b/experimental/CollectiveX/bench/routing.py
similarity index 100%
rename from experimental/CollectiveX/tests/routing.py
rename to experimental/CollectiveX/bench/routing.py
diff --git a/experimental/CollectiveX/tests/run_ep.py b/experimental/CollectiveX/bench/run_ep.py
similarity index 99%
rename from experimental/CollectiveX/tests/run_ep.py
rename to experimental/CollectiveX/bench/run_ep.py
index 0c47cce456..9fc7b3bccb 100644
--- a/experimental/CollectiveX/tests/run_ep.py
+++ b/experimental/CollectiveX/bench/run_ep.py
@@ -16,7 +16,7 @@
import subprocess
import sys
-# Make the sibling tests/ modules importable when run as `tests/run_ep.py` under
+# Make the sibling bench/ modules importable when run as `bench/run_ep.py` under
# torchrun (it executes the file as __main__, not as a package).
HERE = os.path.dirname(os.path.abspath(__file__))
sys.path[:0] = [HERE, os.path.dirname(HERE)]
@@ -424,7 +424,7 @@ def main() -> int:
else:
args.distributed_launcher = "rank-environment"
prefix = f"RANK={rank} WORLD_SIZE={world_size} LOCAL_RANK={local_rank} python3"
- args.reproduction_command = f"{prefix} tests/run_ep.py {shlex.join(sys.argv[1:])}"
+ args.reproduction_command = f"{prefix} bench/run_ep.py {shlex.join(sys.argv[1:])}"
args.image = os.environ.get("COLLECTIVEX_IMAGE", "")
args.image_digest = os.environ.get("COLLECTIVEX_IMAGE_DIGEST", "")
args.image_digest_verified = (
diff --git a/experimental/CollectiveX/tests/workload.py b/experimental/CollectiveX/bench/workload.py
similarity index 100%
rename from experimental/CollectiveX/tests/workload.py
rename to experimental/CollectiveX/bench/workload.py
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index e7a0a8446d..51759b093a 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -1181,7 +1181,7 @@ export RANK="$SLURM_PROCID" WORLD_SIZE="$SLURM_NTASKS"
export LOCAL_RANK="$SLURM_LOCALID" LOCAL_WORLD_SIZE="$CX_GPUS_PER_NODE"
case "${CX_PRECISION_PROBE:-0}" in
1) exec python3 tests/probe_precision.py "$@" ;;
- 0|'') exec python3 tests/run_ep.py "$@" ;;
+ 0|'') exec python3 bench/run_ep.py "$@" ;;
*) exit 67 ;;
esac
BASH
@@ -3261,7 +3261,7 @@ PY
workload_ladder="$ladder"
[ -n "$workload_ladder" ] \
|| workload_ladder="1 2 4 8 16 32 64 128 256 512 1024 2048 4096"
- workload_args=(python3 tests/make_workloads.py --out-dir "$workload_dir"
+ workload_args=(python3 bench/make_workloads.py --out-dir "$workload_dir"
--routing "$routing" --ep "$ep" --hidden "$hidden" --topk "$topk"
--experts "$experts" --seed "${CX_SEED:-67}" --tokens-ladder "$workload_ladder")
workload_log="$(cx_private_log_path "workload-c$(printf '%03d' "$ci")")"
diff --git a/experimental/CollectiveX/runtime/run_in_container.sh b/experimental/CollectiveX/runtime/run_in_container.sh
index 47c439d62d..9cc42b2c8f 100644
--- a/experimental/CollectiveX/runtime/run_in_container.sh
+++ b/experimental/CollectiveX/runtime/run_in_container.sh
@@ -7,7 +7,7 @@
#
# Required env (exported by the adapter): CX_RUNNER CX_NGPUS CX_TS CX_TOPO
# Selector: CX_BENCH = deepep | deepep-v2 | mori | uccl | nccl-ep | deepep-hybrid
-# EP knobs passed to tests/run_ep.py:
+# EP knobs passed to bench/run_ep.py:
# CX_PHASE = decode | prefill | both (default decode) <- picks the token sweep
# CX_TOKENS_LADDER (space/comma sep; blank = phase default)
# CX_HIDDEN CX_TOPK CX_EXPERTS CX_ROUTING CX_SEED CX_ITERS
@@ -30,7 +30,7 @@ cx_apply_timing_profile
cx_log "in-container: runner=$CX_RUNNER ngpus=$CX_NGPUS bench=$CX_BENCH topo=$CX_TOPO"
-# Blank ladders use the phase default in tests/run_ep.py.
+# Blank ladders use the phase default in bench/run_ep.py.
cx_ep_ladder() {
printf '%s' "${CX_TOKENS_LADDER:-}"
}
@@ -48,7 +48,7 @@ cx_stage_canonical() {
# cover both phase ladders when none is given, so either phase finds its files.
[ -z "$ladder" ] && ladder="1 2 4 8 16 32 64 128 256 512 1024 2048 4096"
cx_log "staging canonical workloads (routing=${CX_ROUTING:-uniform} ep=$CX_NGPUS ladder='$ladder')"
- python3 tests/make_workloads.py --out-dir "$dir" --routing "${CX_ROUTING:-uniform}" \
+ python3 bench/make_workloads.py --out-dir "$dir" --routing "${CX_ROUTING:-uniform}" \
--ep "$CX_NGPUS" --hidden "${CX_HIDDEN:-7168}" --topk "${CX_TOPK:-8}" \
--experts "${CX_EXPERTS:-256}" --seed "${CX_SEED:-67}" --tokens-ladder "$ladder" \
|| { cx_log "ERROR: canonical workload staging failed"; return 1; }
@@ -57,7 +57,7 @@ cx_stage_canonical() {
}
# run_ep_suite
-# One tests/run_ep.py invocation per phase (decode/prefill/both); dispatch and
+# One bench/run_ep.py invocation per phase (decode/prefill/both); dispatch and
# combine are timed separately inside it. One JSON per (backend, phase).
# Preserve a failed case with its full scheduled identity instead of letting it vanish.
emit_failed_case() { # backend phase rc
@@ -96,7 +96,7 @@ run_ep_suite() {
[ -n "${CX_WORKLOAD_DIR:-}" ] && EPARGS+=(--workload-dir "$CX_WORKLOAD_DIR")
cx_write_runtime_stage execution || cx_die "cannot record runtime stage"
if timeout -k 30 "${CX_RUN_TIMEOUT:-900}" \
- torchrun --nproc_per_node="$CX_NGPUS" tests/run_ep.py "${EPARGS[@]}"; then
+ torchrun --nproc_per_node="$CX_NGPUS" bench/run_ep.py "${EPARGS[@]}"; then
rc_run=0
else
rc_run=$?
diff --git a/experimental/CollectiveX/sweep_matrix.py b/experimental/CollectiveX/sweep_matrix.py
index 5f0efdb82f..f36aef9061 100644
--- a/experimental/CollectiveX/sweep_matrix.py
+++ b/experimental/CollectiveX/sweep_matrix.py
@@ -20,7 +20,7 @@
HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE))
-sys.path.insert(0, str(HERE / "tests"))
+sys.path.insert(0, str(HERE / "bench"))
try: # Shard extraction on GPU runners is intentionally stdlib-only.
import yaml # type: ignore
diff --git a/experimental/CollectiveX/tests/test_ep_backend.py b/experimental/CollectiveX/tests/test_ep_backend.py
index 97e253da2e..5141f430a6 100644
--- a/experimental/CollectiveX/tests/test_ep_backend.py
+++ b/experimental/CollectiveX/tests/test_ep_backend.py
@@ -21,9 +21,9 @@
from unittest import mock
ROOT = Path(__file__).resolve().parents[1]
-TESTS = ROOT / "tests"
+BENCH = ROOT / "bench"
sys.path.insert(0, str(ROOT))
-sys.path.insert(0, str(TESTS))
+sys.path.insert(0, str(BENCH))
import ep_backend # noqa: E402
from ep_backend import EPBackend, RankInputs # noqa: E402
diff --git a/experimental/CollectiveX/tests/test_ep_precision_adapters.py b/experimental/CollectiveX/tests/test_ep_precision_adapters.py
index d98de83c88..e5f4fd7868 100644
--- a/experimental/CollectiveX/tests/test_ep_precision_adapters.py
+++ b/experimental/CollectiveX/tests/test_ep_precision_adapters.py
@@ -9,9 +9,9 @@
ROOT = Path(__file__).resolve().parents[1]
-TESTS = ROOT / "tests"
+BENCH = ROOT / "bench"
sys.path.insert(0, str(ROOT))
-sys.path.insert(0, str(TESTS))
+sys.path.insert(0, str(BENCH))
import ep_precision # noqa: E402
From e439d1be5171e74ebfce5f321dce30fef8544e13 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Wed, 8 Jul 2026 12:32:11 +0800
Subject: [PATCH 193/210] refactor(collectivex): move backend provenance
evidence to bench/ep_provenance.py
The emitter's implementation-provenance evidence builders (DeepEP MNNVL
resolution, NCCL/RCCL lineage, resource projection, backend-provenance issue
detection, hybrid/deepep-v2 JIT evidence checks, content-manifest hashing) are
cross-field facts a JSON Schema cannot express, but they are an emitter concern,
not part of the neutral document/delivery validation boundary. Relocate the
twelve-function cluster beside the bench modules that call it, so contracts.py
holds only neutral validation. Rewire the five emitter modules (ep_deepep,
ep_deepep_hybrid, ep_harness, ep_nccl, ep_uccl) to ep_provenance. Drop two dead
helpers (backend_version, hybrid_communication_domains) with no callers.
contracts.py now holds only neutral validation (1009 LOC).
---
experimental/CollectiveX/bench/ep_deepep.py | 5 +-
.../CollectiveX/bench/ep_deepep_hybrid.py | 3 +-
experimental/CollectiveX/bench/ep_harness.py | 7 +-
experimental/CollectiveX/bench/ep_nccl.py | 3 +-
.../CollectiveX/bench/ep_provenance.py | 466 ++++++++++++++++++
experimental/CollectiveX/bench/ep_uccl.py | 9 +-
experimental/CollectiveX/contracts.py | 464 +----------------
7 files changed, 485 insertions(+), 472 deletions(-)
create mode 100644 experimental/CollectiveX/bench/ep_provenance.py
diff --git a/experimental/CollectiveX/bench/ep_deepep.py b/experimental/CollectiveX/bench/ep_deepep.py
index 270ece9600..49dcf94692 100644
--- a/experimental/CollectiveX/bench/ep_deepep.py
+++ b/experimental/CollectiveX/bench/ep_deepep.py
@@ -9,6 +9,7 @@
import torch
import torch.distributed as dist
import contracts
+import ep_provenance
import ep_precision
from ep_deepep_family import DeepEPFamilyBackend
@@ -36,7 +37,7 @@ def _mnnvl_buffer_configuration() -> tuple[dict[str, bool], str]:
raise RuntimeError("CX_ALLOW_MNNVL must be unset, 0, or 1")
requested = requested_value == "1"
if not requested:
- return contracts.resolve_deepep_mnnvl(
+ return ep_provenance.resolve_deepep_mnnvl(
requested=False, signature_parameters=(),
deepep_commit=os.environ.get("DEEPEP_COMMIT"),
)
@@ -45,7 +46,7 @@ def _mnnvl_buffer_configuration() -> tuple[dict[str, bool], str]:
except (TypeError, ValueError) as exc:
raise RuntimeError("cannot inspect DeepEP Buffer MNNVL API") from exc
try:
- return contracts.resolve_deepep_mnnvl(
+ return ep_provenance.resolve_deepep_mnnvl(
requested=True, signature_parameters=parameters,
deepep_commit=os.environ.get("DEEPEP_COMMIT"),
)
diff --git a/experimental/CollectiveX/bench/ep_deepep_hybrid.py b/experimental/CollectiveX/bench/ep_deepep_hybrid.py
index 7e1a688bd8..90d251bc50 100644
--- a/experimental/CollectiveX/bench/ep_deepep_hybrid.py
+++ b/experimental/CollectiveX/bench/ep_deepep_hybrid.py
@@ -39,6 +39,7 @@
import torch
import torch.distributed as dist
import contracts
+import ep_provenance
import ep_precision
from ep_backend import EPBackend
@@ -66,7 +67,7 @@ def _hybrid_build_evidence() -> list[dict[str, str]]:
path = getattr(module, "__file__", None)
if not path:
raise RuntimeError(f"{module_name} has no loaded extension path")
- records.append(contracts.content_manifest_evidence(
+ records.append(ep_provenance.content_manifest_evidence(
role=role,
name=module_name,
files=[(os.path.basename(path), path)],
diff --git a/experimental/CollectiveX/bench/ep_harness.py b/experimental/CollectiveX/bench/ep_harness.py
index 6a94523ddd..294366924b 100644
--- a/experimental/CollectiveX/bench/ep_harness.py
+++ b/experimental/CollectiveX/bench/ep_harness.py
@@ -46,6 +46,7 @@
import types
import contracts
+import ep_provenance
import ep_precision
import identity
import workload as workload_contract
@@ -1313,7 +1314,7 @@ def run_sweep(args, backend, torch, dist, device, rank: int, world_size: int) ->
# Setup may materialize deferred provenance such as DeepEP V2 JIT CUBINs.
# Resolve it after conditioning but before correctness or timed measurements.
backend.capture_deferred_provenance()
- provenance_issues = contracts.backend_provenance_issues(
+ provenance_issues = ep_provenance.backend_provenance_issues(
backend.name, backend.backend_provenance
)
if provenance_issues:
@@ -1658,7 +1659,7 @@ def component(percentiles, count, *, derived=False):
# Adapters never self-label official; status is derived from these gates.
prov = backend.backend_provenance
allocation_stratum_sha256 = getattr(args, "allocation_stratum_sha256", None)
- provenance_complete = contracts.provenance_complete(
+ provenance_complete = ep_provenance.provenance_complete(
prov,
backend.name,
getattr(args, "git_run", None),
@@ -1667,7 +1668,7 @@ def component(percentiles, count, *, derived=False):
image_verified=getattr(args, "image_digest_verified", False),
squash_sha256=getattr(args, "squash_sha256", None),
)
- resource_profile = contracts.project_resource_profile(prov)
+ resource_profile = ep_provenance.project_resource_profile(prov)
resource_conformance = resource_profile["conformance_class"]
# record the canonical workload identity consumed (one trace per T -> set of ids/checksums).
if canonical and loaded_workload_ids:
diff --git a/experimental/CollectiveX/bench/ep_nccl.py b/experimental/CollectiveX/bench/ep_nccl.py
index 0b472aec77..e7953f246b 100644
--- a/experimental/CollectiveX/bench/ep_nccl.py
+++ b/experimental/CollectiveX/bench/ep_nccl.py
@@ -20,6 +20,7 @@
import torch
import torch.distributed as dist
import contracts
+import ep_provenance
import ep_precision
from ep_backend import EPBackend
@@ -86,7 +87,7 @@ def __init__(self, args, rank, world_size, local_rank, device):
f"(present={bool(hcas)}, exact={hcas.startswith('=')}, "
f"entries={len(entries)}, shaped={shaped})"
)
- self.kernel_generation = contracts.collective_kernel_generation(_library)
+ self.kernel_generation = ep_provenance.collective_kernel_generation(_library)
self.backend_provenance = {
"backend": f"{_library}-all2all",
"backend_lineage": _library,
diff --git a/experimental/CollectiveX/bench/ep_provenance.py b/experimental/CollectiveX/bench/ep_provenance.py
new file mode 100644
index 0000000000..973b9b7a8a
--- /dev/null
+++ b/experimental/CollectiveX/bench/ep_provenance.py
@@ -0,0 +1,466 @@
+#!/usr/bin/env python3
+"""Backend implementation-provenance evidence and resource projection.
+
+The EP benchmark emitter builds and self-checks the implementation-provenance
+evidence it embeds in each raw attempt document: which backend libraries were
+loaded, that a deepep-v2 attempt carries internally consistent JIT cubins, that
+a hybrid attempt realized its kernels, and so on. Those are cross-field facts a
+JSON Schema cannot express, so they live in Python — but they are an emitter
+concern, not part of the neutral cross-file validation boundary. They are kept
+here, beside the executable bench modules that call them, so ``contracts.py``
+holds only the neutral document/delivery validation.
+"""
+from __future__ import annotations
+
+import hashlib
+import os
+import re
+from pathlib import Path, PurePosixPath
+from typing import Any, Iterable
+
+from contracts import (
+ ContractError,
+ canonical_json_bytes,
+ DEEPEP_V2_DISTRIBUTION_VERSIONS,
+ DEEPEP_V2_JIT_KERNELS,
+ DEEPEP_V2_V1_PROVENANCE,
+ GIT_RUN_FIELDS,
+ REQUIRED_BACKEND_PROVENANCE,
+ UCCL_DEPENDENCY_VERSIONS,
+)
+
+
+def resolve_deepep_mnnvl(
+ *, requested: bool, signature_parameters: Iterable[str], deepep_commit: str | None
+) -> tuple[dict[str, bool], str]:
+ """Resolve one explicit DeepEP MNNVL API mode without signature fallbacks."""
+ if not requested:
+ return {}, "not-requested"
+ if "allow_mnnvl" in set(signature_parameters):
+ return {"allow_mnnvl": True}, "explicit-allow-mnnvl"
+ raise ContractError(
+ f"requested DeepEP MNNVL is unsupported by commit {deepep_commit or 'unknown'}"
+ )
+
+
+def collective_kernel_generation(collective_library: Any) -> str:
+ """Return the public NCCL/RCCL implementation lineage."""
+ if collective_library not in {"nccl", "rccl"}:
+ raise ContractError("reference collective library must be nccl or rccl")
+ return collective_library
+
+
+def project_resource_profile(provenance: dict[str, Any]) -> dict[str, Any]:
+ """Project backend provenance into the canonical cross-backend resource vocabulary."""
+ device_units = provenance.get("device_sms") or provenance.get("device_cus")
+ if provenance.get("num_sms") is not None:
+ kind, configured = "sm", provenance["num_sms"]
+ elif (
+ provenance.get("block_num") is not None
+ and provenance.get("kernel_type") != "AsyncLL"
+ ):
+ kind, configured = "cu_block", provenance["block_num"]
+ else:
+ kind, configured = None, None
+ achieved = configured / device_units if configured and device_units else None
+ fixed = "fixed-kernel" in str(provenance.get("tuned_source", ""))
+ source = str(provenance.get("tuned_source", ""))
+ num_nvl_bytes = provenance.get("num_nvl_bytes")
+ num_rdma_bytes = provenance.get("num_rdma_bytes")
+ persistent_bytes = (
+ (num_nvl_bytes or 0) + (num_rdma_bytes or 0)
+ if num_nvl_bytes is not None or num_rdma_bytes is not None
+ else provenance.get("heap_size")
+ )
+ return {
+ "achieved_fraction": round(achieved, 4) if achieved else None,
+ "comm_units_kind": kind,
+ "configured_units": configured,
+ "conformance_class": (
+ "not-applicable" if fixed else "backend-default" if "default" in source
+ else "pinned-upstream"
+ ),
+ "device_units": device_units,
+ "fixed_kernel": fixed,
+ "nonconforming": False,
+ "pareto_eligible": False,
+ "persistent_bytes": persistent_bytes,
+ "qps_per_rank": provenance.get("num_qps_per_rank"),
+ "requested_fraction": None,
+ "resource_class": "fixed-kernel" if fixed else "fixed-profile",
+ "target_achieved_within_tol": None,
+ "tolerance": 0.10,
+ "tuned_source": provenance.get("tuned_source"),
+ "warps_combine": provenance.get("combine_warps"),
+ "warps_dispatch": provenance.get("dispatch_warps"),
+ }
+
+
+def _resolved_provenance_value(field: str, value: Any) -> bool:
+ if value is None or isinstance(value, (dict, list, tuple, set)) and not value:
+ return False
+ text = str(value).strip().lower()
+ if not text or text in {"unknown", "none", "null", "n/a", "?", "capture-failed"}:
+ return False
+ if "capture-failed" in text:
+ return False
+ if field.endswith("_commit") and (
+ text in {"main", "hybrid-ep", "uccl", "pkg-uccl"}
+ or text.endswith(("-unknown", "-none", "-main", "-hybrid-ep"))
+ ):
+ return False
+ return True
+
+
+def _content_evidence_is_valid(value: Any, required_roles: set[str]) -> bool:
+ if not isinstance(value, list) or not value:
+ return False
+ records: set[tuple[str, str]] = set()
+ roles: set[str] = set()
+ for item in value:
+ if not isinstance(item, dict) or set(item) != {"name", "role", "sha256"}:
+ return False
+ name, role, digest = item["name"], item["role"], item["sha256"]
+ if (
+ not isinstance(name, str)
+ or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.+-]{0,159}", name)
+ or not isinstance(role, str)
+ or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.+-]{0,127}", role)
+ or not isinstance(digest, str)
+ or not re.fullmatch(r"[0-9a-f]{64}", digest)
+ or (role, name) in records
+ ):
+ return False
+ records.add((role, name))
+ roles.add(role)
+ return required_roles <= roles
+
+
+def _deepep_v2_jit_cubins_are_valid(value: Any) -> bool:
+ if not isinstance(value, list) or len(value) != len(DEEPEP_V2_JIT_KERNELS):
+ return False
+ cache_keys = []
+ kernel_names = set()
+ for item in value:
+ if not isinstance(item, dict) or set(item) != {
+ "cache_key", "cubin_sha256", "sass_sha256", "source_sha256",
+ }:
+ return False
+ cache_key = item["cache_key"]
+ match = (
+ re.fullmatch(r"kernel\.([A-Za-z0-9_+-]+)\.[0-9a-f]{32}", cache_key)
+ if isinstance(cache_key, str)
+ else None
+ )
+ if (
+ match is None
+ or any(
+ not isinstance(item[field], str)
+ or not re.fullmatch(r"[0-9a-f]{64}", item[field])
+ for field in ("cubin_sha256", "sass_sha256", "source_sha256")
+ )
+ ):
+ return False
+ cache_keys.append(cache_key)
+ kernel_names.add(match.group(1))
+ return (
+ cache_keys == sorted(set(cache_keys))
+ and kernel_names == DEEPEP_V2_JIT_KERNELS
+ )
+
+
+HYBRID_REALIZED_CONFIG_FIELDS = {
+ "hidden_dim", "max_num_of_tokens_per_rank", "num_of_experts_per_rank",
+ "num_of_ranks_per_node", "num_of_nodes", "pad_multiple",
+ "num_of_tokens_per_chunk_preprocessing_api",
+ "num_of_threads_per_block_preprocessing_api", "num_of_blocks_preprocessing_api",
+ "num_of_blocks_permute", "num_of_blocks_unpermute", "token_data_type",
+ "num_of_stages_dispatch_api", "num_of_stages_permute_block_dispatch_api",
+ "num_of_in_flight_s2g_dispatch_api",
+ "num_of_in_flight_s2g_permute_block_dispatch_api",
+ "num_of_additional_in_flight_s2g_dispatch_api",
+ "num_of_tokens_per_chunk_dispatch_api", "num_of_blocks_dispatch_api",
+ "forward_dispatch_api", "device_side_sync_dispatch_api",
+ "num_of_stages_g2s_combine_api", "num_of_stages_s2g_combine_api",
+ "num_of_tokens_per_chunk_combine_api", "num_of_tokens_per_group_combine_api",
+ "num_of_blocks_combine_api", "num_of_additional_in_flight_s2g_combine_api",
+ "backward_combine_api", "device_side_sync_combine_api",
+}
+HYBRID_REALIZED_BOOL_FIELDS = {
+ "forward_dispatch_api", "device_side_sync_dispatch_api", "backward_combine_api",
+ "device_side_sync_combine_api",
+}
+
+
+def _hybrid_realized_config_is_valid(value: Any) -> bool:
+ if not isinstance(value, dict) or set(value) != HYBRID_REALIZED_CONFIG_FIELDS:
+ return False
+ for field, field_value in value.items():
+ if field in HYBRID_REALIZED_BOOL_FIELDS:
+ if type(field_value) is not bool:
+ return False
+ elif field == "token_data_type":
+ if field_value not in {"UINT8", "UINT16"}:
+ return False
+ elif type(field_value) is not int or field_value < 0:
+ return False
+ return all(value[field] > 0 for field in (
+ "hidden_dim", "max_num_of_tokens_per_rank", "num_of_experts_per_rank",
+ "num_of_ranks_per_node", "num_of_nodes",
+ ))
+
+
+def _hybrid_kernel_keys_are_valid(value: Any) -> bool:
+ return (
+ isinstance(value, list)
+ and len(value) == 3
+ and len(set(value)) == 3
+ and value == sorted(value)
+ and all(
+ isinstance(key, str)
+ and re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.+-]{0,511}", key)
+ for key in value
+ )
+ )
+
+
+def _hybrid_jit_evidence_is_valid(value: Any, kernel_keys: Any) -> bool:
+ if not _hybrid_kernel_keys_are_valid(kernel_keys) or not isinstance(value, list):
+ return False
+ if len(value) != len(kernel_keys):
+ return False
+ rank_sets = []
+ for expected_key, item in zip(kernel_keys, value):
+ if not isinstance(item, dict) or set(item) != {"kernel_key", "rank_artifacts"}:
+ return False
+ rank_artifacts = item["rank_artifacts"]
+ if item["kernel_key"] != expected_key or not isinstance(rank_artifacts, list):
+ return False
+ ranks = []
+ for artifact in rank_artifacts:
+ if not isinstance(artifact, dict) or set(artifact) != {"bytes", "rank", "sha256"}:
+ return False
+ rank, digest, size = artifact["rank"], artifact["sha256"], artifact["bytes"]
+ if (
+ type(rank) is not int
+ or rank < 0
+ or not isinstance(digest, str)
+ or not re.fullmatch(r"[0-9a-f]{64}", digest)
+ or type(size) is not int
+ or size <= 0
+ ):
+ return False
+ ranks.append(rank)
+ if not ranks or ranks != list(range(len(ranks))):
+ return False
+ rank_sets.append(ranks)
+ return all(ranks == rank_sets[0] for ranks in rank_sets)
+
+
+def backend_provenance_issues(backend: str, provenance: dict[str, Any]) -> list[str]:
+ unknown = [
+ field for field, value in provenance.items()
+ if isinstance(value, str) and value.strip().lower() == "unknown"
+ ]
+ unresolved = [
+ field for field in REQUIRED_BACKEND_PROVENANCE.get(backend, ())
+ if not _resolved_provenance_value(field, provenance.get(field))
+ ]
+ if backend == "deepep":
+ mode = provenance.get("mnnvl_comm")
+ allow = provenance.get("allow_mnnvl")
+ valid_modes = {
+ "not-requested": False,
+ "explicit-allow-mnnvl": True,
+ }
+ if type(allow) is not bool or valid_modes.get(mode) is not allow:
+ unresolved.append("mnnvl_comm")
+ if provenance.get("backend_lineage") != "deepep-v1":
+ unresolved.append("backend_lineage")
+ if provenance.get("nvshmem_ibgda_nic_handler") not in {
+ "cpu", "gpu", "not-active",
+ }:
+ unresolved.append("nvshmem_ibgda_nic_handler")
+ if backend in {"deepep", "uccl"}:
+ mode = provenance.get("mode")
+ num_nvl_bytes = provenance.get("num_nvl_bytes")
+ num_rdma_bytes = provenance.get("num_rdma_bytes")
+ if mode not in {"normal", "low-latency"}:
+ unresolved.append("mode")
+ if type(num_nvl_bytes) is not int or num_nvl_bytes < 0:
+ unresolved.append("num_nvl_bytes")
+ if type(num_rdma_bytes) is not int or num_rdma_bytes < 0:
+ unresolved.append("num_rdma_bytes")
+ if mode == "normal" and (type(num_nvl_bytes) is not int or num_nvl_bytes <= 0):
+ unresolved.append("num_nvl_bytes")
+ if mode == "low-latency":
+ if num_nvl_bytes != 0:
+ unresolved.append("num_nvl_bytes")
+ if type(num_rdma_bytes) is not int or num_rdma_bytes <= 0:
+ unresolved.append("num_rdma_bytes")
+ if (
+ type(provenance.get("num_max_tokens_per_rank")) is not int
+ or provenance["num_max_tokens_per_rank"] <= 0
+ ):
+ unresolved.append("num_max_tokens_per_rank")
+ if backend == "deepep" and (
+ type(provenance.get("num_qps_per_rank")) is not int
+ or provenance["num_qps_per_rank"] <= 0
+ ):
+ unresolved.append("num_qps_per_rank")
+ if backend == "deepep-v2":
+ for field in ("num_experts", "tuning_num_experts"):
+ if type(provenance.get(field)) is not int or provenance[field] <= 0:
+ unresolved.append(field)
+ if not _deepep_v2_jit_cubins_are_valid(provenance.get("jit_cubins")):
+ unresolved.append("jit_cubins")
+ if provenance.get("jit_random_seed") != "collectivex-deepep-v2-fa8a9b1":
+ unresolved.append("jit_random_seed")
+ unresolved.extend(
+ field for field, expected in DEEPEP_V2_V1_PROVENANCE.items()
+ if field != "deepep_distribution_version"
+ and provenance.get(field) != expected
+ )
+ if provenance.get("deepep_distribution_version") not in (
+ DEEPEP_V2_DISTRIBUTION_VERSIONS
+ ):
+ unresolved.append("deepep_distribution_version")
+ policy = (
+ provenance.get("allow_hybrid_mode"),
+ provenance.get("gin_enabled"),
+ provenance.get("communication_backend"),
+ )
+ if policy not in {
+ (False, False, "nccl-device-lsa"),
+ (True, True, "nccl-gin"),
+ }:
+ unresolved.extend(
+ ("allow_hybrid_mode", "gin_enabled", "communication_backend")
+ )
+ content_roles = {
+ "deepep-v2": {"deepep-extension", "nccl", "nvshmem"},
+ "deepep-hybrid": {"deepep-extension", "deepep-hybrid-extension"},
+ "uccl": {
+ "uccl-distribution", "uccl-wrapper", "intervaltree-distribution",
+ "sortedcontainers-distribution", "cuda-runtime",
+ },
+ }.get(backend)
+ if content_roles is not None and not _content_evidence_is_valid(
+ provenance.get("loaded_libraries"), content_roles
+ ):
+ unresolved.append("loaded_libraries")
+ if backend in {"deepep-v2", "deepep-hybrid"} and not re.fullmatch(
+ r"[0-9a-f]{40}", str(provenance.get("deepep_tree", ""))
+ ):
+ unresolved.append("deepep_tree")
+ if backend == "deepep-hybrid" and provenance.get("backend_lineage") != "deepep-hybrid":
+ unresolved.append("backend_lineage")
+ if backend == "deepep-hybrid":
+ if not _hybrid_realized_config_is_valid(provenance.get("realized_config")):
+ unresolved.append("realized_config")
+ if not _hybrid_kernel_keys_are_valid(provenance.get("jit_kernel_keys")):
+ unresolved.append("jit_kernel_keys")
+ if not _hybrid_jit_evidence_is_valid(
+ provenance.get("jit_shared_objects"), provenance.get("jit_kernel_keys")
+ ):
+ unresolved.append("jit_shared_objects")
+ if backend == "uccl" and provenance.get("backend_lineage") != "uccl":
+ unresolved.append("backend_lineage")
+ if backend == "uccl" and provenance.get("uccl_dependency_versions") != (
+ UCCL_DEPENDENCY_VERSIONS
+ ):
+ unresolved.append("uccl_dependency_versions")
+ if backend == "nccl-ep":
+ collective = provenance.get("collective_library")
+ if collective not in {"nccl", "rccl"}:
+ unresolved.append("collective_library")
+ if provenance.get("backend_lineage") != collective:
+ unresolved.append("backend_lineage")
+ if backend == "mori" and provenance.get("kernel_type") == "InterNodeV1":
+ expected = {
+ "block_num": 96,
+ "rdma_block_num": 64,
+ "dispatch_warps": 8,
+ "combine_warps": 8,
+ "num_qps": 1,
+ "use_external_inp_buf": True,
+ "gpus_per_node": 8,
+ }
+ unresolved.extend(
+ field for field, value in expected.items()
+ if provenance.get(field) != value
+ )
+ for field, minimum in (
+ ("num_nvl_bytes", 0), ("num_rdma_bytes", 0),
+ ("num_qps_per_rank", 1),
+ ):
+ if field in provenance and (
+ type(provenance[field]) is not int or provenance[field] < minimum
+ ):
+ unresolved.append(field)
+ if "rdma_block_num" in provenance and (
+ type(provenance["rdma_block_num"]) is not int
+ or provenance["rdma_block_num"] < 0
+ ):
+ unresolved.append("rdma_block_num")
+ if "use_external_inp_buf" in provenance and type(
+ provenance["use_external_inp_buf"]
+ ) is not bool:
+ unresolved.append("use_external_inp_buf")
+ return sorted(set(unknown + unresolved))
+
+
+def provenance_complete(
+ provenance: dict[str, Any], backend: str, git_run: dict[str, Any] | None,
+ *, image_digest: Any, image_verified: Any, squash_sha256: Any,
+) -> bool:
+ """Return whether backend provenance and run identity are fully resolved."""
+ image = str(image_digest or "")
+ squash = str(squash_sha256 or "")
+ return (
+ not backend_provenance_issues(backend, provenance)
+ and image_verified is True
+ and bool(re.fullmatch(r"sha256:[0-9a-f]{64}", image))
+ and bool(re.fullmatch(r"[0-9a-f]{64}", squash))
+ and isinstance(git_run, dict)
+ and all(git_run.get(field) for field in GIT_RUN_FIELDS)
+ )
+
+
+def content_manifest_evidence(
+ *, role: str, name: str, files: Iterable[tuple[str, str | os.PathLike[str]]]
+) -> dict[str, str]:
+ """Hash a labeled file set without exposing any host path in provenance."""
+ if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.+-]{0,127}", role):
+ raise ContractError("content evidence role is invalid")
+ if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.+-]{0,159}", name):
+ raise ContractError("content evidence name is invalid")
+ manifest: list[dict[str, Any]] = []
+ labels: set[str] = set()
+ for label, raw_path in files:
+ logical = PurePosixPath(label)
+ if (
+ not label
+ or logical.is_absolute()
+ or ".." in logical.parts
+ or label in labels
+ or any(ord(character) < 0x20 or ord(character) > 0x7E for character in label)
+ ):
+ raise ContractError("content evidence label is invalid or duplicated")
+ path = Path(raw_path)
+ if not path.is_file():
+ raise ContractError("content evidence source is not a file")
+ digest = hashlib.sha256()
+ size = 0
+ with path.open("rb") as handle:
+ for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+ digest.update(chunk)
+ size += len(chunk)
+ labels.add(label)
+ manifest.append({"bytes": size, "label": label, "sha256": digest.hexdigest()})
+ if not manifest:
+ raise ContractError("content evidence cannot be empty")
+ digest = hashlib.sha256(
+ canonical_json_bytes(sorted(manifest, key=lambda item: item["label"]))
+ ).hexdigest()
+ return {"name": name, "role": role, "sha256": digest}
diff --git a/experimental/CollectiveX/bench/ep_uccl.py b/experimental/CollectiveX/bench/ep_uccl.py
index 06b17a3b32..248bc01219 100644
--- a/experimental/CollectiveX/bench/ep_uccl.py
+++ b/experimental/CollectiveX/bench/ep_uccl.py
@@ -12,6 +12,7 @@
import torch
import torch.distributed as dist
import contracts
+import ep_provenance
import ep_precision
from ep_deepep_family import DeepEPFamilyBackend
@@ -67,7 +68,7 @@ def _python_dependency_evidence(package: str, version: str) -> dict[str, str]:
and path.is_file()
):
runtime_files.append((entry.as_posix(), path))
- return contracts.content_manifest_evidence(
+ return ep_provenance.content_manifest_evidence(
role=f"{package}-distribution",
name=f"{package}-{version}",
files=runtime_files,
@@ -118,7 +119,7 @@ def _loaded_libcudart_evidence(
raise RuntimeError(
"expected exactly one mapped libcudart from the pinned CUDA runtime"
)
- return contracts.content_manifest_evidence(
+ return ep_provenance.content_manifest_evidence(
role="cuda-runtime",
name=f"nvidia-cuda-runtime-cu12-{version}",
files=[("libcudart.so", loaded.pop())],
@@ -142,12 +143,12 @@ def _uccl_build_evidence(
if path.is_file()
]
return [
- contracts.content_manifest_evidence(
+ ep_provenance.content_manifest_evidence(
role="uccl-distribution",
name=f"uccl-{version}",
files=distribution_files,
),
- contracts.content_manifest_evidence(
+ ep_provenance.content_manifest_evidence(
role="uccl-wrapper",
name="uccl-deepep-wrapper",
files=wrapper_files,
diff --git a/experimental/CollectiveX/contracts.py b/experimental/CollectiveX/contracts.py
index 449e83e28d..8b27908f03 100644
--- a/experimental/CollectiveX/contracts.py
+++ b/experimental/CollectiveX/contracts.py
@@ -282,7 +282,9 @@ def walk(value, doc_index: int, parent_field: str | None = None) -> None:
# ---------------------------------------------------------------------------
-# Backend emission helpers (called by the bench emitter, not by validation).
+# Scheduled-case profile resolution (shared by the cross-file validation below
+# and by the bench emitter; backend implementation-provenance evidence now lives
+# in bench/ep_provenance.py).
# ---------------------------------------------------------------------------
def scheduled_case_profile(case: dict[str, Any], path: str = "case") -> dict[str, Any]:
"""Resolve an explicit scheduled mode to its immutable measurement profile."""
@@ -292,427 +294,6 @@ def scheduled_case_profile(case: dict[str, Any], path: str = "case") -> dict[str
raise ContractError(f"{path}: {exc}") from exc
-def resolve_deepep_mnnvl(
- *, requested: bool, signature_parameters: Iterable[str], deepep_commit: str | None
-) -> tuple[dict[str, bool], str]:
- """Resolve one explicit DeepEP MNNVL API mode without signature fallbacks."""
- if not requested:
- return {}, "not-requested"
- if "allow_mnnvl" in set(signature_parameters):
- return {"allow_mnnvl": True}, "explicit-allow-mnnvl"
- raise ContractError(
- f"requested DeepEP MNNVL is unsupported by commit {deepep_commit or 'unknown'}"
- )
-
-
-def collective_kernel_generation(collective_library: Any) -> str:
- """Return the public NCCL/RCCL implementation lineage."""
- if collective_library not in {"nccl", "rccl"}:
- raise ContractError("reference collective library must be nccl or rccl")
- return collective_library
-
-
-def project_resource_profile(provenance: dict[str, Any]) -> dict[str, Any]:
- """Project backend provenance into the canonical cross-backend resource vocabulary."""
- device_units = provenance.get("device_sms") or provenance.get("device_cus")
- if provenance.get("num_sms") is not None:
- kind, configured = "sm", provenance["num_sms"]
- elif (
- provenance.get("block_num") is not None
- and provenance.get("kernel_type") != "AsyncLL"
- ):
- kind, configured = "cu_block", provenance["block_num"]
- else:
- kind, configured = None, None
- achieved = configured / device_units if configured and device_units else None
- fixed = "fixed-kernel" in str(provenance.get("tuned_source", ""))
- source = str(provenance.get("tuned_source", ""))
- num_nvl_bytes = provenance.get("num_nvl_bytes")
- num_rdma_bytes = provenance.get("num_rdma_bytes")
- persistent_bytes = (
- (num_nvl_bytes or 0) + (num_rdma_bytes or 0)
- if num_nvl_bytes is not None or num_rdma_bytes is not None
- else provenance.get("heap_size")
- )
- return {
- "achieved_fraction": round(achieved, 4) if achieved else None,
- "comm_units_kind": kind,
- "configured_units": configured,
- "conformance_class": (
- "not-applicable" if fixed else "backend-default" if "default" in source
- else "pinned-upstream"
- ),
- "device_units": device_units,
- "fixed_kernel": fixed,
- "nonconforming": False,
- "pareto_eligible": False,
- "persistent_bytes": persistent_bytes,
- "qps_per_rank": provenance.get("num_qps_per_rank"),
- "requested_fraction": None,
- "resource_class": "fixed-kernel" if fixed else "fixed-profile",
- "target_achieved_within_tol": None,
- "tolerance": 0.10,
- "tuned_source": provenance.get("tuned_source"),
- "warps_combine": provenance.get("combine_warps"),
- "warps_dispatch": provenance.get("dispatch_warps"),
- }
-
-
-def backend_version(provenance: dict[str, Any]) -> str | None:
- """Return the canonical public backend version from implementation provenance."""
- for field in (
- "deepep_version", "uccl_version", "nccl_version",
- "mori_commit", "deepep_commit",
- ):
- value = provenance.get(field)
- if value is not None and str(value).strip():
- return str(value)[:160]
- return None
-
-
-def _resolved_provenance_value(field: str, value: Any) -> bool:
- if value is None or isinstance(value, (dict, list, tuple, set)) and not value:
- return False
- text = str(value).strip().lower()
- if not text or text in {"unknown", "none", "null", "n/a", "?", "capture-failed"}:
- return False
- if "capture-failed" in text:
- return False
- if field.endswith("_commit") and (
- text in {"main", "hybrid-ep", "uccl", "pkg-uccl"}
- or text.endswith(("-unknown", "-none", "-main", "-hybrid-ep"))
- ):
- return False
- return True
-
-
-def _content_evidence_is_valid(value: Any, required_roles: set[str]) -> bool:
- if not isinstance(value, list) or not value:
- return False
- records: set[tuple[str, str]] = set()
- roles: set[str] = set()
- for item in value:
- if not isinstance(item, dict) or set(item) != {"name", "role", "sha256"}:
- return False
- name, role, digest = item["name"], item["role"], item["sha256"]
- if (
- not isinstance(name, str)
- or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.+-]{0,159}", name)
- or not isinstance(role, str)
- or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.+-]{0,127}", role)
- or not isinstance(digest, str)
- or not re.fullmatch(r"[0-9a-f]{64}", digest)
- or (role, name) in records
- ):
- return False
- records.add((role, name))
- roles.add(role)
- return required_roles <= roles
-
-
-def _deepep_v2_jit_cubins_are_valid(value: Any) -> bool:
- if not isinstance(value, list) or len(value) != len(DEEPEP_V2_JIT_KERNELS):
- return False
- cache_keys = []
- kernel_names = set()
- for item in value:
- if not isinstance(item, dict) or set(item) != {
- "cache_key", "cubin_sha256", "sass_sha256", "source_sha256",
- }:
- return False
- cache_key = item["cache_key"]
- match = (
- re.fullmatch(r"kernel\.([A-Za-z0-9_+-]+)\.[0-9a-f]{32}", cache_key)
- if isinstance(cache_key, str)
- else None
- )
- if (
- match is None
- or any(
- not isinstance(item[field], str)
- or not re.fullmatch(r"[0-9a-f]{64}", item[field])
- for field in ("cubin_sha256", "sass_sha256", "source_sha256")
- )
- ):
- return False
- cache_keys.append(cache_key)
- kernel_names.add(match.group(1))
- return (
- cache_keys == sorted(set(cache_keys))
- and kernel_names == DEEPEP_V2_JIT_KERNELS
- )
-
-
-HYBRID_REALIZED_CONFIG_FIELDS = {
- "hidden_dim", "max_num_of_tokens_per_rank", "num_of_experts_per_rank",
- "num_of_ranks_per_node", "num_of_nodes", "pad_multiple",
- "num_of_tokens_per_chunk_preprocessing_api",
- "num_of_threads_per_block_preprocessing_api", "num_of_blocks_preprocessing_api",
- "num_of_blocks_permute", "num_of_blocks_unpermute", "token_data_type",
- "num_of_stages_dispatch_api", "num_of_stages_permute_block_dispatch_api",
- "num_of_in_flight_s2g_dispatch_api",
- "num_of_in_flight_s2g_permute_block_dispatch_api",
- "num_of_additional_in_flight_s2g_dispatch_api",
- "num_of_tokens_per_chunk_dispatch_api", "num_of_blocks_dispatch_api",
- "forward_dispatch_api", "device_side_sync_dispatch_api",
- "num_of_stages_g2s_combine_api", "num_of_stages_s2g_combine_api",
- "num_of_tokens_per_chunk_combine_api", "num_of_tokens_per_group_combine_api",
- "num_of_blocks_combine_api", "num_of_additional_in_flight_s2g_combine_api",
- "backward_combine_api", "device_side_sync_combine_api",
-}
-HYBRID_REALIZED_BOOL_FIELDS = {
- "forward_dispatch_api", "device_side_sync_dispatch_api", "backward_combine_api",
- "device_side_sync_combine_api",
-}
-
-
-def _hybrid_realized_config_is_valid(value: Any) -> bool:
- if not isinstance(value, dict) or set(value) != HYBRID_REALIZED_CONFIG_FIELDS:
- return False
- for field, field_value in value.items():
- if field in HYBRID_REALIZED_BOOL_FIELDS:
- if type(field_value) is not bool:
- return False
- elif field == "token_data_type":
- if field_value not in {"UINT8", "UINT16"}:
- return False
- elif type(field_value) is not int or field_value < 0:
- return False
- return all(value[field] > 0 for field in (
- "hidden_dim", "max_num_of_tokens_per_rank", "num_of_experts_per_rank",
- "num_of_ranks_per_node", "num_of_nodes",
- ))
-
-
-def hybrid_communication_domains(ep_size: int, scale_up_domain: int) -> tuple[int, int]:
- """Return active ranks per fabric domain and the number of such domains."""
- if type(ep_size) is not int or type(scale_up_domain) is not int:
- raise ContractError("hybrid communication topology must be integral")
- if ep_size <= 0 or scale_up_domain <= 0:
- raise ContractError("hybrid communication topology must be positive")
- domain_ranks = min(ep_size, scale_up_domain)
- if ep_size % domain_ranks:
- raise ContractError("hybrid EP size does not divide into communication domains")
- return domain_ranks, ep_size // domain_ranks
-
-
-def _hybrid_kernel_keys_are_valid(value: Any) -> bool:
- return (
- isinstance(value, list)
- and len(value) == 3
- and len(set(value)) == 3
- and value == sorted(value)
- and all(
- isinstance(key, str)
- and re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.+-]{0,511}", key)
- for key in value
- )
- )
-
-
-def _hybrid_jit_evidence_is_valid(value: Any, kernel_keys: Any) -> bool:
- if not _hybrid_kernel_keys_are_valid(kernel_keys) or not isinstance(value, list):
- return False
- if len(value) != len(kernel_keys):
- return False
- rank_sets = []
- for expected_key, item in zip(kernel_keys, value):
- if not isinstance(item, dict) or set(item) != {"kernel_key", "rank_artifacts"}:
- return False
- rank_artifacts = item["rank_artifacts"]
- if item["kernel_key"] != expected_key or not isinstance(rank_artifacts, list):
- return False
- ranks = []
- for artifact in rank_artifacts:
- if not isinstance(artifact, dict) or set(artifact) != {"bytes", "rank", "sha256"}:
- return False
- rank, digest, size = artifact["rank"], artifact["sha256"], artifact["bytes"]
- if (
- type(rank) is not int
- or rank < 0
- or not isinstance(digest, str)
- or not re.fullmatch(r"[0-9a-f]{64}", digest)
- or type(size) is not int
- or size <= 0
- ):
- return False
- ranks.append(rank)
- if not ranks or ranks != list(range(len(ranks))):
- return False
- rank_sets.append(ranks)
- return all(ranks == rank_sets[0] for ranks in rank_sets)
-
-
-def backend_provenance_issues(backend: str, provenance: dict[str, Any]) -> list[str]:
- unknown = [
- field for field, value in provenance.items()
- if isinstance(value, str) and value.strip().lower() == "unknown"
- ]
- unresolved = [
- field for field in REQUIRED_BACKEND_PROVENANCE.get(backend, ())
- if not _resolved_provenance_value(field, provenance.get(field))
- ]
- if backend == "deepep":
- mode = provenance.get("mnnvl_comm")
- allow = provenance.get("allow_mnnvl")
- valid_modes = {
- "not-requested": False,
- "explicit-allow-mnnvl": True,
- }
- if type(allow) is not bool or valid_modes.get(mode) is not allow:
- unresolved.append("mnnvl_comm")
- if provenance.get("backend_lineage") != "deepep-v1":
- unresolved.append("backend_lineage")
- if provenance.get("nvshmem_ibgda_nic_handler") not in {
- "cpu", "gpu", "not-active",
- }:
- unresolved.append("nvshmem_ibgda_nic_handler")
- if backend in {"deepep", "uccl"}:
- mode = provenance.get("mode")
- num_nvl_bytes = provenance.get("num_nvl_bytes")
- num_rdma_bytes = provenance.get("num_rdma_bytes")
- if mode not in {"normal", "low-latency"}:
- unresolved.append("mode")
- if type(num_nvl_bytes) is not int or num_nvl_bytes < 0:
- unresolved.append("num_nvl_bytes")
- if type(num_rdma_bytes) is not int or num_rdma_bytes < 0:
- unresolved.append("num_rdma_bytes")
- if mode == "normal" and (type(num_nvl_bytes) is not int or num_nvl_bytes <= 0):
- unresolved.append("num_nvl_bytes")
- if mode == "low-latency":
- if num_nvl_bytes != 0:
- unresolved.append("num_nvl_bytes")
- if type(num_rdma_bytes) is not int or num_rdma_bytes <= 0:
- unresolved.append("num_rdma_bytes")
- if (
- type(provenance.get("num_max_tokens_per_rank")) is not int
- or provenance["num_max_tokens_per_rank"] <= 0
- ):
- unresolved.append("num_max_tokens_per_rank")
- if backend == "deepep" and (
- type(provenance.get("num_qps_per_rank")) is not int
- or provenance["num_qps_per_rank"] <= 0
- ):
- unresolved.append("num_qps_per_rank")
- if backend == "deepep-v2":
- for field in ("num_experts", "tuning_num_experts"):
- if type(provenance.get(field)) is not int or provenance[field] <= 0:
- unresolved.append(field)
- if not _deepep_v2_jit_cubins_are_valid(provenance.get("jit_cubins")):
- unresolved.append("jit_cubins")
- if provenance.get("jit_random_seed") != "collectivex-deepep-v2-fa8a9b1":
- unresolved.append("jit_random_seed")
- unresolved.extend(
- field for field, expected in DEEPEP_V2_V1_PROVENANCE.items()
- if field != "deepep_distribution_version"
- and provenance.get(field) != expected
- )
- if provenance.get("deepep_distribution_version") not in (
- DEEPEP_V2_DISTRIBUTION_VERSIONS
- ):
- unresolved.append("deepep_distribution_version")
- policy = (
- provenance.get("allow_hybrid_mode"),
- provenance.get("gin_enabled"),
- provenance.get("communication_backend"),
- )
- if policy not in {
- (False, False, "nccl-device-lsa"),
- (True, True, "nccl-gin"),
- }:
- unresolved.extend(
- ("allow_hybrid_mode", "gin_enabled", "communication_backend")
- )
- content_roles = {
- "deepep-v2": {"deepep-extension", "nccl", "nvshmem"},
- "deepep-hybrid": {"deepep-extension", "deepep-hybrid-extension"},
- "uccl": {
- "uccl-distribution", "uccl-wrapper", "intervaltree-distribution",
- "sortedcontainers-distribution", "cuda-runtime",
- },
- }.get(backend)
- if content_roles is not None and not _content_evidence_is_valid(
- provenance.get("loaded_libraries"), content_roles
- ):
- unresolved.append("loaded_libraries")
- if backend in {"deepep-v2", "deepep-hybrid"} and not re.fullmatch(
- r"[0-9a-f]{40}", str(provenance.get("deepep_tree", ""))
- ):
- unresolved.append("deepep_tree")
- if backend == "deepep-hybrid" and provenance.get("backend_lineage") != "deepep-hybrid":
- unresolved.append("backend_lineage")
- if backend == "deepep-hybrid":
- if not _hybrid_realized_config_is_valid(provenance.get("realized_config")):
- unresolved.append("realized_config")
- if not _hybrid_kernel_keys_are_valid(provenance.get("jit_kernel_keys")):
- unresolved.append("jit_kernel_keys")
- if not _hybrid_jit_evidence_is_valid(
- provenance.get("jit_shared_objects"), provenance.get("jit_kernel_keys")
- ):
- unresolved.append("jit_shared_objects")
- if backend == "uccl" and provenance.get("backend_lineage") != "uccl":
- unresolved.append("backend_lineage")
- if backend == "uccl" and provenance.get("uccl_dependency_versions") != (
- UCCL_DEPENDENCY_VERSIONS
- ):
- unresolved.append("uccl_dependency_versions")
- if backend == "nccl-ep":
- collective = provenance.get("collective_library")
- if collective not in {"nccl", "rccl"}:
- unresolved.append("collective_library")
- if provenance.get("backend_lineage") != collective:
- unresolved.append("backend_lineage")
- if backend == "mori" and provenance.get("kernel_type") == "InterNodeV1":
- expected = {
- "block_num": 96,
- "rdma_block_num": 64,
- "dispatch_warps": 8,
- "combine_warps": 8,
- "num_qps": 1,
- "use_external_inp_buf": True,
- "gpus_per_node": 8,
- }
- unresolved.extend(
- field for field, value in expected.items()
- if provenance.get(field) != value
- )
- for field, minimum in (
- ("num_nvl_bytes", 0), ("num_rdma_bytes", 0),
- ("num_qps_per_rank", 1),
- ):
- if field in provenance and (
- type(provenance[field]) is not int or provenance[field] < minimum
- ):
- unresolved.append(field)
- if "rdma_block_num" in provenance and (
- type(provenance["rdma_block_num"]) is not int
- or provenance["rdma_block_num"] < 0
- ):
- unresolved.append("rdma_block_num")
- if "use_external_inp_buf" in provenance and type(
- provenance["use_external_inp_buf"]
- ) is not bool:
- unresolved.append("use_external_inp_buf")
- return sorted(set(unknown + unresolved))
-
-
-def provenance_complete(
- provenance: dict[str, Any], backend: str, git_run: dict[str, Any] | None,
- *, image_digest: Any, image_verified: Any, squash_sha256: Any,
-) -> bool:
- """Return whether backend provenance and run identity are fully resolved."""
- image = str(image_digest or "")
- squash = str(squash_sha256 or "")
- return (
- not backend_provenance_issues(backend, provenance)
- and image_verified is True
- and bool(re.fullmatch(r"sha256:[0-9a-f]{64}", image))
- and bool(re.fullmatch(r"[0-9a-f]{64}", squash))
- and isinstance(git_run, dict)
- and all(git_run.get(field) for field in GIT_RUN_FIELDS)
- )
-
-
# ---------------------------------------------------------------------------
# JSON IO helpers.
# ---------------------------------------------------------------------------
@@ -763,45 +344,6 @@ def canonical_json_bytes(value: Any) -> bytes:
raise ContractError(f"value is not canonical JSON: {exc}") from exc
-def content_manifest_evidence(
- *, role: str, name: str, files: Iterable[tuple[str, str | os.PathLike[str]]]
-) -> dict[str, str]:
- """Hash a labeled file set without exposing any host path in provenance."""
- if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.+-]{0,127}", role):
- raise ContractError("content evidence role is invalid")
- if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.+-]{0,159}", name):
- raise ContractError("content evidence name is invalid")
- manifest: list[dict[str, Any]] = []
- labels: set[str] = set()
- for label, raw_path in files:
- logical = PurePosixPath(label)
- if (
- not label
- or logical.is_absolute()
- or ".." in logical.parts
- or label in labels
- or any(ord(character) < 0x20 or ord(character) > 0x7E for character in label)
- ):
- raise ContractError("content evidence label is invalid or duplicated")
- path = Path(raw_path)
- if not path.is_file():
- raise ContractError("content evidence source is not a file")
- digest = hashlib.sha256()
- size = 0
- with path.open("rb") as handle:
- for chunk in iter(lambda: handle.read(1024 * 1024), b""):
- digest.update(chunk)
- size += len(chunk)
- labels.add(label)
- manifest.append({"bytes": size, "label": label, "sha256": digest.hexdigest()})
- if not manifest:
- raise ContractError("content evidence cannot be empty")
- digest = hashlib.sha256(
- canonical_json_bytes(sorted(manifest, key=lambda item: item["label"]))
- ).hexdigest()
- return {"name": name, "role": role, "sha256": digest}
-
-
def _obj(value: Any, path: str) -> dict[str, Any]:
if not isinstance(value, dict):
raise ContractError(f"{path} must be an object")
From e27685c7b9d4690f8369e892e808ead22ee4107d Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Wed, 8 Jul 2026 12:43:53 +0800
Subject: [PATCH 194/210] fix(collectivex): repair EP emit path and drop
publication_status field
The publication-subsystem strip (b09a270d) removed four emit-side projection
helpers from contracts.py on the premise that their only consumers were the
precision validator and publisher.py. run_sweep's emit tail still calls all of
them, so the real (GPU) emit path raised AttributeError at runtime; the
FakeBackend unit smoke never reaches that tail, so nothing surfaced it.
Relocate the emit-side projection helpers next to the other backend-provenance
projections in bench/ep_provenance.py and rewire the ep_harness call sites:
- series_provenance (privacy projection: drops host path/jit_cache_key/
jit_shared_objects/sm_fraction, projects source-built libs + jit cubins)
- public_series_config / public_series_config_sha256
- routing_implementation_control_sha256
- backend_version (+ a local canonical _sha256_json)
series_id stays load-bearing: validate_samples_document derives the detached
sample point path from it (identity.point_id(series=...)), and identity + the
schemas still require it. This is completed relocation, not removal.
Also remove the emitter's publication-disposition logic: _derive_publication_status
and the outcome.publication_status field. The raw-case schema's outcome object is
additionalProperties:false over {reasons,status,validity}, so any real emission
carrying publication_status would have failed validation; deciding publication
disposition is out of scope for the sweep backend.
---
experimental/CollectiveX/bench/ep_harness.py | 27 +----
.../CollectiveX/bench/ep_provenance.py | 105 ++++++++++++++++++
2 files changed, 109 insertions(+), 23 deletions(-)
diff --git a/experimental/CollectiveX/bench/ep_harness.py b/experimental/CollectiveX/bench/ep_harness.py
index 294366924b..dfc72b4de7 100644
--- a/experimental/CollectiveX/bench/ep_harness.py
+++ b/experimental/CollectiveX/bench/ep_harness.py
@@ -333,11 +333,6 @@ def _sha256_json(value) -> str:
return hashlib.sha256(payload).hexdigest()
-def _series_provenance(provenance: dict) -> dict:
- """Retain stable semantic build identity while keeping raw binaries diagnostic."""
- return contracts.series_provenance(provenance)
-
-
def _write_bytes_atomic(path: str, payload: bytes) -> tuple[str, int]:
os.makedirs(os.path.dirname(os.path.abspath(path)), exist_ok=True)
temporary = f"{path}.tmp-{os.getpid()}"
@@ -1198,18 +1193,6 @@ def _histogram(xs: list[float], nbins: int = 40) -> dict:
return {"n": len(xs), "min": round(lo, 3), "max": round(hi, 3), "bins": nbins, "counts": counts}
-def _derive_publication_status(v: dict) -> str:
- """Classify raw attempts; only the isolated coverage publisher may promote evidence."""
- if v["execution_status"] != "complete":
- return "failed"
- if v["semantic_correctness"] != "pass" or v["measurement_conformance"] != "conformant" \
- or v["workload_identity"] == "inconsistent":
- return "invalid"
- # Per-case producers cannot prove exact matrix coverage, repeat stability, or controlled
- # cohorts. Keep even sound attempts diagnostic until the isolated publisher validates them.
- return "diagnostic"
-
-
def run_sweep(args, backend, torch, dist, device, rank: int, world_size: int) -> int:
"""Drive the source-tokens-per-rank sweep for one fully-specified line."""
mode = getattr(args, "mode", "normal")
@@ -1703,7 +1686,6 @@ def component(percentiles, count, *, derived=False):
# anomaly-free unless a contract-level timing anomaly fired (then diagnostic, see above).
"anomaly_free": anomaly_free,
}
- publication_status = _derive_publication_status(validity)
shape = { # FIXED line identity (no T, no per-backend resource knobs)
"hidden": args.hidden, "topk": args.topk, "experts": args.experts,
@@ -1788,10 +1770,10 @@ def component(percentiles, count, *, derived=False):
implementation_contract = {
"kernel_generation": kernel_generation(backend),
"name": backend.name,
- "provenance": _series_provenance(backend.backend_provenance),
+ "provenance": ep_provenance.series_provenance(backend.backend_provenance),
"resource_profile": resource_profile,
}
- public_config = contracts.public_series_config(
+ public_config = ep_provenance.public_series_config(
kernel_generation=implementation_contract["kernel_generation"],
provenance=backend.backend_provenance,
resource_profile=resource_profile,
@@ -1801,8 +1783,8 @@ def component(percentiles, count, *, derived=False):
series_factors = {
"backend": backend.name,
"implementation_contract_sha256": _sha256_json(implementation_contract),
- "public_config_sha256": contracts.public_series_config_sha256(public_config),
- "routing_control_sha256": contracts.routing_implementation_control_sha256(
+ "public_config_sha256": ep_provenance.public_series_config_sha256(public_config),
+ "routing_control_sha256": ep_provenance.routing_implementation_control_sha256(
implementation_contract
),
"case_id": case_identifier,
@@ -2016,7 +1998,6 @@ def component(percentiles, count, *, derived=False):
},
"sample_artifact": sample_artifact,
"outcome": {
- "publication_status": publication_status,
"reasons": [] if all_ok else ["semantic correctness or routing identity failed"],
"status": "success" if all_ok else "invalid",
"validity": validity,
diff --git a/experimental/CollectiveX/bench/ep_provenance.py b/experimental/CollectiveX/bench/ep_provenance.py
index 973b9b7a8a..a637f27c2e 100644
--- a/experimental/CollectiveX/bench/ep_provenance.py
+++ b/experimental/CollectiveX/bench/ep_provenance.py
@@ -464,3 +464,108 @@ def content_manifest_evidence(
canonical_json_bytes(sorted(manifest, key=lambda item: item["label"]))
).hexdigest()
return {"name": name, "role": role, "sha256": digest}
+
+
+# Source-built backend libraries whose raw build tree is projected to a stable
+# public source identity (private cache keys / host paths are dropped).
+SOURCE_BUILT_LIBRARY_ROLES = frozenset({
+ "deepep-extension", "deepep-hybrid-extension",
+})
+
+
+def series_provenance(provenance: dict[str, Any]) -> dict[str, Any]:
+ """Project stable semantic build identity while dropping private binaries and host paths."""
+ projected = {
+ key: value for key, value in provenance.items()
+ if key not in {"jit_cache_key", "jit_shared_objects", "path", "sm_fraction"}
+ }
+ libraries = provenance.get("loaded_libraries")
+ if isinstance(libraries, list):
+ projected["loaded_libraries"] = [
+ {
+ "name": item.get("name"),
+ "role": item.get("role"),
+ "source_tree": provenance.get("deepep_tree"),
+ }
+ if isinstance(item, dict) and item.get("role") in SOURCE_BUILT_LIBRARY_ROLES
+ else item
+ for item in libraries
+ ]
+ jit_cubins = provenance.get("jit_cubins")
+ if isinstance(jit_cubins, list):
+ projected["jit_cubins"] = [
+ {
+ "cache_key": item.get("cache_key"),
+ "sass_sha256": item.get("sass_sha256"),
+ "source_sha256": item.get("source_sha256"),
+ }
+ if isinstance(item, dict)
+ else item
+ for item in jit_cubins
+ ]
+ return projected
+
+
+def _sha256_json(value: Any) -> str:
+ """Canonical sha256 of a finite JSON value (matches contracts.canonical_json_bytes)."""
+ return hashlib.sha256(canonical_json_bytes(value)).hexdigest()
+
+
+def backend_version(provenance: dict[str, Any]) -> str | None:
+ """Return the canonical public backend version from implementation provenance."""
+ for field in (
+ "deepep_version", "uccl_version", "nccl_version",
+ "mori_commit", "deepep_commit",
+ ):
+ value = provenance.get(field)
+ if value is not None and str(value).strip():
+ return str(value)[:160]
+ return None
+
+
+def public_series_config(
+ *, kernel_generation: Any, provenance: dict[str, Any],
+ resource_profile: dict[str, Any], resource_mode: Any, device_product: Any,
+) -> dict[str, Any]:
+ """Project raw implementation facts into the exact public configuration fields."""
+ generation = None if kernel_generation == "n-a" else kernel_generation
+ profile = "profile-" + _sha256_json(resource_profile)[:16]
+ return {
+ "backend": {
+ "generation": generation,
+ "version": backend_version(provenance),
+ },
+ "resource": {
+ "mode": resource_mode,
+ "profile": profile,
+ "comm_units_kind": resource_profile.get("comm_units_kind"),
+ "configured_units": resource_profile.get("configured_units"),
+ },
+ "system": {"label": str(device_product)[:160]},
+ }
+
+
+def public_series_config_sha256(config: dict[str, Any]) -> str:
+ """Commit the canonical public configuration projection into series identity."""
+ return _sha256_json(config)
+
+
+def routing_implementation_control_sha256(implementation: dict[str, Any]) -> str:
+ """Bind routing cohorts to the same static build/generator and non-treatment configuration."""
+ provenance = implementation.get("provenance")
+ if not isinstance(provenance, dict):
+ raise ContractError("implementation provenance is unavailable")
+ semantic = series_provenance(provenance)
+ treatment_fields = {
+ "jit_cache_key", "jit_cubins", "jit_kernel_keys", "jit_shared_objects",
+ "local_experts", "num_experts", "path", "realized_config", "sm_fraction",
+ }
+ return _sha256_json({
+ "kernel_generation": implementation.get("kernel_generation"),
+ "name": implementation.get("name"),
+ "provenance": {
+ key: value for key, value in semantic.items()
+ if key not in treatment_fields
+ },
+ "resource_profile": implementation.get("resource_profile"),
+ })
From 7809962e278338b1f15c12d7f8b524b807e3ed72 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Wed, 8 Jul 2026 12:52:31 +0800
Subject: [PATCH 195/210] refactor(collectivex): prune precision-probe and
required-publication shell
The MVP sweep schedules, executes, validates, and uploads neutral
artifacts only; it does not run the swept precision-probe path and no
longer carries a required-publication signal. Remove the precision-probe
dispatch, control-field parsing, manifest validation, and failure-class
branches from runtime/common.sh and runtime/run_in_container.sh, and drop
the CX_REQUIRED_PUBLICATION / --required-publication plumbing (the arg no
longer exists on the harness). Keep the BF16-default --precision-profile
and --qualification-index paths (FP8 stays callable-but-dormant), and
preserve every cluster execution fix untouched.
---
experimental/CollectiveX/runtime/common.sh | 153 +++---------------
.../CollectiveX/runtime/run_in_container.sh | 33 +---
2 files changed, 22 insertions(+), 164 deletions(-)
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 51759b093a..612ae62403 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -36,11 +36,9 @@ cx_set_failure_stage() {
}
cx_fail_stage() {
- local stage="$1" log_path="${2:-}" diagnostic="unknown" probe_stage=""
+ local stage="$1" log_path="${2:-}" diagnostic="unknown"
cx_set_failure_stage "$stage"
if [ -n "$log_path" ] && [ -f "$log_path" ]; then
- probe_stage="$(grep -aoE 'precision-probe-stage=(distributed-init|runtime-context|backend-construction|construction-consensus|native-operation|operation-consensus|evidence-aggregation)' "$log_path" \
- | tail -n 1 | cut -d= -f2 || true)"
if grep -aEqi 'no space left|disk quota|quota exceeded' "$log_path"; then
diagnostic="storage-capacity"
elif grep -aEqi 'permission denied|operation not permitted|read-only file system|source mount (creation|ownership validation|permission inspection|permission normalization|permission validation) failed' "$log_path"; then
@@ -59,9 +57,6 @@ cx_fail_stage() {
diagnostic="nccl-device-api"
elif grep -aEqi 'NVCC (PTX )?compilation failed|cuobjdump failed|invalid device (kernel )?image|no kernel image is available' "$log_path"; then
diagnostic="jit-toolchain"
- elif [ -n "$probe_stage" ] \
- && grep -aEqi 'cuda out of memory|CUDA_ERROR_OUT_OF_MEMORY|out of memory.*cuda' "$log_path"; then
- diagnostic="${probe_stage}-accelerator-memory"
elif grep -aEqi 'cuda out of memory|CUDA_ERROR_OUT_OF_MEMORY|out of memory.*cuda' "$log_path"; then
diagnostic="accelerator-memory"
elif grep -aEqi 'does not match its pinned image contract|requires the exact pinned|version mismatch' "$log_path"; then
@@ -101,9 +96,6 @@ cx_fail_stage() {
diagnostic="network-or-timeout"
elif grep -aEqi 'salloc:|srun:.*(unable to create step|step creation|invalid partition|invalid account)|unable to create step|job allocation' "$log_path"; then
diagnostic="scheduler"
- elif [ -n "$probe_stage" ] \
- && grep -aEqi 'Traceback \(most recent call last\)|[A-Za-z]+(Error|Exception):' "$log_path"; then
- diagnostic="${probe_stage}-failed"
elif grep -aEqi 'ModuleNotFoundError|ImportError:' "$log_path"; then
diagnostic="python-import"
elif grep -aEqi 'AttributeError:|TypeError:.*(unexpected|argument|operand)|has no attribute' "$log_path"; then
@@ -132,35 +124,6 @@ cx_fail_stage() {
diagnostic="python-assertion"
elif grep -aEqi 'RuntimeError:' "$log_path"; then
diagnostic="python-runtime"
- elif grep -aEqi 'ValueError:.*(fields differ from collectivex[.]precision-probe|probe format, record type, schema, or contract differs)' "$log_path"; then
- diagnostic="probe-schema-value"
- elif grep -aEqi 'ValueError:.*probe target|ValueError:.*probe topology' "$log_path"; then
- diagnostic="probe-target-value"
- elif grep -aEqi 'ValueError:.*probe result|ValueError:.*statically promote the registry' "$log_path"; then
- diagnostic="probe-result-value"
- elif grep -aEqi 'ValueError:.*probe privacy' "$log_path"; then
- diagnostic="probe-privacy-value"
- elif grep -aEqi 'ValueError:.*probe API' "$log_path"; then
- diagnostic="probe-api-value"
- elif grep -aEqi 'ValueError:.*probe completion' "$log_path"; then
- diagnostic="probe-completion-value"
- elif grep -aEqi 'ValueError:.*probe (source SHA|image digest|backend provenance|backend component)' "$log_path"; then
- diagnostic="probe-identity-value"
- elif grep -aEqi 'ValueError:.*probe precision correctness did not pass' "$log_path"; then
- diagnostic="probe-correctness-failed"
- elif grep -aEqi 'ValueError:.*probe scale shapes are invalid' "$log_path"; then
- diagnostic="probe-scale-shape"
- elif grep -aEqi 'ValueError:.*probe precision .* is not finite' "$log_path"; then
- diagnostic="probe-nonfinite"
- elif grep -aEqi 'ValueError:.*probe precision .* shapes are empty|ValueError:.*probe precision .* shape is invalid' "$log_path"; then
- diagnostic="probe-tensor-shape"
- elif grep -aEqi 'ValueError:.*probe (precision correctness|scale shapes|precision .* (input|output|semantic|scales))' "$log_path"; then
- diagnostic="probe-correctness-value"
- elif grep -aEqi 'ValueError:.*probe transport' "$log_path"; then
- diagnostic="probe-transport-value"
- elif grep -aEqi 'ValueError:' "$log_path" \
- && grep -aEqi 'probe_precision[.]py' "$log_path"; then
- diagnostic="probe-manifest-value"
elif grep -aEqi 'ValueError:' "$log_path" \
&& grep -aEqi 'ep_harness[.]py' "$log_path"; then
diagnostic="harness-value"
@@ -751,7 +714,7 @@ PY
# Explicit Slurm export boundary. Operator config, runner credentials, HOME,
# workspace paths, and unrelated service secrets never enter the container.
cx_container_exports() {
- printf '%s' 'COLLECTIVEX_SOURCE_SHA,COLLECTIVEX_ARTIFACT_NAME,COLLECTIVEX_EXECUTION_ID,COLLECTIVEX_CONTROL_SHA256,COLLECTIVEX_IMAGE,COLLECTIVEX_IMAGE_DIGEST,COLLECTIVEX_IMAGE_DIGEST_VERIFIED,COLLECTIVEX_SQUASH_SHA256,GITHUB_REF_NAME,GITHUB_REF,GITHUB_REPOSITORY,GITHUB_JOB,GITHUB_RUN_ID,GITHUB_RUN_ATTEMPT,GITHUB_SHA,CX_RUNNER,CX_BENCH,CX_NODES,CX_GPUS_PER_NODE,CX_SCALE_UP_DOMAIN,CX_SHARD_FILE,CX_SHARD_SKU,CX_PRECISION_PROBE,CX_NGPUS,CX_TS,CX_TOPO,CX_SCOPE,CX_TRANSPORT,CX_SCALE_UP_TRANSPORT,CX_SCALE_OUT_TRANSPORT,CX_MODE,CX_PHASE,CX_ROUTING,CX_EPLB,CX_CASE_ID,CX_SUITE,CX_WORKLOAD_NAME,CX_REQUIRED_PUBLICATION,CX_PRECISION_PROFILE,CX_QUALIFICATION_INDEX,CX_HIDDEN,CX_TOPK,CX_EXPERTS,CX_TOKENS_LADDER,CX_CANONICAL,CX_ITERS,CX_TRIALS,CX_WARMUP,CX_SAMPLES_PER_POINT,CX_WARMUP_SEMANTICS,CX_SEED,CX_RUN_TIMEOUT,CX_NCCL_HOME,CX_ALLOW_MNNVL,CX_ATTEMPT_ID,CX_RUNTIME_MARKER,CX_MORI_KERNEL_TYPE,CX_WORKLOAD_DIR,CX_BACKEND_CACHE_ROOT,CX_BACKEND_CACHE_SENTINEL_SHA256,CX_BACKEND_SOURCE_ROOT,CX_AUDIT_SALT,CX_SOCKET_IFNAME,CX_RDMA_DEVICES,CX_IB_GID_INDEX,CX_RDMA_SERVICE_LEVEL,CX_RDMA_TRAFFIC_CLASS,CX_RDMA_LINK_LAYER,MASTER_ADDR,MASTER_PORT,RANK,WORLD_SIZE,LOCAL_RANK,LOCAL_WORLD_SIZE,NCCL_NET,NCCL_SOCKET_IFNAME,GLOO_SOCKET_IFNAME,NCCL_IB_HCA,NCCL_IB_GID_INDEX,NCCL_IB_SL,NVSHMEM_DISABLE_IB,NVSHMEM_ENABLE_NIC_PE_MAPPING,NVSHMEM_HCA_LIST,NVSHMEM_IB_GID_INDEX,NVSHMEM_IB_SL,NVSHMEM_IB_ENABLE_IBGDA,NVSHMEM_IBGDA_NIC_HANDLER,EP_NIC_NAME,EP_OVERRIDE_RDMA_SL,UCCL_SOCKET_IFNAME,UCCL_IB_GID_INDEX,UCCL_IB_SL,MORI_RDMA_DEVICES,MORI_RDMA_TC,MORI_IO_TC,MORI_RDMA_SL,MORI_IO_SL,HYBRID_EP_MULTINODE,USE_NIXL,RDMA_CORE_HOME,DEEPEP_HYBRID_BUILD_MODE,NCCL_CUMEM_ENABLE,NCCL_MNNVL_ENABLE,MC_FORCE_MNNVL,MORI_DISABLE_AUTO_XGMI,MORI_ENABLE_SDMA,MORI_APP_LOG_LEVEL,MORI_SHMEM_LOG_LEVEL,MORI_IO_LOG_LEVEL'
+ printf '%s' 'COLLECTIVEX_SOURCE_SHA,COLLECTIVEX_ARTIFACT_NAME,COLLECTIVEX_EXECUTION_ID,COLLECTIVEX_CONTROL_SHA256,COLLECTIVEX_IMAGE,COLLECTIVEX_IMAGE_DIGEST,COLLECTIVEX_IMAGE_DIGEST_VERIFIED,COLLECTIVEX_SQUASH_SHA256,GITHUB_REF_NAME,GITHUB_REF,GITHUB_REPOSITORY,GITHUB_JOB,GITHUB_RUN_ID,GITHUB_RUN_ATTEMPT,GITHUB_SHA,CX_RUNNER,CX_BENCH,CX_NODES,CX_GPUS_PER_NODE,CX_SCALE_UP_DOMAIN,CX_SHARD_FILE,CX_SHARD_SKU,CX_NGPUS,CX_TS,CX_TOPO,CX_SCOPE,CX_TRANSPORT,CX_SCALE_UP_TRANSPORT,CX_SCALE_OUT_TRANSPORT,CX_MODE,CX_PHASE,CX_ROUTING,CX_EPLB,CX_CASE_ID,CX_SUITE,CX_WORKLOAD_NAME,CX_PRECISION_PROFILE,CX_QUALIFICATION_INDEX,CX_HIDDEN,CX_TOPK,CX_EXPERTS,CX_TOKENS_LADDER,CX_CANONICAL,CX_ITERS,CX_TRIALS,CX_WARMUP,CX_SAMPLES_PER_POINT,CX_WARMUP_SEMANTICS,CX_SEED,CX_RUN_TIMEOUT,CX_NCCL_HOME,CX_ALLOW_MNNVL,CX_ATTEMPT_ID,CX_RUNTIME_MARKER,CX_MORI_KERNEL_TYPE,CX_WORKLOAD_DIR,CX_BACKEND_CACHE_ROOT,CX_BACKEND_CACHE_SENTINEL_SHA256,CX_BACKEND_SOURCE_ROOT,CX_AUDIT_SALT,CX_SOCKET_IFNAME,CX_RDMA_DEVICES,CX_IB_GID_INDEX,CX_RDMA_SERVICE_LEVEL,CX_RDMA_TRAFFIC_CLASS,CX_RDMA_LINK_LAYER,MASTER_ADDR,MASTER_PORT,RANK,WORLD_SIZE,LOCAL_RANK,LOCAL_WORLD_SIZE,NCCL_NET,NCCL_SOCKET_IFNAME,GLOO_SOCKET_IFNAME,NCCL_IB_HCA,NCCL_IB_GID_INDEX,NCCL_IB_SL,NVSHMEM_DISABLE_IB,NVSHMEM_ENABLE_NIC_PE_MAPPING,NVSHMEM_HCA_LIST,NVSHMEM_IB_GID_INDEX,NVSHMEM_IB_SL,NVSHMEM_IB_ENABLE_IBGDA,NVSHMEM_IBGDA_NIC_HANDLER,EP_NIC_NAME,EP_OVERRIDE_RDMA_SL,UCCL_SOCKET_IFNAME,UCCL_IB_GID_INDEX,UCCL_IB_SL,MORI_RDMA_DEVICES,MORI_RDMA_TC,MORI_IO_TC,MORI_RDMA_SL,MORI_IO_SL,HYBRID_EP_MULTINODE,USE_NIXL,RDMA_CORE_HOME,DEEPEP_HYBRID_BUILD_MODE,NCCL_CUMEM_ENABLE,NCCL_MNNVL_ENABLE,MC_FORCE_MNNVL,MORI_DISABLE_AUTO_XGMI,MORI_ENABLE_SDMA,MORI_APP_LOG_LEVEL,MORI_SHMEM_LOG_LEVEL,MORI_IO_LOG_LEVEL'
printf '%s' ',MORI_COMMIT'
}
@@ -1179,11 +1142,7 @@ fi
cx_write_runtime_stage execution || exit 68
export RANK="$SLURM_PROCID" WORLD_SIZE="$SLURM_NTASKS"
export LOCAL_RANK="$SLURM_LOCALID" LOCAL_WORLD_SIZE="$CX_GPUS_PER_NODE"
-case "${CX_PRECISION_PROBE:-0}" in
- 1) exec python3 tests/probe_precision.py "$@" ;;
- 0|'') exec python3 bench/run_ep.py "$@" ;;
- *) exit 67 ;;
-esac
+exec python3 bench/run_ep.py "$@"
BASH
}
@@ -1202,61 +1161,27 @@ cx_validate_shard_control() {
[ -f "$path" ] || path="${cx_root%/}/$shard"
[ -f "$path" ] || cx_die "shard control does not exist"
[ -s "$path" ] || cx_die "shard control is empty"
- if [ "${CX_PRECISION_PROBE:-0}" = 1 ]; then
- python3 "${cx_root%/}/tests/probe_precision.py" \
- --validate-control "$path" --expect-sku "$expected_sku" \
- --expect-backend "$CX_BENCH" --expect-nodes "$CX_NODES" >/dev/null 2>&1 \
- || cx_die "invalid precision probe control"
- else
- python3 "${cx_root%/}/sweep_matrix.py" \
- --validate-control "$path" --expect-sku "$expected_sku" \
- --expect-backend "$CX_BENCH" --expect-nodes "$CX_NODES" >/dev/null 2>&1 \
- || cx_die "invalid shard control"
- fi
+ python3 "${cx_root%/}/sweep_matrix.py" \
+ --validate-control "$path" --expect-sku "$expected_sku" \
+ --expect-backend "$CX_BENCH" --expect-nodes "$CX_NODES" >/dev/null 2>&1 \
+ || cx_die "invalid shard control"
control_sha256="$(sha256sum "$path" | awk '{print $1}')"
[[ "$control_sha256" =~ ^[0-9a-f]{64}$ ]] \
|| cx_die "cannot hash shard control"
export COLLECTIVEX_CONTROL_SHA256="$control_sha256"
}
-cx_precision_probe_control_fields() {
- local cx_root="$1" shard="${CX_SHARD_FILE:-}" path
- [ "${CX_PRECISION_PROBE:-0}" = 1 ] || return 1
- path="$shard"
- [ -f "$path" ] || path="${cx_root%/}/$shard"
- python3 - "$path" <<'PY'
-import json
-import pathlib
-import sys
-
-path = pathlib.Path(sys.argv[1])
-document = json.loads(path.read_text())
-target = document["target"]
-values = (
- document["id"], target["backend"], target["sku"], target["ep"],
- target["mode"], target["precision_profile"],
-)
-if any("|" in str(value) or "\n" in str(value) for value in values):
- raise SystemExit("unsafe precision probe control field")
-print("|".join(map(str, values)))
-PY
-}
-
# Load only the case mode needed to choose the allocation/network profile. A
# consolidated shard may contain normal and low-latency cases; selecting the
# strictest mode here makes allocation preflight prove every required device.
# The in-container dispatcher reapplies the profile for each individual case.
cx_load_network_control_mode() {
- local cx_root="$1" shard="${CX_SHARD_FILE:-}" path fields mode
+ local cx_root="$1" shard="${CX_SHARD_FILE:-}" path mode
[ -n "$shard" ] || return 0
path="$shard"
[ -f "$path" ] || path="${cx_root%/}/$shard"
[ -f "$path" ] || return 1
- if [ "${CX_PRECISION_PROBE:-0}" = 1 ]; then
- fields="$(cx_precision_probe_control_fields "$cx_root")" || return 1
- IFS='|' read -r _ _ _ _ mode _ <<< "$fields"
- else
- mode="$(python3 - "$path" <<'PY'
+ mode="$(python3 - "$path" <<'PY'
import json
import sys
@@ -1268,7 +1193,6 @@ if not modes or modes - {"normal", "low-latency"}:
print("low-latency" if "low-latency" in modes else "normal")
PY
)" || return 1
- fi
case "$mode" in
normal|low-latency) export CX_MODE="$mode" ;;
*) return 1 ;;
@@ -3033,7 +2957,7 @@ with open(sys.argv[1]) as handle:
for case in cases:
fields = (
case["phase"], case["mode"], case["case_id"], case["suite"], case["workload"],
- case["required_publication"], case["routing"], "1" if case["eplb"] else "",
+ case["routing"], "1" if case["eplb"] else "",
case["ep"], case["hidden"], case["topk"], case["experts"], case["nodes"],
case["gpus_per_node"], case["scale_up_domain"], case["scope"],
case["scale_up_transport"], case.get("scale_out_transport") or "",
@@ -3051,11 +2975,11 @@ PY
fi
expected="$(wc -l < "$cases_file" | tr -d ' ')"
[ "$expected" -gt 0 ] || { rm -f "$cases_file"; unset CX_FAILURE_MODE; return 1; }
- while IFS='|' read -r phase mode case_id suite workload required routing eplb ep hidden topk \
+ while IFS='|' read -r phase mode case_id suite workload routing eplb ep hidden topk \
experts nodes gpn domain scope scale_up_transport scale_out_transport transport \
topology_class ladder warmup_semantics canonical timing precision_profile; do
export CX_CASE_ID="$case_id" CX_SUITE="$suite" CX_WORKLOAD_NAME="$workload"
- export CX_REQUIRED_PUBLICATION="$required" CX_ROUTING="$routing" CX_EPLB="$eplb"
+ export CX_ROUTING="$routing" CX_EPLB="$eplb"
export CX_EP="$ep" CX_NGPUS="$ep" CX_HIDDEN="$hidden" CX_TOPK="$topk" CX_EXPERTS="$experts"
export CX_MODE="$mode" CX_NODES="$nodes" CX_GPUS_PER_NODE="$gpn"
export CX_SCALE_UP_DOMAIN="$domain" CX_SCOPE="$scope"
@@ -3087,7 +3011,7 @@ PY
# shellcheck disable=SC2153
cx_run_distributed_shard() {
local build_log build_rc cases_file expected_cases ci=0 failed_cases=0
- local ph mode routing eplb hidden topk experts ladder suite workload required_pub
+ local ph mode routing eplb hidden topk experts ladder suite workload
local canonical case_id ep timing case_iters case_trials case_warmup case_stem
local scope scale_up_transport scale_out_transport transport topology_class nodes gpn domain
local precision_profile
@@ -3131,46 +3055,12 @@ cx_run_distributed_shard() {
set -e
if [ "$build_rc" != 0 ]; then
cx_fail_stage backend-setup "$build_log" || true
- [ "${CX_PRECISION_PROBE:-0}" != 1 ] || return "$build_rc"
cx_emit_setup_failures "$CX_DIR" "$MOUNT_SRC/experimental/CollectiveX/results" \
"$CX_BENCH" "$build_rc"
return "$build_rc"
fi
cx_set_failure_stage execution
- if [ "${CX_PRECISION_PROBE:-0}" = 1 ]; then
- local fields probe_id backend sku ep mode profile
- fields="$(cx_precision_probe_control_fields "$CX_DIR")" || return 1
- IFS='|' read -r probe_id backend sku ep mode profile <<< "$fields"
- [ "$backend" = "$CX_BENCH" ] && [ "$sku" = "$RUNNER" ] && [ "$ep" = "$NGPUS" ] \
- || cx_die "precision probe control differs from runtime placement"
- out="results/${probe_id}.json"
- expected_out="$MOUNT_SRC/experimental/CollectiveX/$out"
- runtime_log="$(cx_private_log_path precision-probe)"
- set +e
- timeout -k 30 "${CX_RUN_TIMEOUT:-900}" srun --jobid="$JOB_ID" --nodes="$NODES" \
- --ntasks="$NGPUS" --ntasks-per-node="$GPN" --chdir=/tmp \
- --container-name="$container_name" --container-image="$SQUASH_FILE" \
- "${container_args[@]}" \
- --export="$(cx_container_exports)" \
- bash -c "$WRAP" _ --backend "$backend" --sku "$sku" --ep "$ep" \
- --mode "$mode" --precision-profile "$profile" --out "$out" \
- "$runtime_log" 2>&1
- run_rc=$?
- set -e
- if [ "$run_rc" = 124 ] || [ "$run_rc" = 137 ]; then
- printf '[collectivex] precision probe timed out rc=%s limit=%ss\n' \
- "$run_rc" "${CX_RUN_TIMEOUT:-900}" >> "$runtime_log"
- fi
- if [ "$run_rc" != 0 ] || ! python3 "$CX_DIR/tests/probe_precision.py" \
- --validate-manifest "$expected_out" >/dev/null 2>&1; then
- [ "$run_rc" != 0 ] || run_rc=1
- cx_fail_stage execution "$runtime_log" || true
- return "$run_rc"
- fi
- return 0
- fi
-
cases_file="$(mktemp)" || return 1
local shard="${CX_SHARD_FILE:-}"
[ -z "$shard" ] || [ -f "$shard" ] || shard="$CX_DIR/$shard"
@@ -3187,7 +3077,7 @@ for case in cases:
get("phase", "decode"), get("mode", "normal"), get("routing", "uniform"),
"1" if case.get("eplb") else "", get("hidden", "7168"),
get("topk", "8"), get("experts", "256"), get("ladder"),
- get("suite"), get("workload"), get("required_publication"),
+ get("suite"), get("workload"),
"1" if case.get("canonical") else "", get("case_id"), get("ep"),
get("timing", "8:64:32"), get("nodes"), get("gpus_per_node"),
get("scale_up_domain"), get("scope"), get("scale_up_transport"),
@@ -3206,14 +3096,14 @@ PY
cx_require_record_safe "$phases" "${CX_MODE:-normal}" "${CX_ROUTING:-uniform}" \
"${CX_EPLB:-}" "${CX_HIDDEN:-7168}" "${CX_TOPK:-8}" "${CX_EXPERTS:-256}" \
"${CX_TOKENS_LADDER:-}" "${CX_SUITE:-}" "${CX_WORKLOAD_NAME:-}" \
- "${CX_REQUIRED_PUBLICATION:-}" "${CX_CANONICAL:-}" "${CX_CASE_ID:-}" \
+ "${CX_CANONICAL:-}" "${CX_CASE_ID:-}" \
"${CX_PRECISION_PROFILE:-}" \
"${CX_ITERS:-8}" "${CX_TRIALS:-64}" "${CX_WARMUP:-32}" \
"${CX_SCOPE:-scale-up}" \
"${CX_SCALE_UP_TRANSPORT:-unknown}" "${CX_SCALE_OUT_TRANSPORT:-}" \
"${CX_TRANSPORT:-unknown}" "${CX_TOPO:-manual}"
for phase in $phases; do
- (IFS='|'; printf '%s\n' "$phase|${CX_MODE:-normal}|${CX_ROUTING:-uniform}|${CX_EPLB:-}|${CX_HIDDEN:-7168}|${CX_TOPK:-8}|${CX_EXPERTS:-256}|${CX_TOKENS_LADDER:-}|${CX_SUITE:-}|${CX_WORKLOAD_NAME:-}|${CX_REQUIRED_PUBLICATION:-}|${CX_CANONICAL:-}|${CX_CASE_ID:-}|$NGPUS|${CX_ITERS:-8}:${CX_TRIALS:-64}:${CX_WARMUP:-32}|$NODES|$GPN|$SCALE_UP_DOMAIN|${CX_SCOPE:-scale-up}|${CX_SCALE_UP_TRANSPORT:-unknown}|${CX_SCALE_OUT_TRANSPORT:-}|${CX_TRANSPORT:-unknown}|${CX_TOPO:-manual}|${CX_PRECISION_PROFILE:-}")
+ (IFS='|'; printf '%s\n' "$phase|${CX_MODE:-normal}|${CX_ROUTING:-uniform}|${CX_EPLB:-}|${CX_HIDDEN:-7168}|${CX_TOPK:-8}|${CX_EXPERTS:-256}|${CX_TOKENS_LADDER:-}|${CX_SUITE:-}|${CX_WORKLOAD_NAME:-}|${CX_CANONICAL:-}|${CX_CASE_ID:-}|$NGPUS|${CX_ITERS:-8}:${CX_TRIALS:-64}:${CX_WARMUP:-32}|$NODES|$GPN|$SCALE_UP_DOMAIN|${CX_SCOPE:-scale-up}|${CX_SCALE_UP_TRANSPORT:-unknown}|${CX_SCALE_OUT_TRANSPORT:-}|${CX_TRANSPORT:-unknown}|${CX_TOPO:-manual}|${CX_PRECISION_PROFILE:-}")
done > "$cases_file"
fi
expected_cases="$(wc -l < "$cases_file" | tr -d ' ')"
@@ -3221,7 +3111,7 @@ PY
|| { rm -f "$cases_file"; cx_die "distributed case list is empty"; }
while IFS='|' read -r ph mode routing eplb hidden topk experts ladder suite workload \
- required_pub canonical case_id ep timing nodes gpn domain scope scale_up_transport \
+ canonical case_id ep timing nodes gpn domain scope scale_up_transport \
scale_out_transport transport topology_class precision_profile; do
[ -n "$ph" ] || continue
ci=$((ci + 1))
@@ -3233,7 +3123,7 @@ PY
ep="${ep:-$NGPUS}"
export CX_MODE="$mode" CX_PHASE="$ph" CX_CASE_ID="$case_id" CX_SUITE="$suite"
export CX_WORKLOAD_NAME="$workload"
- export CX_REQUIRED_PUBLICATION="$required_pub" CX_CANONICAL="$canonical" CX_EP="$ep"
+ export CX_CANONICAL="$canonical" CX_EP="$ep"
export CX_PRECISION_PROFILE="$precision_profile"
export CX_ROUTING="$routing" CX_EPLB="$eplb" CX_TOKENS_LADDER="$ladder"
export CX_HIDDEN="$hidden" CX_TOPK="$topk" CX_EXPERTS="$experts"
@@ -3290,7 +3180,7 @@ PY
--warmup "$case_warmup" --iters "$case_iters" --trials "$case_trials"
--seed "${CX_SEED:-67}" --runner "$RUNNER" --topology-class "$topology_class"
--transport "$transport" --case-id "$case_id" --suite "$suite"
- --workload-name "$workload" --required-publication "$required_pub"
+ --workload-name "$workload"
--qualification-index "${CX_QUALIFICATION_INDEX:-1}")
cx_bool_enabled "$eplb" && ep_args+=(--eplb)
[ -z "$workload_dir" ] || ep_args+=(--workload-dir "$workload_dir")
@@ -3369,10 +3259,7 @@ cx_launcher_cleanup() {
cx_write_cleanup_guard unsafe || true
fi
[ "$allocation_stopped" = 1 ] || source_root="${REPO_ROOT:-$source_root}"
- if [ "$rc" != 0 ] && [ "${CX_PRECISION_PROBE:-0}" = 1 ]; then
- cx_log "ERROR: precision-probe-failure-class=${CX_FAILSAFE_MODE:-setup}"
- fi
- if [ "$rc" != 0 ] && [ "${CX_PRECISION_PROBE:-0}" != 1 ] \
+ if [ "$rc" != 0 ] \
&& [ -n "${REPO_ROOT:-}" ] && [ -n "${CX_BENCH:-}" ]; then
cx_log "ERROR: terminal-failure-class=${CX_FAILSAFE_MODE:-setup}"
[ -d "$source_root/experimental/CollectiveX" ] || source_root="$REPO_ROOT"
diff --git a/experimental/CollectiveX/runtime/run_in_container.sh b/experimental/CollectiveX/runtime/run_in_container.sh
index 9cc42b2c8f..856a904989 100644
--- a/experimental/CollectiveX/runtime/run_in_container.sh
+++ b/experimental/CollectiveX/runtime/run_in_container.sh
@@ -89,7 +89,6 @@ run_ep_suite() {
--scope "${CX_SCOPE:-scale-up}" --scale-up-transport "${CX_SCALE_UP_TRANSPORT:-unknown}"
--scale-out-transport "${CX_SCALE_OUT_TRANSPORT:-}"
--case-id "${CX_CASE_ID:-}" --suite "${CX_SUITE:-}" --workload-name "${CX_WORKLOAD_NAME:-}"
- --required-publication "${CX_REQUIRED_PUBLICATION:-}"
--qualification-index "${CX_QUALIFICATION_INDEX:-1}"
--runner "$CX_RUNNER" --topology-class "$CX_TOPO" --transport "$CX_TRANSPORT"
--out "$out")
@@ -994,25 +993,6 @@ dispatch_bench() {
esac
}
-run_precision_probe() {
- local fields probe_id backend sku ep mode profile out rc_run
- fields="$(cx_precision_probe_control_fields "$PWD")" || return 1
- IFS='|' read -r probe_id backend sku ep mode profile <<< "$fields"
- [ "$backend" = "$CX_BENCH" ] && [ "$sku" = "$CX_RUNNER" ] && [ "$ep" = "$CX_NGPUS" ] \
- || { cx_log "ERROR: precision probe control differs from runtime placement"; return 1; }
- out="results/${probe_id}.json"
- cx_write_runtime_stage execution || return 1
- if timeout -k 30 "${CX_RUN_TIMEOUT:-900}" torchrun --nproc_per_node="$CX_NGPUS" \
- tests/probe_precision.py --backend "$backend" --sku "$sku" --ep "$ep" \
- --mode "$mode" --precision-profile "$profile" --out "$out"; then
- rc_run=0
- else
- rc_run=$?
- fi
- [ "$rc_run" = 0 ] || return "$rc_run"
- python3 tests/probe_precision.py --validate-manifest "$out"
-}
-
rc=0
cx_validate_shard_control "$PWD"
cx_load_network_control_mode "$PWD" || cx_die "cannot resolve network control mode"
@@ -1028,13 +1008,7 @@ if [ -n "${CX_BUILD_ONLY:-}" ]; then
cx_log "backend preparation: bench=${CX_BENCH:-unknown} rc=$rc"
exit "$rc"
fi
-if [ "${CX_PRECISION_PROBE:-0}" = 1 ]; then
- if cx_probe_scaleout_network && cx_prepare_backend "${CX_BENCH:-}"; then
- run_precision_probe || rc=$?
- else
- rc=1
- fi
-elif [ -n "${CX_SHARD_FILE:-}" ]; then
+if [ -n "${CX_SHARD_FILE:-}" ]; then
# SHARD/SWEEP mode (collectivex-sweep.yml): run EVERY case of this shard in THIS one allocation.
# All cases share (sku, backend, nodes), so backend preparation is paid once and cached.
ncases="$(python3 -c "import json;print(len(json.load(open('$CX_SHARD_FILE'))['cases']))")"
@@ -1059,7 +1033,6 @@ env = {
"CX_EP": g("ep", "1"),
"CX_EPLB": "1" if c.get("eplb") else "",
"CX_CASE_ID": g("case_id"), "CX_SUITE": g("suite"), "CX_WORKLOAD_NAME": g("workload"),
- "CX_REQUIRED_PUBLICATION": g("required_publication"),
"CX_PRECISION_PROFILE": g("precision_profile"),
"CX_HIDDEN": g("hidden"), "CX_TOPK": g("topk"), "CX_EXPERTS": g("experts"),
"CX_TOKENS_LADDER": g("ladder"), "CX_CANONICAL": ("1" if c.get("canonical") else ""),
@@ -1119,7 +1092,5 @@ else
fi
# Summary table for the log; also fails the job if no valid results were produced.
-if [ "${CX_PRECISION_PROBE:-0}" != 1 ]; then
- python3 summarize.py --results-dir results --runner "$CX_RUNNER" --ts "$CX_TS" || rc=1
-fi
+python3 summarize.py --results-dir results --runner "$CX_RUNNER" --ts "$CX_TS" || rc=1
exit "$rc"
From c80ea097fc551f3a313fcac7fa6970f4de0d99eb Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Wed, 8 Jul 2026 12:55:26 +0800
Subject: [PATCH 196/210] docs(collectivex): rewrite README and methodology to
sweep-only MVP
CollectiveX now schedules, executes, validates, and uploads neutral
artifacts and no longer promotes, ranks, recommends, or decides display.
Rewrite README.md and docs/methodology.md to describe the setup+sweep
workflow, BF16-default measurement (FP8 codec callable but not swept),
per-dispatch matrix (no frozen digest/qualification), and neutral
validate-delivery + artifact upload. Drop all publication/promotion/
dev-latest/publisher/cohort/recommendation content; preserve the
measurement, correctness, execution-isolation, and image-pinning
contracts unchanged. Remove the language switcher from the English docs
(existing _zh files left untouched per the English-only override).
---
experimental/CollectiveX/README.md | 200 +++++------
experimental/CollectiveX/docs/methodology.md | 329 +++++++------------
2 files changed, 190 insertions(+), 339 deletions(-)
diff --git a/experimental/CollectiveX/README.md b/experimental/CollectiveX/README.md
index 3091b0e62b..2d356f763b 100644
--- a/experimental/CollectiveX/README.md
+++ b/experimental/CollectiveX/README.md
@@ -1,50 +1,46 @@
# CollectiveX
-
-
-**English** | [中文](./README_zh.md)
-
-
-
CollectiveX is an experimental MoE expert-parallel communication benchmark. It measures dispatch,
-combine, and paired roundtrip latency across EP libraries and accelerator systems.
+combine, and paired roundtrip latency across EP libraries and accelerator systems, then uploads
+neutral result artifacts.
-> Publication hold: historical schema 3-5 data is diagnostic. No current dataset is approved for
-> rankings, recommendations, or regression baselines.
+CollectiveX schedules benchmarks, executes them on real allocations, validates what each run emits,
+and uploads neutral artifacts. It does not promote, rank, recommend, select, or decide what a
+consumer displays. Any downstream display or comparison is the consumer's responsibility. The full
+measurement and validation contract is in [docs/methodology.md](docs/methodology.md).
-> Development status: the seven-SKU V1 matrix, precision, point-level publication, branch-only
-> delivery, and frontend contracts are frozen and ready for qualification. MI325X is deferred to a
-> later version because its intended two-node runner pool is unavailable.
+## Execution Profile
-## Implemented Pre-V1 Execution Profile
-
-The BF16/BF16 control and endpoint precision suites use packed placement and one pinned
-`fixed-profile` resource configuration per backend/topology/precision. V1 performs no tuning sweep.
-The explicit mode selects one of two contracts:
+The workload uses packed placement and one pinned `fixed-profile` resource configuration per
+backend/topology; there is no tuning sweep. The default sweep runs the portable BF16 dispatch / BF16
+combine control. The native FP8 codec paths stay in the code (`bench/ep_precision.py` and each
+adapter's FP8 dispatch/combine) and remain callable but are not a swept dimension. The explicit mode
+selects one of two contracts:
- Normal mode uses `layout-and-dispatch-v1`, rank-deduplicated token payloads, and activation-only
combine. Uniform core coverage and one Zipf sensitivity remain; EPLB is measured only as the Zipf
remedy.
- Low-latency mode uses `expert-packed-weighted-combine-v1`, token-expert payloads, and gate-weighted
- combine through genuine DeepEP V1 or UCCL low-latency APIs. It is decode-only and never shares a
- ranking cohort with normal mode. Other backends are explicitly unsupported for this suite.
+ combine through genuine DeepEP V1 or UCCL low-latency APIs. It is decode-only. Other backends are
+ recorded as unsupported for this suite.
Both modes use `fixed-512-v1`: 64 trials x 8 timed iterations with 32 synchronized full roundtrip
warmups before each measured component at every trial/point. Roundtrip is measured first; each
iteration takes the cross-rank maximum before nearest-rank p50/p90/p95/p99, and roundtrip p99 is the
headline latency. A stdlib integer counter produces byte-identical routing and gate weights.
-Correctness is checked against the semantic value after the declared communication codec. The
-frozen combine gates are `rtol=0.05, atol=0.02` for BF16, `rtol=0.06, atol=0.03` for native
-logfmt10, and `rtol=0.08, atol=0.04` for native FP8 direct-cast combine. These are fail-closed
-publication thresholds, not estimates of codec error. FP8 direct-cast evidence also counts
-saturation on the exact transformed native combine input; any saturated value fails the point.
+Correctness is checked against the semantic value after the declared communication codec. The combine
+gates are `rtol=0.05, atol=0.02` for BF16 (the default sweep), `rtol=0.06, atol=0.03` for native
+logfmt10, and `rtol=0.08, atol=0.04` for native FP8 direct-cast combine when those codec paths are
+exercised. FP8 direct-cast evidence also counts saturation on the exact transformed native combine
+input; any saturated value fails the point. Any failed rank or point makes the case ineligible and is
+recorded as a terminal outcome.
-The frozen V1 graph covers H100, H200, B200, B300, GB200, GB300, and MI355X. It requests 664 cases /
-1,532 token points: 393 runnable cases / 898 points, emitted as 50 executable workflow
-shards/allocation cells, plus 271 explicit unsupported cases / 634 points. `sweep_matrix.py`
-materializes every token ladder and rejects missing, stale, malformed, or altered shard controls.
-Shards are emitted round-robin by SKU so the bounded GHA matrix uses every runner pool early.
+The matrix covers H100, H200, B200, B300, GB200, GB300, and MI355X. `sweep_matrix.py` materializes
+the requested SKUs, backends, EP sizes, and token ladders, then extracts strict per-shard controls
+and rejects missing, stale, malformed, or altered shard controls. `--only-sku`, `--exclude-skus`, and
+`--ep-sizes` select a subset; the matrix is generated per dispatch, with no frozen digest or locked
+case count.
| Systems | EP8 | EP16 |
|---|---|---|
@@ -53,7 +49,8 @@ Shards are emitted round-robin by SKU so the bounded GHA matrix uses every runne
| GB200/GB300 | 2x4 MNNVL, scale-up | 4x4 MNNVL, scale-up |
Physical host count does not determine scope: both GB topologies stay inside one 72-GPU MNNVL
-scale-up domain.
+scale-up domain. The MI325X launcher/configuration path is retained for future versions but is not
+referenced by any current suite or shard.
| Backend | Current scope |
|---|---|
@@ -64,9 +61,6 @@ scale-up domain.
| NCCL/RCCL A2A | Portable rank-deduplicated payload plus expert/routing-metadata reference |
| MoRI | MI355X EP8 uses IntraNode; EP16 pins InterNodeV1 over 2x8 XGMI + RDMA |
-FlashInfer is outside v1 because its exercised EP path failed intermittently at runtime. It is not
-misreported as a platform capability limitation and can return after a stable pinned path is proven.
-
DeepEP V2 means the `ElasticBuffer` implementation introduced by
[DeepEP PR #605](https://github.com/deepseek-ai/DeepEP/pull/605), not a newer legacy `Buffer` build.
The pinned source is the [PR #630](https://github.com/deepseek-ai/DeepEP/pull/630) head, whose parent
@@ -77,47 +71,27 @@ misclassified as duplicate NCCL libraries. Scale-up cases request NCCL Device AP
unless the realized LSA team covers the full EP world. x86 EP16 scale-out cases instead require the
hybrid path with GIN, two logical scale-out domains represented by two physical RDMA ranks, and eight
scale-up ranks per domain; GB EP16 remains MNNVL scale-up and therefore uses LSA. The isolated build
-records the API, source, loaded libraries, generated JIT source, executable SASS, and raw CUBIN
-diagnostics. H100 is explicitly unsupported for V2 because NCCL 2.30.4
-reports that its EP8 communicator lacks Device API symmetric-memory support; re-enabling that pool
-requires an all-rank CUDA P2P/LSA-capable runtime. Every other NVIDIA precision cell now records an
-exact native supported or unsupported outcome from the pinned runtime.
-
-H100 EP16 is supported on the healthy runner subset. The private overlay excludes pods whose network
-namespaces lack the host RDMA devices, and every allocation must validate the complete pinned RoCE
-profile before image import. EP8 remains in scope and derives its private stage beside the existing
-shared container directory because the runner account home is not mounted on compute nodes. B300
-EP16 is terminal unsupported for V1 publication because its functional fallback HCAs are not the
-GPU-adjacent product fabric; this is an operational topology decision, not a library limitation.
-
-Axes not implemented in this baseline include cached-layout `[cl]`, runtime-visible `[rv]`, precision
-formats outside the exact native allowlist, extra routing distributions, activation profiles, uneven
-allocation, placement permutations, model envelopes, and scaling studies.
+records the API, source, loaded libraries, generated JIT source, and executable SASS; raw CUBIN bytes
+stay private diagnostics. Whether a given SKU/backend/EP cell is attempted is a capability fact;
+whether it succeeded is decided only by the emitted artifact.
## Workflow And Artifacts
-`.github/workflows/collectivex-sweep.yml` generates a public-SKU matrix, extracts a strict ignored
-`.shards/.json` control, executes one allocation per shard, privacy-checks result JSON, and uploads
-raw GitHub artifacts. Runs default to `release_tag=unversioned` and are diagnostic-only. A V1 run
-must explicitly select `release_tag=v1`; setup then requires the locked full-matrix digest and emits
-a run/attempt/source-bound `cxrelease-v1-*` marker. Partial and filtered runs cannot receive it.
-
-The main-registered `.github/workflows/collectivex-sweep.yml` provides `probe-precision`, `sweep`,
-`publish-v1`, and `refresh-v1` operations, so its branch revision can be dispatched with
-`--ref collectivex` without a standalone branch-only workflow. Probes are unversioned, bounded,
-native-runtime checks and cannot publish. Publication accepts one successful first-attempt tagged
-sweep run ID at qualification index 1, revalidates its metadata and release marker, and runs
-`publisher.py` in a disposable runner-local workspace. Refresh revalidates and reuploads only the
-exact content-addressed sanitized dataset. Raw artifacts and the private publisher workspace are
-never exposed to the frontend.
-
-There is no results server, attached store, Vercel storage, GCP, Neon, managed database, or managed
-object store. With the existing server-side `GITHUB_TOKEN`, the frontend discovers the latest
-successful version-scoped publication workflow, downloads its NDJSON artifact just in time, verifies
-the ZIP layout, UTF-8/NDJSON shape, schema, promotion state, and SHA-256, then serves versioned channel
-and immutable dataset URLs. The UI keeps an explicit benchmark-version selector; V2 and later
-versions must use separate release tags and publication identities. The full validation contract is
-in [docs/methodology.md](docs/methodology.md).
+`.github/workflows/collectivex-sweep.yml` has two jobs. `setup` generates a public-SKU matrix
+(`backend`, `suites`, `only_sku`, `exclude_skus`, `ep_sizes` inputs), fetches the pinned backend
+source archive, emits a neutral `collectivex.frontend-catalog.v1` projection, and uploads the matrix.
+`sweep` extracts a strict ignored `.shards/.json` control per matrix entry, executes one
+allocation per shard, runs `contracts.py validate-delivery`, and uploads the result artifacts with
+`always()` so a red or partial run still uploads.
+
+Each shard emits per-case result and terminal JSON, detached sample JSON, and a small mechanical
+summary. `validate-delivery` asserts strict schemas, one terminal record per scheduled case,
+detached-sample path/SHA-256 consistency, and privacy before upload. No step promotes a run, builds a
+dataset, or advances a channel; the neutral artifacts are the output. A consumer downloads them and
+decides what to display.
+
+Private host, address, device, NIC, credential, workspace, and path data stays in encrypted config,
+ignored operator notes, or bounded mode-0600 runner logs; it is never uploaded.
## Runner Configuration
@@ -126,8 +100,8 @@ Runner-local Slurm and storage values use a strict per-SKU JSON document at
same-owner, non-symlink file is outside the checkout and never uploaded. Unknown runners, fields,
duplicate keys, endpoint literals, unsafe paths, and non-JSON input fail closed; configuration is
never evaluated as shell. GHA passes encrypted `COLLECTIVEX_OPERATOR_CONFIG_V1` content only to the
-launcher, which validates it, exports the selected SKU's allowlisted values, and deletes the
-temporary copy before allocation. Required JSON fields are:
+launcher, which validates it, exports the selected SKU's allowlisted values, and deletes the temporary
+copy before allocation. Required JSON fields are:
| SKU | Variables |
|---|---|
@@ -139,17 +113,13 @@ temporary copy before allocation. Required JSON fields are:
| `gb300` | `partition`, `account`, `squash_dir`, `enroot_cache_path` |
| `mi325x`, `mi355x` | `partition`, `squash_dir`, `stage_dir` |
-The MI325X launcher/configuration path is retained for future versions but is not referenced by any
-V1 suite, capability target, workflow shard, or publication cohort.
-
-Every selected non-MNNVL EP16 placement additionally requires `socket_ifname` and `rdma_devices`
-for its operator-approved fabric; optional
-`ib_gid_index`, `rdma_service_level`, and `rdma_traffic_class` are also allowlisted. Service level
-and traffic class are mapped into MoRI's RDMA/IO QoS environment. CollectiveX does not heuristically
-select a management route or HCA. After allocation, every non-MNNVL scale-out node must prove that
-all configured interfaces and active HCA ports exist before backend setup. Scale-up and MNNVL jobs
-clear these overrides. Scale-out NCCL/RCCL is pinned to `IB` with exact-match HCA selectors so a
-socket fallback fails instead of being mislabeled as RDMA.
+Every selected non-MNNVL EP16 placement additionally requires `socket_ifname` and `rdma_devices` for
+its operator-approved fabric; optional `ib_gid_index`, `rdma_service_level`, and `rdma_traffic_class`
+are also allowlisted. Service level and traffic class are mapped into MoRI's RDMA/IO QoS environment.
+CollectiveX does not heuristically select a management route or HCA. After allocation, every
+non-MNNVL scale-out node must prove that all configured interfaces and active HCA ports exist before
+backend setup. Scale-up and MNNVL jobs clear these overrides. Scale-out NCCL/RCCL is pinned to `IB`
+with exact-match HCA selectors so a socket fallback fails instead of being mislabeled as RDMA.
`ib_gid_index` is applied only when every selected HCA port reports an Ethernet link layer, where it
selects the operator-approved RoCE GID. Native InfiniBand profiles retain explicit HCA and service
@@ -160,45 +130,29 @@ Mixed Ethernet and InfiniBand HCA lists are rejected.
workspace. It is not group- or world-writable and is visible at the same path on the runner and every
allocated node. Jobs create only a marked mode-0700 execution child, prove cross-node read/write
visibility, and remove that exact child after allocation teardown; they never mount the runner
-checkout or create a stage beneath image storage on AMD.
-
-When an AMD operator row omits `stage_dir`, the self-hosted runner derives a private base beside its
-standard `_work` directory on the shared runner filesystem. The root-owned squash cache is never
-used as a repository stage. The derived parent must be runner-owned, non-symlinked, and not writable
-by group or world before any benchmark source is copied.
-
-H200, B200, and B300 runners may omit `stage_dir`. Their isolated execution child is then created
-under a runner-owned mode-0700 base in the validated operating-system account home, independent of
-the workflow's temporary `HOME`. A symlinked account-home entry is resolved once to its canonical,
-runner-owned target; the single hidden staging base directly beneath it must itself be non-symlinked
-and not writable by other users. The workflow still proves that base is visible from every allocated
-node before launch. The execution child is validated, marked, cross-node checked, and removed using
-the same contract. H100 may also omit `stage_dir`; its private base is created beside, never beneath,
-the configured shared container directory so it is compute-visible. All other runners require the
-dedicated `stage_dir` above; no canonical run stages source beneath shared container storage.
-Canonical B300 execution ignores any legacy configured `stage_dir` and always uses the validated
-compute-visible account-home base. Its UID-mapped Actions shell may accept that exact base when its
-owner matches the private parent owner; this exception is not available to configured stages or
-other runners. A hashed execution-ID suffix isolates parallel B300 workers without exposing private
-runner identity. On this NFS export, a newly created hashed base may realize as UID 0; only that
-creation path is accepted, while a pre-existing root-owned base is rejected.
-Canonical GB300 execution likewise ignores its legacy group-writable `stage_dir` and derives an
-execution-hashed private base beneath the validated compute-visible account home.
+checkout or create a stage beneath image storage on AMD. When an AMD operator row omits `stage_dir`,
+the runner derives a private base beside its standard `_work` directory on the shared runner
+filesystem; the root-owned squash cache is never used as a repository stage.
+
+H200, B200, and B300 runners may omit `stage_dir`; their isolated execution child is created under a
+runner-owned mode-0700 base in the validated operating-system account home, independent of the
+workflow's temporary `HOME`. H100 may also omit `stage_dir`; its private base is created beside, never
+beneath, the configured shared container directory so it is compute-visible. Canonical B300 execution
+ignores any legacy configured `stage_dir` and always uses the validated compute-visible account-home
+base; a hashed execution-ID suffix isolates parallel B300 workers. Canonical GB300 execution likewise
+ignores its legacy group-writable `stage_dir` and derives an execution-hashed private base beneath the
+validated compute-visible account home. The workflow proves every derived base is visible from all
+allocated nodes before launch.
Before import, each Docker Hub tag is resolved with bounded registry requests and must match its
pinned digest; digest-qualified overrides are rejected. Enroot imports use a fixed filesystem epoch
-and a versioned, registry-digest-bound cache key. Every mounted squash is freshly hashed. The
-verified registry digest and local squash hash are both recorded. Image-provided DeepEP is checked
-against exact wheel and installed-file fingerprints; source-built backends use pinned commits and
-runtime-verified GPU targets. DeepEP V2's mode-0700 cluster-local build cache is keyed by a versioned
-build recipe, verified image, architecture, upstream trees, and dependency pins; only its fixed
-`/cx-cache` mount reaches the container, and it never enters result artifacts.
-Pinned V2 and Hybrid sources are fetched once per workflow. Each job validates the complete archive,
-extracts only its exact backend root, permits only contained relative leaf symlinks to archived
-regular files, and revalidates the Git tree and submodule pins before staging.
-Compute containers receive an explicit environment allowlist. Private host, address, device, NIC,
-credential, workspace, and path data stays in encrypted config, ignored operator notes, or bounded
-mode-0600 runner logs; it is never uploaded.
+and a versioned, registry-digest-bound cache key. Every mounted squash is freshly hashed. Image-provided
+DeepEP is checked against exact wheel and installed-file fingerprints; source-built backends use pinned
+commits and runtime-verified GPU targets. DeepEP V2's mode-0700 cluster-local build cache is keyed by a
+versioned build recipe, verified image, architecture, upstream trees, and dependency pins; only its
+fixed `/cx-cache` mount reaches the container, and it never enters result artifacts. Pinned V2 and
+Hybrid sources are fetched once per workflow, validated whole, and extracted to their exact backend
+root before staging.
## Local Checks
@@ -207,10 +161,8 @@ uv run --with-requirements experimental/CollectiveX/requirements.txt \
python -m unittest discover experimental/CollectiveX/tests -p 'test_*.py'
uv run --with-requirements experimental/CollectiveX/requirements.txt \
python experimental/CollectiveX/sweep_matrix.py --backends all --out /tmp/cx-matrix.json >/dev/null
-uv run --with-requirements experimental/CollectiveX/requirements.txt \
- python experimental/CollectiveX/publisher.py --store-root "$COLLECTIVEX_STORE_ROOT" verify
bash -n experimental/CollectiveX/runtime/*.sh experimental/CollectiveX/launchers/*.sh
```
-Core paths are `capability.py`, `configs/`, `contracts.py`, `schemas/`, `sweep_matrix.py`,
-`publisher.py`, `runtime/`, `launchers/`, and `tests/`.
+Core paths are `capability.py`, `configs/`, `contracts.py`, `identity.py`, `schemas/`,
+`sweep_matrix.py`, `summarize.py`, `bench/`, `runtime/`, `launchers/`, and `tests/`.
diff --git a/experimental/CollectiveX/docs/methodology.md b/experimental/CollectiveX/docs/methodology.md
index 7d95a99aa2..a554c54d2b 100644
--- a/experimental/CollectiveX/docs/methodology.md
+++ b/experimental/CollectiveX/docs/methodology.md
@@ -1,15 +1,11 @@
-# CollectiveX EP V1 Methodology
+# CollectiveX EP Benchmark Methodology
-
-
-**English** | [中文](./methodology_zh.md)
-
-
-
-This document is the frozen V1 qualification contract. No V1 dataset is approved until one
-complete first-attempt execution passes publication and frontend audit. Chinese documentation
-synchronization is explicitly deferred for the V1 implementation phase. MI325X is deferred to a
-later version because its intended two-node runner pool is unavailable.
+CollectiveX schedules expert-parallel (EP) communication benchmarks, executes them on real
+accelerator allocations, validates the results each run emits, and uploads neutral artifacts. It
+does **not** promote, rank, recommend, select, hide, or decide what any consumer displays. The
+frontend reads the neutral matrix, result, summary, and catalog artifacts and makes its own coverage
+and display decisions. This document describes how a case is scheduled, measured, checked, and
+recorded — not a publication or qualification contract.
## Product Boundary
@@ -17,36 +13,27 @@ CollectiveX is a communication microbenchmark for:
- comparing EP libraries on one chip/topology;
- comparing EP latency and logical payload bandwidth across systems under the same workload; and
-- exposing unsupported, failed, invalid, and unstable evidence without contaminating decisions.
+- recording unsupported, failed, invalid, and unstable cases as explicit terminal evidence.
It does not predict serving throughput without a separate correlation study.
-## Implemented Matrix
+## Matrix
The implemented workload is `deepseek-v3-v1`: hidden 7168, top-k 8, 256 routed experts, packed
-placement, and one pinned fixed resource profile per backend/topology/precision. The BF16/BF16
-control is portable; endpoint precision suites schedule only allowlisted native communication-format
-profiles after real-hardware probes resolve their capability cells. Each case explicitly selects
-normal `layout-and-dispatch-v1` or low-latency `expert-packed-weighted-combine-v1` semantics.
+placement, and one pinned fixed resource profile per backend/topology. The default sweep runs the
+portable BF16 dispatch / BF16 combine control; the native FP8 codec paths remain in the code
+(`bench/ep_precision.py` and each adapter's FP8 dispatch/combine) and stay callable but are not a
+swept dimension. Each case explicitly selects normal `layout-and-dispatch-v1` or low-latency
+`expert-packed-weighted-combine-v1` semantics.
- `ep-core-v1`: uniform routing; decode T=1..128 powers of two; prefill T=256/512.
-- `ep-routing-v1`: Zipf with EPLB off/on; decode T=128; prefill T=512.
-- `ep-low-latency-v1`: DeepEP V1/UCCL native low-latency APIs; uniform decode T=1..128 powers of
- two; the capability contract rejects every other backend instead of fabricating a low-latency path.
-- `ep-precision-normal-v1`: nonbaseline native dispatch/combine profiles at decode T=128 and prefill
- T=512; BF16/BF16 endpoint controls are referenced rather than duplicated.
-- `ep-precision-low-latency-v1`: nonbaseline native low-latency profiles at decode T=128.
-- Frozen V1 matrix: 664 requested cases / 1,532 token points; 393 runnable cases / 898 points in 50
- executable workflow shards/allocation cells; 271 unsupported cases / 634 points.
-- Canonical matrix SHA-256: `c6988c5e81239ace699541322a88a37bfd80819d8bc1a2446f928665cd3ebba0`.
-- Case/disposition catalog SHA-256: `98b0c5e85c223a2edcbf9ea580a48bea115294c83691650bb25cb85904ed8be4`.
-- Qualification execution-plan SHA-256 values for repeats 1/2/3 are respectively
- `dd35bee6909a85761e069ebe83444a6f982eb571a4e61ad604a845ea7e4d10dc`,
- `fbc7262b3399225493f6c025c3bd2ceca52e90762bda1b6f6002683037094caf`, and
- `5edc837858df2d7498229db30ec0d364f5b6cd7ca703790a8c83aabb09e6aff4`.
-- The backend-produced frontend catalog is 601,970 bytes with SHA-256
- `86ac7bb3f9f6310385db52f6f77554d705985bd99e387ecb5fedd627c11994d7`, below the 1 MiB fixture
- budget and the 32 MiB publication limit.
+- `ep-low-latency-v1`: DeepEP V1 / UCCL native low-latency APIs; uniform decode T=1..128 powers of
+ two; other backends are recorded as unsupported rather than fabricating a low-latency path.
+
+`sweep_matrix.py` materializes the requested SKUs, backends, EP sizes, and token ladders into a
+matrix document, then extracts strict per-shard controls. `--only-sku`, `--exclude-skus`, and
+`--ep-sizes` select a subset; a subset produces a smaller matrix, not a different contract. The
+matrix is generated per dispatch; there is no frozen matrix digest or locked case count.
| Systems | EP8 | EP16 |
|---|---|---|
@@ -55,32 +42,20 @@ normal `layout-and-dispatch-v1` or low-latency `expert-packed-weighted-combine-v
| GB200/GB300 | 2x4 MNNVL, scale-up | 4x4 MNNVL, scale-up |
Physical host count does not define scope. Both GB cells remain inside one 72-GPU MNNVL scale-up
-domain.
+domain. The MI325X launcher/configuration path is retained for future versions but is not referenced
+by any current suite or shard.
Unsupported combinations are terminal outcomes, not silently skipped coverage. DeepEP V2 is the
`ElasticBuffer` introduced by PR #605, pinned with upstream PR #630's minimal pure-scale-up fix and
-the exact upstream PR #640 library matcher that excludes NCCL shared-memory mappings.
-Scale-up cases request NCCL Device API LSA and fail closed unless the realized LSA team covers the
-full EP world. x86 EP16 scale-out uses the hybrid path with GIN and requires two logical scale-out
-domains represented by two physical RDMA ranks, with eight scale-up ranks per domain. GB EP16
-remains MNNVL scale-up and uses LSA. NVIDIA capabilities declared in source remain unvalidated until
-GPU outcomes pass the native oracle and publisher gates. H100 V2 on the current runner pool is a
-declared unsupported combination in v1 because NCCL 2.30.4 reports no Device API symmetric-memory
-support for its EP8 communicator; that pool can return only after all-rank CUDA P2P/LSA support is
-restored. This baseline omits `[cl]`, `[rv]`, quantization, alternate activation/routing profiles,
-uneven allocation, placement permutations, model envelopes, and scaling.
-H100 EP16 is supported on the healthy runner subset. The private runner overlay excludes pods that
-do not expose the host RDMA devices in their network namespace, and allocation validation requires
-the complete operator-pinned RoCE profile on every selected node before image import. H100 EP8
-remains in scope with a private stage beside its configured shared container directory; unlike the
-B300 runner account home, the H100 account home is not compute-visible. B300 EP16 is terminal
-unsupported for V1 publication because its currently functional fallback HCAs are not the
-GPU-adjacent product fabric; this is an operational topology decision, not a library limitation.
-FlashInfer is excluded from v1 after repeatable intermittent execution failures; those failures are
-not converted into planned-unsupported coverage.
-MoRI EP8 uses MI355X IntraNode in normal mode. EP16 uses pinned InterNodeV1 over
-2x8 XGMI + RDMA with 96 blocks, 64 RDMA blocks, 8 warps, one QP per PE, and external input. MoRI's
-AsyncLL transport is not the genuine low-latency suite contract and is never labeled as such.
+the exact upstream PR #640 library matcher that excludes NCCL shared-memory mappings. Scale-up cases
+request NCCL Device API LSA and fail closed unless the realized LSA team covers the full EP world.
+x86 EP16 scale-out uses the hybrid path with GIN and requires two logical scale-out domains
+represented by two physical RDMA ranks, with eight scale-up ranks per domain. GB EP16 remains MNNVL
+scale-up and uses LSA. MoRI EP8 uses MI355X IntraNode in normal mode; EP16 uses pinned InterNodeV1
+over 2x8 XGMI + RDMA with 96 blocks, 64 RDMA blocks, 8 warps, one QP per PE, and external input.
+MoRI's AsyncLL transport is not the low-latency suite contract and is never labeled as such. Whether
+a given SKU/backend/EP cell is attempted is a capability fact; whether it succeeded is decided only
+by the emitted artifact.
## Workload Identity
@@ -101,12 +76,12 @@ Adapters may not generate routing or reinterpret one quantity as the other.
Normal mode uses `layout-and-dispatch-v1`: dispatch timing includes layout plus communication, and
combine returns activation payload through an unweighted rank-sum path. Low-latency mode uses
-`expert-packed-weighted-combine-v1`: native DeepEP V1/UCCL APIs dispatch token-expert assignments and
-perform gate-weighted combine. Expert-output staging is outside isolated combine timing and inside
-measured paired roundtrip. Each component declares availability, origin, start/end states, stage
-scope, and sample count. A paired-only API reports null isolated components. `isolated_sum` is
-derived and never used for throughput or recommendations. Mode is series identity, and normal and
-low-latency evidence cannot share a ranking cohort.
+`expert-packed-weighted-combine-v1`: native DeepEP V1 / UCCL APIs dispatch token-expert assignments
+and perform gate-weighted combine. Expert-output staging is outside isolated combine timing and
+inside the measured paired roundtrip. Each component declares availability, origin, start/end states,
+stage scope, and sample count. A paired-only API reports null isolated components. `isolated_sum` is
+derived. Normal and low-latency evidence describe different measurement contracts and are not
+directly comparable; the artifact records the mode so a reader can keep them separate.
Every measured component uses `fixed-512-v1`:
@@ -122,8 +97,8 @@ MoE-layer collective; they do not change the timed primitive at an otherwise ide
The NCCL/RCCL reference is an end-to-end Python adapter, not a bare fabric primitive. Its dispatch
boundary includes layout, count exchange, a device-to-host split synchronization, fresh receive
-allocation, and four payload/metadata all-to-all calls; activation-only combine adds one all-to-all plus
-scatter/reduction. Its p99 therefore measures the complete reference-adapter boundary and can be
+allocation, and four payload/metadata all-to-all calls; activation-only combine adds one all-to-all
+plus scatter/reduction. Its p99 therefore measures the complete reference-adapter boundary and can be
host/scheduler-sensitive. It is useful for portable system controls but must not be labeled fabric,
link, bus, or single-collective latency.
@@ -136,17 +111,17 @@ Logical payload bandwidth is:
Normal-mode payload bytes use rank-deduplicated token-rank activations; low-latency bytes use
token-expert assignments. Both add required scale bytes at the named boundary and exclude expert
-metadata, padding, and backend buffer capacity. Algorithm bandwidth, bus bandwidth, wire utilization,
-and physical-link utilization are not published without a defined primitive model or transport
-counters. Logical bandwidth must never be labeled physical bandwidth. Published payload and token
+metadata, padding, and backend buffer capacity. Algorithm bandwidth, bus bandwidth, wire
+utilization, and physical-link utilization are not emitted without a defined primitive model or
+transport counters. Logical bandwidth must never be labeled physical bandwidth. Payload and token
rates are named `rate_at_latency_percentile`: bytes or tokens divided by the matching latency
-percentile. They are lower-tail service rates at p99 latency, not p99 percentiles of an inverted rate
-distribution.
+percentile. They are lower-tail service rates at p99 latency, not p99 percentiles of an inverted
+rate distribution.
## Correctness
-An implementation-independent oracle uses an expert-specific deterministic transform so wrong
-expert routing cannot pass an identity roundtrip. For every rank and point it verifies:
+An implementation-independent oracle uses an expert-specific deterministic transform so wrong expert
+routing cannot pass an identity roundtrip. For every rank and point it verifies:
1. destination rank/expert, source token, multiplicity, gate weight, and receive counts;
2. dispatched payload and metadata before timing;
@@ -155,37 +130,40 @@ expert routing cannot pass an identity roundtrip. For every rank and point it ve
5. dispatched payload/metadata and combined output again after timing.
Normal-mode adapters use activation-only, unweighted rank-sum combine. The oracle builds each rank's
-gate-weighted expert aggregate before combine, independently derives `sum(gate * expert(token))`,
-and checks the dispatch metadata and transformed output. Low-latency adapters separately verify the
+gate-weighted expert aggregate before combine, independently derives `sum(gate * expert(token))`, and
+checks the dispatch metadata and transformed output. Low-latency adapters separately verify the
expert-packed source/expert assignment, native gate weights, and gate-weighted combined output. Both
-contracts compare against the semantic value after the declared communication codec. The frozen
-combine gates are `rtol=0.05, atol=0.02` for BF16, `rtol=0.06, atol=0.03` for native logfmt10, and
-`rtol=0.08, atol=0.04` for native FP8 direct-cast combine. These thresholds are correctness gates,
-not estimates of codec error. Direct-cast saturation is measured on the exact transformed native
-combine input, and the required saturation count is zero. Any failed rank or point makes the case
-ineligible.
+contracts compare against the semantic value after the declared communication codec. The combine
+gates are `rtol=0.05, atol=0.02` for BF16 (the default sweep), `rtol=0.06, atol=0.03` for native
+logfmt10, and `rtol=0.08, atol=0.04` for native FP8 direct-cast combine when those codec paths are
+exercised. These thresholds are correctness gates, not estimates of codec error. Direct-cast
+saturation is measured on the exact transformed native combine input, and the required saturation
+count is zero. Any failed rank or point makes the case ineligible and is recorded as such.
Pre/post dispatch evidence is hashed in canonical source-token order. Native receive slots may be
assigned nondeterministically, so physical receive order is not treated as a correctness property.
-## Native Result
+## Result Artifact
-One raw case document uses `format: "collectivex.ep.v1"`, rejects unknown fields, and contains:
+One raw case document uses `format: "collectivex.ep.v1"`, carries `schema_version: 1`, rejects
+unknown fields, and contains:
-- `case`: stable case ID, suite, required tier, and coordinate;
+- `case`: stable case ID, suite, and coordinate;
- `workload`: canonical identity and logical MoE shape;
- `measurement`: sampling, component states, timing, and byte accounting;
- `implementation`: instantiated class/API, pinned source, loaded libraries, and resources;
- `topology`: requested and realized SKU, devices, placement, scale-up domain, and transport;
- `provenance`: source SHA, image/squash hashes, allocation, run, and attempt;
-- `rows`: point latency, byte accounting, token rate, correctness, load, fanout, and anomaly evidence; and
-- `outcome`: `success`, `failed`, `invalid`, `diagnostic`, or `unsupported`, with reasons.
+- `rows`: point latency, byte accounting, token rate, correctness, load, fanout, and anomaly
+ evidence; and
+- `outcome`: `success`, `failed`, `invalid`, `unsupported`, with `diagnostic` and reasons.
-Raw result documents and exact samples pass through transient GitHub delivery artifacts before the
-publisher archives them in the private bundle; they never enter the public tree. Private environment
-details remain in local mode-0600 logs and ignored operator notes; they are never archived or
-published. Every expected case has one terminal selected outcome while every attempt remains retained.
+Exact per-point samples are emitted as detached `collectivex.samples.v1` documents referenced by path
+and SHA-256, so the raw document stays compact. Every scheduled case produces exactly one terminal
+record; every attempt is retained (a superseded retry does not overwrite an earlier failure). Private
+environment details (hosts, addresses, device selectors, credentials, workspace paths) remain in
+local mode-0600 logs and ignored operator notes and never enter an emitted artifact.
-## Identity And Comparisons
+## Identity
Canonical JSON produces three full SHA-256 IDs:
@@ -194,57 +172,17 @@ Canonical JSON produces three full SHA-256 IDs:
- `evidence_id`: `point_id` plus allocation/run/attempt/sample checksum.
Locked factors include workload bytes, measurement and sampling contract, resources, realized
-topology, implementation/build, loaded libraries, image/squash, runtime, and source SHA.
-Deferred code generation is captured before measurement and recaptured afterward. DeepEP V2 uses a
-fixed NVCC random seed and binds final cache keys plus generated-source and executable-SASS hashes;
-raw CUBIN bytes remain private diagnostics. Hybrid binds its realized auto-tuned config and complete
-kernel-key set while retaining rank-local shared-object hashes as private diagnostics. Locally built
-extension hashes are diagnostic; their pinned source trees, build recipe, runtime, and dependencies
-remain series-bound.
-The series identity includes the case ID, which binds the complete scheduled token ladder and the
-frozen percentile, rank-reduction, conditioning, warmup, and correctness semantics.
-
-A controlled comparison declares one contrast:
-
-- `library`: backend implementation and its pinned fixed resource profile may differ; the realized system,
- workload, EP, resource policy, source, and measurement remain matched;
-- `chip`: a controlled platform contrast. The full realized system/topology and pinned resource
- profile may differ while workload, EP, placement class, resource policy, backend lineage, source,
- and measurement remain matched. It is not a silicon-only comparison;
-- `system`: all hardware/backend differences stay visible while workload, EP, and measurement match;
-- `routing`: routing distribution/EPLB differs while the static implementation build/generator,
- system, model shape, resource profile, and measurement remain matched. Uniform and Zipf without
- EPLB reuse the same generated implementation; EPLB's physical-expert/JIT configuration remains an
- explicit treatment difference.
-
-Any undeclared mismatch rejects the overlay. Chip/system results describe measured systems, not
-silicon alone.
-
-## Evidence Policy
-
-Capability declarations say what may be attempted; artifacts determine evidence status. Promotion
-requires exact expected coverage with no missing, extra, duplicate, malformed, or heterogeneous
-case. Public coverage preserves each matrix disposition; promotion requires every runnable case to
-succeed and every planned-unsupported case to remain unsupported in every selected run. Only the
-pinned canonical full-v1 matrix, with a decision-grade library, chip, system, and routing cohort,
-may advance `dev-latest`; partial matrices remain diagnostic. The full-matrix digest intentionally
-pins the exact workflow shard grouping as well as the requested cases, so changing `--max-cases`
-or the SKU round-robin scheduling order produces diagnostic-only runs even when case coverage is
-unchanged. Superseded retries,
-planned-unsupported outcomes, and non-decision-grade (diagnostic) series and cohorts may render
-diagnostically but cannot rank or recommend; a promoted dataset ships them labelled with their
-verdict (`status` / `eligibility.reasons`) rather than dropping them, and need only hold at least
-one decision-grade series. The publisher computes and attaches each series' and cohort's verdict;
-the consumer decides which to surface and may reveal a diagnostic series.
-Any failed, invalid, or diagnostic retry of a runnable case still blocks promotion even if a later
-retry succeeds — the honesty floor that every runnable case physically produced clean data is
-never relaxed. Routing cohorts are comparable-experimental sensitivities and never produce configuration
-recommendations; official library/platform/system cohorts own actionable recommendations.
-
-A point becomes decision-grade only after a single workflow run and allocation ID passes
-correctness, identity, provenance, tail gates, within-run trial drift/outlier diagnostics, and stable
-ordering. The publisher, not the frontend, computes eligibility, controlled cohorts, sensitivity pairs,
-and recommendations.
+topology, implementation/build, loaded libraries, image/squash, runtime, and source SHA. Deferred
+code generation is captured before measurement and recaptured afterward. DeepEP V2 uses a fixed NVCC
+random seed and binds final cache keys plus generated-source and executable-SASS hashes; raw CUBIN
+bytes remain private diagnostics and are stripped before they enter an artifact. Hybrid binds its
+realized auto-tuned config and complete kernel-key set while retaining rank-local shared-object
+hashes as private diagnostics. Locally built extension hashes are diagnostic; their pinned source
+trees, build recipe, runtime, and dependencies remain series-bound.
+
+These IDs let a consumer group matched configurations and separate distinct ones. The backend does
+not itself compute cohorts, controlled comparisons, sensitivity pairs, eligibility, or
+recommendations — a reader decides which series to surface and how to compare them.
## Execution Isolation
@@ -263,10 +201,10 @@ including a safely identified partial claim. The same-run V2/Hybrid source archi
under fixed member and expanded-size bounds, and only the selected pinned root is extracted; a
symlink is accepted only when it is a relative leaf pointing to a regular member inside the same
backend root, followed by exact Git tree/submodule validation.
-H200, B200, and B300 may derive that private base beneath the validated operating-system account
-home when it is compute-visible. H100 instead derives a sibling of its shared container directory,
-never a child of image storage. The launcher still proves cross-node visibility before any benchmark
-starts.
+
+H200, B200, and B300 may derive that private base beneath the validated operating-system account home
+when it is compute-visible. H100 instead derives a sibling of its shared container directory, never a
+child of image storage. The launcher still proves cross-node visibility before any benchmark starts.
Canonical B300 execution ignores the legacy operator `stage_dir` field and always derives the base
from the validated shared account home. Its UID-mapped Actions shell may accept that exact base when
its owner matches the private parent owner; explicit stages and all other runners retain the strict
@@ -276,78 +214,39 @@ UID 0; only that creation path is accepted, while a pre-existing root-owned base
Canonical GB300 execution likewise ignores its legacy group-writable `stage_dir` and derives an
execution-hashed private base beneath the validated compute-visible account home.
-## Artifact Validation And JIT Delivery
-
-There is no self-hosted service, Vercel storage, GCP, Neon, managed database, or managed object
-store. The publication workflow uses runner-local temporary storage only as a disposable validation
-and promotion workspace:
-
-```text
-$COLLECTIVEX_STORE_ROOT/
- private/incoming/ # write-once downloaded GHA attempts
- private/bundles// # immutable source archives, native results/samples, matrix, checksums
- private/quarantine/ # rejected attempts plus machine-readable reasons
- public/datasets// # immutable sanitized frontend datasets
- public/channels/ # promoted dev-latest pointer; never served from persistent storage
- locks/
-```
-
-Private and public trees use separate permissions. JSON manifests and checksums are authoritative;
-a rebuildable catalog is only an index. Raw sweep artifacts are transient publisher input; only the
-sanitized promoted NDJSON is retained as a frontend publication artifact.
+## Image Pinning And Build Isolation
Container tags are checked against pinned registry digests. Enroot imports use a fixed
-`SOURCE_DATE_EPOCH` and versioned cache generation; every mounted squash is freshly hashed into
-series identity. Image-provided DeepEP is also checked against exact per-architecture wheel and
-installed-file fingerprints, so a stale cache cannot inherit the pinned source identity.
-Source-built DeepEP V2 uses a separate mode-0700 cluster-local cache mounted only as `/cx-cache`.
-Its content key binds a versioned build recipe, verified image digest, CPU/GPU architecture,
-upstream source trees, and pinned build dependencies. The cache is never an artifact or publisher
-input; per-execution source/results stages remain isolated and disposable, and marker plus runtime
-probes fail closed before reuse. The runner UID is inside the trusted cluster boundary: this cache
-guards against stale or accidental mutation, not hostile same-UID jobs. Only an unpublished partial
-build may be reset automatically; a published cache that fails integrity or runtime checks is left
-intact and rejected so a concurrent allocation cannot lose files it is using.
-
-Publication is fail-closed:
-
-1. acquire an exclusive filesystem lock and stage on the destination filesystem;
-2. archive source bytes before parsing;
-3. require the exact matrix-declared artifact set and reject every unconsumed archive member;
-4. validate strict schemas, privacy, checksums, identities, timing, and exact matrix outcomes;
-5. write checksums and `COMPLETE`, fsync, then atomically rename the private bundle;
-6. build and validate the sanitized content-addressed dataset, fsync, then atomically rename it;
-7. atomically replace `dev-latest.json` only when every promotion gate passes.
-
-Rejected attempts remain only in the disposable private workspace and short-lived source artifacts;
-they never advance `dev-latest` or enter a production channel. The workspace is destroyed with the
-publication runner and is never attached to the frontend. No publication artifact is emitted unless
-the single selected bundle advances `dev-latest`.
-
-`publisher.py ingest` accepts the exact matrix plus one `--artifact` directory or ZIP per GitHub
-artifact. `promote` accepts explicit immutable bundle IDs. Default `verify` requires
-the private workspace; it also verifies `dev-latest` when present, while an explicit
-`--channel dev-latest` requires it. The workflow copies only the verified sanitized dataset to a
-one-record `collectivex_public_v1_.ndjson` artifact. Raw artifacts and private workspace
-content are never bundled into the application.
-
-Sweeps default to `release_tag=unversioned`. The main-registered `collectivex-sweep.yml` owns
-`sweep`, `publish-v1`, and `refresh-v1`, so its branch revision remains dispatchable. V1 emits a
-marker bound to the run ID, first attempt, qualification index, source SHA, and locked matrix digest.
-Publication accepts one successful first-attempt run ID at qualification index 1, downloads its
-immutable artifacts, and passes the same provenance assertions to `publisher.py ingest`. Refresh
-requires an exact source run and dataset digest and reuploads the same validated sanitized bytes.
-Partial, filtered, untagged, cross-source, rerun, failed, expired, or digest-mismatched inputs fail
-closed.
-
-Using a server-side GitHub read token, the frontend discovers the latest successful version-scoped
-publication run and downloads the publication artifact just in time. It requires exactly one root
-NDJSON entry, validates UTF-8, schema, promotion status, and filename/body SHA-256, then exposes a
-short-lived versioned channel pointer and immutable versioned dataset URL. The benchmark-version
-selector currently exposes V1; later versions require separate release and publication identities.
-The frontend never invents missing values, selects retries, or recomputes decision eligibility.
+`SOURCE_DATE_EPOCH` and versioned cache generation; every mounted squash is freshly hashed into series
+identity. Image-provided DeepEP is also checked against exact per-architecture wheel and installed-file
+fingerprints, so a stale cache cannot inherit the pinned source identity. Source-built DeepEP V2 uses
+a separate mode-0700 cluster-local cache mounted only as `/cx-cache`. Its content key binds a
+versioned build recipe, verified image digest, CPU/GPU architecture, upstream source trees, and pinned
+build dependencies. The cache is never an artifact; per-execution source/results stages remain
+isolated and disposable, and marker plus runtime probes fail closed before reuse. The runner UID is
+inside the trusted cluster boundary: this cache guards against stale or accidental mutation, not
+hostile same-UID jobs. Only an unpublished partial build may be reset automatically; a cache that
+fails integrity or runtime checks is left intact and rejected so a concurrent allocation cannot lose
+files it is using.
+
+## Neutral Artifact Delivery
+
+There is no results server, attached store, or managed object store. Each shard runs one allocation,
+emits per-case result and terminal JSON plus detached sample JSON and a small mechanical summary, and
+uploads them as GitHub artifacts with `always()` so a red or partial run still uploads. Before upload,
+`contracts.py validate-delivery` checks, for the scheduled shard:
+
+- strict schema validity of every result, terminal, and samples document;
+- exactly one terminal record per scheduled case, with no missing, extra, or duplicate case;
+- detached-sample path and SHA-256 consistency (`point_id` / `evidence_id`); and
+- privacy — no private host, address, selector, credential, or path leaks into any artifact.
+
+The `setup` job also emits a neutral `collectivex.frontend-catalog.v1` projection of the matrix so a
+consumer can enumerate scheduled coverage without parsing shard internals. No step promotes a run,
+builds a dataset, or advances a channel; the artifacts are the output. Any downstream display or
+comparison is the consumer's responsibility.
## Legacy Data
-Numeric schemas 3-5 are outside the v1 publisher and frontend reader. They remain historical
-diagnostic evidence and cannot seed `dev-latest` or drive v1 decisions.
+Historical numeric schemas 3-5 are outside this benchmark's artifacts. They remain historical
+diagnostic evidence and are not produced or consumed by the current sweep.
From 2e94fd4b94effdb25d2e5d0a1fa983d6484e3a86 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Wed, 8 Jul 2026 13:14:18 +0800
Subject: [PATCH 197/210] fix(collectivex): strip drift fields from emitted raw
document
The P1 schema strip removed publication/precision/qualification fields
from raw-case-v1, but the emitter in bench/ep_harness.py still wrote
them, so contracts.validate_raw_document would reject every emitted case
(including the default BF16 sweep) on the cluster.
Remove the fields the stripped schema no longer accepts:
- case.shape: precision_profile, dispatch_precision, combine_precision
- identity.series_factors: implementation_contract_sha256,
public_config_sha256, routing_control_sha256, runtime_fingerprint_sha256
- identity.allocation_factors / measurement / samples: qualification_index
- measurement.execution_order_sha256 (+ dead observed_component_orders)
- provenance.allocation_stratum_sha256
- provenance.git_run.qualification_index (bench/run_ep.py)
Add tests/test_raw_document_roundtrip.py: a hermetic regression test that
builds a full raw + samples document from the real identity module and
drives contracts.validate_raw_document, asserting the BF16 fixed facts
and rejecting each removed drift field. Closes the FakeBackend coverage
gap where no test exercised the emitter->validate contract on CPU.
---
experimental/CollectiveX/bench/ep_harness.py | 33 --
experimental/CollectiveX/bench/run_ep.py | 1 -
.../tests/test_raw_document_roundtrip.py | 494 ++++++++++++++++++
3 files changed, 494 insertions(+), 34 deletions(-)
create mode 100644 experimental/CollectiveX/tests/test_raw_document_roundtrip.py
diff --git a/experimental/CollectiveX/bench/ep_harness.py b/experimental/CollectiveX/bench/ep_harness.py
index dfc72b4de7..219b118fad 100644
--- a/experimental/CollectiveX/bench/ep_harness.py
+++ b/experimental/CollectiveX/bench/ep_harness.py
@@ -1371,7 +1371,6 @@ def run_sweep(args, backend, torch, dist, device, rank: int, world_size: int) ->
"runner": args.runner,
"suite": args.suite,
})
- observed_component_orders = []
for trial_index in range(args.trials):
order = qualification_order(
list(ladder), args.qualification_index, trial_index,
@@ -1387,11 +1386,6 @@ def run_sweep(args, backend, torch, dist, device, rank: int, world_size: int) ->
trial_index,
identity_key=f"{order_identity}:components:{T}",
)
- observed_component_orders.append({
- "components": component_order,
- "tokens_per_rank": T,
- "trial_index": trial_index,
- })
measured = {name: [] for name in ("dispatch", "stage", "combine", "roundtrip")}
for component_name in component_order:
# The base template gives every component the same synchronized
@@ -1690,9 +1684,6 @@ def component(percentiles, count, *, derived=False):
shape = { # FIXED line identity (no T, no per-backend resource knobs)
"hidden": args.hidden, "topk": args.topk, "experts": args.experts,
"experts_per_rank": experts_per_rank,
- "precision_profile": resolved_precision_id,
- "dispatch_precision": communication_precision["dispatch"],
- "combine_precision": communication_precision["combine"],
"routing": args.routing, "eplb": bool(eplb_plan), "num_logical_experts": num_logical,
# V2 is reserved for the PR #605 ElasticBuffer adapter; package versions never imply it.
"kernel_gen": kernel_generation(backend),
@@ -1749,7 +1740,6 @@ def component(percentiles, count, *, derived=False):
"artifact": git_run.get("artifact"),
"execution_id": getattr(args, "allocation_execution_id", None),
"job": git_run.get("job"),
- "qualification_index": args.qualification_index,
"repo": git_run.get("repo"),
"run_attempt": git_run.get("run_attempt"),
"run_id": git_run.get("run_id"),
@@ -1767,29 +1757,10 @@ def component(percentiles, count, *, derived=False):
allocation=allocation_identifier, case=case_identifier, ordinal=attempt_ordinal
)
runtime_fingerprint = getattr(args, "runtime_fingerprint", None) or {}
- implementation_contract = {
- "kernel_generation": kernel_generation(backend),
- "name": backend.name,
- "provenance": ep_provenance.series_provenance(backend.backend_provenance),
- "resource_profile": resource_profile,
- }
- public_config = ep_provenance.public_series_config(
- kernel_generation=implementation_contract["kernel_generation"],
- provenance=backend.backend_provenance,
- resource_profile=resource_profile,
- resource_mode=args.resource_mode,
- device_product=getattr(args, "runtime_device_product", None),
- )
series_factors = {
"backend": backend.name,
- "implementation_contract_sha256": _sha256_json(implementation_contract),
- "public_config_sha256": ep_provenance.public_series_config_sha256(public_config),
- "routing_control_sha256": ep_provenance.routing_implementation_control_sha256(
- implementation_contract
- ),
"case_id": case_identifier,
"image_digest": getattr(args, "image_digest", None),
- "runtime_fingerprint_sha256": _sha256_json(runtime_fingerprint),
"source_sha": git_run.get("source_sha"),
"squash_sha256": getattr(args, "squash_sha256", None),
"workload_id": getattr(args, "workload_id", None) or trace_sig,
@@ -1840,7 +1811,6 @@ def component(percentiles, count, *, derived=False):
"case_id": case_identifier,
"format": "collectivex.samples.v1",
"points": sample_points,
- "qualification_index": args.qualification_index,
"sampling": {
"iterations_per_trial": args.iters,
"reduction": case_profile["rank_reduction"],
@@ -1945,8 +1915,6 @@ def component(percentiles, count, *, derived=False):
"roundtrips_per_shape": CONDITIONING_ROUNDS_PER_SHAPE,
},
"contract": case_profile["contract"],
- "execution_order_sha256": _sha256_json(observed_component_orders),
- "qualification_index": args.qualification_index,
"rows": rows,
"sampling": {
"contract": SAMPLING_CONTRACT,
@@ -1983,7 +1951,6 @@ def component(percentiles, count, *, derived=False):
},
"runtime_fingerprint": runtime_fingerprint,
"provenance": {
- "allocation_stratum_sha256": allocation_stratum_sha256,
"command": getattr(args, "reproduction_command", ""),
"distributed_launcher": getattr(args, "distributed_launcher", None),
"git_run": getattr(args, "git_run", None),
diff --git a/experimental/CollectiveX/bench/run_ep.py b/experimental/CollectiveX/bench/run_ep.py
index 9fc7b3bccb..0983ee904c 100644
--- a/experimental/CollectiveX/bench/run_ep.py
+++ b/experimental/CollectiveX/bench/run_ep.py
@@ -446,7 +446,6 @@ def main() -> int:
"artifact": os.environ.get("COLLECTIVEX_ARTIFACT_NAME"),
}
if any(_run.values()):
- _run["qualification_index"] = args.qualification_index
args.git_run = _run
else:
args.git_run = None
diff --git a/experimental/CollectiveX/tests/test_raw_document_roundtrip.py b/experimental/CollectiveX/tests/test_raw_document_roundtrip.py
new file mode 100644
index 0000000000..f0523926f9
--- /dev/null
+++ b/experimental/CollectiveX/tests/test_raw_document_roundtrip.py
@@ -0,0 +1,494 @@
+"""End-to-end raw-document contract test (no GPU, no torch).
+
+`test_ep_backend.py` exercises the mocked benchmark up to the measurement rows, and
+`test_schema_v1_contract.py` asserts facts about the schema files, but nothing drove a
+complete raw attempt document plus its detached samples through
+`contracts.validate_raw_document`. That left the emitter tail of `ep_harness.run_sweep`
+(the doc assembly that only runs on a real allocation) unguarded: a field the emitter
+writes but the frozen schema forbids, or vice versa, would only surface on a live sweep.
+
+This builds a document whose field sets mirror `ep_harness.run_sweep`'s assembly exactly,
+using the real `identity` module for every typed id and the real
+`identity.profile_for_case` for the case profile, then validates it. The negative cases
+pin the drift fields that were removed from the emitter so a re-introduction fails here
+instead of on the cluster.
+"""
+import copy
+import os
+import sys
+import unittest
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "bench"))
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+
+import identity # noqa: E402
+import contracts # noqa: E402
+
+_HEX = "ab" * 32 # a syntactically valid lowercase sha256 digest (64 hex)
+_SHA40 = "abcdef0123456789abcdef0123456789abcdef01" # a git-style 40-hex sha
+_WORKLOAD_ID = identity.workload_id({"name": "deepseek-v3-v1", "trace": _HEX})
+
+# A representative EP8 H100 scheduled case, identical in shape to what sweep_matrix
+# materializes and ep_harness rebuilds from args (no case_id key: it is computed from
+# these factors).
+SCHEDULED_CASE = {
+ "backend": "deepep",
+ "canonical": True,
+ "ep": 8,
+ "eplb": False,
+ "experts": 256,
+ "gpus_per_node": 8,
+ "hidden": 7168,
+ "ladder": "1 2 4 8 16 32 64 128",
+ "mode": "normal",
+ "nodes": 1,
+ "phase": "decode",
+ "routing": "uniform",
+ "samples_per_point": 512,
+ "scale_out_transport": None,
+ "scale_up_domain": 8,
+ "scale_up_transport": "nvlink",
+ "scope": "scale-up",
+ "suite": "ep-core-v1",
+ "timing": "8:64:32",
+ "topk": 8,
+ "topology_class": "h100-nvlink-island",
+ "transport": "nvlink",
+ "warmup_semantics": "full-roundtrip-before-each-component-trial-point-v1",
+ "workload": "deepseek-v3-v1",
+}
+
+
+def _percentiles():
+ return {"p50": 10.0, "p90": 20.0, "p95": 25.0, "p99": 30.0}
+
+
+def _component():
+ return {
+ "availability": "measured",
+ "origin": "measured",
+ "percentiles_us": _percentiles(),
+ "sample_count": 512,
+ }
+
+
+def _samples_component():
+ # 64 trials x 8 iterations of non-negative numbers, per samples-v1 trials def.
+ return {
+ "availability": "measured",
+ "sample_count": 512,
+ "trials": [[1.0] * 8 for _ in range(64)],
+ }
+
+
+def _byte_accounting():
+ return {
+ "accounting_contract": "activation-data-plus-scales-v1",
+ "activation_data_bytes": 1024,
+ "scale_bytes": 0,
+ "total_logical_bytes": 1024,
+ }
+
+
+def _histogram():
+ return {"bins": 4, "counts": [1, 2, 3, 4], "max": 30.0, "min": 10.0, "n": 10}
+
+
+def _oracle():
+ return {
+ "atol": 0.02,
+ "checks": {
+ "combine_values": True,
+ "counts": True,
+ "metadata": True,
+ "multiplicity": True,
+ "payload": True,
+ "source_set": True,
+ "weights": True,
+ },
+ "combine_weight_semantics": "unweighted-rank-sum",
+ "contract": "expert-specific-transform-v1",
+ "dispatch_sha256": None,
+ "max_absolute_error": 0.0,
+ "max_elementwise_relative_error": 0.0,
+ "max_relative_error": 0.0,
+ "max_weight_error": None,
+ "order_sha256": None,
+ "ordering_contract": "roundtrip-first-then-components-v1",
+ "passed": True,
+ "receive_count": 8,
+ "rtol": 0.05,
+ }
+
+
+def _routing():
+ return {
+ "empty_expert_count": 0,
+ "empty_rank_count": 0,
+ "expert_assignment_rank_cv": 0.0,
+ "expert_assignments_per_rank": [32, 32, 32, 32, 32, 32, 32, 32],
+ "expert_load_cv": 0.0,
+ "expert_load_max": 8,
+ "expert_load_mean": 8.0,
+ "expert_load_min": 8,
+ "fanout_histogram": [0, 8],
+ "fanout_max": 1,
+ "fanout_mean": 1.0,
+ "fanout_min": 1,
+ "hash": _HEX,
+ "hotspot_ratio": 1.0,
+ "locality": None,
+ "payload_copies_per_rank": [8, 8, 8, 8, 8, 8, 8, 8],
+ "payload_rank_cv": 0.0,
+ "routed_copies": 64,
+ "source_token_stats": None,
+ }
+
+
+def _correctness():
+ return {
+ "contract": "expert-specific-transform-v1",
+ "max_relative_error": 0.0,
+ "passed": True,
+ "rank_evidence": [
+ {
+ "input_unchanged": True,
+ "order_stable": True,
+ "post_timing": _oracle(),
+ "pre_timing": _oracle(),
+ "rank": 0,
+ }
+ ],
+ "scope": "dispatch-metadata-and-transformed-combine",
+ }
+
+
+def _build_documents():
+ """Return (raw_document, samples_document) with fully reconciled identities."""
+ profile = identity.profile_for_case({"mode": SCHEDULED_CASE["mode"]})
+ case_factors = {"case": SCHEDULED_CASE, "profile": profile, "sku": "h100-dgxc"}
+ case_id = identity.digest("case", case_factors)
+
+ allocation_factors = {
+ "artifact": "collectivex-results-h100-dgxc-deepep-n1",
+ "execution_id": "exec-0001",
+ "job": "sweep",
+ "repo": "SemiAnalysisAI/InferenceX",
+ "run_attempt": "1",
+ "run_id": "100200300",
+ "runner": "h100-dgxc",
+ "source_sha": _SHA40,
+ }
+ allocation_id = identity.allocation_id(allocation_factors)
+ attempt_ordinal = 1
+ attempt_id = identity.attempt_id(
+ allocation=allocation_id, case=case_id, ordinal=attempt_ordinal
+ )
+
+ series_factors = {
+ "backend": "deepep",
+ "case_id": case_id,
+ "image_digest": "sha256:" + _HEX,
+ "source_sha": _SHA40,
+ "squash_sha256": _HEX,
+ "workload_id": _WORKLOAD_ID,
+ }
+ series_id = identity.series_id(series_factors)
+
+ rows = []
+ sample_points = []
+ for token_count in (64, 128):
+ sample_sha256 = identity.digest("point", {"t": token_count})[-64:]
+ point_id = identity.point_id(series=series_id, tokens_per_rank=token_count)
+ evidence_id = identity.evidence_id(
+ point=point_id,
+ allocation=allocation_id,
+ attempt=attempt_id,
+ sample_sha256=sample_sha256,
+ )
+ rows.append({
+ "anomalies": [],
+ "byte_provenance": {c: _byte_accounting() for c in
+ ("combine", "dispatch", "roundtrip", "stage")},
+ "components": {
+ "combine": _component(),
+ "dispatch": _component(),
+ "isolated_sum": _component(),
+ "roundtrip": _component(),
+ "stage": _component(),
+ },
+ "correctness": _correctness(),
+ "evidence_id": evidence_id,
+ "global_tokens": token_count * 8,
+ "point_id": point_id,
+ "receive": {"max": 8, "mean": 8.0, "min": 8, "total": 64},
+ "routing": _routing(),
+ "sample_histograms": {
+ "combine": None,
+ "dispatch": None,
+ "roundtrip": _histogram(),
+ "stage": None,
+ },
+ "sample_sha256": sample_sha256,
+ "token_rate_at_latency_percentile": _percentiles(),
+ "tokens_per_rank": token_count,
+ })
+ sample_points.append({
+ "components": {
+ "combine": _samples_component(),
+ "dispatch": _samples_component(),
+ "roundtrip": _samples_component(),
+ "stage": _samples_component(),
+ },
+ "evidence_id": evidence_id,
+ "point_id": point_id,
+ "sample_sha256": sample_sha256,
+ "tokens_per_rank": token_count,
+ })
+
+ samples_document = {
+ "allocation_id": allocation_id,
+ "attempt_id": attempt_id,
+ "case_id": case_id,
+ "format": "collectivex.samples.v1",
+ "points": sample_points,
+ "sampling": {
+ "iterations_per_trial": 8,
+ "reduction": "cross-rank-max-per-iteration",
+ "trials": 64,
+ },
+ "schema_version": 1,
+ "series_id": series_id,
+ }
+
+ eplb_record = {
+ "calibration_token_offset": None,
+ "calibration_trace_sha256": None,
+ "calibration_window": None,
+ "calibration_workload_id": None,
+ "enabled": False,
+ "imbalance_after": None,
+ "imbalance_before": None,
+ "mapping_hash": None,
+ "max_replicas": None,
+ "num_logical_experts": 256,
+ "num_physical_experts": 256,
+ "num_redundant": 0,
+ "planner": None,
+ "reference_tokens_per_rank": None,
+ "replicated_experts": 0,
+ }
+
+ document = {
+ "format": "collectivex.ep.v1",
+ "schema_version": 1,
+ "record_type": "case-attempt",
+ "generated_at": "2026-07-08T00:00:00Z",
+ "identity": {
+ "allocation_factors": allocation_factors,
+ "allocation_id": allocation_id,
+ "attempt_id": attempt_id,
+ "attempt_ordinal": attempt_ordinal,
+ "case_factors": case_factors,
+ "case_id": case_id,
+ "series_factors": series_factors,
+ "series_id": series_id,
+ },
+ "case": {
+ "attempt_ordinal": attempt_ordinal,
+ "backend": "deepep",
+ "eplb": eplb_record,
+ "ep_size": 8,
+ "mode": "normal",
+ "phase": "decode",
+ "resource_mode": "fixed-profile",
+ "runner": "h100-dgxc",
+ "shape": {
+ "activation_profile": "canonical-counter-source-v4",
+ "eplb": False,
+ "experts": 256,
+ "experts_per_rank": 32,
+ "hidden": 7168,
+ "kernel_gen": "image-pinned-v1",
+ "num_logical_experts": 256,
+ "routing": "uniform",
+ "topk": 8,
+ },
+ "suite": "ep-core-v1",
+ "workload_name": "deepseek-v3-v1",
+ },
+ "workload": {
+ "activation_generator": "collectivex-activation-counter-v4",
+ "activation_identity": _HEX,
+ "activation_profile": "canonical-counter-source-v4",
+ "cross_rank_consistent": True,
+ "manifest_checksums": None,
+ "members": None,
+ "routing_generator": "collectivex-routing-counter-v3",
+ "source": "seeded-runtime",
+ "trace_hashes": [_HEX],
+ "trace_signature": _HEX,
+ "workload_id": _WORKLOAD_ID,
+ },
+ "measurement": {
+ "component_order_contract": "qualification-hash-rotated-components-v1",
+ "conditioning": {
+ "contract": "fixed-phase-ramp-8-roundtrips-v1",
+ "ladder": [1, 2, 4, 8, 16, 32, 64, 128],
+ "roundtrips_per_shape": 8,
+ },
+ "contract": "layout-and-dispatch-v1",
+ "rows": rows,
+ "sampling": {
+ "contract": "fixed-512-v1",
+ "iterations_per_trial": 8,
+ "percentile_method": "nearest-rank",
+ "reduction": "cross-rank-max-per-iteration",
+ "samples_per_component": 512,
+ "trials": 64,
+ "warmup_iterations": 32,
+ "warmup_semantics": "full-roundtrip-before-each-component-trial-point-v1",
+ },
+ "source_allocation": "even",
+ },
+ "implementation": {
+ "kernel_generation": "image-pinned-v1",
+ "name": "deepep",
+ "provenance": {},
+ "resource_profile": {
+ key: None for key in (
+ "achieved_fraction", "comm_units_kind", "configured_units",
+ "conformance_class", "device_units", "fixed_kernel",
+ "nonconforming", "pareto_eligible", "persistent_bytes",
+ "qps_per_rank", "requested_fraction", "resource_class",
+ "target_achieved_within_tol", "tolerance", "tuned_source",
+ "warps_combine", "warps_dispatch",
+ )
+ },
+ },
+ "topology": {
+ "device_count": 8,
+ "device_product": "NVIDIA H100 80GB HBM3",
+ "gpus_per_node": 8,
+ "nodes": 1,
+ "placement": "packed",
+ "realized_placement": {
+ "gpus_per_node": 8,
+ "nodes": 1,
+ "ranks_per_node": 8,
+ "unique_local_ranks": True,
+ "valid": True,
+ },
+ "scale_up_domain": 8,
+ "scale_up_transport": "nvlink",
+ "scale_out_transport": None,
+ "scope": "scale-up",
+ "topology_class": "h100-nvlink-island",
+ "transport": "nvlink",
+ "world_size": 8,
+ },
+ "runtime_fingerprint": {
+ "accelerator_runtime": {"kind": "cuda", "version": "12.4"},
+ "collective_library": {"kind": "nccl", "version": "2.21.5"},
+ "device": {
+ "arch": "sm90",
+ "compute_units": 132,
+ "memory_bytes": 85899345920,
+ "product": "NVIDIA H100 80GB HBM3",
+ "warp_size": 32,
+ },
+ "driver_version": "550.54.15",
+ "framework": {"kind": "torch", "version": "2.5.1"},
+ "machine": "x86_64",
+ "python_version": "3.11.9",
+ "vendor": "nvidia",
+ },
+ "provenance": {
+ "command": "run_ep.py ...",
+ "distributed_launcher": None,
+ "git_run": {
+ "artifact": "collectivex-results-h100-dgxc-deepep-n1",
+ "job": "sweep",
+ "ref": "collectivex",
+ "repo": "SemiAnalysisAI/InferenceX",
+ "run_attempt": "1",
+ "run_id": "100200300",
+ "source_sha": _SHA40,
+ },
+ "image": {
+ "arch": "x86_64",
+ "digest": "sha256:" + _HEX,
+ "digest_verified": True,
+ "reference": "example/image:tag",
+ "squash_sha256": _HEX,
+ },
+ "redaction": "sanitized-v1",
+ },
+ "sample_artifact": {
+ "bytes": 1,
+ "format": "collectivex.samples.v1",
+ "path": "case.samples.json",
+ "sha256": _HEX,
+ },
+ "outcome": {
+ "reasons": [],
+ "status": "success",
+ "validity": {
+ "anomaly_free": True,
+ "execution_status": "complete",
+ "measurement_conformance": "conformant",
+ "provenance_complete": True,
+ "resource_conformance": "conformant",
+ "sampling_conformance": "conformant",
+ "semantic_correctness": "pass",
+ "workload_identity": "consistent-across-ranks",
+ "workload_source": "seeded-runtime",
+ },
+ },
+ }
+ return document, samples_document
+
+
+class RawDocumentRoundTripTest(unittest.TestCase):
+ def test_representative_raw_document_validates(self):
+ document, samples = _build_documents()
+ # Must not raise: schema shape + all cross-file identity checks pass.
+ result = contracts.validate_raw_document(document, samples)
+ self.assertEqual(result["identity"]["case_id"], document["identity"]["case_id"])
+
+ def test_profile_is_bf16_fixed_fact(self):
+ profile = identity.profile_for_case({"mode": "normal"})
+ self.assertEqual(profile["combine_dtype"], "bf16")
+ self.assertEqual(profile["combine_quant_mode"], "none")
+
+ def test_removed_drift_fields_are_rejected(self):
+ # Each of these keys was emitted by an earlier build but is forbidden by the
+ # frozen schema. Re-introducing any of them must fail closed here.
+ base_doc, base_samples = _build_documents()
+ cases = [
+ ("identity", "allocation_factors", "qualification_index"),
+ ("identity", "series_factors", "runtime_fingerprint_sha256"),
+ ("measurement", None, "execution_order_sha256"),
+ ("measurement", None, "qualification_index"),
+ ("provenance", None, "allocation_stratum_sha256"),
+ ("provenance", "git_run", "qualification_index"),
+ ("case", "shape", "precision_profile"),
+ ("case", "shape", "dispatch_precision"),
+ ]
+ for top, sub, key in cases:
+ with self.subTest(field=f"{top}.{sub or ''}.{key}"):
+ doc = copy.deepcopy(base_doc)
+ target = doc[top] if sub is None else doc[top][sub]
+ target[key] = 1
+ with self.assertRaises(contracts.ContractError):
+ contracts.validate_raw_document(doc, copy.deepcopy(base_samples))
+
+ def test_samples_qualification_index_is_rejected(self):
+ doc, samples = _build_documents()
+ samples["qualification_index"] = 1
+ with self.assertRaises(contracts.ContractError):
+ contracts.validate_raw_document(doc, samples)
+
+
+if __name__ == "__main__":
+ unittest.main()
From 19cac7222596e265515188ced23c94eee1ce1890 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Wed, 8 Jul 2026 13:19:47 +0800
Subject: [PATCH 198/210] fix(collectivex): keep shard-control path importable
without jsonschema
sweep_matrix.py imports contracts unconditionally, and its
--validate-control / --extract-from paths are documented (and relied on
by cx_validate_shard_control in runtime/common.sh) to run on a minimal
GPU-runner python3 with only the standard library. The P2 rewrite added
a module-level `from jsonschema import Draft202012Validator` to
contracts.py, which made importing contracts (hence sweep_matrix) fail on
any host without jsonschema, breaking host-side shard validation before
allocation.
Defer the jsonschema import into contracts._validator(), which is the
only runtime user of the symbol (all annotations are strings under
`from __future__ import annotations`). Importing contracts no longer
requires jsonschema; document validation still imports it on first use.
---
experimental/CollectiveX/contracts.py | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/experimental/CollectiveX/contracts.py b/experimental/CollectiveX/contracts.py
index 8b27908f03..4746014427 100644
--- a/experimental/CollectiveX/contracts.py
+++ b/experimental/CollectiveX/contracts.py
@@ -29,8 +29,6 @@
import sys
from typing import Any, Iterable
-from jsonschema import Draft202012Validator
-
import identity
RAW_FORMAT = "collectivex.ep.v1"
@@ -382,6 +380,11 @@ def _integer(value: Any, path: str, *, minimum: int = 0) -> int:
def _validator(schema_file: str) -> Draft202012Validator:
+ # Imported lazily so that importing this module (e.g. for the stdlib-only
+ # shard-control path in sweep_matrix.py) does not require jsonschema; it is
+ # only needed once a document is actually validated.
+ from jsonschema import Draft202012Validator
+
validator = _VALIDATOR_CACHE.get(schema_file)
if validator is None:
try:
From d23a25501a6da72369201ab956f0abef0d34587f Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Wed, 8 Jul 2026 15:00:15 +0800
Subject: [PATCH 199/210] fix(collectivex): reconcile raw emission and launcher
with trimmed schema
The schema/publication trim removed the precision-evidence key and the
allocation_stratum_sha256 provenance parameter, but the raw-document emitter
still wrote correctness.precision (rejected by raw-case schema
additionalProperties) and still passed allocation_stratum_sha256 to
provenance_complete() (now a TypeError). It also left two orphaned probe_stage
branches in cx_fail_stage() that abort the launcher under set -u.
These desyncs are invisible to the CPU unit battery (run_sweep imports torch and
never executes off-GPU); a live 8xH100 nccl-ep sweep surfaced all three. Fix:
drop the dead precision gather + correctness.precision emit, drop the
allocation_stratum_sha256 kwarg, and remove the probe_stage branches. Full
offline schema reconciliation confirms zero remaining forbidden keys.
Verified green on 8xH100 (nccl-ep, EP8, BF16): both prefill and decode emit
status=success case-attempt docs + detached samples; summarize produces the
neutral table.
---
experimental/CollectiveX/bench/ep_harness.py | 9 ---------
experimental/CollectiveX/runtime/common.sh | 5 -----
2 files changed, 14 deletions(-)
diff --git a/experimental/CollectiveX/bench/ep_harness.py b/experimental/CollectiveX/bench/ep_harness.py
index 219b118fad..7052b1e87d 100644
--- a/experimental/CollectiveX/bench/ep_harness.py
+++ b/experimental/CollectiveX/bench/ep_harness.py
@@ -1493,12 +1493,6 @@ def component(percentiles, count, *, derived=False):
"rank": rank,
},
)
- precision_rank_evidence = [None] * world_size
- dist.all_gather_object(
- precision_rank_evidence,
- {"pre": g["precision_pre"], "post": g["precision_post"]},
- )
- precision_evidence = aggregate_precision_evidence(precision_rank_evidence)
# Canonical LOGICAL payload byte contracts (from the routing trace, NOT backend recv
# tensors): token-rank = one copy per unique (token,dest-rank); token-expert = one copy
# per routed (token,expert). routed_copies = token-rank copies; gt*topk = token-expert.
@@ -1564,7 +1558,6 @@ def component(percentiles, count, *, derived=False):
"contract": case_profile["oracle_contract"],
"max_relative_error": max_rel,
"passed": point_ok,
- "precision": precision_evidence,
"rank_evidence": rank_evidence,
"scope": case_profile["correctness_scope"],
},
@@ -1635,12 +1628,10 @@ def component(percentiles, count, *, derived=False):
# Adapters never self-label official; status is derived from these gates.
prov = backend.backend_provenance
- allocation_stratum_sha256 = getattr(args, "allocation_stratum_sha256", None)
provenance_complete = ep_provenance.provenance_complete(
prov,
backend.name,
getattr(args, "git_run", None),
- allocation_stratum_sha256=allocation_stratum_sha256,
image_digest=getattr(args, "image_digest", None),
image_verified=getattr(args, "image_digest_verified", False),
squash_sha256=getattr(args, "squash_sha256", None),
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 612ae62403..dd9daf7bdd 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -73,9 +73,6 @@ cx_fail_stage() {
diagnostic="missing-runtime"
elif grep -aEqi 'too many requests|rate.?limit' "$log_path"; then
diagnostic="registry-rate-limit"
- elif [ -n "$probe_stage" ] \
- && grep -aEqi 'timed out|operation timeout|wait timeout after|watchdog.*timeout|timeout: sending signal' "$log_path"; then
- diagnostic="${probe_stage}-timeout"
elif grep -aEqi 'ncclRemoteError|remote process exited|connection closed by peer' "$log_path"; then
diagnostic="collective-remote"
elif grep -aEqi 'ncclSystemError|unhandled system error' "$log_path"; then
@@ -153,8 +150,6 @@ cx_fail_stage() {
diagnostic="python-subprocess"
elif grep -aEqi 'Traceback \(most recent call last\)' "$log_path"; then
diagnostic="python-exception"
- elif [ -n "$probe_stage" ]; then
- diagnostic="${probe_stage}-failed"
elif grep -aEqi 'SHARD done: [0-9]+/[0-9]+ case\(s\) failed|WARN: .* run failed rc=|completed with invalid semantic evidence' "$log_path"; then
diagnostic="benchmark-case-failure"
elif [ -s "$log_path" ]; then
From b2ba96e764c23a7dce0c5d528240f25b386a690c Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Wed, 8 Jul 2026 15:00:15 +0800
Subject: [PATCH 200/210] ci(collectivex): provision host jsonschema for sweep
contract validation
contracts.py now depends on jsonschema (P2 moved document validation onto it).
The setup job and self-hosted sweep runners ship a bare python3 with no system
pip, so validate-delivery / emit-terminal / summarize invocations of bare
python3 could not resolve jsonschema. Add jsonschema to the matrix-deps install
and build a host venv (venv+ensurepip, available where system pip is not) on
PATH for every sweep step.
---
.github/workflows/collectivex-sweep.yml | 22 +++++++++++++++++++++-
1 file changed, 21 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml
index 18e83e5b9a..daec48c6a7 100644
--- a/.github/workflows/collectivex-sweep.yml
+++ b/.github/workflows/collectivex-sweep.yml
@@ -49,7 +49,10 @@ jobs:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v5.0.0
with: { clean: true, persist-credentials: false }
- name: Install matrix dependencies
- run: python3 -m pip install --quiet PyYAML==6.0.2
+ # jsonschema is required by contracts.py (make_terminal_document via
+ # --emit-unsupported-from, and validate-delivery) once it is exercised;
+ # PyYAML is used by sweep_matrix.py to read the suites config.
+ run: python3 -m pip install --quiet PyYAML==6.0.2 jsonschema==4.25.1
- id: gen
working-directory: experimental/CollectiveX
env:
@@ -188,6 +191,23 @@ jobs:
CX_SOURCE_ROOT: ${{ matrix.launcher == 'mi-amds' && format('/tmp/inferencex-collectivex-parent-{0}-{1}-{2}/inferencex-collectivex-{0}-{1}-{2}/source', github.run_id, github.run_attempt, matrix.id) || format('/tmp/inferencex-collectivex-{0}-{1}-{2}/source', github.run_id, github.run_attempt, matrix.id) }}
HOME: ${{ matrix.launcher == 'mi-amds' && format('/tmp/inferencex-collectivex-parent-{0}-{1}-{2}/inferencex-collectivex-{0}-{1}-{2}/home', github.run_id, github.run_attempt, matrix.id) || format('/tmp/inferencex-collectivex-{0}-{1}-{2}/home', github.run_id, github.run_attempt, matrix.id) }}
steps:
+ - name: Provision host Python for contract validation
+ # P2 moved contracts.py onto jsonschema (host-only publisher schemas that,
+ # per requirements.txt, are never imported by GPU execution). The self-hosted
+ # runners ship a bare python3 with no system pip, so build a venv and put it on
+ # PATH for every later step: the launcher fail-safe (emit-terminal),
+ # validate-delivery, and summarize.py all invoke bare `python3` and must resolve
+ # jsonschema. venv + ensurepip are available even where system pip is not.
+ env:
+ HOME: ${{ runner.temp }}
+ run: |
+ set -euo pipefail
+ venv="$RUNNER_TEMP/cx-hostpy"
+ python3 -m venv --clear "$venv"
+ "$venv/bin/python3" -m pip install --quiet --no-cache-dir --upgrade pip
+ "$venv/bin/python3" -m pip install --quiet --no-cache-dir \
+ jsonschema==4.25.1 PyYAML==6.0.2 "numpy>=1.26,<3"
+ echo "$venv/bin" >> "$GITHUB_PATH"
- name: Prepare isolated source
id: source
env:
From 4511b2bec5a44a3e4960d3a2596816ab3a7566df Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Wed, 8 Jul 2026 18:07:33 +0800
Subject: [PATCH 201/210] =?UTF-8?q?refactor(collectivex):=20neutral=20sche?=
=?UTF-8?q?dule=E2=86=92execute=E2=86=92validate=E2=86=92upload=20backend?=
=?UTF-8?q?=20(MVP=20cleanup)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Backend now only schedules, executes, validates, and uploads neutral artifacts;
it never promotes, ranks, recommends, selects, hides, or decides frontend display.
- B1: iterable benchmark `version` in one config (configs/suites.yaml), threaded
through matrix, shard control, raw+terminal docs, and the summary; mismatches
and missing/non-int versions are rejected across Python + JSON Schema + shell.
- B2: drop frontend-facing `v1` suffixes from suite/workload names (ep-core,
ep-low-latency, deepseek-v3); keep schema_version, format strings, internal names.
- B3: remove precision profiles (BF16-only); delete ep_precision.py.
- B4: readable case IDs with minimal hashing in identity.py.
- B5/B11: trim contracts.py to its neutral public surface, de-duplicate the
allocation-factors projection and the double delivery-validation pass; privacy
boundary and all cross-file checks preserved.
- B6: delete frontend projection/ranking from sweep_matrix.py.
- B7: remove max_nodes coupling; clean capability.py.
- B9: consolidate tests into test_contracts.py + test_matrix.py; mechanical summarize.py.
- B8: confirm common.sh maps cleanly to its six responsibilities (no dead code).
CPU battery: 13 passed / 6 subtests. All modules compile; runtime shell bash -n clean.
---
experimental/CollectiveX/README.md | 15 +-
experimental/CollectiveX/bench/ep_backend.py | 47 +-
experimental/CollectiveX/bench/ep_deepep.py | 23 +-
.../CollectiveX/bench/ep_deepep_family.py | 103 +---
.../CollectiveX/bench/ep_deepep_hybrid.py | 83 +--
.../CollectiveX/bench/ep_deepep_v2.py | 91 +--
experimental/CollectiveX/bench/ep_harness.py | 250 +-------
experimental/CollectiveX/bench/ep_mori.py | 159 +----
experimental/CollectiveX/bench/ep_nccl.py | 14 -
.../CollectiveX/bench/ep_precision.py | 543 ------------------
.../CollectiveX/bench/ep_provenance.py | 111 +++-
experimental/CollectiveX/bench/ep_uccl.py | 30 +-
.../CollectiveX/bench/make_workloads.py | 6 +-
experimental/CollectiveX/bench/run_ep.py | 2 +-
experimental/CollectiveX/bench/workload.py | 2 +-
experimental/CollectiveX/capability.py | 8 +-
experimental/CollectiveX/configs/suites.yaml | 15 +-
.../CollectiveX/configs/workloads.yaml | 4 +-
experimental/CollectiveX/contracts.py | 265 +++------
experimental/CollectiveX/docs/methodology.md | 61 +-
experimental/CollectiveX/identity.py | 141 +++--
experimental/CollectiveX/runtime/common.sh | 49 +-
.../CollectiveX/runtime/run_in_container.sh | 10 +-
.../schemas/raw-case-v1.schema.json | 49 +-
.../schemas/samples-v1.schema.json | 18 +-
.../schemas/terminal-outcome-v1.schema.json | 17 +-
experimental/CollectiveX/summarize.py | 10 +-
experimental/CollectiveX/sweep_matrix.py | 87 +--
.../CollectiveX/tests/test_contracts.py | 74 +++
.../CollectiveX/tests/test_ep_backend.py | 307 ++--------
.../tests/test_ep_precision_adapters.py | 80 ---
experimental/CollectiveX/tests/test_matrix.py | 99 ++++
.../tests/test_qualification_planning.py | 202 -------
.../tests/test_raw_document_roundtrip.py | 494 ----------------
.../tests/test_schema_v1_contract.py | 146 -----
35 files changed, 730 insertions(+), 2885 deletions(-)
delete mode 100644 experimental/CollectiveX/bench/ep_precision.py
create mode 100644 experimental/CollectiveX/tests/test_contracts.py
delete mode 100644 experimental/CollectiveX/tests/test_ep_precision_adapters.py
create mode 100644 experimental/CollectiveX/tests/test_matrix.py
delete mode 100644 experimental/CollectiveX/tests/test_qualification_planning.py
delete mode 100644 experimental/CollectiveX/tests/test_raw_document_roundtrip.py
delete mode 100644 experimental/CollectiveX/tests/test_schema_v1_contract.py
diff --git a/experimental/CollectiveX/README.md b/experimental/CollectiveX/README.md
index 2d356f763b..ef8d508909 100644
--- a/experimental/CollectiveX/README.md
+++ b/experimental/CollectiveX/README.md
@@ -12,10 +12,8 @@ measurement and validation contract is in [docs/methodology.md](docs/methodology
## Execution Profile
The workload uses packed placement and one pinned `fixed-profile` resource configuration per
-backend/topology; there is no tuning sweep. The default sweep runs the portable BF16 dispatch / BF16
-combine control. The native FP8 codec paths stay in the code (`bench/ep_precision.py` and each
-adapter's FP8 dispatch/combine) and remain callable but are not a swept dimension. The explicit mode
-selects one of two contracts:
+backend/topology; there is no tuning sweep. Dispatch and combine are fixed BF16 on every backend;
+precision is not a swept dimension. The explicit mode selects one of two contracts:
- Normal mode uses `layout-and-dispatch-v1`, rank-deduplicated token payloads, and activation-only
combine. Uniform core coverage and one Zipf sensitivity remain; EPLB is measured only as the Zipf
@@ -29,11 +27,8 @@ warmups before each measured component at every trial/point. Roundtrip is measur
iteration takes the cross-rank maximum before nearest-rank p50/p90/p95/p99, and roundtrip p99 is the
headline latency. A stdlib integer counter produces byte-identical routing and gate weights.
-Correctness is checked against the semantic value after the declared communication codec. The combine
-gates are `rtol=0.05, atol=0.02` for BF16 (the default sweep), `rtol=0.06, atol=0.03` for native
-logfmt10, and `rtol=0.08, atol=0.04` for native FP8 direct-cast combine when those codec paths are
-exercised. FP8 direct-cast evidence also counts saturation on the exact transformed native combine
-input; any saturated value fails the point. Any failed rank or point makes the case ineligible and is
+Correctness is checked against the reference activation. The combine gate is `rtol=0.05, atol=0.02`
+for the BF16 communication path. Any failed rank or point makes the case ineligible and is
recorded as a terminal outcome.
The matrix covers H100, H200, B200, B300, GB200, GB300, and MI355X. `sweep_matrix.py` materializes
@@ -79,7 +74,7 @@ whether it succeeded is decided only by the emitted artifact.
`.github/workflows/collectivex-sweep.yml` has two jobs. `setup` generates a public-SKU matrix
(`backend`, `suites`, `only_sku`, `exclude_skus`, `ep_sizes` inputs), fetches the pinned backend
-source archive, emits a neutral `collectivex.frontend-catalog.v1` projection, and uploads the matrix.
+source archive, and uploads the matrix.
`sweep` extracts a strict ignored `.shards/.json` control per matrix entry, executes one
allocation per shard, runs `contracts.py validate-delivery`, and uploads the result artifacts with
`always()` so a red or partial run still uploads.
diff --git a/experimental/CollectiveX/bench/ep_backend.py b/experimental/CollectiveX/bench/ep_backend.py
index 6dd879c8e8..3dd9cf7246 100644
--- a/experimental/CollectiveX/bench/ep_backend.py
+++ b/experimental/CollectiveX/bench/ep_backend.py
@@ -39,7 +39,6 @@
import types
from dataclasses import dataclass, field
-import ep_precision
from ep_harness import (
CONDITIONING_LADDERS,
CONDITIONING_ROUNDS_PER_SHAPE,
@@ -108,10 +107,8 @@ class EPBackend(abc.ABC):
Subclasses implement the transport (``create_buffer``, ``dispatch``, ``stage``,
``combine``, ``recv_tokens``, ``inspect_dispatch``, ``combine_transformed``);
- everything the driver and the oracles need beyond that is provided here. Each
- adapter resolves its own precision profile in ``__init__`` (passing the profile
- set inline to ``ep_precision.resolve_precision``), so there is no abstract
- ``supported_profiles`` hook.
+ everything the driver and the oracles need beyond that is provided here.
+ Dispatch and combine are fixed BF16, so no adapter selects a precision codec.
"""
# ---- Contract flags (class defaults; subclasses / low-latency override) ----
@@ -126,9 +123,6 @@ class EPBackend(abc.ABC):
oracle_layout = "token-rank"
payload_unit = "token-rank"
roundtrip_only = False
- # Precision-evidence knob: adapters that hand scales back as uint8-viewed
- # storage (deepep-hybrid) flip this so the evidence pass reads them correctly.
- _uint8_scale_storage = False
def __init_subclass__(cls, **kwargs):
super().__init_subclass__(**kwargs)
@@ -290,21 +284,16 @@ def _build_rank_inputs(self, args, tokens_per_rank, *, canonical, retain_global)
)
def make_problem(self, T, idx, weights, x):
- """Encode the dispatch payload and assemble the per-shape problem namespace."""
+ """Assemble the per-shape problem namespace (BF16 dispatch sends ``x`` directly)."""
import torch
- encoding = ep_precision.encode_dispatch(torch, x, self.communication_precision)
- problem = types.SimpleNamespace(
+ return types.SimpleNamespace(
T=T,
x=x,
- dispatch_x=encoding.native_input,
- oracle_x=encoding.semantic,
- dispatch_precision_evidence=encoding.evidence,
+ dispatch_x=x,
topk_idx=idx.to(self._topk_idx_dtype()),
topk_weights=weights.to(torch.float32),
)
- self._extend_problem(problem, encoding)
- return problem
def _topk_idx_dtype(self):
"""Integer dtype the backend's kernels expect for top-k routing indices."""
@@ -312,9 +301,6 @@ def _topk_idx_dtype(self):
return torch.int64
- def _extend_problem(self, problem, encoding):
- """Hook: subclasses attach backend-specific problem fields (scales, layout, ...)."""
-
# ---- Timing template methods -----------------------------------------------------
def timed_components(self):
@@ -412,33 +398,12 @@ def prep_combine(p=problem):
torch.cuda.synchronize()
return time_us(torch, lambda p=problem, hx=hh: self.combine(p, hx), 0, iters)
- # ---- Correctness / precision hooks (shared) --------------------------------------
+ # ---- Correctness hooks (shared) --------------------------------------------------
def inspect_expert_dispatch(self, problem, handle):
"""Expert-packed post-dispatch view; only low-latency backends provide one."""
raise RuntimeError("expert-packed inspection requires low-latency mode")
- def oracle_dispatch_payload(self, payload):
- """Project an oracle-built payload through the dispatch encoding's semantics."""
- import torch
-
- return ep_precision.encode_dispatch(
- torch, payload, self.communication_precision
- ).semantic
-
- def precision_evidence(self, problem, view=None):
- """Per-direction precision evidence for the raw document."""
- import torch
-
- return ep_precision.precision_evidence(
- torch,
- profile_id=self.precision_profile_id,
- profile=self.communication_precision,
- problem=problem,
- view=view,
- uint8_storage=self._uint8_scale_storage,
- )
-
def capture_deferred_provenance(self):
"""Resolve provenance materialized only after conditioning (e.g. JIT CUBINs).
diff --git a/experimental/CollectiveX/bench/ep_deepep.py b/experimental/CollectiveX/bench/ep_deepep.py
index 49dcf94692..6346a421e1 100644
--- a/experimental/CollectiveX/bench/ep_deepep.py
+++ b/experimental/CollectiveX/bench/ep_deepep.py
@@ -1,5 +1,5 @@
#!/usr/bin/env python3
-"""CollectiveX DeepEP adapter for native V1 dispatch/combine precision profiles."""
+"""CollectiveX DeepEP adapter: native BF16 dispatch/combine over deep_ep."""
from __future__ import annotations
import inspect
@@ -10,7 +10,6 @@
import torch.distributed as dist
import contracts
import ep_provenance
-import ep_precision
from ep_deepep_family import DeepEPFamilyBackend
try:
@@ -70,8 +69,8 @@ def _normal_buffer_sizes(hidden: int, world_size: int) -> tuple[int, int]:
class DeepEPBackend(DeepEPFamilyBackend):
- # Mode handling, dispatch/combine, and fp8 dequantization are shared with UCCL in
- # DeepEPFamilyBackend; only the native deep_ep buffer construction and teardown live here.
+ # Mode handling and dispatch/combine are shared with UCCL in DeepEPFamilyBackend;
+ # only the native deep_ep buffer construction and teardown live here.
name = "deepep"
def create_buffer(self, spec):
@@ -81,12 +80,12 @@ def create_buffer(self, spec):
mnnvl_kwargs, mnnvl_comm = _mnnvl_buffer_configuration()
if self.mode == "low-latency":
assert spec.max_tokens_per_rank <= 128
- ep_precision.require_keyword(
+ ep_provenance.require_keyword(
Buffer.low_latency_dispatch,
"use_fp8",
api="deep_ep.Buffer.low_latency_dispatch",
)
- ep_precision.require_keyword(
+ ep_provenance.require_keyword(
Buffer.low_latency_combine,
"use_logfmt",
api="deep_ep.Buffer.low_latency_combine",
@@ -119,12 +118,12 @@ def create_buffer(self, spec):
"num_qps_per_rank": num_qps_per_rank,
}
else:
- ep_precision.require_keyword(
+ ep_provenance.require_keyword(
Buffer.dispatch,
"async_finish",
api="deep_ep.Buffer.dispatch",
)
- ep_precision.require_keyword(
+ ep_provenance.require_keyword(
Buffer.combine,
"async_finish",
api="deep_ep.Buffer.combine",
@@ -161,12 +160,8 @@ def create_buffer(self, spec):
"deepep_commit": os.environ.get("DEEPEP_COMMIT") or f"pkg-{version}",
"backend_lineage": "deepep-v1",
"mode": self.mode,
- "dispatch_dtype": ep_precision.communication_format(
- self.communication_precision, "dispatch"
- ),
- "combine_dtype": ep_precision.communication_format(
- self.communication_precision, "combine"
- ),
+ "dispatch_dtype": "bf16",
+ "combine_dtype": "bf16",
"resource_mode": "fixed-profile",
"device_sms": device_sms,
"allow_mnnvl": bool(mnnvl_kwargs),
diff --git a/experimental/CollectiveX/bench/ep_deepep_family.py b/experimental/CollectiveX/bench/ep_deepep_family.py
index 1e069d8b3b..9cb0e95ea3 100644
--- a/experimental/CollectiveX/bench/ep_deepep_family.py
+++ b/experimental/CollectiveX/bench/ep_deepep_family.py
@@ -2,15 +2,18 @@
"""Shared DeepEP-API dispatch/combine surface for the DeepEP and UCCL adapters.
UCCL's ``uccl_deepep.Buffer`` is a drop-in clone of DeepEP's ``deep_ep.Buffer`` low-latency and
-normal API, so both adapters run byte-identical mode handling, dispatch/combine, expert-packed
-inspection, and fp8 dequantization. That shared operation lives here; each concrete backend keeps
-only what is genuinely vendor-specific: its native buffer import, ``create_buffer`` provenance, and
-process teardown.
+normal API, so both adapters run byte-identical mode handling, dispatch/combine, and expert-packed
+inspection. That shared operation lives here; each concrete backend keeps only what is genuinely
+vendor-specific: its native buffer import, ``create_buffer`` provenance, and process teardown.
This base is deliberately free of any ``deep_ep``/``uccl`` import. The UCCL benchmark image installs
uccl WITHOUT deep_ep, so importing ``ep_uccl`` must never transitively require ``deep_ep`` — an
inherited method resolves module globals from where it is *defined*, so keeping this file
vendor-agnostic is what makes the shared base safe for both images.
+
+Communication is fixed BF16: dispatch and combine move BF16 activations, so the native
+``use_fp8``/``use_logfmt`` controls are always driven off and the received buffer is the
+semantic payload directly.
"""
from __future__ import annotations
@@ -18,7 +21,6 @@
import torch
import torch.distributed as dist
-import ep_precision
from ep_backend import EPBackend
@@ -41,31 +43,6 @@ class DeepEPFamilyBackend(EPBackend):
def __init__(self, args, rank, world_size, local_rank, device):
# Base validates mode against SUPPORTED_MODES (normal / low-latency).
super().__init__(args, rank, world_size, local_rank, device)
- supported_profiles = {
- "normal": {
- "d-bf16.c-bf16",
- "d-fp8-e4m3fn-b128-f32-prequantized.c-bf16",
- },
- "low-latency": {
- "d-bf16.c-bf16",
- "d-fp8-e4m3fn-b128-f32-fused.c-bf16",
- },
- }
- self.precision_profile_id, self.communication_precision = (
- ep_precision.resolve_precision(
- args,
- backend=self.name,
- mode=self.mode,
- supported_profiles=supported_profiles[self.mode],
- )
- )
- self._fp8_dispatch = ep_precision.is_low_precision_dispatch(
- self.communication_precision
- )
- self._use_logfmt = ep_precision.uses_logfmt_combine(
- self.communication_precision
- )
- self.stage_device_work = self._fp8_dispatch
self.group = dist.group.WORLD
# Low-latency flips the contract flags and fixes the per-rank cap so
# buffer_cap can report it to make_inputs before create_buffer runs.
@@ -94,7 +71,7 @@ def dispatch(self, p):
p.topk_idx,
self.max_tokens_per_rank,
self.args.experts,
- use_fp8=self._fp8_dispatch, # BF16 control realizes use_fp8=False.
+ use_fp8=False, # BF16 communication path.
async_finish=False,
return_recv_hook=False,
)
@@ -129,7 +106,8 @@ def dispatch(self, p):
)
def stage(self, p, h):
- h.combine_input = self._semantic_recv(h, p)
+ # BF16: the received buffer is already the semantic payload to combine.
+ h.combine_input = h.recv_x
def combine(self, p, h):
if self.mode == "low-latency":
@@ -138,7 +116,7 @@ def combine(self, p, h):
p.topk_idx,
p.topk_weights,
h.handle,
- use_logfmt=self._use_logfmt,
+ use_logfmt=False, # BF16 communication path.
async_finish=False,
return_recv_hook=False,
)
@@ -156,9 +134,7 @@ def inspect_dispatch(self, p, h):
h.recv_topk_idx,
)
return types.SimpleNamespace(
- payload=self._semantic_recv(h, p),
- encoded_payload=self._encoded_recv(h),
- scales=self._recv_scales(h),
+ payload=h.recv_x,
expert_ids=expert_ids,
weights=h.recv_topk_weights.masked_fill(~valid, 0),
local_expert_counts=torch.tensor(h.recv_counts, device=self.device, dtype=torch.int64),
@@ -170,9 +146,7 @@ def inspect_expert_dispatch(self, p, h):
raise RuntimeError("expert-packed inspection requires low-latency mode")
p.recv_counts = tuple(int(value) for value in h.recv_counts.tolist())
return types.SimpleNamespace(
- payload=self._semantic_recv(h, p),
- encoded_payload=self._encoded_recv(h),
- scales=self._recv_scales(h),
+ payload=h.recv_x,
local_expert_counts=h.recv_counts,
source_info=h.handle[0],
layout_range=h.handle[1],
@@ -181,9 +155,9 @@ def inspect_expert_dispatch(self, p, h):
def combine_transformed(self, p, h, transformed):
if self.mode == "low-latency":
packed = torch.zeros(
- self._encoded_recv(h).shape,
+ h.recv_x.shape,
dtype=torch.bfloat16,
- device=self._encoded_recv(h).device,
+ device=h.recv_x.device,
)
packed[h.oracle_local_expert_slots, h.oracle_packed_positions] = transformed.to(
packed.dtype
@@ -193,58 +167,17 @@ def combine_transformed(self, p, h, transformed):
p.topk_idx,
p.topk_weights,
h.handle,
- use_logfmt=self._use_logfmt,
+ use_logfmt=False, # BF16 communication path.
async_finish=False,
return_recv_hook=False,
)
return combined
- semantic = self._semantic_recv(h, p)
combined, _, _ = self.buffer.combine(
- transformed.to(semantic.dtype), h.handle, async_finish=False
+ transformed.to(h.recv_x.dtype), h.handle, async_finish=False
)
return combined
def recv_tokens(self, h):
if self.mode == "low-latency":
return int(h.recv_counts.to(torch.int64).sum().item())
- return int(self._encoded_recv(h).shape[0])
-
- def _encoded_recv(self, h):
- return h.recv_x[0] if isinstance(h.recv_x, tuple) else h.recv_x
-
- def _recv_scales(self, h):
- return h.recv_x[1] if isinstance(h.recv_x, tuple) else None
-
- def _semantic_recv(self, h, problem=None):
- if not self._fp8_dispatch:
- return h.recv_x
- if not hasattr(h, "recv_semantic"):
- if self.mode == "low-latency":
- counts = getattr(problem, "recv_counts", None)
- if counts is None:
- counts = tuple(int(value) for value in h.recv_counts.tolist())
- if problem is not None:
- problem.recv_counts = counts
- workspace = getattr(self, "_ll_semantic_workspace", None)
- if workspace is None:
- encoded = self._encoded_recv(h)
- workspace = torch.empty(
- encoded.shape, dtype=torch.bfloat16, device=encoded.device
- )
- self._ll_semantic_workspace = workspace
- h.recv_semantic = ep_precision.dequantize_expert_prefixes(
- torch,
- self._encoded_recv(h),
- self._recv_scales(h),
- self.communication_precision["dispatch"],
- counts,
- workspace,
- )
- else:
- h.recv_semantic = ep_precision.dequantize_dispatch(
- torch,
- self._encoded_recv(h),
- self._recv_scales(h),
- self.communication_precision["dispatch"],
- )
- return h.recv_semantic
+ return int(h.recv_x.shape[0])
diff --git a/experimental/CollectiveX/bench/ep_deepep_hybrid.py b/experimental/CollectiveX/bench/ep_deepep_hybrid.py
index 90d251bc50..3e7f3f0b1c 100644
--- a/experimental/CollectiveX/bench/ep_deepep_hybrid.py
+++ b/experimental/CollectiveX/bench/ep_deepep_hybrid.py
@@ -40,7 +40,6 @@
import torch.distributed as dist
import contracts
import ep_provenance
-import ep_precision
from ep_backend import EPBackend
try:
@@ -199,34 +198,16 @@ class DeepEPHybridBackend(EPBackend):
# before a timed combine); the harness times dispatch and combine separately (like ep_deepep).
combine_needs_redispatch = False
combine_weight_semantics = "unweighted-rank-sum"
- # Hybrid hands scales back as uint8-viewed storage; the precision-evidence pass
- # must read them that way.
- _uint8_scale_storage = True
def __init__(self, args, rank, world_size, local_rank, device):
# deepep-hybrid is normal-mode only; base SUPPORTED_MODES=("normal",) enforces it.
super().__init__(args, rank, world_size, local_rank, device)
- self.precision_profile_id, self.communication_precision = (
- ep_precision.resolve_precision(
- args,
- backend=self.name,
- mode=self.mode,
- supported_profiles={
- "d-bf16.c-bf16",
- "d-fp8-e4m3fn-b128-f32-prequantized.c-bf16",
- },
- )
- )
- self._fp8_dispatch = ep_precision.is_low_precision_dispatch(
- self.communication_precision
- )
- self.stage_device_work = self._fp8_dispatch
- ep_precision.require_keyword(
+ ep_provenance.require_keyword(
HybridEPBuffer.__init__,
"use_fp8",
api="deep_ep.HybridEPBuffer.__init__",
)
- ep_precision.require_keyword(
+ ep_provenance.require_keyword(
HybridEPBuffer.dispatch,
"scaling_factor",
api="deep_ep.HybridEPBuffer.dispatch",
@@ -293,7 +274,7 @@ def create_buffer(self, spec):
self.group, hidden_dim=self.hidden,
max_num_of_tokens_per_rank=self.max_tokens,
num_local_experts=self.local_experts,
- use_fp8=self._fp8_dispatch,
+ use_fp8=False, # BF16 communication path.
)
realized_geometry = (
int(self.buffer.num_of_hybrid_ep_ranks_per_nvlink_domain),
@@ -340,10 +321,10 @@ def tracked_update_template_config(*call_args, **call_kwargs):
or realized["num_of_nodes"] != self.communication_domains
):
raise RuntimeError("DeepEP Hybrid realized topology changed within one case")
- expected_token_type = "UINT8" if self._fp8_dispatch else "UINT16"
- if realized["token_data_type"] != expected_token_type:
+ # BF16 dispatch realizes the 2-byte UINT16 token wire type.
+ if realized["token_data_type"] != "UINT16":
raise RuntimeError(
- "DeepEP Hybrid realized token dtype differs from the precision profile"
+ "DeepEP Hybrid realized token dtype is not the BF16 UINT16 wire type"
)
if self._realized_config is not None and realized != self._realized_config:
raise RuntimeError("DeepEP Hybrid realized autotune config changed within one case")
@@ -367,12 +348,8 @@ def tracked_update_template_config(*call_args, **call_kwargs):
"loaded_libraries": loaded_libraries,
"impl": "deep_ep.HybridEPBuffer (NVIDIA TMA + warp-pipeline)",
"mode": "normal", "transport": topology["transport"],
- "dispatch_dtype": ep_precision.communication_format(
- self.communication_precision, "dispatch"
- ),
- "combine_dtype": ep_precision.communication_format(
- self.communication_precision, "combine"
- ),
+ "dispatch_dtype": "bf16",
+ "combine_dtype": "bf16",
"resource_mode": "fixed-profile",
"num_sms": None, "device_sms": dev_sms,
"tuned_source": "deepep-hybrid-configurer-autotune-v1",
@@ -386,21 +363,12 @@ def buffer_cap(self, args):
return self.max_tokens
def make_problem(self, T, idx, weights, x):
- encoding = ep_precision.encode_dispatch(
- torch, x, self.communication_precision
- )
- dispatch_x = (
- encoding.encoded_payload.view(torch.uint8)
- if self._fp8_dispatch
- else encoding.native_input
- )
+ # BF16 dispatch sends x directly; scaling_factor stays None (no separate scale payload).
return types.SimpleNamespace(
T=int(T),
x=x,
- dispatch_x=dispatch_x,
- dispatch_scales=encoding.scales,
- oracle_x=encoding.semantic,
- dispatch_precision_evidence=encoding.evidence,
+ dispatch_x=x,
+ dispatch_scales=None,
topk_idx=idx.to(torch.int64),
topk_weights=weights.to(torch.float32),
)
@@ -416,7 +384,6 @@ def dispatch(self, p):
return types.SimpleNamespace(
recv=recv,
recv_payload=recv,
- recv_scales=_scales,
recv_probs=recv_probs,
handle=handle,
combine_input=None,
@@ -424,7 +391,7 @@ def dispatch(self, p):
def stage(self, p, h):
# Identity expert: the recv hidden IS the "expert output". combine reduces it per source token.
- h.combine_input = self._semantic_recv(h, p.recv_tokens)
+ h.combine_input = h.recv_payload
return None
def combine(self, p, h):
@@ -487,9 +454,7 @@ def inspect_dispatch(self, p, h):
expert_ids[rows, positions] = local_expert_ids + self.rank * self.local_experts
weights[rows, positions] = h.recv_probs[:count][rows, probability_columns]
return types.SimpleNamespace(
- payload=self._semantic_recv(h, count)[:count],
- encoded_payload=h.recv_payload[:count],
- scales=(h.recv_scales[:count] if h.recv_scales is not None else None),
+ payload=h.recv_payload[:count],
expert_ids=expert_ids,
weights=weights,
local_expert_counts=routing_map.sum(dim=0, dtype=torch.int64),
@@ -505,28 +470,6 @@ def combine_transformed(self, p, h, transformed):
def recv_tokens(self, h):
return int(h.handle[3].item())
- def _semantic_recv(self, h, rows):
- if not self._fp8_dispatch:
- return h.recv_payload
- if not hasattr(h, "recv_semantic"):
- semantic = torch.empty(
- h.recv_payload.shape,
- dtype=torch.bfloat16,
- device=h.recv_payload.device,
- )
- semantic[:rows].copy_(ep_precision.dequantize_dispatch(
- torch,
- h.recv_payload[:rows],
- h.recv_scales[:rows],
- self.communication_precision["dispatch"],
- uint8_storage=True,
- ))
- h.recv_semantic = semantic
- h.recv_semantic_rows = rows
- elif h.recv_semantic_rows != rows:
- raise RuntimeError("DeepEP Hybrid receive count changed for one dispatch handle")
- return h.recv_semantic
-
def finalize(self, rc):
try:
dist.barrier()
diff --git a/experimental/CollectiveX/bench/ep_deepep_v2.py b/experimental/CollectiveX/bench/ep_deepep_v2.py
index 6d366370bd..f9ce05e799 100644
--- a/experimental/CollectiveX/bench/ep_deepep_v2.py
+++ b/experimental/CollectiveX/bench/ep_deepep_v2.py
@@ -16,9 +16,8 @@
import torch
import torch.distributed as dist
-import contracts
import ep_harness
-import ep_precision
+import ep_provenance
from ep_backend import EPBackend
try:
@@ -42,7 +41,7 @@
TORCH_VERSION = "2.10.0+cu130"
NCCL_VERSION = "2.30.4"
NVSHMEM_VERSION = "3.3.9"
-DEEPEP_V2_JIT_KERNELS = contracts.DEEPEP_V2_JIT_KERNELS
+DEEPEP_V2_JIT_KERNELS = ep_provenance.DEEPEP_V2_JIT_KERNELS
def _sha256(path: str) -> str:
@@ -164,18 +163,8 @@ def _jit_cache_key(
max_tokens: int,
allow_hybrid_mode: bool,
realized: dict[str, int | bool],
- precision_profile_id: str = "d-bf16.c-bf16",
- communication_precision: dict[str, object] | None = None,
) -> str:
"""Key generated kernels by codegen inputs, not routing data or case identity."""
- if communication_precision is None:
- communication_precision = {
- "dispatch": {
- "communication_format": "bf16",
- "api_input_dtype": "bf16",
- },
- "combine": {"communication_format": "bf16"},
- }
payload = {
"contract": "deepep-v2-jit-config-v3",
"runner": args.runner,
@@ -185,10 +174,9 @@ def _jit_cache_key(
"physical_experts": args.experts,
"tuning_experts": getattr(args, "num_logical_experts", args.experts),
"max_tokens": max_tokens,
- "precision_profile": precision_profile_id,
- "dispatch_dtype": communication_precision["dispatch"]["communication_format"],
- "combine_dtype": communication_precision["combine"]["communication_format"],
- "input_layout": communication_precision["dispatch"]["api_input_dtype"],
+ "dispatch_dtype": "bf16",
+ "combine_dtype": "bf16",
+ "input_layout": "bf16",
"expert_alignment": 1,
"do_cpu_sync": True,
"cached_mode": False,
@@ -338,22 +326,7 @@ class DeepEPV2Backend(EPBackend):
def __init__(self, args, rank, world_size, local_rank, device):
# deepep-v2 is normal-mode only; base SUPPORTED_MODES=("normal",) enforces it.
super().__init__(args, rank, world_size, local_rank, device)
- self.precision_profile_id, self.communication_precision = (
- ep_precision.resolve_precision(
- args,
- backend=self.name,
- mode=self.mode,
- supported_profiles={
- "d-bf16.c-bf16",
- "d-fp8-e4m3fn-b128-f32-prequantized.c-bf16",
- },
- )
- )
- self._fp8_dispatch = ep_precision.is_low_precision_dispatch(
- self.communication_precision
- )
- self.stage_device_work = self._fp8_dispatch
- ep_precision.require_keyword(
+ ep_provenance.require_keyword(
ElasticBuffer.__init__,
"use_fp8_dispatch",
api="deep_ep.ElasticBuffer.__init__",
@@ -382,7 +355,7 @@ def create_buffer(self, spec):
num_max_tokens_per_rank=self.max_tokens,
hidden=args.hidden,
num_topk=args.topk,
- use_fp8_dispatch=self._fp8_dispatch,
+ use_fp8_dispatch=False, # BF16 communication path.
deterministic=False,
allow_hybrid_mode=allow_hybrid_mode,
allow_multiple_reduction=True,
@@ -428,8 +401,6 @@ def create_buffer(self, spec):
self.max_tokens,
allow_hybrid_mode,
jit_config,
- self.precision_profile_id,
- self.communication_precision,
)
os.environ["EP_JIT_CACHE_DIR"] = str(jit_root / self.jit_cache_key)
realized_config = {
@@ -472,12 +443,8 @@ def create_buffer(self, spec):
"jit_random_seed": DEEPEP_V2_JIT_RANDOM_SEED,
"num_experts": int(args.experts),
"mode": "normal",
- "dispatch_dtype": ep_precision.communication_format(
- self.communication_precision, "dispatch"
- ),
- "combine_dtype": ep_precision.communication_format(
- self.communication_precision, "combine"
- ),
+ "dispatch_dtype": "bf16",
+ "combine_dtype": "bf16",
"deterministic": False,
"resource_mode": "fixed-profile",
"requested_num_sms": self.num_sms,
@@ -527,7 +494,8 @@ def dispatch(self, p):
)
def stage(self, p, h):
- h.combine_input = self._semantic_recv(h, p.recv_tokens)
+ # BF16: the received buffer is already the semantic payload to combine.
+ h.combine_input = h.recv_x
def combine(self, p, h):
combined_x, _, _ = self.buffer.combine(
@@ -565,13 +533,7 @@ def inspect_dispatch(self, p, h):
)
local = local_idx[valid].to(torch.int64)
return types.SimpleNamespace(
- payload=self._semantic_recv(h, count)[:count],
- encoded_payload=self._encoded_recv(h)[:count],
- scales=(
- self._recv_scales(h)[:count]
- if self._recv_scales(h) is not None
- else None
- ),
+ payload=h.recv_x[:count],
expert_ids=expert_ids,
weights=h.recv_topk_weights[:count].masked_fill(~valid, 0),
local_expert_counts=torch.bincount(
@@ -581,8 +543,7 @@ def inspect_dispatch(self, p, h):
)
def combine_transformed(self, p, h, transformed):
- semantic = self._semantic_recv(h, self.recv_tokens(h))
- combine_input = torch.zeros_like(semantic)
+ combine_input = torch.zeros_like(h.recv_x)
combine_input[: transformed.shape[0]].copy_(transformed.to(combine_input.dtype))
combined, _, _ = self.buffer.combine(
combine_input,
@@ -596,32 +557,6 @@ def combine_transformed(self, p, h, transformed):
def recv_tokens(self, h):
return int(h.handle.psum_num_recv_tokens_per_scaleup_rank[-1].item())
- def _encoded_recv(self, h):
- return h.recv_x[0] if isinstance(h.recv_x, tuple) else h.recv_x
-
- def _recv_scales(self, h):
- return h.recv_x[1] if isinstance(h.recv_x, tuple) else None
-
- def _semantic_recv(self, h, rows):
- if not self._fp8_dispatch:
- return h.recv_x
- if not hasattr(h, "recv_semantic"):
- encoded = self._encoded_recv(h)
- semantic = torch.empty(
- encoded.shape, dtype=torch.bfloat16, device=encoded.device
- )
- semantic[:rows].copy_(ep_precision.dequantize_dispatch(
- torch,
- encoded[:rows],
- self._recv_scales(h)[:rows],
- self.communication_precision["dispatch"],
- ))
- h.recv_semantic = semantic
- h.recv_semantic_rows = rows
- elif h.recv_semantic_rows != rows:
- raise RuntimeError("DeepEP V2 receive count changed for one dispatch handle")
- return h.recv_semantic
-
def finalize(self, rc):
try:
dist.barrier()
diff --git a/experimental/CollectiveX/bench/ep_harness.py b/experimental/CollectiveX/bench/ep_harness.py
index 7052b1e87d..4032c806c0 100644
--- a/experimental/CollectiveX/bench/ep_harness.py
+++ b/experimental/CollectiveX/bench/ep_harness.py
@@ -47,7 +47,6 @@
import contracts
import ep_provenance
-import ep_precision
import identity
import workload as workload_contract
@@ -88,21 +87,16 @@
CONDITIONING_ROUNDS_PER_SHAPE = contracts.V1_CONDITIONING_ROUNDS_PER_SHAPE
CONDITIONING_CONTRACT = identity.V1_CASE_PROFILE["conditioning_contract"]
ORACLE_CONTRACT = identity.V1_CASE_PROFILE["oracle_contract"]
+# Dispatch and combine are fixed BF16, so the combine oracle uses one frozen gate.
ORACLE_RTOL = 5e-2
ORACLE_ATOL = 2e-2
-
-def _oracle_tolerances(backend) -> tuple[float, float]:
- tolerance = ep_precision.combine_oracle_tolerances(backend.communication_precision)
- return tolerance["atol"], tolerance["rtol"]
-
EPLB_REDUNDANT_EXPERTS = 32
EPLB_REFERENCE_TOKENS_PER_RANK = 2048
EPLB_PLANNER = "greedy-rank-major-v1"
V1_PROFILE = {
"dispatch_dtype": "bf16",
"combine_dtype": "bf16",
- "combine_quant_mode": "none",
"mode": "normal",
"measurement_contract": "layout-and-dispatch-v1",
"resource_mode": "fixed-profile",
@@ -120,33 +114,20 @@ def _oracle_tolerances(backend) -> tuple[float, float]:
}
-def precision_byte_provenance(
- axis: dict, logical_copies: int, hidden: int
-) -> dict[str, int | str]:
- """Return comparable logical activation and required scale bytes for one direction."""
+def logical_byte_provenance(logical_copies: int, hidden: int) -> dict[str, int | str]:
+ """Return comparable logical BF16 activation bytes for one direction.
+
+ Dispatch and combine both move BF16 (2 bytes/value) with no separate scale
+ payload, so ``scale_bytes`` is always zero.
+ """
if logical_copies < 0 or hidden < 0:
- raise ValueError("logical precision byte dimensions must be non-negative")
- bits_per_value = {
- "bf16": 16,
- "fp8-e4m3fn": 8,
- "fp8-e4m3fnuz": 8,
- }.get(axis["communication_format"])
- if bits_per_value is None:
- raise ValueError(f"unknown communication format {axis['communication_format']!r}")
- activation_data_bytes = logical_copies * math.ceil(hidden * bits_per_value / 8)
- scale_bytes_per_value = {None: 0, "f32": 4}.get(
- axis["scale_dtype"]
- )
- if scale_bytes_per_value is None:
- raise ValueError(f"unknown communication scale dtype {axis['scale_dtype']!r}")
- group_size = axis["scale_group_size"]
- scale_groups = math.ceil(hidden / group_size) if group_size is not None else 0
- scale_bytes = logical_copies * scale_groups * scale_bytes_per_value
+ raise ValueError("logical byte dimensions must be non-negative")
+ activation_data_bytes = logical_copies * hidden * 2
return {
"accounting_contract": "activation-data-plus-scales-v1",
"activation_data_bytes": activation_data_bytes,
- "scale_bytes": scale_bytes,
- "total_logical_bytes": activation_data_bytes + scale_bytes,
+ "scale_bytes": 0,
+ "total_logical_bytes": activation_data_bytes,
}
def format_collective_version(raw) -> str:
@@ -164,12 +145,6 @@ def add_common_args(ap: argparse.ArgumentParser) -> None:
"""Add the varying v1 inputs; fixed profile values are not CLI axes."""
ap.set_defaults(**V1_PROFILE)
ap.add_argument("--mode", default="normal", choices=["normal", LOW_LATENCY_MODE])
- ap.add_argument(
- "--precision-profile",
- default="",
- choices=("", *ep_precision.V1_PRECISION_PROFILES),
- help="exact native dispatch/combine communication profile; blank selects BF16 control",
- )
ap.add_argument("--phase", default="decode", choices=["decode", "prefill"],
help="token-size regime: decode (small T) / prefill (large T) — picks the default ladder")
ap.add_argument("--tokens-ladder", default="",
@@ -194,6 +169,12 @@ def add_common_args(ap: argparse.ArgumentParser) -> None:
default=os.environ.get("CX_QUALIFICATION_INDEX", "1"),
help="one-based qualification repeat used for deterministic measurement ordering",
)
+ ap.add_argument(
+ "--version",
+ type=int,
+ default=os.environ.get("CX_VERSION", "1"),
+ help="iterable benchmark version copied verbatim into the emitted result",
+ )
# 32: B300/Blackwell needs ~30 untimed iters to reach steady-state GPU clocks +
# establish NVLink/NVSHMEM connections — at warmup=8 its dispatch read ~1787us
# (cold), at warmup>=30 it settles to ~85us (faster than H100, reproducible within
@@ -627,126 +608,6 @@ def _expected_transformed_combine(torch, problem):
return expected
-def _baseline_precision_axis() -> dict:
- return {
- "encoded_payload_valid": True,
- "scales_finite": None,
- "scales_positive": None,
- "dequantized_semantics": True,
- "saturation_count": 0,
- "saturation_rate": 0.0,
- "max_abs_error": 0.0,
- "max_rel_error": 0.0,
- "passed": True,
- }
-
-
-def _precision_evidence(backend, problem, view, combined, expected_combined) -> dict:
- method = getattr(backend, "precision_evidence", None)
- if method is not None:
- evidence = method(problem, view)
- combine_axis = backend.communication_precision["combine"]
- if combine_axis["communication_format"] != "bf16":
- oracle_atol, oracle_rtol = _oracle_tolerances(backend)
- if combined.shape == expected_combined.shape and combined.numel():
- absolute = (combined.float() - expected_combined.float()).abs()
- max_abs_error = float(absolute.max().item())
- max_rel_error = max_abs_error / (
- float(expected_combined.float().abs().max().item()) + 1e-6
- )
- tolerance = oracle_atol + oracle_rtol * expected_combined.float().abs()
- semantics = bool((absolute <= tolerance).all().item())
- elif combined.shape == expected_combined.shape:
- max_abs_error = max_rel_error = 0.0
- semantics = True
- else:
- max_abs_error = max_rel_error = 1e30
- semantics = False
- direction = evidence["combine"]
- direction.update({
- "dequantized_semantics": semantics,
- "max_abs_error": max_abs_error,
- "max_rel_error": max_rel_error,
- })
- scale_ok = (
- direction["scales_finite"] is not False
- and direction["scales_positive"] is not False
- )
- direction["passed"] = bool(
- direction["encoded_payload_valid"]
- and semantics
- and scale_ok
- and direction["saturation_count"] == 0
- )
- evidence["passed"] = bool(
- evidence["dispatch"]["passed"] and direction["passed"]
- )
- return evidence
- profile_id = getattr(backend, "precision_profile_id", None)
- if profile_id != ep_precision.V1_CONTROL_PRECISION_PROFILE:
- failed = _baseline_precision_axis()
- failed.update({"encoded_payload_valid": False, "dequantized_semantics": False,
- "passed": False})
- return {"profile_id": profile_id, "dispatch": failed, "combine": dict(failed),
- "passed": False}
- return {
- "profile_id": profile_id,
- "dispatch": _baseline_precision_axis(),
- "combine": _baseline_precision_axis(),
- "passed": True,
- }
-
-
-def _failed_precision_evidence(backend) -> dict:
- failed = _baseline_precision_axis()
- failed.update({"encoded_payload_valid": False, "dequantized_semantics": False,
- "passed": False})
- return {
- "profile_id": getattr(backend, "precision_profile_id", None),
- "dispatch": failed,
- "combine": dict(failed),
- "passed": False,
- }
-
-
-def aggregate_precision_evidence(evidence_by_rank: list[dict]) -> dict:
- """Collapse pre/post rank evidence without hiding any direction's worst observation."""
- records = [record[phase] for record in evidence_by_rank for phase in ("pre", "post")]
- profile_ids = {record["profile_id"] for record in records}
- if len(profile_ids) != 1:
- raise ValueError("precision evidence profiles differ across ranks or oracle passes")
- result = {"profile_id": profile_ids.pop()}
- for direction in ("dispatch", "combine"):
- axes = [record[direction] for record in records]
- rank_counts = []
- for rank_index in range(len(evidence_by_rank)):
- rank_counts.append(max(
- evidence_by_rank[rank_index][phase][direction]["saturation_count"]
- for phase in ("pre", "post")
- ))
- scale_finite = [axis["scales_finite"] for axis in axes]
- scale_positive = [axis["scales_positive"] for axis in axes]
- result[direction] = {
- "encoded_payload_valid": all(axis["encoded_payload_valid"] for axis in axes),
- "scales_finite": (
- None if all(value is None for value in scale_finite)
- else all(value is True for value in scale_finite)
- ),
- "scales_positive": (
- None if all(value is None for value in scale_positive)
- else all(value is True for value in scale_positive)
- ),
- "dequantized_semantics": all(axis["dequantized_semantics"] for axis in axes),
- "saturation_count": sum(rank_counts),
- "saturation_rate": max(axis["saturation_rate"] for axis in axes),
- "max_abs_error": max(axis["max_abs_error"] for axis in axes),
- "max_rel_error": max(axis["max_rel_error"] for axis in axes),
- "passed": all(axis["passed"] for axis in axes),
- }
- result["passed"] = all(result[direction]["passed"] for direction in ("dispatch", "combine"))
- return result
-
-
def _run_expert_packed_oracle(
torch,
routing,
@@ -760,7 +621,7 @@ def _run_expert_packed_oracle(
):
"""Verify an expert-packed dispatch and native gate-weighted combine."""
contract = LOW_LATENCY_ORACLE_CONTRACT
- oracle_atol, oracle_rtol = _oracle_tolerances(backend)
+ oracle_atol, oracle_rtol = ORACLE_ATOL, ORACLE_RTOL
handle = backend.dispatch(problem)
torch.cuda.synchronize()
try:
@@ -786,7 +647,6 @@ def _run_expert_packed_oracle(
except Exception as cleanup_error:
raise inspection_error from cleanup_error
return {
- "_precision": _failed_precision_evidence(backend),
"contract": contract,
"passed": False,
"ordering_contract": "adapter-inspection-failed",
@@ -847,9 +707,6 @@ def _run_expert_packed_oracle(
if source_range
else torch.empty_like(view.payload)
)
- normalize_payload = getattr(backend, "oracle_dispatch_payload", None)
- if source_range and normalize_payload is not None:
- expected_payload = normalize_payload(expected_payload)
payload_ok = bool(
source_range
and torch.equal(decoded_source_ids.to(torch.int64), view.source_ids)
@@ -941,7 +798,6 @@ def _run_expert_packed_oracle(
max_relative_error = None
combine_values_ok = False
tolerance = oracle_rtol
- precision = _precision_evidence(backend, problem, view, combined, expected_combined)
checks = {
"combine_values": combine_values_ok,
"counts": counts_ok,
@@ -952,11 +808,9 @@ def _run_expert_packed_oracle(
"weights": weights_ok,
}
return {
- "_precision": precision,
"contract": contract,
"passed": bool(
all(checks.values())
- and precision["passed"]
and ordering_contract
and max_relative_error is not None
and max_relative_error < tolerance
@@ -1000,7 +854,7 @@ def _run_expert_oracle(
experts_per_rank,
seed,
)
- oracle_atol, oracle_rtol = _oracle_tolerances(backend)
+ oracle_atol, oracle_rtol = ORACLE_ATOL, ORACLE_RTOL
handle = backend.dispatch(problem)
torch.cuda.synchronize()
try:
@@ -1015,7 +869,6 @@ def _run_expert_oracle(
except Exception as cleanup_error:
raise inspection_error from cleanup_error
return {
- "_precision": _failed_precision_evidence(backend),
"contract": ORACLE_CONTRACT,
"passed": False,
"ordering_contract": "adapter-inspection-failed",
@@ -1061,9 +914,6 @@ def _run_expert_oracle(
expected_payload = routing.activations_for_source_ids(
source_ids, problem.x.shape[1], seed, problem.x.dtype
)
- normalize_payload = getattr(backend, "oracle_dispatch_payload", None)
- if normalize_payload is not None:
- expected_payload = normalize_payload(expected_payload)
else:
expected_ids = torch.full_like(view.expert_ids, -1)
expected_weights = torch.zeros_like(view.weights)
@@ -1143,7 +993,6 @@ def _run_expert_oracle(
max_relative_error = None
combine_values_ok = False
tolerance = oracle_rtol
- precision = _precision_evidence(backend, problem, view, combined, expected_combined)
checks = {
"combine_values": combine_values_ok,
"counts": counts_ok,
@@ -1154,11 +1003,9 @@ def _run_expert_oracle(
"weights": weights_ok,
}
return {
- "_precision": precision,
"contract": ORACLE_CONTRACT,
"passed": bool(
all(checks.values())
- and precision["passed"]
and ordering_contract
and max_relative_error is not None
and max_relative_error < tolerance
@@ -1196,12 +1043,9 @@ def _histogram(xs: list[float], nbins: int = 40) -> dict:
def run_sweep(args, backend, torch, dist, device, rank: int, world_size: int) -> int:
"""Drive the source-tokens-per-rank sweep for one fully-specified line."""
mode = getattr(args, "mode", "normal")
- requested_precision = getattr(args, "precision_profile", "") or None
- resolved_precision_id = requested_precision or ep_precision.V1_CONTROL_PRECISION_PROFILE
try:
case_profile = identity.profile_for_case({"mode": mode})
- communication_precision = ep_precision.precision_profile(resolved_precision_id)
- except (identity.IdentityError, ep_precision.PrecisionError) as exc:
+ except identity.IdentityError as exc:
if rank == 0:
print(f"ERROR: {exc}")
return 2
@@ -1224,13 +1068,6 @@ def run_sweep(args, backend, torch, dist, device, rank: int, world_size: int) ->
if rank == 0:
print(f"ERROR: backend mode {getattr(backend, 'mode', None)!r} != {mode!r}")
return 2
- if (
- getattr(backend, "precision_profile_id", None) != resolved_precision_id
- or getattr(backend, "communication_precision", None) != communication_precision
- ):
- if rank == 0:
- print("ERROR: backend did not realize the requested communication precision")
- return 2
expected_weight_semantics = (
"gate-weighted-sum"
if case_profile["combine_semantics"] == "gate-weighted"
@@ -1333,7 +1170,6 @@ def run_sweep(args, backend, torch, dist, device, rank: int, world_size: int) ->
torch, routing, backend, problem, idx_g, w_g, rank, experts_per_rank,
args.seed,
)
- precision_pre = oracle.pop("_precision")
before_x, before_idx, before_weights = input_snapshots[T]
pre_input_unchanged = (
torch.equal(problem.x, before_x)
@@ -1348,7 +1184,6 @@ def run_sweep(args, backend, torch, dist, device, rank: int, world_size: int) ->
"max_rel": oracle["max_relative_error"] or 0.0,
"local_ok": int(oracle["passed"]),
"oracle_pre": oracle,
- "precision_pre": precision_pre,
"pre_input_unchanged": pre_input_unchanged,
}
@@ -1367,7 +1202,6 @@ def run_sweep(args, backend, torch, dist, device, rank: int, world_size: int) ->
"ep_size": ep_size,
"mode": mode,
"phase": args.phase,
- "precision_profile": resolved_precision_id,
"runner": args.runner,
"suite": args.suite,
})
@@ -1428,7 +1262,6 @@ def run_sweep(args, backend, torch, dist, device, rank: int, world_size: int) ->
torch, routing, backend, problem, idx_g, w_g, rank, experts_per_rank,
args.seed,
)
- precision_post = post.pop("_precision")
pre = gate[T]["oracle_pre"]
order_stable = (
pre["ordering_contract"] == post["ordering_contract"]
@@ -1440,7 +1273,6 @@ def run_sweep(args, backend, torch, dist, device, rank: int, world_size: int) ->
"local_ok": int(pre["passed"] and post["passed"] and input_unchanged and order_stable),
"max_rel": max(pre["max_relative_error"] or 0.0, post["max_relative_error"] or 0.0),
"oracle_post": post,
- "precision_post": precision_post,
"order_stable": order_stable,
})
@@ -1507,12 +1339,8 @@ def component(percentiles, count, *, derived=False):
percentile_name: gt / (latency_us * 1e-6)
for percentile_name, latency_us in rtp.items()
}
- dispatch_bytes = precision_byte_provenance(
- communication_precision["dispatch"], logical_copies, H
- )
- combine_bytes = precision_byte_provenance(
- communication_precision["combine"], logical_copies, H
- )
+ dispatch_bytes = logical_byte_provenance(logical_copies, H)
+ combine_bytes = logical_byte_provenance(logical_copies, H)
stage_bytes = {
"accounting_contract": "activation-data-plus-scales-v1",
"activation_data_bytes": 0,
@@ -1713,14 +1541,12 @@ def component(percentiles, count, *, derived=False):
"warmup_semantics": WARMUP_SEMANTICS,
"workload": args.workload_name or "manual",
}
- if requested_precision is not None:
- scheduled_case["precision_profile"] = requested_precision
case_factors = {
"case": scheduled_case,
"profile": case_profile,
"sku": args.runner,
}
- computed_case_id = identity.digest("case", case_factors)
+ computed_case_id = identity.case_id_from_factors(case_factors)
if args.case_id and args.case_id != computed_case_id:
raise ValueError(
f"scheduled case ID does not match realized factors: {args.case_id} != {computed_case_id}"
@@ -1737,7 +1563,6 @@ def component(percentiles, count, *, derived=False):
"runner": args.runner,
"source_sha": git_run.get("source_sha"),
}
- allocation_identifier = identity.allocation_id(allocation_factors)
try:
attempt_ordinal = int(os.environ.get("CX_ATTEMPT_ID", "1"))
except ValueError:
@@ -1745,18 +1570,9 @@ def component(percentiles, count, *, derived=False):
if attempt_ordinal <= 0:
raise ValueError("CX_ATTEMPT_ID must be a positive integer")
attempt_identifier = identity.attempt_id(
- allocation=allocation_identifier, case=case_identifier, ordinal=attempt_ordinal
+ case=case_identifier, ordinal=attempt_ordinal
)
runtime_fingerprint = getattr(args, "runtime_fingerprint", None) or {}
- series_factors = {
- "backend": backend.name,
- "case_id": case_identifier,
- "image_digest": getattr(args, "image_digest", None),
- "source_sha": git_run.get("source_sha"),
- "squash_sha256": getattr(args, "squash_sha256", None),
- "workload_id": getattr(args, "workload_id", None) or trace_sig,
- }
- series_identifier = identity.series_id(series_factors)
sample_points = []
for row in rows:
@@ -1773,31 +1589,22 @@ def component(percentiles, count, *, derived=False):
}
sample_sha256 = _sha256_json(sample_point)
point_identifier = identity.point_id(
- series=series_identifier, tokens_per_rank=token_count
- )
- evidence_identifier = identity.evidence_id(
- point=point_identifier,
- allocation=allocation_identifier,
- attempt=attempt_identifier,
- sample_sha256=sample_sha256,
+ case=case_identifier, tokens_per_rank=token_count
)
sample_point.update(
{
- "evidence_id": evidence_identifier,
"point_id": point_identifier,
"sample_sha256": sample_sha256,
}
)
sample_points.append(sample_point)
row.update({
- "evidence_id": evidence_identifier,
"point_id": point_identifier,
"sample_sha256": sample_sha256,
})
samples_path = args.out[:-5] + ".samples.json" if args.out.endswith(".json") else args.out + ".samples.json"
samples_document = {
- "allocation_id": allocation_identifier,
"attempt_id": attempt_identifier,
"case_id": case_identifier,
"format": "collectivex.samples.v1",
@@ -1808,9 +1615,8 @@ def component(percentiles, count, *, derived=False):
"trials": args.trials,
},
"schema_version": 1,
- "series_id": series_identifier,
}
- samples_payload = contracts.canonical_json_bytes(samples_document)
+ samples_payload = ep_provenance.canonical_json_bytes(samples_document)
samples_sha256 = hashlib.sha256(samples_payload).hexdigest()
samples_bytes = len(samples_payload)
sample_artifact = {
@@ -1860,17 +1666,15 @@ def component(percentiles, count, *, derived=False):
doc = {
"format": "collectivex.ep.v1",
"schema_version": SCHEMA_VERSION,
+ "version": args.version,
"record_type": "case-attempt",
"generated_at": generated_at,
"identity": {
"allocation_factors": allocation_factors,
- "allocation_id": allocation_identifier,
"attempt_id": attempt_identifier,
"attempt_ordinal": attempt_ordinal,
"case_factors": case_factors,
"case_id": case_identifier,
- "series_factors": series_factors,
- "series_id": series_identifier,
},
"case": {
"attempt_ordinal": attempt_ordinal,
diff --git a/experimental/CollectiveX/bench/ep_mori.py b/experimental/CollectiveX/bench/ep_mori.py
index 30e0d264e2..ffe174d336 100644
--- a/experimental/CollectiveX/bench/ep_mori.py
+++ b/experimental/CollectiveX/bench/ep_mori.py
@@ -1,5 +1,5 @@
#!/usr/bin/env python3
-"""CollectiveX MoRI adapter for native V1 dispatch/combine precision profiles."""
+"""CollectiveX MoRI adapter: native BF16 dispatch/combine over mori.ops."""
from __future__ import annotations
import os
@@ -14,7 +14,6 @@
import torch
import torch.distributed as dist
-import ep_precision
from ep_backend import EPBackend
try:
@@ -71,68 +70,25 @@ class MoRIBackend(EPBackend):
def __init__(self, args, rank, world_size, local_rank, device):
# mori is normal-mode only; base SUPPORTED_MODES=("normal",) enforces it.
super().__init__(args, rank, world_size, local_rank, device)
+ # The runner pins the AMD GPU architecture MoRI is built against. This is a
+ # hardware-lineage contract, independent of the (fixed BF16) communication path.
runner = str(getattr(args, "runner", ""))
if runner.startswith("mi355x"):
- fp8_format = "fp8-e4m3fn"
- supported_profiles = {
- "d-bf16.c-bf16",
- "d-fp8-e4m3fn-b128-f32-prequantized.c-bf16",
- }
- if world_size == 8:
- supported_profiles.update({
- "d-bf16.c-fp8-e4m3fn-direct-cast-noscale",
- "d-fp8-e4m3fn-b128-f32-prequantized.c-fp8-e4m3fn-direct-cast-noscale",
- })
+ expected_arch = "gfx950"
elif runner.startswith(("mi300x", "mi325x")):
- fp8_format = "fp8-e4m3fnuz"
- supported_profiles = {
- "d-bf16.c-bf16",
- "d-fp8-e4m3fnuz-b128-f32-prequantized.c-bf16",
- }
- if world_size == 8:
- supported_profiles.update({
- "d-bf16.c-fp8-e4m3fnuz-direct-cast-noscale",
- "d-fp8-e4m3fnuz-b128-f32-prequantized.c-fp8-e4m3fnuz-direct-cast-noscale",
- })
+ expected_arch = "gfx942"
else:
- raise ep_precision.PrecisionError(
- f"MoRI precision contract has no pinned FP8 format for runner {runner!r}"
- )
- self.precision_profile_id, self.communication_precision = (
- ep_precision.resolve_precision(
- args,
- backend=self.name,
- mode=self.mode,
- supported_profiles=supported_profiles,
- )
- )
- self._fp8_dispatch = ep_precision.is_low_precision_dispatch(
- self.communication_precision
- )
- self._direct_cast_combine = ep_precision.uses_direct_cast_combine(
- self.communication_precision
- )
- if self._fp8_dispatch and ep_precision.communication_format(
- self.communication_precision, "dispatch"
- ) != fp8_format:
- raise ep_precision.PrecisionError(
- "MoRI dispatch FP8 format differs from the pinned GPU architecture"
+ raise RuntimeError(
+ f"MoRI has no pinned GPU architecture for runner {runner!r}"
)
- if self._direct_cast_combine:
- quant_enum = getattr(mori.ops, "EpDispatchCombineQuantType", None)
- if quant_enum is None or not hasattr(quant_enum, "Fp8DirectCast"):
- raise ep_precision.PrecisionError(
- "pinned MoRI API omits EpDispatchCombineQuantType.Fp8DirectCast"
- )
self.ep_size = world_size
self.experts_per_rank = args.experts // self.ep_size
device_properties = torch.cuda.get_device_properties(device)
device_cus = device_properties.multi_processor_count
realized_arch = str(getattr(device_properties, "gcnArchName", "")).split(":", 1)[0]
- expected_arch = "gfx950" if runner.startswith("mi355x") else "gfx942"
if realized_arch != expected_arch:
- raise ep_precision.PrecisionError(
+ raise RuntimeError(
f"MoRI runner {runner!r} realized architecture {realized_arch!r}, "
f"expected {expected_arch!r}"
)
@@ -226,21 +182,17 @@ def __init__(self, args, rank, world_size, local_rank, device):
else "async-ll" if self._async_ll
else "intranode"
)
- self._external_input = (
- self._async_ll or self._inter_node or self._direct_cast_combine
- )
+ self._external_input = self._async_ll or self._inter_node
# Registered-input MoRI copies expert output into a device-side symmetric buffer. External
# input kernels consume the dispatch output directly, so their stage is not applicable.
- self.stage_device_work = self._fp8_dispatch or not self._external_input
+ self.stage_device_work = not self._external_input
# Stash the __init__-only locals the moved create_buffer body reads back.
- self._fp8_format = fp8_format
self._gpus_per_node = gpus_per_node
def create_buffer(self, spec):
# Local aliases re-expose the __init__ names so the moved shmem-init /
# config / op / provenance body below stays byte-verbatim.
args, world_size, rank, device = self.args, self.world_size, self.rank, self.device
- fp8_format = self._fp8_format
gpus_per_node = self._gpus_per_node
device_cus = torch.cuda.get_device_properties(device).multi_processor_count
@@ -255,33 +207,15 @@ def create_buffer(self, spec):
self._cap = self.buffer_cap(args)
assert spec.max_tokens_per_rank <= self._cap
- dispatch_dtype = (
- getattr(
- torch,
- "float8_e4m3fn"
- if fp8_format == "fp8-e4m3fn"
- else "float8_e4m3fnuz",
- None,
- )
- if self._fp8_dispatch
- else torch.bfloat16
- )
- if dispatch_dtype is None:
- raise ep_precision.PrecisionError(
- f"active torch build does not expose {fp8_format}"
- )
- scale_dim = args.hidden // 128 if self._fp8_dispatch else 0
- if self._fp8_dispatch and args.hidden % 128:
- raise ep_precision.PrecisionError(
- "MoRI native FP8 dispatch requires hidden divisible by 128"
- )
+ # BF16 communication path: no FP8 dispatch dtype, no scale payload, no combine quantization.
+ dispatch_dtype = torch.bfloat16
config_kwargs = {
"data_type": dispatch_dtype,
"rank": rank,
"world_size": world_size,
"hidden_dim": args.hidden,
- "scale_dim": scale_dim,
- "scale_type_size": 4 if self._fp8_dispatch else 1,
+ "scale_dim": 0,
+ "scale_type_size": 1,
"max_token_type_size": (
torch.tensor([], dtype=torch.bfloat16).element_size()
if self._inter_node
@@ -291,9 +225,7 @@ def create_buffer(self, spec):
"num_experts_per_rank": self.experts_per_rank,
"num_experts_per_token": args.topk,
"use_external_inp_buf": self._external_input,
- "quant_type": (
- "fp8_direct_cast" if self._direct_cast_combine else "none"
- ),
+ "quant_type": "none",
}
if self._kernel_type is not None:
config_kwargs["kernel_type"] = self._kernel_type
@@ -311,8 +243,8 @@ def create_buffer(self, spec):
self.config = mori.ops.EpDispatchCombineConfig(**config_kwargs)
expected_config = {
"data_type": dispatch_dtype,
- "scale_dim": scale_dim,
- "scale_type_size": 4 if self._fp8_dispatch else 1,
+ "scale_dim": 0,
+ "scale_type_size": 1,
"use_external_inp_buf": self._external_input,
"quant_type": config_kwargs["quant_type"],
}
@@ -348,12 +280,8 @@ def create_buffer(self, spec):
else "mori.ops.EpDispatchCombineOp/registered-input"
),
"mode": "normal",
- "dispatch_dtype": ep_precision.communication_format(
- self.communication_precision, "dispatch"
- ),
- "combine_dtype": ep_precision.communication_format(
- self.communication_precision, "combine"
- ),
+ "dispatch_dtype": "bf16",
+ "combine_dtype": "bf16",
"kernel_type": self._kernel_type_label,
"enable_sdma": os.environ.get("MORI_ENABLE_SDMA"),
"heap_size": os.environ.get("MORI_SHMEM_HEAP_SIZE"),
@@ -378,26 +306,17 @@ def buffer_cap(self, args):
return _mori_input_capacity(args, self.world_size)
def make_problem(self, T, idx, weights, x):
- encoding = ep_precision.encode_dispatch(
- torch, x, self.communication_precision
- )
indices = idx.to(torch.int32)
gate_weights = weights.to(torch.float32)
return types.SimpleNamespace(
T=T,
x=x,
- dispatch_x=encoding.native_input[0] if self._fp8_dispatch else x,
- oracle_x=encoding.semantic,
- dispatch_precision_evidence=encoding.evidence,
+ dispatch_x=x,
topk_idx=indices,
topk_weights=gate_weights,
indices=indices,
weights=gate_weights,
- scales=(
- encoding.scales
- if encoding.scales is not None
- else torch.empty((T, 0), dtype=torch.uint8, device=self.device)
- ),
+ scales=torch.empty((T, 0), dtype=torch.uint8, device=self.device),
)
def dispatch(self, p):
@@ -417,7 +336,6 @@ def dispatch(self, p):
return types.SimpleNamespace(
dispatch_output=dispatch_output,
dispatch_weights=dispatch_weights,
- dispatch_scales=_scales,
dispatch_indices=dispatch_indices,
recv_num=recv_num[0],
combine_input=None,
@@ -427,7 +345,7 @@ def stage(self, p, h):
rows = getattr(p, "recv_tokens", None)
if not isinstance(rows, int) or rows < 0 or rows > h.dispatch_output.size(0):
raise RuntimeError("MoRI receive count was not validated before staging")
- h.combine_input = self._semantic_recv(h, rows)
+ h.combine_input = h.dispatch_output
if self._external_input:
return None
buffer = self.op.get_registered_combine_input_buffer(
@@ -468,13 +386,7 @@ def inspect_dispatch(self, p, h):
self.experts_per_rank,
)
return types.SimpleNamespace(
- payload=self._semantic_recv(h, count)[:count],
- encoded_payload=h.dispatch_output[:count],
- scales=(
- h.dispatch_scales[:count]
- if h.dispatch_scales is not None
- else None
- ),
+ payload=h.dispatch_output[:count],
expert_ids=expert_ids,
weights=weights,
local_expert_counts=torch.bincount(
@@ -499,31 +411,6 @@ def combine_transformed(self, p, h, transformed):
def recv_tokens(self, h):
return int(h.recv_num.item())
- def _semantic_recv(self, h, rows):
- if not self._fp8_dispatch:
- return h.dispatch_output
- if not hasattr(h, "recv_semantic"):
- if h.dispatch_scales is None:
- raise ep_precision.PrecisionError(
- "MoRI FP8 dispatch did not return scaling factors"
- )
- semantic = torch.empty(
- h.dispatch_output.shape,
- dtype=torch.bfloat16,
- device=h.dispatch_output.device,
- )
- semantic[:rows].copy_(ep_precision.dequantize_dispatch(
- torch,
- h.dispatch_output[:rows],
- h.dispatch_scales[:rows],
- self.communication_precision["dispatch"],
- ))
- h.recv_semantic = semantic
- h.recv_semantic_rows = rows
- elif h.recv_semantic_rows != rows:
- raise RuntimeError("MoRI receive count changed for one dispatch handle")
- return h.recv_semantic
-
def finalize(self, rc):
try:
dist.barrier()
diff --git a/experimental/CollectiveX/bench/ep_nccl.py b/experimental/CollectiveX/bench/ep_nccl.py
index e7953f246b..4dac5f4cc7 100644
--- a/experimental/CollectiveX/bench/ep_nccl.py
+++ b/experimental/CollectiveX/bench/ep_nccl.py
@@ -21,7 +21,6 @@
import torch.distributed as dist
import contracts
import ep_provenance
-import ep_precision
from ep_backend import EPBackend
@@ -50,14 +49,6 @@ def __init__(self, args, rank, world_size, local_rank, device):
# bf16 normal only, so an unsupported mode raises there.
super().__init__(args, rank, world_size, local_rank, device)
self.experts = args.experts
- self.precision_profile_id, self.communication_precision = (
- ep_precision.resolve_precision(
- args,
- backend=self.name,
- mode=self.mode,
- supported_profiles={"d-bf16.c-bf16"},
- )
- )
if args.experts % world_size:
raise ValueError(f"experts({args.experts}) must divide world_size({world_size})")
self.experts_per_rank = args.experts // world_size
@@ -202,8 +193,3 @@ def combine_transformed(self, p, h, transformed):
def recv_tokens(self, h):
return int(h.total_recv)
-
- def oracle_dispatch_payload(self, payload):
- # NCCL is the identity reference: the wire payload is exactly what the oracle
- # built, so no dispatch-encoding projection (unlike the base) applies.
- return payload
diff --git a/experimental/CollectiveX/bench/ep_precision.py b/experimental/CollectiveX/bench/ep_precision.py
deleted file mode 100644
index e01b5a0353..0000000000
--- a/experimental/CollectiveX/bench/ep_precision.py
+++ /dev/null
@@ -1,543 +0,0 @@
-#!/usr/bin/env python3
-"""Native communication-precision helpers for CollectiveX EP adapters."""
-from __future__ import annotations
-
-import inspect
-from copy import deepcopy
-from dataclasses import dataclass
-from typing import Any, Iterable
-
-
-class PrecisionError(RuntimeError):
- """A requested precision profile cannot be realized by the pinned API."""
-
-
-# ---------------------------------------------------------------------------
-# Native communication-precision catalog.
-#
-# Dispatch and combine are fixed BF16 benchmark facts for the default sweep, so
-# no scheduled case selects an FP8 profile. The FP8 axes below stay callable so
-# an adapter can still exercise its native codec directly; they are dormant, not
-# removed. This catalog lives with the codec (not in ``identity``) because it is
-# a property of the communication kernels, not of the neutral artifact identity.
-# ---------------------------------------------------------------------------
-
-
-def _communication_axis(
- *,
- api_input_dtype: str,
- api_output_dtype: str,
- communication_format: str,
- scale_dtype: str | None,
- scale_layout: str,
- scale_group_size: int | None,
- padding_contract: str,
- alignment_contract: str,
- quantization_origin: str,
- conversion_boundary: str,
-) -> dict[str, Any]:
- return {
- "api_input_dtype": api_input_dtype,
- "api_output_dtype": api_output_dtype,
- "communication_format": communication_format,
- "scale_dtype": scale_dtype,
- "scale_layout": scale_layout,
- "scale_group_size": scale_group_size,
- "padding_contract": padding_contract,
- "alignment_contract": alignment_contract,
- "quantization_origin": quantization_origin,
- "conversion_boundary": conversion_boundary,
- }
-
-
-_BF16_AXIS = _communication_axis(
- api_input_dtype="bf16",
- api_output_dtype="bf16",
- communication_format="bf16",
- scale_dtype=None,
- scale_layout="none",
- scale_group_size=None,
- padding_contract="none",
- alignment_contract="native-bf16-vector-alignment",
- quantization_origin="none",
- conversion_boundary="none",
-)
-_FP8_E4M3FN_PREQUANTIZED_DISPATCH = _communication_axis(
- api_input_dtype="fp8-e4m3fn-with-f32-scale",
- api_output_dtype="fp8-e4m3fn-with-f32-scale",
- communication_format="fp8-e4m3fn",
- scale_dtype="f32",
- scale_layout="per-token-hidden-block",
- scale_group_size=128,
- padding_contract="right-zero-pad-hidden-to-128",
- alignment_contract="hidden-block-128",
- quantization_origin="caller-prequantized",
- conversion_boundary="before-dispatch-timing",
-)
-_FP8_E4M3FNUZ_PREQUANTIZED_DISPATCH = _communication_axis(
- api_input_dtype="fp8-e4m3fnuz-with-f32-scale",
- api_output_dtype="fp8-e4m3fnuz-with-f32-scale",
- communication_format="fp8-e4m3fnuz",
- scale_dtype="f32",
- scale_layout="per-token-hidden-block",
- scale_group_size=128,
- padding_contract="right-zero-pad-hidden-to-128",
- alignment_contract="hidden-block-128",
- quantization_origin="caller-prequantized",
- conversion_boundary="before-dispatch-timing",
-)
-_FP8_E4M3FN_FUSED_DISPATCH = _communication_axis(
- api_input_dtype="bf16",
- api_output_dtype="fp8-e4m3fn-with-f32-scale",
- communication_format="fp8-e4m3fn",
- scale_dtype="f32",
- scale_layout="per-token-hidden-block",
- scale_group_size=128,
- padding_contract="right-zero-pad-hidden-to-128",
- alignment_contract="hidden-block-128",
- quantization_origin="backend-fused",
- conversion_boundary="inside-dispatch-timing",
-)
-_FP8_E4M3FN_DIRECT_CAST_COMBINE = _communication_axis(
- api_input_dtype="bf16",
- api_output_dtype="bf16",
- communication_format="fp8-e4m3fn",
- scale_dtype=None,
- scale_layout="none",
- scale_group_size=None,
- padding_contract="none",
- alignment_contract="native-fp8-vector-alignment",
- quantization_origin="backend-internal-direct-cast",
- conversion_boundary="inside-combine-timing",
-)
-_FP8_E4M3FNUZ_DIRECT_CAST_COMBINE = _communication_axis(
- api_input_dtype="bf16",
- api_output_dtype="bf16",
- communication_format="fp8-e4m3fnuz",
- scale_dtype=None,
- scale_layout="none",
- scale_group_size=None,
- padding_contract="none",
- alignment_contract="native-fp8-vector-alignment",
- quantization_origin="backend-internal-direct-cast",
- conversion_boundary="inside-combine-timing",
-)
-
-V1_CONTROL_PRECISION_PROFILE = "d-bf16.c-bf16"
-V1_NORMAL_PRECISION_PROFILE_IDS = (
- "d-fp8-e4m3fn-b128-f32-prequantized.c-bf16",
- "d-fp8-e4m3fnuz-b128-f32-prequantized.c-fp8-e4m3fnuz-direct-cast-noscale",
-)
-V1_LOW_LATENCY_PRECISION_PROFILE_IDS = (
- "d-fp8-e4m3fn-b128-f32-fused.c-bf16",
-)
-
-V1_PRECISION_PROFILES: dict[str, dict[str, Any]] = {
- V1_CONTROL_PRECISION_PROFILE: {
- "modes": ["normal", "low-latency"],
- "dispatch": _BF16_AXIS,
- "combine": _BF16_AXIS,
- },
- "d-fp8-e4m3fn-b128-f32-prequantized.c-bf16": {
- "modes": ["normal"],
- "dispatch": _FP8_E4M3FN_PREQUANTIZED_DISPATCH,
- "combine": _BF16_AXIS,
- },
- "d-fp8-e4m3fn-b128-f32-fused.c-bf16": {
- "modes": ["low-latency"],
- "dispatch": _FP8_E4M3FN_FUSED_DISPATCH,
- "combine": _BF16_AXIS,
- },
- "d-bf16.c-fp8-e4m3fn-direct-cast-noscale": {
- "modes": ["normal"],
- "dispatch": _BF16_AXIS,
- "combine": _FP8_E4M3FN_DIRECT_CAST_COMBINE,
- },
- "d-fp8-e4m3fn-b128-f32-prequantized.c-fp8-e4m3fn-direct-cast-noscale": {
- "modes": ["normal"],
- "dispatch": _FP8_E4M3FN_PREQUANTIZED_DISPATCH,
- "combine": _FP8_E4M3FN_DIRECT_CAST_COMBINE,
- },
- "d-fp8-e4m3fnuz-b128-f32-prequantized.c-bf16": {
- "modes": ["normal"],
- "dispatch": _FP8_E4M3FNUZ_PREQUANTIZED_DISPATCH,
- "combine": _BF16_AXIS,
- },
- "d-bf16.c-fp8-e4m3fnuz-direct-cast-noscale": {
- "modes": ["normal"],
- "dispatch": _BF16_AXIS,
- "combine": _FP8_E4M3FNUZ_DIRECT_CAST_COMBINE,
- },
- "d-fp8-e4m3fnuz-b128-f32-prequantized.c-fp8-e4m3fnuz-direct-cast-noscale": {
- "modes": ["normal"],
- "dispatch": _FP8_E4M3FNUZ_PREQUANTIZED_DISPATCH,
- "combine": _FP8_E4M3FNUZ_DIRECT_CAST_COMBINE,
- },
-}
-
-V1_COMBINE_ORACLE_TOLERANCES = {
- "bf16": {"atol": 2e-2, "rtol": 5e-2},
- "fp8-direct-cast": {"atol": 4e-2, "rtol": 8e-2},
-}
-
-
-def precision_profile(name: str) -> dict[str, Any]:
- """Return one exact dispatch/combine communication-format profile."""
- try:
- profile = V1_PRECISION_PROFILES[name]
- except KeyError as exc:
- raise PrecisionError(f"unknown CollectiveX precision profile {name!r}") from exc
- return {"profile_id": name, **deepcopy(profile)}
-
-
-def combine_oracle_tolerances(communication_precision: dict[str, Any]) -> dict[str, float]:
- """Return the frozen combine-oracle gate for one exact native codec."""
- combine = communication_precision["combine"]
- fmt = combine["communication_format"]
- if fmt == "bf16":
- key = "bf16"
- elif (
- fmt in {"fp8-e4m3fn", "fp8-e4m3fnuz"}
- and combine["quantization_origin"] == "backend-internal-direct-cast"
- ):
- key = "fp8-direct-cast"
- else:
- raise PrecisionError("precision profile has no frozen combine-oracle tolerance")
- return dict(V1_COMBINE_ORACLE_TOLERANCES[key])
-
-
-@dataclass(frozen=True)
-class DispatchEncoding:
- """One dispatch input plus its post-codec semantic representation."""
-
- native_input: Any
- encoded_payload: Any | None
- scales: Any | None
- semantic: Any
- evidence: dict[str, Any]
-
-
-def resolve_precision(
- args,
- *,
- backend: str,
- mode: str,
- supported_profiles: Iterable[str],
-) -> tuple[str, dict[str, Any]]:
- """Resolve and validate the exact profile requested for one adapter."""
- profile_id = (
- getattr(args, "precision_profile", "")
- or V1_CONTROL_PRECISION_PROFILE
- )
- profile = precision_profile(profile_id) # raises PrecisionError on unknown
- if mode not in profile["modes"]:
- raise PrecisionError(
- f"precision profile {profile_id!r} is not valid in mode {mode!r}"
- )
- supported = frozenset(supported_profiles)
- if profile_id not in supported:
- raise PrecisionError(
- f"{backend} does not realize precision profile {profile_id!r} in mode {mode!r}"
- )
- return profile_id, profile
-
-
-def require_keyword(callable_object, keyword: str, *, api: str) -> None:
- """Fail closed when a pinned Python API does not expose a required control."""
- try:
- parameters = inspect.signature(callable_object).parameters
- except (TypeError, ValueError) as exc:
- raise PrecisionError(f"cannot inspect required precision API {api}") from exc
- if keyword not in parameters:
- raise PrecisionError(f"required precision API {api} omits {keyword!r}")
-
-
-def communication_format(profile: dict[str, Any], component: str) -> str:
- """Return the exact wire format for dispatch or combine."""
- return str(profile[component]["communication_format"])
-
-
-def is_low_precision_dispatch(profile: dict[str, Any]) -> bool:
- return communication_format(profile, "dispatch").startswith("fp8-")
-
-
-def is_caller_prequantized(profile: dict[str, Any]) -> bool:
- return profile["dispatch"]["quantization_origin"] == "caller-prequantized"
-
-
-def uses_logfmt_combine(profile: dict[str, Any]) -> bool:
- return communication_format(profile, "combine") == "logfmt10"
-
-
-def uses_direct_cast_combine(profile: dict[str, Any]) -> bool:
- return profile["combine"]["quantization_origin"] == "backend-internal-direct-cast"
-
-
-def _fp8_dtype(torch_module, axis: dict[str, Any]):
- fmt = axis["communication_format"]
- attribute = {
- "fp8-e4m3fn": "float8_e4m3fn",
- "fp8-e4m3fnuz": "float8_e4m3fnuz",
- }.get(fmt)
- if attribute is None:
- raise PrecisionError(f"unsupported FP8 communication format {fmt!r}")
- dtype = getattr(torch_module, attribute, None)
- if dtype is None:
- raise PrecisionError(f"active torch build does not expose torch.{attribute}")
- return dtype
-
-
-def _axis_evidence(
- *,
- dequantized_semantics: bool,
- encoded_payload_valid: bool,
- max_abs_error: float,
- max_rel_error: float,
- saturation_count: int,
- saturation_rate: float,
- scales_finite: bool | None,
- scales_positive: bool | None,
- passed: bool,
-) -> dict[str, Any]:
- return {
- "encoded_payload_valid": bool(encoded_payload_valid),
- "scales_finite": scales_finite,
- "scales_positive": scales_positive,
- "dequantized_semantics": bool(dequantized_semantics),
- "saturation_count": int(saturation_count),
- "saturation_rate": float(saturation_rate),
- "max_abs_error": float(max_abs_error),
- "max_rel_error": float(max_rel_error),
- "passed": bool(passed),
- }
-
-
-def exact_axis_evidence() -> dict[str, Any]:
- """Evidence for an unquantized BF16 communication axis."""
- return _axis_evidence(
- encoded_payload_valid=True,
- scales_finite=None,
- scales_positive=None,
- dequantized_semantics=True,
- saturation_count=0,
- saturation_rate=0.0,
- max_abs_error=0.0,
- max_rel_error=0.0,
- passed=True,
- )
-
-
-def _quantize_fp8(torch_module, x, axis: dict[str, Any]) -> DispatchEncoding:
- group_size = axis["scale_group_size"]
- if group_size != 128 or axis["scale_dtype"] != "f32":
- raise PrecisionError("v1 FP8 dispatch requires block-128 FP32 scales")
- if x.ndim != 2 or x.shape[1] % group_size:
- raise PrecisionError(
- "v1 native FP8 dispatch requires a 2D hidden dimension divisible by 128"
- )
- dtype = _fp8_dtype(torch_module, axis)
- fp8_max = float(torch_module.finfo(dtype).max)
- blocks = x.float().reshape(x.shape[0], x.shape[1] // group_size, group_size)
- amax = blocks.abs().amax(dim=-1)
- # Match the pinned DeepEP/HybridEP block codec, including its nonzero scale floor.
- scales = (amax.clamp_min(1e-4) / fp8_max).to(torch_module.float32)
- normalized = blocks / scales.unsqueeze(-1)
- saturation_mask = normalized.abs() > fp8_max
- encoded = normalized.clamp(min=-fp8_max, max=fp8_max).to(dtype).reshape_as(x).contiguous()
- semantic = dequantize_dispatch(
- torch_module, encoded, scales, axis
- ).to(x.dtype).contiguous()
- absolute = (semantic.float() - x.float()).abs()
- max_abs = float(absolute.max().item()) if absolute.numel() else 0.0
- reference_max = float(x.float().abs().max().item()) if x.numel() else 0.0
- max_rel = max_abs / (reference_max + 1e-6)
- saturation_count = int(saturation_mask.sum().item())
- saturation_rate = saturation_count / max(1, int(x.numel()))
- finite = bool(torch_module.isfinite(scales).all().item())
- positive = bool((scales > 0).all().item())
- semantic_ok = bool(
- torch_module.isfinite(semantic.float()).all().item()
- and torch_module.allclose(
- semantic.float(), x.float(), rtol=0.05, atol=0.02
- )
- )
- valid = encoded.dtype == dtype and encoded.shape == x.shape
- evidence = _axis_evidence(
- encoded_payload_valid=valid,
- scales_finite=finite,
- scales_positive=positive,
- dequantized_semantics=semantic_ok,
- saturation_count=saturation_count,
- saturation_rate=saturation_rate,
- max_abs_error=max_abs,
- max_rel_error=max_rel,
- passed=valid and finite and positive and semantic_ok,
- )
- return DispatchEncoding(
- native_input=(encoded, scales),
- encoded_payload=encoded,
- scales=scales,
- semantic=semantic,
- evidence=evidence,
- )
-
-
-def encode_dispatch(torch_module, x, profile: dict[str, Any]) -> DispatchEncoding:
- """Build caller-prequantized input or a fused-codec oracle outside timing."""
- axis = profile["dispatch"]
- origin = axis["quantization_origin"]
- if origin == "none":
- return DispatchEncoding(
- native_input=x,
- encoded_payload=None,
- scales=None,
- semantic=x,
- evidence=exact_axis_evidence(),
- )
- if origin not in {"caller-prequantized", "backend-fused"}:
- raise PrecisionError(f"unsupported dispatch quantization origin {origin!r}")
- encoded = _quantize_fp8(torch_module, x, axis)
- if origin == "backend-fused":
- return DispatchEncoding(
- native_input=x,
- encoded_payload=encoded.encoded_payload,
- scales=encoded.scales,
- semantic=encoded.semantic,
- evidence=encoded.evidence,
- )
- return encoded
-
-
-def dequantize_dispatch(
- torch_module,
- encoded_payload,
- scales,
- axis: dict[str, Any],
- *,
- uint8_storage: bool = False,
-):
- """Decode one native block-scaled FP8 payload to BF16 semantics."""
- group_size = axis["scale_group_size"]
- if group_size != 128 or scales is None:
- raise PrecisionError("FP8 dispatch payload is missing block-128 scales")
- dtype = _fp8_dtype(torch_module, axis)
- payload = encoded_payload.view(dtype) if uint8_storage else encoded_payload
- if payload.dtype != dtype or payload.ndim < 2 or payload.shape[-1] % group_size:
- raise PrecisionError("native FP8 dispatch payload has an invalid dtype or shape")
- expected_scale_shape = (*payload.shape[:-1], payload.shape[-1] // group_size)
- if tuple(scales.shape) != expected_scale_shape or scales.dtype != torch_module.float32:
- raise PrecisionError("native FP8 dispatch scales have an invalid dtype or shape")
- values = payload.float().reshape(
- *payload.shape[:-1], payload.shape[-1] // group_size, group_size
- )
- return (values * scales.float().reshape(*expected_scale_shape, 1)).reshape(
- payload.shape
- ).to(torch_module.bfloat16).contiguous()
-
-
-def validate_received_encoding(
- torch_module,
- *,
- encoded_payload,
- scales,
- semantic,
- axis: dict[str, Any],
- uint8_storage: bool = False,
-) -> bool:
- """Validate that received bytes/scales exactly decode to the semantic view."""
- try:
- decoded = dequantize_dispatch(
- torch_module,
- encoded_payload,
- scales,
- axis,
- uint8_storage=uint8_storage,
- )
- except PrecisionError:
- return False
- return bool(torch_module.equal(decoded, semantic))
-
-
-def dequantize_expert_prefixes(
- torch_module,
- encoded_payload,
- scales,
- axis: dict[str, Any],
- counts: tuple[int, ...],
- output,
-):
- """Decode only valid expert-packed prefixes into a reusable BF16 workspace."""
- if encoded_payload.ndim != 3 or len(counts) != encoded_payload.shape[0]:
- raise PrecisionError("expert-packed FP8 receive counts have an invalid shape")
- if output.shape != encoded_payload.shape or output.dtype != torch_module.bfloat16:
- raise PrecisionError("expert-packed BF16 stage workspace has an invalid shape")
- capacity = encoded_payload.shape[1]
- for expert, count in enumerate(counts):
- if count < 0 or count > capacity:
- raise PrecisionError("expert-packed FP8 receive count exceeds capacity")
- if count:
- output[expert, :count].copy_(dequantize_dispatch(
- torch_module,
- encoded_payload[expert, :count],
- scales[expert, :count],
- axis,
- ))
- return output
-
-
-def _direct_cast_saturation(torch_module, profile: dict[str, Any], view) -> tuple[int, float]:
- """Count values clipped in the exact native BF16-to-FP8 combine input."""
- if not hasattr(view, "combine_input"):
- return 0, 0.0
- transformed = view.combine_input.float()
- dtype = _fp8_dtype(torch_module, profile["combine"])
- fp8_max = float(torch_module.finfo(dtype).max)
- count = int((transformed.abs() > fp8_max).sum().item())
- return count, count / max(1, int(transformed.numel()))
-
-
-def precision_evidence(
- torch_module,
- *,
- profile_id: str,
- profile: dict[str, Any],
- problem,
- view=None,
- uint8_storage: bool = False,
-) -> dict[str, Any]:
- """Return schema-shaped codec evidence; the harness adds combine-oracle errors."""
- dispatch = deepcopy(problem.dispatch_precision_evidence)
- if (
- is_low_precision_dispatch(profile)
- and view is not None
- and all(hasattr(view, field) for field in ("encoded_payload", "scales", "payload"))
- ):
- valid = validate_received_encoding(
- torch_module,
- encoded_payload=view.encoded_payload,
- scales=view.scales,
- semantic=view.payload,
- axis=profile["dispatch"],
- uint8_storage=uint8_storage,
- )
- dispatch["encoded_payload_valid"] = (
- dispatch["encoded_payload_valid"] and valid
- )
- dispatch["passed"] = dispatch["passed"] and valid
- combine = exact_axis_evidence()
- # Internal quantizers are validated by native configuration here. The harness
- # replaces the error fields and pass bit with the transformed-combine oracle.
- if communication_format(profile, "combine") != "bf16":
- combine["scales_finite"] = None
- combine["scales_positive"] = None
- if uses_direct_cast_combine(profile) and view is not None:
- count, rate = _direct_cast_saturation(torch_module, profile, view)
- combine["saturation_count"] = count
- combine["saturation_rate"] = rate
- combine["passed"] = combine["passed"] and count == 0
- return {
- "profile_id": profile_id,
- "dispatch": dispatch,
- "combine": combine,
- "passed": bool(dispatch["passed"] and combine["passed"]),
- }
diff --git a/experimental/CollectiveX/bench/ep_provenance.py b/experimental/CollectiveX/bench/ep_provenance.py
index a637f27c2e..82a74c96fe 100644
--- a/experimental/CollectiveX/bench/ep_provenance.py
+++ b/experimental/CollectiveX/bench/ep_provenance.py
@@ -13,21 +13,110 @@
from __future__ import annotations
import hashlib
+import inspect
+import json
+import math
import os
import re
from pathlib import Path, PurePosixPath
from typing import Any, Iterable
-from contracts import (
- ContractError,
- canonical_json_bytes,
- DEEPEP_V2_DISTRIBUTION_VERSIONS,
- DEEPEP_V2_JIT_KERNELS,
- DEEPEP_V2_V1_PROVENANCE,
- GIT_RUN_FIELDS,
- REQUIRED_BACKEND_PROVENANCE,
- UCCL_DEPENDENCY_VERSIONS,
-)
+from contracts import ContractError, GIT_RUN_FIELDS
+
+
+def _finite_tree(value: Any, path: str = "$") -> None:
+ if isinstance(value, float) and not math.isfinite(value):
+ raise ContractError(f"{path} contains a non-finite number")
+ if isinstance(value, list):
+ for index, item in enumerate(value):
+ _finite_tree(item, f"{path}[{index}]")
+ elif isinstance(value, dict):
+ for key, item in value.items():
+ _finite_tree(item, f"{path}.{key}")
+
+
+def canonical_json_bytes(value: Any) -> bytes:
+ """Canonical finite JSON bytes for checksums and immutable artifacts."""
+ _finite_tree(value)
+ try:
+ return json.dumps(
+ value, allow_nan=False, ensure_ascii=False, sort_keys=True,
+ separators=(",", ":"),
+ ).encode("utf-8")
+ except (TypeError, ValueError) as exc:
+ raise ContractError(f"value is not canonical JSON: {exc}") from exc
+
+
+# Backend implementation-provenance constants: the required evidence fields per
+# backend and the pinned native versions a self-check compares against. They
+# describe executable code, so they live beside the emitter that consumes them
+# rather than in the neutral document/delivery validator.
+DEEPEP_V2_JIT_KERNELS = frozenset({
+ "barrier", "combine", "combine_reduce_epilogue", "dispatch",
+ "dispatch_copy_epilogue",
+})
+DEEPEP_V2_V1_PROVENANCE = {
+ "deepep_version": "2.0.0",
+ "deepep_distribution_version": "2.0.0+fa8a9b1",
+ "deepep_commit": "fa8a9b16898204afd347c663b89e65ef87dc6ce6",
+ "deepep_tree": "29809e75c5874e6609dac4804e7b651d5226959f",
+ "deepep_pr": 605,
+ "deepep_fix_pr": 630,
+ "deepep_nccl_check_fix_pr": 640,
+ "deepep_nccl_check_commit": "93d0564188f7a0a6288c6e316484861b0efa042e",
+ "fmt_commit": "a4c7e17133ee9cb6a2f45545f6e974dd3c393efa",
+ "torch_version": "2.10.0+cu130",
+ "nccl_package_version": "2.30.4",
+ "nccl_version": "2.30.4",
+ "nvshmem_package_version": "3.3.9",
+}
+DEEPEP_V2_DISTRIBUTION_VERSIONS = frozenset({
+ "2.0.0+fa8a9b1", "2.0.0+local",
+})
+UCCL_DEPENDENCY_VERSIONS = {
+ "intervaltree": "3.1.0",
+ "nvidia-cuda-runtime-cu12": "12.9.79",
+ "sortedcontainers": "2.4.0",
+}
+REQUIRED_BACKEND_PROVENANCE = {
+ "deepep": (
+ "deepep_version", "deepep_commit", "backend_lineage", "allow_mnnvl",
+ "mnnvl_comm", "mode", "num_nvl_bytes", "num_rdma_bytes",
+ "nvshmem_ibgda_nic_handler",
+ ),
+ "deepep-v2": (
+ *DEEPEP_V2_V1_PROVENANCE, "api_signature_sha256", "loaded_libraries",
+ "jit_cubins", "jit_random_seed", "deterministic", "num_experts",
+ "tuning_num_experts", "allow_hybrid_mode", "gin_enabled",
+ "communication_backend",
+ ),
+ "deepep-hybrid": (
+ "deepep_commit", "deepep_tree", "branch", "backend_lineage",
+ "loaded_libraries", "realized_config", "jit_kernel_keys", "jit_shared_objects",
+ ),
+ "uccl": (
+ "uccl_version", "uccl_commit", "uccl_wrapper_commit", "backend_lineage",
+ "loaded_libraries", "uccl_dependency_versions", "mode", "num_nvl_bytes",
+ "num_rdma_bytes",
+ ),
+ "mori": ("mori_commit",),
+ "nccl-ep": ("nccl_version", "collective_library", "backend_lineage"),
+}
+
+
+def require_keyword(callable_object, keyword: str, *, api: str) -> None:
+ """Fail closed when a pinned native API does not expose a required control.
+
+ A runtime-API contract guard: it verifies the loaded kernel build still accepts
+ the keyword the adapter drives (e.g. the BF16 ``use_fp8=False`` control), so a
+ version mismatch surfaces here instead of as an opaque call-site TypeError.
+ """
+ try:
+ parameters = inspect.signature(callable_object).parameters
+ except (TypeError, ValueError) as exc:
+ raise ContractError(f"cannot inspect required native API {api}") from exc
+ if keyword not in parameters:
+ raise ContractError(f"required native API {api} omits {keyword!r}")
def resolve_deepep_mnnvl(
@@ -507,7 +596,7 @@ def series_provenance(provenance: dict[str, Any]) -> dict[str, Any]:
def _sha256_json(value: Any) -> str:
- """Canonical sha256 of a finite JSON value (matches contracts.canonical_json_bytes)."""
+ """Canonical sha256 of a finite JSON value."""
return hashlib.sha256(canonical_json_bytes(value)).hexdigest()
diff --git a/experimental/CollectiveX/bench/ep_uccl.py b/experimental/CollectiveX/bench/ep_uccl.py
index 248bc01219..55d07bdce2 100644
--- a/experimental/CollectiveX/bench/ep_uccl.py
+++ b/experimental/CollectiveX/bench/ep_uccl.py
@@ -1,5 +1,5 @@
#!/usr/bin/env python3
-"""CollectiveX UCCL adapter for native V1 dispatch/combine precision profiles."""
+"""CollectiveX UCCL adapter: native BF16 dispatch/combine over uccl_deepep."""
from __future__ import annotations
import importlib.metadata as metadata
@@ -11,9 +11,7 @@
import torch
import torch.distributed as dist
-import contracts
import ep_provenance
-import ep_precision
from ep_deepep_family import DeepEPFamilyBackend
try:
@@ -35,9 +33,9 @@ def _uccl_version() -> str:
def _uccl_dependency_versions() -> dict[str, str]:
versions = {
package: metadata.version(package)
- for package in contracts.UCCL_DEPENDENCY_VERSIONS
+ for package in ep_provenance.UCCL_DEPENDENCY_VERSIONS
}
- if versions != contracts.UCCL_DEPENDENCY_VERSIONS:
+ if versions != ep_provenance.UCCL_DEPENDENCY_VERSIONS:
raise RuntimeError(
"UCCL runtime dependency versions differ from the v1 contract"
)
@@ -185,9 +183,9 @@ def _normal_buffer_sizes(hidden: int, world_size: int) -> tuple[int, int]:
class UCCLBackend(DeepEPFamilyBackend):
- # uccl_deepep.Buffer is a DeepEP-API clone, so mode handling, dispatch/combine, and fp8
- # dequantization are shared in DeepEPFamilyBackend; only the native uccl_deepep buffer
- # construction, its content-provenance, and process teardown are UCCL-specific.
+ # uccl_deepep.Buffer is a DeepEP-API clone, so mode handling and dispatch/combine are
+ # shared in DeepEPFamilyBackend; only the native uccl_deepep buffer construction, its
+ # content-provenance, and process teardown are UCCL-specific.
name = "uccl"
_vendor = "UCCL"
@@ -197,12 +195,12 @@ def create_buffer(self, spec):
device_sms = torch.cuda.get_device_properties(device).multi_processor_count
if self.mode == "low-latency":
assert spec.max_tokens_per_rank <= 128
- ep_precision.require_keyword(
+ ep_provenance.require_keyword(
Buffer.low_latency_dispatch,
"use_fp8",
api="uccl_deepep.Buffer.low_latency_dispatch",
)
- ep_precision.require_keyword(
+ ep_provenance.require_keyword(
Buffer.low_latency_combine,
"use_logfmt",
api="uccl_deepep.Buffer.low_latency_combine",
@@ -232,12 +230,12 @@ def create_buffer(self, spec):
"num_rdma_bytes": num_rdma_bytes,
}
else:
- ep_precision.require_keyword(
+ ep_provenance.require_keyword(
Buffer.dispatch,
"async_finish",
api="uccl_deepep.Buffer.dispatch",
)
- ep_precision.require_keyword(
+ ep_provenance.require_keyword(
Buffer.combine,
"async_finish",
api="uccl_deepep.Buffer.combine",
@@ -278,12 +276,8 @@ def create_buffer(self, spec):
"uccl_dependency_versions": dependency_versions,
"loaded_libraries": loaded_libraries,
"mode": self.mode,
- "dispatch_dtype": ep_precision.communication_format(
- self.communication_precision, "dispatch"
- ),
- "combine_dtype": ep_precision.communication_format(
- self.communication_precision, "combine"
- ),
+ "dispatch_dtype": "bf16",
+ "combine_dtype": "bf16",
"resource_mode": "fixed-profile",
"device_sms": device_sms,
**resource_provenance,
diff --git a/experimental/CollectiveX/bench/make_workloads.py b/experimental/CollectiveX/bench/make_workloads.py
index 65dd2064d9..f92d20c923 100644
--- a/experimental/CollectiveX/bench/make_workloads.py
+++ b/experimental/CollectiveX/bench/make_workloads.py
@@ -8,13 +8,13 @@
--routing uniform --ep 8 --hidden 7168 --topk 8 --experts 256 --seed 67 \\
--tokens-ladder "1 2 4 8 16 32 64 128 256 512"
-Or by the named v1 workload in configs/workloads.yaml. Explicit dimension flags still override it:
+Or by the named workload in configs/workloads.yaml. Explicit dimension flags still override it:
- python3 bench/make_workloads.py --out-dir /path/to/cx_workloads --workload deepseek-v3-v1 --routing uniform --ep 8
+ python3 bench/make_workloads.py --out-dir /path/to/cx_workloads --workload deepseek-v3 --routing uniform --ep 8
--id-only prints the content-bound workload_id per ladder point without torch/numpy:
- python3 bench/make_workloads.py --workload deepseek-v3-v1 --ep 8 --id-only
+ python3 bench/make_workloads.py --workload deepseek-v3 --ep 8 --id-only
Generate every routing the suites need by running once per --routing. Idempotent (same id => same
file). The dir is the cross-hardware artifact: copy it to each cluster so all consume identical bytes.
diff --git a/experimental/CollectiveX/bench/run_ep.py b/experimental/CollectiveX/bench/run_ep.py
index 0983ee904c..ae43f4dbc5 100644
--- a/experimental/CollectiveX/bench/run_ep.py
+++ b/experimental/CollectiveX/bench/run_ep.py
@@ -297,7 +297,7 @@ def main() -> int:
if args.phase != "decode":
print("ERROR: low-latency mode requires --phase decode", file=sys.stderr)
return 2
- if args.case_id and not identity.is_typed_id(args.case_id, "case"):
+ if args.case_id and not identity.is_case_id(args.case_id):
print(f"ERROR: invalid native case ID {args.case_id!r}", file=sys.stderr)
return 2
if args.case_id and args.seed != ep_harness.ROUTING_SEED:
diff --git a/experimental/CollectiveX/bench/workload.py b/experimental/CollectiveX/bench/workload.py
index 1b121e17f8..bb9b3f18c3 100644
--- a/experimental/CollectiveX/bench/workload.py
+++ b/experimental/CollectiveX/bench/workload.py
@@ -318,7 +318,7 @@ def verify_workload(manifest, idx_np, weights_np):
or manifest["routing_profile"] != "uniform"):
return False, "manifest version or generator is unsupported"
if (isinstance(manifest["seed"], bool) or not isinstance(manifest["seed"], int)
- or not identity.is_typed_id(manifest["workload_id"], "workload")):
+ or not identity.is_workload_id(manifest["workload_id"])):
return False, "manifest seed or workload ID is invalid"
dims = manifest["dims"]
dim_fields = {"hidden", "topk", "experts", "ep_size", "tokens_per_rank",
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
index 59f0794716..bdc9b2cafb 100644
--- a/experimental/CollectiveX/capability.py
+++ b/experimental/CollectiveX/capability.py
@@ -1,13 +1,11 @@
#!/usr/bin/env python3
-"""Public runner and backend capability registry for CollectiveX v1."""
+"""Public runner and backend capability registry for CollectiveX."""
from __future__ import annotations
import re
from typing import Any
-import identity
-
DEEPEP_V2_COMMIT = "fa8a9b16898204afd347c663b89e65ef87dc6ce6"
DEEPEP_V2_SKU_CAPABILITIES = {
@@ -272,7 +270,7 @@ def _resolve_base(
if topology is None or (nodes is not None and nodes != topology["nodes"]):
return False, f"{sku} does not register EP{ep} on {nodes} nodes"
if routing != "uniform" or eplb:
- return False, "v1 routing is uniform; EPLB is unavailable"
+ return False, "core routing is uniform; EPLB is unavailable"
if platform["vendor"] not in implementation["vendors"]:
return False, f"{backend} does not support {platform['vendor']}"
sku_capability = implementation.get("sku_capabilities", {}).get(sku)
@@ -326,7 +324,7 @@ def resolve(
eplb: bool = False,
mode: str = "normal",
) -> tuple[bool, str]:
- """Return whether one fixed-v1 case can run on a public GHA runner label."""
+ """Return whether one fixed-profile case can run on a public GHA runner label."""
disposition, detail = resolve_disposition(
sku,
backend,
diff --git a/experimental/CollectiveX/configs/suites.yaml b/experimental/CollectiveX/configs/suites.yaml
index 411d0914a1..882c6779b0 100644
--- a/experimental/CollectiveX/configs/suites.yaml
+++ b/experimental/CollectiveX/configs/suites.yaml
@@ -1,20 +1,25 @@
-# CollectiveX v1 comparison suites.
+# CollectiveX EP comparison suites.
schema_version: 1
+# Iterable benchmark version. Bump when a methodology change makes new results
+# incomparable with prior ones; it is copied verbatim into the matrix, every
+# emitted case result, every extracted shard control, and the run summary so the
+# frontend can group and pick runs by it. This is the only place it is set.
+version: 1
suites:
- ep-core-v1:
+ ep-core:
mode: normal
- workloads: [deepseek-v3-v1]
+ workloads: [deepseek-v3]
platforms: [h100-dgxc, h200-dgxc, b300, b200-dgxc, gb300, gb200, mi300x, mi355x]
ep_degrees: [8, 16]
routings: [uniform]
phases: [decode, prefill]
token_points_prefill: [256, 512]
- ep-low-latency-v1:
+ ep-low-latency:
mode: low-latency
backends: [deepep, uccl]
- workloads: [deepseek-v3-v1]
+ workloads: [deepseek-v3]
platforms: [h100-dgxc, h200-dgxc, b300, b200-dgxc, gb300, gb200, mi300x, mi355x]
ep_degrees: [8, 16]
routings: [uniform]
diff --git a/experimental/CollectiveX/configs/workloads.yaml b/experimental/CollectiveX/configs/workloads.yaml
index b5b68334c4..ba750b9100 100644
--- a/experimental/CollectiveX/configs/workloads.yaml
+++ b/experimental/CollectiveX/configs/workloads.yaml
@@ -1,8 +1,8 @@
-# CollectiveX v1 canonical workload and phase metadata.
+# CollectiveX EP canonical workload and phase metadata.
schema_version: 1
model_derived:
- deepseek-v3-v1:
+ deepseek-v3:
hidden: 7168
topk: 8
routed_experts: 256
diff --git a/experimental/CollectiveX/contracts.py b/experimental/CollectiveX/contracts.py
index 4746014427..ea12270ccf 100644
--- a/experimental/CollectiveX/contracts.py
+++ b/experimental/CollectiveX/contracts.py
@@ -3,17 +3,18 @@
JSON Schema (``schemas/*.schema.json``, Draft 2020-12) owns document shape and
field constraints. The Python here covers only the facts JSON Schema cannot
-express: typed-identity recomputation, detached-sample path/hash joins, one
-terminal result per scheduled case, complete shard delivery, and the artifact
-privacy boundary. The public surface is intentionally small:
+express: recomputing the readable identities from their factors, joining detached
+samples by point identity and digest, one terminal result per scheduled case,
+complete shard delivery, and the artifact privacy boundary. The public surface is
+intentionally small:
- strict_json_load() - duplicate/non-finite-rejecting JSON reader
- validate_result() - validate one emitted result file (raw+samples or terminal)
+ strict_json_load() - duplicate/non-finite-rejecting JSON reader
+ validate_result() - validate one emitted result file (raw+samples or terminal)
make_terminal_result() - build and self-validate one non-success outcome
validate_delivery() - reconcile a shard/matrix with its complete attempt set
-The backend-provenance and resource-projection helpers stay here so the bench
-emitter can call them; they move to the bench layer with the executable modules.
+Backend implementation-provenance constants and canonical-JSON helpers now live in
+``bench/ep_provenance.py``, beside the executable modules that emit them.
"""
from __future__ import annotations
@@ -22,29 +23,17 @@
import hashlib
import ipaddress
import json
-import math
import os
-from pathlib import Path, PurePosixPath
+from pathlib import Path
import re
import sys
-from typing import Any, Iterable
+from typing import Any
import identity
RAW_FORMAT = "collectivex.ep.v1"
SAMPLES_FORMAT = "collectivex.samples.v1"
TERMINAL_FORMAT = "collectivex.terminal.v1"
-TERMINAL_CASE_FIELDS = {
- "backend", "canonical", "eplb", "ep", "experts", "gpus_per_node", "hidden",
- "ladder", "mode", "nodes", "phase", "routing",
- "samples_per_point", "scale_out_transport", "scale_up_domain", "scale_up_transport",
- "scope", "suite", "timing", "topk", "topology_class", "transport",
- "warmup_semantics", "workload",
-}
-ALLOCATION_FACTOR_FIELDS = {
- "artifact", "execution_id", "job", "repo", "run_attempt", "run_id", "runner",
- "source_sha",
-}
GIT_RUN_FIELDS = {
"artifact", "job", "ref", "repo", "run_attempt", "run_id", "source_sha",
}
@@ -84,58 +73,7 @@
"prefill": (1, 2, 4, 8, 16, 32, 64, 128, 256, 512),
}
V1_CONDITIONING_ROUNDS_PER_SHAPE = 8
-DEEPEP_V2_JIT_KERNELS = frozenset({
- "barrier", "combine", "combine_reduce_epilogue", "dispatch",
- "dispatch_copy_epilogue",
-})
-DEEPEP_V2_V1_PROVENANCE = {
- "deepep_version": "2.0.0",
- "deepep_distribution_version": "2.0.0+fa8a9b1",
- "deepep_commit": "fa8a9b16898204afd347c663b89e65ef87dc6ce6",
- "deepep_tree": "29809e75c5874e6609dac4804e7b651d5226959f",
- "deepep_pr": 605,
- "deepep_fix_pr": 630,
- "deepep_nccl_check_fix_pr": 640,
- "deepep_nccl_check_commit": "93d0564188f7a0a6288c6e316484861b0efa042e",
- "fmt_commit": "a4c7e17133ee9cb6a2f45545f6e974dd3c393efa",
- "torch_version": "2.10.0+cu130",
- "nccl_package_version": "2.30.4",
- "nccl_version": "2.30.4",
- "nvshmem_package_version": "3.3.9",
-}
-DEEPEP_V2_DISTRIBUTION_VERSIONS = frozenset({
- "2.0.0+fa8a9b1", "2.0.0+local",
-})
-UCCL_DEPENDENCY_VERSIONS = {
- "intervaltree": "3.1.0",
- "nvidia-cuda-runtime-cu12": "12.9.79",
- "sortedcontainers": "2.4.0",
-}
SCHEMA_DIR = Path(__file__).resolve().parent / "schemas"
-REQUIRED_BACKEND_PROVENANCE = {
- "deepep": (
- "deepep_version", "deepep_commit", "backend_lineage", "allow_mnnvl",
- "mnnvl_comm", "mode", "num_nvl_bytes", "num_rdma_bytes",
- "nvshmem_ibgda_nic_handler",
- ),
- "deepep-v2": (
- *DEEPEP_V2_V1_PROVENANCE, "api_signature_sha256", "loaded_libraries",
- "jit_cubins", "jit_random_seed", "deterministic", "num_experts",
- "tuning_num_experts", "allow_hybrid_mode", "gin_enabled",
- "communication_backend",
- ),
- "deepep-hybrid": (
- "deepep_commit", "deepep_tree", "branch", "backend_lineage",
- "loaded_libraries", "realized_config", "jit_kernel_keys", "jit_shared_objects",
- ),
- "uccl": (
- "uccl_version", "uccl_commit", "uccl_wrapper_commit", "backend_lineage",
- "loaded_libraries", "uccl_dependency_versions", "mode", "num_nvl_bytes",
- "num_rdma_bytes",
- ),
- "mori": ("mori_commit",),
- "nccl-ep": ("nccl_version", "collective_library", "backend_lineage"),
-}
class ContractError(ValueError):
@@ -319,29 +257,6 @@ def constant(value):
strict_load = strict_json_load
-def _finite_tree(value: Any, path: str = "$") -> None:
- if isinstance(value, float) and not math.isfinite(value):
- raise ContractError(f"{path} contains a non-finite number")
- if isinstance(value, list):
- for index, item in enumerate(value):
- _finite_tree(item, f"{path}[{index}]")
- elif isinstance(value, dict):
- for key, item in value.items():
- _finite_tree(item, f"{path}.{key}")
-
-
-def canonical_json_bytes(value: Any) -> bytes:
- """Canonical finite JSON bytes for checksums and immutable artifacts."""
- _finite_tree(value)
- try:
- return json.dumps(
- value, allow_nan=False, ensure_ascii=False, sort_keys=True,
- separators=(",", ":"),
- ).encode("utf-8")
- except (TypeError, ValueError) as exc:
- raise ContractError(f"value is not canonical JSON: {exc}") from exc
-
-
def _obj(value: Any, path: str) -> dict[str, Any]:
if not isinstance(value, dict):
raise ContractError(f"{path} must be an object")
@@ -367,12 +282,6 @@ def _text(value: Any, path: str, *, nullable: bool = False) -> str | None:
return value
-def _integer(value: Any, path: str, *, minimum: int = 0) -> int:
- if type(value) is not int or value < minimum:
- raise ContractError(f"{path} must be an integer >= {minimum}")
- return value
-
-
# ---------------------------------------------------------------------------
# Schema validation core (JSON Schema owns document shape).
# ---------------------------------------------------------------------------
@@ -428,17 +337,11 @@ def validate_raw_document(document: Any, samples: Any) -> dict[str, Any]:
_schema_validate(document, "raw-case-v1.schema.json", "raw")
_schema_validate(samples, "samples-v1.schema.json", "samples")
ids = document["identity"]
- case_id = identity.digest("case", ids["case_factors"])
- allocation_id = identity.allocation_id(ids["allocation_factors"])
- attempt_id = identity.attempt_id(
- allocation=allocation_id, case=case_id, ordinal=ids["attempt_ordinal"]
- )
- if (case_id, allocation_id, attempt_id) != (
- ids["case_id"], ids["allocation_id"], ids["attempt_id"]
- ):
- raise ContractError("raw typed identities do not match their factors")
- series_id = ids["series_id"]
- for field in ("case_id", "series_id", "allocation_id", "attempt_id"):
+ case_id = identity.case_id_from_factors(ids["case_factors"])
+ attempt_id = identity.attempt_id(case=case_id, ordinal=ids["attempt_ordinal"])
+ if (case_id, attempt_id) != (ids["case_id"], ids["attempt_id"]):
+ raise ContractError("raw case identity does not match its factors")
+ for field in ("case_id", "attempt_id"):
if samples.get(field) != ids[field]:
raise ContractError(f"detached samples {field} differs from raw identity")
rows = document["measurement"]["rows"]
@@ -448,17 +351,13 @@ def validate_raw_document(document: Any, samples: Any) -> dict[str, Any]:
success = document["outcome"]["status"] == "success"
for row in rows:
expected_point = identity.point_id(
- series=series_id, tokens_per_rank=row["tokens_per_rank"]
+ case=case_id, tokens_per_rank=row["tokens_per_rank"]
)
- expected_evidence = identity.evidence_id(
- point=expected_point, allocation=allocation_id, attempt=attempt_id,
- sample_sha256=row["sample_sha256"],
- )
- if row["point_id"] != expected_point or row["evidence_id"] != expected_evidence:
- raise ContractError("raw point/evidence identity does not match its factors")
+ if row["point_id"] != expected_point:
+ raise ContractError("raw point identity does not match its factors")
point = sample_points[row["point_id"]]
- if (point["sample_sha256"], point["evidence_id"], point["tokens_per_rank"]) != (
- row["sample_sha256"], row["evidence_id"], row["tokens_per_rank"]
+ if (point["sample_sha256"], point["tokens_per_rank"]) != (
+ row["sample_sha256"], row["tokens_per_rank"]
):
raise ContractError("detached sample point differs from its raw row")
if success and row["correctness"]["passed"] is not True:
@@ -499,6 +398,22 @@ def validate_result(path: str | os.PathLike[str]) -> dict[str, Any]:
# ---------------------------------------------------------------------------
# Terminal outcome construction and validation.
# ---------------------------------------------------------------------------
+_ALLOCATION_GIT_FIELDS = ("artifact", "job", "repo", "run_attempt", "run_id", "source_sha")
+
+
+def _allocation_factors(
+ git_run: dict[str, Any] | None, runner: str, execution_id: str | None
+) -> dict[str, Any]:
+ """Project the CI provenance into the eight canonical allocation factors."""
+ factors: dict[str, Any] = {
+ name: (git_run[name] if git_run is not None else None)
+ for name in _ALLOCATION_GIT_FIELDS
+ }
+ factors["execution_id"] = execution_id
+ factors["runner"] = runner
+ return factors
+
+
def make_terminal_result(
*,
allocation_factors: dict[str, Any],
@@ -513,26 +428,26 @@ def make_terminal_result(
return_code: int,
source: str,
status: str,
+ version: int,
expected_case_id: str | None = None,
) -> dict[str, Any]:
"""Build and self-validate one attributable non-success attempt."""
- case_id = identity.digest("case", case_factors)
+ case_id = identity.case_id_from_factors(case_factors)
if expected_case_id is not None and expected_case_id != case_id:
raise ContractError(
f"scheduled case ID differs from terminal factors: {expected_case_id} != {case_id}"
)
- allocation_id = identity.allocation_id(allocation_factors)
- attempt_id = identity.attempt_id(
- allocation=allocation_id, case=case_id, ordinal=attempt_ordinal
- )
+ if type(version) is not int or version < 1:
+ raise ContractError("terminal benchmark version must be a positive integer")
+ attempt_id = identity.attempt_id(case=case_id, ordinal=attempt_ordinal)
document = {
"format": TERMINAL_FORMAT,
"schema_version": 1,
+ "version": version,
"record_type": "terminal-outcome",
"generated_at": generated_at,
"identity": {
"allocation_factors": allocation_factors,
- "allocation_id": allocation_id,
"attempt_id": attempt_id,
"attempt_ordinal": attempt_ordinal,
"case_factors": case_factors,
@@ -573,15 +488,10 @@ def validate_terminal_document(document: Any) -> dict[str, Any]:
raise ContractError("terminal case factors differ from the scheduled case/profile")
allocation = ids["allocation_factors"]
ordinal = ids["attempt_ordinal"]
- expected_case = identity.digest("case", factors)
- expected_allocation = identity.allocation_id(allocation)
- expected_attempt = identity.attempt_id(
- allocation=expected_allocation, case=expected_case, ordinal=ordinal
- )
- if (ids["case_id"], ids["allocation_id"], ids["attempt_id"]) != (
- expected_case, expected_allocation, expected_attempt
- ):
- raise ContractError("terminal typed identities do not match their factors")
+ expected_case = identity.case_id_from_factors(factors)
+ expected_attempt = identity.attempt_id(case=expected_case, ordinal=ordinal)
+ if (ids["case_id"], ids["attempt_id"]) != (expected_case, expected_attempt):
+ raise ContractError("terminal case identity does not match its factors")
provenance = document["provenance"]
git_run = provenance["git_run"]
outcome = document["outcome"]
@@ -610,16 +520,9 @@ def validate_terminal_document(document: Any) -> dict[str, Any]:
raise ContractError("terminal provenance source is not registered")
if not valid_outcome:
raise ContractError("terminal source and outcome are not registered")
- expected_allocation_factors = {
- "artifact": git_run["artifact"] if git_run is not None else None,
- "execution_id": allocation["execution_id"],
- "job": git_run["job"] if git_run is not None else None,
- "repo": git_run["repo"] if git_run is not None else None,
- "run_attempt": git_run["run_attempt"] if git_run is not None else None,
- "run_id": git_run["run_id"] if git_run is not None else None,
- "runner": expected_runner,
- "source_sha": git_run["source_sha"] if git_run is not None else None,
- }
+ expected_allocation_factors = _allocation_factors(
+ git_run, expected_runner, allocation["execution_id"]
+ )
if allocation != expected_allocation_factors:
raise ContractError("terminal allocation factors differ from provenance or source")
assert_publication_safe([document])
@@ -723,21 +626,6 @@ def value(name: str) -> str | None:
return git_run
-def _allocation_factors_from_environment(
- runner: str, git_run: dict[str, Any] | None
-) -> dict[str, Any]:
- return {
- "artifact": git_run["artifact"] if git_run is not None else None,
- "execution_id": os.environ.get("COLLECTIVEX_EXECUTION_ID") or None,
- "job": git_run["job"] if git_run is not None else None,
- "repo": git_run["repo"] if git_run is not None else None,
- "run_attempt": git_run["run_attempt"] if git_run is not None else None,
- "run_id": git_run["run_id"] if git_run is not None else None,
- "runner": runner,
- "source_sha": git_run["source_sha"] if git_run is not None else None,
- }
-
-
def make_terminal_from_environment(
*, backend: str, phase: str, return_code: int, failure_mode: str | None = None
) -> dict[str, Any]:
@@ -756,7 +644,9 @@ def make_terminal_from_environment(
git_run = _git_run_from_environment()
control = os.environ.get("COLLECTIVEX_CONTROL_SHA256") or None
return make_terminal_result(
- allocation_factors=_allocation_factors_from_environment(runner, git_run),
+ allocation_factors=_allocation_factors(
+ git_run, runner, os.environ.get("COLLECTIVEX_EXECUTION_ID") or None
+ ),
attempt_ordinal=_env_integer("CX_ATTEMPT_ID", 1),
case=case,
case_factors=case_factors,
@@ -768,6 +658,7 @@ def make_terminal_from_environment(
return_code=return_code,
source="runtime-emitter",
status="failed",
+ version=_env_integer("CX_VERSION", 1),
expected_case_id=os.environ.get("CX_CASE_ID") or None,
)
@@ -788,12 +679,15 @@ def demote_raw_attempt(path: str | os.PathLike[str], return_code: int) -> dict[s
raise ContractError("only a raw attempt can be demoted")
ids = _obj(raw.get("identity"), "raw.identity")
required = {
- "allocation_factors", "allocation_id", "attempt_id", "attempt_ordinal",
+ "allocation_factors", "attempt_id", "attempt_ordinal",
"case_factors", "case_id",
}
if not required.issubset(ids):
raise ContractError("raw identity lacks terminal factors")
mode = RETURN_CODE_FAILURE_MODES.get(return_code, "execution")
+ version = raw.get("version")
+ if type(version) is not int or version < 1:
+ raise ContractError("raw attempt lacks a valid benchmark version to demote")
git_run = _obj(raw.get("provenance"), "raw.provenance").get("git_run")
if git_run is not None:
git_run = _keys(git_run, GIT_RUN_FIELDS, "raw.provenance.git_run")
@@ -810,6 +704,7 @@ def demote_raw_attempt(path: str | os.PathLike[str], return_code: int) -> dict[s
return_code=return_code,
source="post-emit-command",
status="failed",
+ version=version,
expected_case_id=ids["case_id"],
)
artifact = raw.get("sample_artifact") or {}
@@ -823,33 +718,33 @@ def demote_raw_attempt(path: str | os.PathLike[str], return_code: int) -> dict[s
# ---------------------------------------------------------------------------
# Shard / matrix delivery reconciliation.
# ---------------------------------------------------------------------------
-def _validate_attempt_paths(paths: list[str]) -> int:
- """Fully validate a result directory's attempts and paired sample artifacts."""
+def _load_delivery_attempts(paths: list[str]) -> list[dict[str, Any]]:
+ """Fully validate every result path once and return its raw/terminal attempts."""
if not paths or len(paths) != len(set(paths)):
raise ContractError("delivery requires unique result paths")
+ attempts: list[dict[str, Any]] = []
sample_paths: set[Path] = set()
referenced_samples: set[Path] = set()
- attempt_count = 0
for raw_path in paths:
path = Path(raw_path).resolve()
document = strict_json_load(path)
- if isinstance(document, dict) and document.get("format") == RAW_FORMAT:
+ kind = document.get("format") if isinstance(document, dict) else None
+ if kind == RAW_FORMAT:
document = load_raw_attempt(path)
referenced_samples.add(path.with_name(document["sample_artifact"]["path"]))
- attempt_count += 1
- elif isinstance(document, dict) and document.get("format") == TERMINAL_FORMAT:
- validate_terminal_document(document)
- attempt_count += 1
- elif isinstance(document, dict) and document.get("format") == SAMPLES_FORMAT:
+ attempts.append(document)
+ elif kind == TERMINAL_FORMAT:
+ attempts.append(validate_terminal_document(document))
+ elif kind == SAMPLES_FORMAT:
validate_samples_document(document)
sample_paths.add(path)
else:
raise ContractError(f"unknown result artifact {path.name}")
if sample_paths != referenced_samples:
raise ContractError("sample artifacts are missing, orphaned, or outside the validated set")
- if attempt_count == 0:
+ if not attempts:
raise ContractError("result set contains no attempts")
- return attempt_count
+ return attempts
def validate_delivery(
@@ -882,25 +777,27 @@ def validate_delivery(
raise ContractError("delivery source is not a matrix or shard control")
if not expected or len(expected) != expected_count:
raise ContractError("delivery source has empty or duplicate case coverage")
+ source_version = source.get("version")
+ if type(source_version) is not int or source_version < 1:
+ raise ContractError("delivery source lacks a valid benchmark version")
- _validate_attempt_paths(paths)
- attempts = []
- for raw_path in paths:
- document = strict_json_load(raw_path)
- if isinstance(document, dict) and document.get("format") in {RAW_FORMAT, TERMINAL_FORMAT}:
- attempts.append(load_attempt(raw_path))
+ attempts = _load_delivery_attempts(paths)
assert_publication_safe(attempts)
by_case: dict[str, list[dict[str, Any]]] = {}
attempt_ids = set()
- allocation_ids = set()
+ allocation_keys = set()
source_sha256 = hashlib.sha256(source_file.read_bytes()).hexdigest()
for document in attempts:
ids = document["identity"]
case_id = ids["case_id"]
if case_id not in expected or ids["attempt_id"] in attempt_ids:
raise ContractError("delivery contains an extra case or duplicate attempt")
+ if document.get("version") != source_version:
+ raise ContractError("delivery attempt benchmark version differs from its source")
attempt_ids.add(ids["attempt_id"])
- allocation_ids.add(ids["allocation_id"])
+ allocation_keys.add(
+ json.dumps(ids["allocation_factors"], sort_keys=True, separators=(",", ":"))
+ )
sku, scheduled = expected[case_id]
scheduled_case = {key: value for key, value in scheduled.items() if key != "case_id"}
if ids["case_factors"] != {
@@ -941,8 +838,8 @@ def validate_delivery(
ordinals = sorted(document["identity"]["attempt_ordinal"] for document in documents)
if ordinals != list(range(1, len(ordinals) + 1)):
raise ContractError(f"delivery attempt ordinals are not contiguous for {case_id}")
- if require_one_allocation and len(allocation_ids) != 1:
- raise ContractError("one shard must use exactly one allocation identity")
+ if require_one_allocation and len(allocation_keys) != 1:
+ raise ContractError("one shard must use exactly one allocation")
return len(attempts)
diff --git a/experimental/CollectiveX/docs/methodology.md b/experimental/CollectiveX/docs/methodology.md
index a554c54d2b..33fa4c64e9 100644
--- a/experimental/CollectiveX/docs/methodology.md
+++ b/experimental/CollectiveX/docs/methodology.md
@@ -19,15 +19,13 @@ It does not predict serving throughput without a separate correlation study.
## Matrix
-The implemented workload is `deepseek-v3-v1`: hidden 7168, top-k 8, 256 routed experts, packed
-placement, and one pinned fixed resource profile per backend/topology. The default sweep runs the
-portable BF16 dispatch / BF16 combine control; the native FP8 codec paths remain in the code
-(`bench/ep_precision.py` and each adapter's FP8 dispatch/combine) and stay callable but are not a
-swept dimension. Each case explicitly selects normal `layout-and-dispatch-v1` or low-latency
-`expert-packed-weighted-combine-v1` semantics.
-
-- `ep-core-v1`: uniform routing; decode T=1..128 powers of two; prefill T=256/512.
-- `ep-low-latency-v1`: DeepEP V1 / UCCL native low-latency APIs; uniform decode T=1..128 powers of
+The implemented workload is `deepseek-v3`: hidden 7168, top-k 8, 256 routed experts, packed
+placement, and one pinned fixed resource profile per backend/topology. Dispatch and combine are
+fixed BF16 on every backend; precision is not a swept dimension. Each case explicitly selects normal
+`layout-and-dispatch-v1` or low-latency `expert-packed-weighted-combine-v1` semantics.
+
+- `ep-core`: uniform routing; decode T=1..128 powers of two; prefill T=256/512.
+- `ep-low-latency`: DeepEP V1 / UCCL native low-latency APIs; uniform decode T=1..128 powers of
two; other backends are recorded as unsupported rather than fabricating a low-latency path.
`sweep_matrix.py` materializes the requested SKUs, backends, EP sizes, and token ladders into a
@@ -133,12 +131,9 @@ Normal-mode adapters use activation-only, unweighted rank-sum combine. The oracl
gate-weighted expert aggregate before combine, independently derives `sum(gate * expert(token))`, and
checks the dispatch metadata and transformed output. Low-latency adapters separately verify the
expert-packed source/expert assignment, native gate weights, and gate-weighted combined output. Both
-contracts compare against the semantic value after the declared communication codec. The combine
-gates are `rtol=0.05, atol=0.02` for BF16 (the default sweep), `rtol=0.06, atol=0.03` for native
-logfmt10, and `rtol=0.08, atol=0.04` for native FP8 direct-cast combine when those codec paths are
-exercised. These thresholds are correctness gates, not estimates of codec error. Direct-cast
-saturation is measured on the exact transformed native combine input, and the required saturation
-count is zero. Any failed rank or point makes the case ineligible and is recorded as such.
+contracts compare against the reference activation. The combine gate is `rtol=0.05, atol=0.02` for
+the BF16 communication path. This threshold is a correctness gate, not an estimate of transport
+error. Any failed rank or point makes the case ineligible and is recorded as such.
Pre/post dispatch evidence is hashed in canonical source-token order. Native receive slots may be
assigned nondeterministically, so physical receive order is not treated as a correctness property.
@@ -165,24 +160,26 @@ local mode-0600 logs and ignored operator notes and never enter an emitted artif
## Identity
-Canonical JSON produces three full SHA-256 IDs:
+Identifiers are readable factor strings with a short digest suffix, not opaque hashes:
-- `series_id`: all locked factors except token coordinate and repeat allocation;
-- `point_id`: `series_id` plus token coordinate; and
-- `evidence_id`: `point_id` plus allocation/run/attempt/sample checksum.
+- `case_id`: `{sku}-{backend}-{workload}-{mode}-{phase}-ep{ep}-{routing}[-eplb]` followed by a
+ 12-hex digest of the full case factors (SKU, profile, and case), so the body is human-readable
+ while the suffix makes the ID collision-proof and a tamper-evident join key;
+- `attempt_id`: `case_id` plus `-a{ordinal:02d}`; and
+- `point_id`: `case_id` plus `-t{tokens_per_rank}`.
-Locked factors include workload bytes, measurement and sampling contract, resources, realized
-topology, implementation/build, loaded libraries, image/squash, runtime, and source SHA. Deferred
-code generation is captured before measurement and recaptured afterward. DeepEP V2 uses a fixed NVCC
-random seed and binds final cache keys plus generated-source and executable-SASS hashes; raw CUBIN
-bytes remain private diagnostics and are stripped before they enter an artifact. Hybrid binds its
-realized auto-tuned config and complete kernel-key set while retaining rank-local shared-object
-hashes as private diagnostics. Locally built extension hashes are diagnostic; their pinned source
-trees, build recipe, runtime, and dependencies remain series-bound.
+Content SHA-256 is retained only where it identifies separately-stored bytes or executable code: the
+workload manifest (`workload_id` = `cxwork-v1-{sha256}`), detached sample files (`sample_sha256`),
+source commits, container images and squash layers, and deferred/JIT-generated source. DeepEP V2
+uses a fixed NVCC random seed and binds final cache keys plus generated-source and executable-SASS
+hashes; raw CUBIN bytes remain private diagnostics and are stripped before they enter an artifact.
+Hybrid binds its realized auto-tuned config and complete kernel-key set while retaining rank-local
+shared-object hashes as private diagnostics. Locally built extension hashes are diagnostic; their
+pinned source trees, build recipe, runtime, and dependencies remain bound to the case factors.
These IDs let a consumer group matched configurations and separate distinct ones. The backend does
not itself compute cohorts, controlled comparisons, sensitivity pairs, eligibility, or
-recommendations — a reader decides which series to surface and how to compare them.
+recommendations — a reader decides which cases to surface and how to compare them.
## Execution Isolation
@@ -238,13 +235,11 @@ uploads them as GitHub artifacts with `always()` so a red or partial run still u
- strict schema validity of every result, terminal, and samples document;
- exactly one terminal record per scheduled case, with no missing, extra, or duplicate case;
-- detached-sample path and SHA-256 consistency (`point_id` / `evidence_id`); and
+- detached-sample path and SHA-256 consistency (`point_id` / `sample_sha256`); and
- privacy — no private host, address, selector, credential, or path leaks into any artifact.
-The `setup` job also emits a neutral `collectivex.frontend-catalog.v1` projection of the matrix so a
-consumer can enumerate scheduled coverage without parsing shard internals. No step promotes a run,
-builds a dataset, or advances a channel; the artifacts are the output. Any downstream display or
-comparison is the consumer's responsibility.
+No step promotes a run, builds a dataset, or advances a channel; the artifacts are the output. Any
+downstream display or comparison is the consumer's responsibility.
## Legacy Data
diff --git a/experimental/CollectiveX/identity.py b/experimental/CollectiveX/identity.py
index 7b691fb411..7d22fefd4c 100644
--- a/experimental/CollectiveX/identity.py
+++ b/experimental/CollectiveX/identity.py
@@ -1,5 +1,18 @@
#!/usr/bin/env python3
-"""Canonical, cross-runtime identities for CollectiveX v1."""
+"""Canonical cross-runtime identities for CollectiveX.
+
+A case identity is a readable, factor-derived join between a scheduled case and
+its emitted result: a human-readable body (SKU, backend, workload, mode, phase,
+EP size, routing) plus a short digest of the exact case factors. The suffix keeps
+the identity collision-proof and tamper-evident without hiding what the case is.
+Attempt and point identities are readable derivations of the case identity.
+
+Content SHA-256 survives only where it identifies separately-stored bytes: the
+workload manifest whose exact routing/gate data a result consumes. Detached
+sample files, source commits, container images, and generated kernels carry their
+own SHA-256 fields in the artifacts; this module does not mint identifiers for
+catalogs, series, evidence, allocations, or any other publisher-side grouping.
+"""
from __future__ import annotations
import hashlib
@@ -7,22 +20,17 @@
import re
from typing import Any
-IDENTITY_VERSION = 1
MAX_SAFE_INTEGER = (1 << 53) - 1
-PREFIXES = {
- "case": "cxcase-v1-",
- "workload": "cxwork-v1-",
- "series": "cxseries-v1-",
- "point": "cxpoint-v1-",
- "evidence": "cxevidence-v1-",
- "allocation": "cxallocation-v1-",
- "attempt": "cxattempt-v1-",
-}
+WORKLOAD_PREFIX = "cxwork-v1-"
+_CASE_SUFFIX_LEN = 12
+_NON_SLUG = re.compile(r"[^a-z0-9]+")
+_CASE_ID = re.compile(r"^[a-z0-9][a-z0-9_.-]*-[0-9a-f]{%d}$" % _CASE_SUFFIX_LEN)
+_WORKLOAD_ID = re.compile(r"^cxwork-v1-[0-9a-f]{64}$")
+
V1_NORMAL_CASE_PROFILE = {
"activation_generator": "collectivex-activation-counter-v4",
"activation_profile": "canonical-counter-source-v4",
"combine_dtype": "bf16",
- "combine_quant_mode": "none",
"combine_semantics": "activation-only",
"component_order_contract": "qualification-hash-rotated-components-v1",
"conditioning_contract": "fixed-phase-ramp-8-roundtrips-v1",
@@ -34,7 +42,6 @@
"eplb_reference_tokens_per_rank": 2048,
"mode": "normal",
"oracle_contract": "expert-specific-transform-v1",
- "oracle_tolerances": "codec-specific-combine-v1",
"payload_unit": "token-rank",
"placement": "packed",
"percentile_method": "nearest-rank",
@@ -66,6 +73,10 @@
}
+class IdentityError(ValueError):
+ """An identity payload cannot be represented consistently across runtimes."""
+
+
def case_profile(mode: str) -> dict[str, Any]:
"""Return the immutable measurement profile for one scheduled mode."""
try:
@@ -86,10 +97,6 @@ def profile_for_case(case: dict[str, Any]) -> dict[str, Any]:
return case_profile(mode)
-class IdentityError(ValueError):
- """An identity payload cannot be represented consistently across runtimes."""
-
-
def _validate(value: Any, path: str = "$") -> None:
if value is None or isinstance(value, bool):
return
@@ -128,79 +135,65 @@ def canonical_bytes(value: Any) -> bytes:
).encode("utf-8")
-def digest(kind: str, value: Any) -> str:
- """Hash a typed v1 identity payload and return its typed identifier."""
- try:
- prefix = PREFIXES[kind]
- except KeyError as exc:
- raise IdentityError(f"unknown identity kind {kind!r}") from exc
- body = {"kind": kind, "value": value, "version": IDENTITY_VERSION}
- return prefix + hashlib.sha256(canonical_bytes(body)).hexdigest()
+def _content_sha256(value: Any) -> str:
+ return hashlib.sha256(canonical_bytes(value)).hexdigest()
-def is_typed_id(value: Any, kind: str) -> bool:
- prefix = PREFIXES.get(kind)
- return bool(
- isinstance(value, str)
- and prefix
- and re.fullmatch(re.escape(prefix) + r"[0-9a-f]{64}", value)
- )
+def _slug(value: Any) -> str:
+ text = _NON_SLUG.sub("-", str(value).strip().lower()).strip("-")
+ if not text:
+ raise IdentityError("case factor has no printable identity component")
+ return text
-def case_id(*, sku: str, profile: dict[str, Any], case: dict[str, Any]) -> str:
- return digest("case", {"case": case, "profile": profile, "sku": sku})
+def _case_body(sku: str, case: dict[str, Any]) -> str:
+ parts = [
+ _slug(sku),
+ _slug(case["backend"]),
+ _slug(case.get("workload") or "manual"),
+ _slug(case["mode"]),
+ _slug(case["phase"]),
+ "ep%d" % int(case["ep"]),
+ _slug(case["routing"]),
+ ]
+ if case.get("eplb"):
+ parts.append("eplb")
+ return "-".join(parts)
-def workload_id(value: dict[str, Any]) -> str:
- return digest("workload", value)
+def case_id(*, sku: str, profile: dict[str, Any], case: dict[str, Any]) -> str:
+ """Readable case identity: factor body plus a digest of the exact factors."""
+ factors = {"case": case, "profile": profile, "sku": sku}
+ return f"{_case_body(sku, case)}-{_content_sha256(factors)[:_CASE_SUFFIX_LEN]}"
-def series_id(value: dict[str, Any]) -> str:
- return digest("series", value)
+def case_id_from_factors(factors: dict[str, Any]) -> str:
+ """Recompute a case identity from an emitted `case_factors` object."""
+ return case_id(sku=factors["sku"], profile=factors["profile"], case=factors["case"])
-def point_id(*, series: str, tokens_per_rank: int) -> str:
- return digest("point", {"series_id": series, "tokens_per_rank": tokens_per_rank})
+def attempt_id(*, case: str, ordinal: int) -> str:
+ if not isinstance(ordinal, int) or isinstance(ordinal, bool) or ordinal < 1:
+ raise IdentityError("attempt ordinal must be a positive integer")
+ return f"{case}-a{ordinal:02d}"
-def allocation_id(value: dict[str, Any]) -> str:
- return digest("allocation", value)
+def point_id(*, case: str, tokens_per_rank: int) -> str:
+ if not isinstance(tokens_per_rank, int) or isinstance(tokens_per_rank, bool) or tokens_per_rank < 1:
+ raise IdentityError("tokens_per_rank must be a positive integer")
+ return f"{case}-t{tokens_per_rank}"
-def attempt_id(*, allocation: str, case: str, ordinal: int) -> str:
- return digest(
- "attempt", {"allocation_id": allocation, "case_id": case, "ordinal": ordinal}
- )
+def is_case_id(value: Any) -> bool:
+ return bool(isinstance(value, str) and _CASE_ID.fullmatch(value))
-def evidence_id(
- *, point: str, allocation: str, attempt: str, sample_sha256: str
-) -> str:
- return digest(
- "evidence",
- {
- "allocation_id": allocation,
- "attempt_id": attempt,
- "point_id": point,
- "sample_sha256": sample_sha256,
- },
+def workload_id(value: dict[str, Any]) -> str:
+ """Content identity of a workload manifest's canonical routing/gate data."""
+ return WORKLOAD_PREFIX + _content_sha256(
+ {"kind": "workload", "value": value, "version": 1}
)
-IDENTITY_TEST_VECTOR = {
- "payload": {"backend": "deepep", "ep": 8, "shape": [7168, 8, 256]},
- "series_id": "cxseries-v1-a79bf758488e3edd50f5531f3af825f371bf42aae7c4097e461fd2a32615af81",
-}
-
-
-def verify_test_vector() -> None:
- observed = series_id(IDENTITY_TEST_VECTOR["payload"])
- if observed != IDENTITY_TEST_VECTOR["series_id"]:
- raise IdentityError(
- f"identity implementation differs: {observed} != {IDENTITY_TEST_VECTOR['series_id']}"
- )
-
-
-if __name__ == "__main__":
- verify_test_vector()
- print(IDENTITY_TEST_VECTOR["series_id"])
+def is_workload_id(value: Any) -> bool:
+ return bool(isinstance(value, str) and _WORKLOAD_ID.fullmatch(value))
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index dd9daf7bdd..0f91fe9d98 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -97,26 +97,6 @@ cx_fail_stage() {
diagnostic="python-import"
elif grep -aEqi 'AttributeError:|TypeError:.*(unexpected|argument|operand)|has no attribute' "$log_path"; then
diagnostic="backend-api"
- elif grep -aEqi 'FP8 dispatch payload is missing block-128 scales' "$log_path"; then
- diagnostic="precision-dispatch-scales-missing"
- elif grep -aEqi 'native FP8 dispatch payload has an invalid dtype or shape' "$log_path"; then
- diagnostic="precision-dispatch-payload-shape"
- elif grep -aEqi 'native FP8 dispatch scales have an invalid dtype or shape' "$log_path"; then
- diagnostic="precision-dispatch-scale-shape"
- elif grep -aEqi 'expert-packed FP8 receive count exceeds capacity' "$log_path"; then
- diagnostic="precision-receive-capacity"
- elif grep -aEqi 'expert-packed (FP8 receive counts|BF16 stage workspace) has an invalid shape' "$log_path"; then
- diagnostic="precision-receive-shape"
- elif grep -aEqi 'active torch build does not expose (torch[.])?float8|active torch build does not expose fp8-' "$log_path"; then
- diagnostic="precision-runtime-dtype"
- elif grep -aEqi 'omits EpDispatchCombineQuantType[.]Fp8DirectCast' "$log_path"; then
- diagnostic="precision-combine-api"
- elif grep -aEqi 'dispatch FP8 format differs from the pinned GPU architecture' "$log_path"; then
- diagnostic="precision-architecture-format"
- elif grep -aEqi 'native FP8 dispatch requires hidden divisible by 128' "$log_path"; then
- diagnostic="precision-hidden-alignment"
- elif grep -aEqi 'PrecisionError:|unsupported precision|precision profile.*(invalid|unsupported|differs)' "$log_path"; then
- diagnostic="precision-contract"
elif grep -aEqi 'AssertionError:' "$log_path"; then
diagnostic="python-assertion"
elif grep -aEqi 'RuntimeError:' "$log_path"; then
@@ -709,8 +689,7 @@ PY
# Explicit Slurm export boundary. Operator config, runner credentials, HOME,
# workspace paths, and unrelated service secrets never enter the container.
cx_container_exports() {
- printf '%s' 'COLLECTIVEX_SOURCE_SHA,COLLECTIVEX_ARTIFACT_NAME,COLLECTIVEX_EXECUTION_ID,COLLECTIVEX_CONTROL_SHA256,COLLECTIVEX_IMAGE,COLLECTIVEX_IMAGE_DIGEST,COLLECTIVEX_IMAGE_DIGEST_VERIFIED,COLLECTIVEX_SQUASH_SHA256,GITHUB_REF_NAME,GITHUB_REF,GITHUB_REPOSITORY,GITHUB_JOB,GITHUB_RUN_ID,GITHUB_RUN_ATTEMPT,GITHUB_SHA,CX_RUNNER,CX_BENCH,CX_NODES,CX_GPUS_PER_NODE,CX_SCALE_UP_DOMAIN,CX_SHARD_FILE,CX_SHARD_SKU,CX_NGPUS,CX_TS,CX_TOPO,CX_SCOPE,CX_TRANSPORT,CX_SCALE_UP_TRANSPORT,CX_SCALE_OUT_TRANSPORT,CX_MODE,CX_PHASE,CX_ROUTING,CX_EPLB,CX_CASE_ID,CX_SUITE,CX_WORKLOAD_NAME,CX_PRECISION_PROFILE,CX_QUALIFICATION_INDEX,CX_HIDDEN,CX_TOPK,CX_EXPERTS,CX_TOKENS_LADDER,CX_CANONICAL,CX_ITERS,CX_TRIALS,CX_WARMUP,CX_SAMPLES_PER_POINT,CX_WARMUP_SEMANTICS,CX_SEED,CX_RUN_TIMEOUT,CX_NCCL_HOME,CX_ALLOW_MNNVL,CX_ATTEMPT_ID,CX_RUNTIME_MARKER,CX_MORI_KERNEL_TYPE,CX_WORKLOAD_DIR,CX_BACKEND_CACHE_ROOT,CX_BACKEND_CACHE_SENTINEL_SHA256,CX_BACKEND_SOURCE_ROOT,CX_AUDIT_SALT,CX_SOCKET_IFNAME,CX_RDMA_DEVICES,CX_IB_GID_INDEX,CX_RDMA_SERVICE_LEVEL,CX_RDMA_TRAFFIC_CLASS,CX_RDMA_LINK_LAYER,MASTER_ADDR,MASTER_PORT,RANK,WORLD_SIZE,LOCAL_RANK,LOCAL_WORLD_SIZE,NCCL_NET,NCCL_SOCKET_IFNAME,GLOO_SOCKET_IFNAME,NCCL_IB_HCA,NCCL_IB_GID_INDEX,NCCL_IB_SL,NVSHMEM_DISABLE_IB,NVSHMEM_ENABLE_NIC_PE_MAPPING,NVSHMEM_HCA_LIST,NVSHMEM_IB_GID_INDEX,NVSHMEM_IB_SL,NVSHMEM_IB_ENABLE_IBGDA,NVSHMEM_IBGDA_NIC_HANDLER,EP_NIC_NAME,EP_OVERRIDE_RDMA_SL,UCCL_SOCKET_IFNAME,UCCL_IB_GID_INDEX,UCCL_IB_SL,MORI_RDMA_DEVICES,MORI_RDMA_TC,MORI_IO_TC,MORI_RDMA_SL,MORI_IO_SL,HYBRID_EP_MULTINODE,USE_NIXL,RDMA_CORE_HOME,DEEPEP_HYBRID_BUILD_MODE,NCCL_CUMEM_ENABLE,NCCL_MNNVL_ENABLE,MC_FORCE_MNNVL,MORI_DISABLE_AUTO_XGMI,MORI_ENABLE_SDMA,MORI_APP_LOG_LEVEL,MORI_SHMEM_LOG_LEVEL,MORI_IO_LOG_LEVEL'
- printf '%s' ',MORI_COMMIT'
+ printf '%s' 'COLLECTIVEX_SOURCE_SHA,COLLECTIVEX_ARTIFACT_NAME,COLLECTIVEX_EXECUTION_ID,COLLECTIVEX_CONTROL_SHA256,COLLECTIVEX_IMAGE,COLLECTIVEX_IMAGE_DIGEST,COLLECTIVEX_IMAGE_DIGEST_VERIFIED,COLLECTIVEX_SQUASH_SHA256,GITHUB_REF_NAME,GITHUB_REF,GITHUB_REPOSITORY,GITHUB_JOB,GITHUB_RUN_ID,GITHUB_RUN_ATTEMPT,GITHUB_SHA,CX_RUNNER,CX_BENCH,CX_NODES,CX_GPUS_PER_NODE,CX_SCALE_UP_DOMAIN,CX_SHARD_FILE,CX_SHARD_SKU,CX_NGPUS,CX_TS,CX_TOPO,CX_SCOPE,CX_TRANSPORT,CX_SCALE_UP_TRANSPORT,CX_SCALE_OUT_TRANSPORT,CX_MODE,CX_PHASE,CX_ROUTING,CX_EPLB,CX_CASE_ID,CX_SUITE,CX_WORKLOAD_NAME,CX_QUALIFICATION_INDEX,CX_VERSION,CX_HIDDEN,CX_TOPK,CX_EXPERTS,CX_TOKENS_LADDER,CX_CANONICAL,CX_ITERS,CX_TRIALS,CX_WARMUP,CX_SAMPLES_PER_POINT,CX_WARMUP_SEMANTICS,CX_SEED,CX_RUN_TIMEOUT,CX_NCCL_HOME,CX_ALLOW_MNNVL,CX_ATTEMPT_ID,CX_RUNTIME_MARKER,CX_MORI_KERNEL_TYPE,CX_WORKLOAD_DIR,CX_BACKEND_CACHE_ROOT,CX_BACKEND_CACHE_SENTINEL_SHA256,CX_BACKEND_SOURCE_ROOT,CX_AUDIT_SALT,CX_SOCKET_IFNAME,CX_RDMA_DEVICES,CX_IB_GID_INDEX,CX_RDMA_SERVICE_LEVEL,CX_RDMA_TRAFFIC_CLASS,CX_RDMA_LINK_LAYER,MASTER_ADDR,MASTER_PORT,RANK,WORLD_SIZE,LOCAL_RANK,LOCAL_WORLD_SIZE,NCCL_NET,NCCL_SOCKET_IFNAME,GLOO_SOCKET_IFNAME,NCCL_IB_HCA,NCCL_IB_GID_INDEX,NCCL_IB_SL,NVSHMEM_DISABLE_IB,NVSHMEM_ENABLE_NIC_PE_MAPPING,NVSHMEM_HCA_LIST,NVSHMEM_IB_GID_INDEX,NVSHMEM_IB_SL,NVSHMEM_IB_ENABLE_IBGDA,NVSHMEM_IBGDA_NIC_HANDLER,EP_NIC_NAME,EP_OVERRIDE_RDMA_SL,UCCL_SOCKET_IFNAME,UCCL_IB_GID_INDEX,UCCL_IB_SL,MORI_RDMA_DEVICES,MORI_RDMA_TC,MORI_IO_TC,MORI_RDMA_SL,MORI_IO_SL,HYBRID_EP_MULTINODE,USE_NIXL,RDMA_CORE_HOME,DEEPEP_HYBRID_BUILD_MODE,NCCL_CUMEM_ENABLE,NCCL_MNNVL_ENABLE,MC_FORCE_MNNVL,MORI_DISABLE_AUTO_XGMI,MORI_ENABLE_SDMA,MORI_APP_LOG_LEVEL,MORI_SHMEM_LOG_LEVEL,MORI_IO_LOG_LEVEL,MORI_COMMIT'
}
# Host-side utility steps need only the basic login paths. They never receive
@@ -2911,7 +2890,7 @@ cx_emit_setup_failures() {
local root="$1" out_dir="$2" backend="$3" rc="$4" shard="${CX_SHARD_FILE:-}" path
local phase case_id suite workload required routing eplb ep hidden topk experts nodes
local gpn domain ladder canonical timing mode scope scale_up_transport scale_out_transport
- local warmup_semantics precision_profile
+ local warmup_semantics
local transport topology_class
local cases_file expected emitted=0 covered=0
mkdir -p "$out_dir" || return 1
@@ -2943,6 +2922,9 @@ cx_emit_setup_failures() {
cx_log "ERROR: cannot hash shard for setup-failure records"
return 1
}
+ # Setup-failure terminal outcomes carry the shard's iterable benchmark version.
+ CX_VERSION="$(python3 -c "import json,sys;print(json.load(open(sys.argv[1]))['version'])" "$path")"
+ export CX_VERSION
cases_file="$(mktemp)" || return 1
if ! python3 - "$path" > "$cases_file" <<'PY'
import json, sys
@@ -2959,7 +2941,6 @@ for case in cases:
case["transport"], case["topology_class"], case["ladder"],
case["warmup_semantics"],
"1" if case["canonical"] else "", case["timing"],
- case.get("precision_profile") or "",
)
print("|".join(map(str, fields)))
PY
@@ -2972,7 +2953,7 @@ PY
[ "$expected" -gt 0 ] || { rm -f "$cases_file"; unset CX_FAILURE_MODE; return 1; }
while IFS='|' read -r phase mode case_id suite workload routing eplb ep hidden topk \
experts nodes gpn domain scope scale_up_transport scale_out_transport transport \
- topology_class ladder warmup_semantics canonical timing precision_profile; do
+ topology_class ladder warmup_semantics canonical timing; do
export CX_CASE_ID="$case_id" CX_SUITE="$suite" CX_WORKLOAD_NAME="$workload"
export CX_ROUTING="$routing" CX_EPLB="$eplb"
export CX_EP="$ep" CX_NGPUS="$ep" CX_HIDDEN="$hidden" CX_TOPK="$topk" CX_EXPERTS="$experts"
@@ -2982,7 +2963,6 @@ PY
export CX_SCALE_OUT_TRANSPORT="$scale_out_transport"
export CX_TRANSPORT="$transport" CX_TOPO="$topology_class"
export CX_TOKENS_LADDER="$ladder" CX_CANONICAL="$canonical"
- export CX_PRECISION_PROFILE="$precision_profile"
export CX_WARMUP_SEMANTICS="$warmup_semantics"
IFS=: read -r CX_ITERS CX_TRIALS CX_WARMUP <<< "$timing"
export CX_ITERS CX_TRIALS CX_WARMUP CX_SAMPLES_PER_POINT="$((CX_ITERS * CX_TRIALS))"
@@ -3009,7 +2989,6 @@ cx_run_distributed_shard() {
local ph mode routing eplb hidden topk experts ladder suite workload
local canonical case_id ep timing case_iters case_trials case_warmup case_stem
local scope scale_up_transport scale_out_transport transport topology_class nodes gpn domain
- local precision_profile
local workload_dir workload_ladder workload_log stage_rc attempt_tag out failure_out
local runtime_log run_rc expected_out case_ok summary_log
local -a container_args workload_args ep_args
@@ -3059,6 +3038,12 @@ cx_run_distributed_shard() {
cases_file="$(mktemp)" || return 1
local shard="${CX_SHARD_FILE:-}"
[ -z "$shard" ] || [ -f "$shard" ] || shard="$CX_DIR/$shard"
+ # Iterable benchmark version is a shard-level scalar; export it once so the
+ # harness copies it verbatim into every emitted result for this shard.
+ if [ -n "$shard" ] && [ -f "$shard" ]; then
+ CX_VERSION="$(python3 -c "import json,sys;print(json.load(open(sys.argv[1]))['version'])" "$shard")"
+ export CX_VERSION
+ fi
if [ -n "$shard" ]; then
if [ ! -f "$shard" ] || ! python3 - "$shard" > "$cases_file" <<'PY'
import json
@@ -3077,7 +3062,6 @@ for case in cases:
get("timing", "8:64:32"), get("nodes"), get("gpus_per_node"),
get("scale_up_domain"), get("scope"), get("scale_up_transport"),
get("scale_out_transport"), get("transport"), get("topology_class"),
- get("precision_profile"),
)
print("|".join(fields))
PY
@@ -3092,13 +3076,12 @@ PY
"${CX_EPLB:-}" "${CX_HIDDEN:-7168}" "${CX_TOPK:-8}" "${CX_EXPERTS:-256}" \
"${CX_TOKENS_LADDER:-}" "${CX_SUITE:-}" "${CX_WORKLOAD_NAME:-}" \
"${CX_CANONICAL:-}" "${CX_CASE_ID:-}" \
- "${CX_PRECISION_PROFILE:-}" \
"${CX_ITERS:-8}" "${CX_TRIALS:-64}" "${CX_WARMUP:-32}" \
"${CX_SCOPE:-scale-up}" \
"${CX_SCALE_UP_TRANSPORT:-unknown}" "${CX_SCALE_OUT_TRANSPORT:-}" \
"${CX_TRANSPORT:-unknown}" "${CX_TOPO:-manual}"
for phase in $phases; do
- (IFS='|'; printf '%s\n' "$phase|${CX_MODE:-normal}|${CX_ROUTING:-uniform}|${CX_EPLB:-}|${CX_HIDDEN:-7168}|${CX_TOPK:-8}|${CX_EXPERTS:-256}|${CX_TOKENS_LADDER:-}|${CX_SUITE:-}|${CX_WORKLOAD_NAME:-}|${CX_CANONICAL:-}|${CX_CASE_ID:-}|$NGPUS|${CX_ITERS:-8}:${CX_TRIALS:-64}:${CX_WARMUP:-32}|$NODES|$GPN|$SCALE_UP_DOMAIN|${CX_SCOPE:-scale-up}|${CX_SCALE_UP_TRANSPORT:-unknown}|${CX_SCALE_OUT_TRANSPORT:-}|${CX_TRANSPORT:-unknown}|${CX_TOPO:-manual}|${CX_PRECISION_PROFILE:-}")
+ (IFS='|'; printf '%s\n' "$phase|${CX_MODE:-normal}|${CX_ROUTING:-uniform}|${CX_EPLB:-}|${CX_HIDDEN:-7168}|${CX_TOPK:-8}|${CX_EXPERTS:-256}|${CX_TOKENS_LADDER:-}|${CX_SUITE:-}|${CX_WORKLOAD_NAME:-}|${CX_CANONICAL:-}|${CX_CASE_ID:-}|$NGPUS|${CX_ITERS:-8}:${CX_TRIALS:-64}:${CX_WARMUP:-32}|$NODES|$GPN|$SCALE_UP_DOMAIN|${CX_SCOPE:-scale-up}|${CX_SCALE_UP_TRANSPORT:-unknown}|${CX_SCALE_OUT_TRANSPORT:-}|${CX_TRANSPORT:-unknown}|${CX_TOPO:-manual}")
done > "$cases_file"
fi
expected_cases="$(wc -l < "$cases_file" | tr -d ' ')"
@@ -3107,7 +3090,7 @@ PY
while IFS='|' read -r ph mode routing eplb hidden topk experts ladder suite workload \
canonical case_id ep timing nodes gpn domain scope scale_up_transport \
- scale_out_transport transport topology_class precision_profile; do
+ scale_out_transport transport topology_class; do
[ -n "$ph" ] || continue
ci=$((ci + 1))
case_stem="${RUNNER}_${CX_BENCH}_${ph}_${TS}-c$(printf '%03d' "$ci")"
@@ -3119,7 +3102,6 @@ PY
export CX_MODE="$mode" CX_PHASE="$ph" CX_CASE_ID="$case_id" CX_SUITE="$suite"
export CX_WORKLOAD_NAME="$workload"
export CX_CANONICAL="$canonical" CX_EP="$ep"
- export CX_PRECISION_PROFILE="$precision_profile"
export CX_ROUTING="$routing" CX_EPLB="$eplb" CX_TOKENS_LADDER="$ladder"
export CX_HIDDEN="$hidden" CX_TOPK="$topk" CX_EXPERTS="$experts"
export CX_NODES="$nodes" CX_GPUS_PER_NODE="$gpn" CX_SCALE_UP_DOMAIN="$domain"
@@ -3168,7 +3150,6 @@ PY
fi
ep_args=(--backend "$CX_BENCH" --mode "$mode" --phase "$ph" --routing "$routing"
- --precision-profile "$precision_profile"
--gpus-per-node "$gpn" --scale-up-domain "$domain" --scope "$scope"
--scale-up-transport "$scale_up_transport" --scale-out-transport "$scale_out_transport"
--tokens-ladder "$ladder" --hidden "$hidden" --topk "$topk" --experts "$experts"
@@ -3176,7 +3157,7 @@ PY
--seed "${CX_SEED:-67}" --runner "$RUNNER" --topology-class "$topology_class"
--transport "$transport" --case-id "$case_id" --suite "$suite"
--workload-name "$workload"
- --qualification-index "${CX_QUALIFICATION_INDEX:-1}")
+ --qualification-index "${CX_QUALIFICATION_INDEX:-1}" --version "${CX_VERSION:-1}")
cx_bool_enabled "$eplb" && ep_args+=(--eplb)
[ -z "$workload_dir" ] || ep_args+=(--workload-dir "$workload_dir")
export CX_ATTEMPT_ID=1
diff --git a/experimental/CollectiveX/runtime/run_in_container.sh b/experimental/CollectiveX/runtime/run_in_container.sh
index 856a904989..16b06d0704 100644
--- a/experimental/CollectiveX/runtime/run_in_container.sh
+++ b/experimental/CollectiveX/runtime/run_in_container.sh
@@ -80,7 +80,6 @@ run_ep_suite() {
cx_log "ep backend=$backend phase=$phase ngpus=$CX_NGPUS ladder='${ladder:-}'"
local out="results/${CX_RUNNER}_${backend}_${phase}_${CX_TS}.json"
local -a EPARGS=(--backend "$backend" --mode "${CX_MODE:-normal}" --phase "$phase"
- --precision-profile "${CX_PRECISION_PROFILE:-}"
--tokens-ladder "$ladder"
--hidden "${CX_HIDDEN:-7168}" --topk "${CX_TOPK:-8}" --experts "${CX_EXPERTS:-256}"
--routing "${CX_ROUTING:-uniform}" --seed "${CX_SEED:-67}" --iters "${CX_ITERS:-8}"
@@ -89,7 +88,7 @@ run_ep_suite() {
--scope "${CX_SCOPE:-scale-up}" --scale-up-transport "${CX_SCALE_UP_TRANSPORT:-unknown}"
--scale-out-transport "${CX_SCALE_OUT_TRANSPORT:-}"
--case-id "${CX_CASE_ID:-}" --suite "${CX_SUITE:-}" --workload-name "${CX_WORKLOAD_NAME:-}"
- --qualification-index "${CX_QUALIFICATION_INDEX:-1}"
+ --qualification-index "${CX_QUALIFICATION_INDEX:-1}" --version "${CX_VERSION:-1}"
--runner "$CX_RUNNER" --topology-class "$CX_TOPO" --transport "$CX_TRANSPORT"
--out "$out")
[ -n "${CX_WORKLOAD_DIR:-}" ] && EPARGS+=(--workload-dir "$CX_WORKLOAD_DIR")
@@ -1012,7 +1011,11 @@ if [ -n "${CX_SHARD_FILE:-}" ]; then
# SHARD/SWEEP mode (collectivex-sweep.yml): run EVERY case of this shard in THIS one allocation.
# All cases share (sku, backend, nodes), so backend preparation is paid once and cached.
ncases="$(python3 -c "import json;print(len(json.load(open('$CX_SHARD_FILE'))['cases']))")"
- cx_log "SHARD mode: $ncases case(s) in one allocation (shard=$CX_SHARD_FILE)"
+ # The iterable benchmark version is a shard-level scalar (identical for every case);
+ # export it once so run_ep copies it verbatim into each emitted result.
+ CX_VERSION="$(python3 -c "import json;print(json.load(open('$CX_SHARD_FILE'))['version'])")"
+ export CX_VERSION
+ cx_log "SHARD mode: $ncases case(s) in one allocation (shard=$CX_SHARD_FILE, version=$CX_VERSION)"
_cx_ts_base="$CX_TS" # per-case CX_TS suffix below keeps each case's result file UNIQUE (else
# cases sharing backend+phase overwrite each other at the same timestamp).
ci=0
@@ -1033,7 +1036,6 @@ env = {
"CX_EP": g("ep", "1"),
"CX_EPLB": "1" if c.get("eplb") else "",
"CX_CASE_ID": g("case_id"), "CX_SUITE": g("suite"), "CX_WORKLOAD_NAME": g("workload"),
- "CX_PRECISION_PROFILE": g("precision_profile"),
"CX_HIDDEN": g("hidden"), "CX_TOPK": g("topk"), "CX_EXPERTS": g("experts"),
"CX_TOKENS_LADDER": g("ladder"), "CX_CANONICAL": ("1" if c.get("canonical") else ""),
"CX_NODES": g("nodes"), "CX_GPUS_PER_NODE": g("gpus_per_node"),
diff --git a/experimental/CollectiveX/schemas/raw-case-v1.schema.json b/experimental/CollectiveX/schemas/raw-case-v1.schema.json
index 5f6c83977f..279c6da909 100644
--- a/experimental/CollectiveX/schemas/raw-case-v1.schema.json
+++ b/experimental/CollectiveX/schemas/raw-case-v1.schema.json
@@ -57,7 +57,7 @@
"num_of_tokens_per_chunk_preprocessing_api": {"minimum":0,"type":"integer"},
"num_of_tokens_per_group_combine_api": {"minimum":0,"type":"integer"},
"pad_multiple": {"minimum":0,"type":"integer"},
- "token_data_type": {"enum":["UINT8","UINT16"]}
+ "token_data_type": {"const":"UINT16"}
},
"required": [
"backward_combine_api","device_side_sync_combine_api","device_side_sync_dispatch_api",
@@ -120,7 +120,6 @@
"activation_generator": {"const":"collectivex-activation-counter-v4"},
"activation_profile": {"const":"canonical-counter-source-v4"},
"combine_dtype": {"const":"bf16"},
- "combine_quant_mode": {"const":"none"},
"combine_semantics": {"enum":["activation-only","gate-weighted"]},
"component_order_contract": {"const":"qualification-hash-rotated-components-v1"},
"conditioning_contract": {"const":"fixed-phase-ramp-8-roundtrips-v1"},
@@ -132,7 +131,6 @@
"eplb_reference_tokens_per_rank": {"const":2048},
"mode": {"enum":["normal","low-latency"]},
"oracle_contract": {"enum":["expert-specific-transform-v1","expert-assignment-transform-v1"]},
- "oracle_tolerances": {"const":"codec-specific-combine-v1"},
"payload_unit": {"enum":["token-rank","token-expert"]},
"placement": {"const":"packed"},
"percentile_method": {"const":"nearest-rank"},
@@ -144,10 +142,10 @@
"source_identity_contract": {"const":"bounded-sign-bit-source-v1"}
},
"required": [
- "activation_generator","activation_profile","combine_dtype","combine_quant_mode",
+ "activation_generator","activation_profile","combine_dtype",
"combine_semantics","component_order_contract","conditioning_contract","contract",
"correctness_scope","dtype","eplb_planner","eplb_redundant_experts",
- "eplb_reference_tokens_per_rank","mode","oracle_contract","oracle_tolerances",
+ "eplb_reference_tokens_per_rank","mode","oracle_contract",
"payload_unit","placement","percentile_method","rank_reduction","resource_mode",
"routing_generator","sampling_contract","seed","source_identity_contract"
],
@@ -170,7 +168,7 @@
"required": ["combine_values","counts","metadata","multiplicity","payload","source_set","weights"],
"type": "object"
},
- "atol": {"enum":[0.02,0.04]},
+ "atol": {"const":0.02},
"combine_weight_semantics": {"enum":["unweighted-rank-sum","gate-weighted-sum"]},
"contract": {"enum":["expert-specific-transform-v1","expert-assignment-transform-v1"]},
"dispatch_sha256": {"$ref":"#/$defs/nullable_sha256"},
@@ -182,7 +180,7 @@
"ordering_contract": {"minLength":1,"type":"string"},
"passed": {"type":"boolean"},
"receive_count": {"minimum":0,"type":"integer"},
- "rtol": {"enum":[0.05,0.08]}
+ "rtol": {"const":0.05}
},
"required": [
"atol",
@@ -463,8 +461,7 @@
"required": ["artifact","execution_id","job","repo","run_attempt","run_id","runner","source_sha"],
"type": "object"
},
- "allocation_id": {"pattern":"^cxallocation-v1-[0-9a-f]{64}$","type":"string"},
- "attempt_id": {"pattern":"^cxattempt-v1-[0-9a-f]{64}$","type":"string"},
+ "attempt_id": {"maxLength":180,"pattern":"^[a-z0-9][a-z0-9_.-]*$","type":"string"},
"attempt_ordinal": {"minimum":1,"type":"integer"},
"case_factors": {
"additionalProperties": false,
@@ -476,38 +473,14 @@
"required": ["case","profile","sku"],
"type": "object"
},
- "case_id": {"pattern":"^cxcase-v1-[0-9a-f]{64}$","type":"string"},
- "series_factors": {
- "additionalProperties": false,
- "properties": {
- "backend": {"maxLength":128,"pattern":"^[a-z0-9][a-z0-9_.-]*$","type":"string"},
- "case_id": {"pattern":"^cxcase-v1-[0-9a-f]{64}$","type":"string"},
- "image_digest": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]},
- "source_sha": {"oneOf":[{"type":"null"},{"pattern":"^[0-9a-f]{40}$","type":"string"}]},
- "squash_sha256": {"oneOf":[{"type":"null"},{"pattern":"^[0-9a-f]{64}$","type":"string"}]},
- "workload_id": {"pattern":"^cxwork-v1-[0-9a-f]{64}$","type":"string"}
- },
- "required": [
- "backend",
- "case_id",
- "image_digest",
- "source_sha",
- "squash_sha256",
- "workload_id"
- ],
- "type": "object"
- },
- "series_id": {"pattern":"^cxseries-v1-[0-9a-f]{64}$","type":"string"}
+ "case_id": {"maxLength":180,"pattern":"^[a-z0-9][a-z0-9_.-]*$","type":"string"}
},
"required": [
"allocation_factors",
- "allocation_id",
"attempt_id",
"attempt_ordinal",
"case_factors",
- "case_id",
- "series_factors",
- "series_id"
+ "case_id"
],
"type": "object"
},
@@ -796,7 +769,6 @@
"required": ["contract","max_relative_error","passed","rank_evidence","scope"],
"type": "object"
},
- "evidence_id": {"pattern":"^cxevidence-v1-[0-9a-f]{64}$","type":"string"},
"global_tokens": {"minimum":1,"type":"integer"},
"byte_provenance": {
"additionalProperties": false,
@@ -809,7 +781,7 @@
"required": ["combine","dispatch","roundtrip","stage"],
"type": "object"
},
- "point_id": {"pattern":"^cxpoint-v1-[0-9a-f]{64}$","type":"string"},
+ "point_id": {"maxLength":180,"pattern":"^[a-z0-9][a-z0-9_.-]*$","type":"string"},
"receive": {
"additionalProperties": false,
"properties": {
@@ -935,7 +907,6 @@
"components",
"correctness",
"byte_provenance",
- "evidence_id",
"global_tokens",
"point_id",
"receive",
@@ -1160,6 +1131,7 @@
],
"type": "object"
},
+ "version": {"minimum":1,"type":"integer"},
"workload": {
"additionalProperties": false,
"properties": {
@@ -1238,6 +1210,7 @@
"sample_artifact",
"schema_version",
"topology",
+ "version",
"workload"
],
"title": "CollectiveX raw case attempt v1",
diff --git a/experimental/CollectiveX/schemas/samples-v1.schema.json b/experimental/CollectiveX/schemas/samples-v1.schema.json
index e1c34fda51..496dcea998 100644
--- a/experimental/CollectiveX/schemas/samples-v1.schema.json
+++ b/experimental/CollectiveX/schemas/samples-v1.schema.json
@@ -4,9 +4,8 @@
"title": "CollectiveX exact private samples v1",
"type": "object",
"additionalProperties": false,
- "required": ["allocation_id","attempt_id","case_id","format","points","sampling","schema_version","series_id"],
+ "required": ["attempt_id","case_id","format","points","sampling","schema_version"],
"properties": {
- "allocation_id": {"$ref": "#/$defs/allocationId"},
"attempt_id": {"$ref": "#/$defs/attemptId"},
"case_id": {"$ref": "#/$defs/caseId"},
"format": {"const": "collectivex.samples.v1"},
@@ -16,7 +15,7 @@
"items": {
"type": "object",
"additionalProperties": false,
- "required": ["components","evidence_id","point_id","sample_sha256","tokens_per_rank"],
+ "required": ["components","point_id","sample_sha256","tokens_per_rank"],
"properties": {
"components": {
"type": "object",
@@ -29,7 +28,6 @@
"stage": {"$ref":"#/$defs/component"}
}
},
- "evidence_id": {"$ref": "#/$defs/evidenceId"},
"point_id": {"$ref": "#/$defs/pointId"},
"sample_sha256": {"$ref": "#/$defs/sha256"},
"tokens_per_rank": {"type": "integer","minimum": 1}
@@ -46,17 +44,13 @@
"trials": {"const": 64}
}
},
- "schema_version": {"const": 1},
- "series_id": {"$ref": "#/$defs/seriesId"}
+ "schema_version": {"const": 1}
},
"$defs": {
"sha256": {"type": "string","pattern": "^[0-9a-f]{64}$"},
- "caseId": {"type": "string","pattern": "^cxcase-v1-[0-9a-f]{64}$"},
- "seriesId": {"type": "string","pattern": "^cxseries-v1-[0-9a-f]{64}$"},
- "pointId": {"type": "string","pattern": "^cxpoint-v1-[0-9a-f]{64}$"},
- "evidenceId": {"type": "string","pattern": "^cxevidence-v1-[0-9a-f]{64}$"},
- "allocationId": {"type": "string","pattern": "^cxallocation-v1-[0-9a-f]{64}$"},
- "attemptId": {"type": "string","pattern": "^cxattempt-v1-[0-9a-f]{64}$"},
+ "caseId": {"type": "string","pattern": "^[a-z0-9][a-z0-9_.-]*$","maxLength": 180},
+ "pointId": {"type": "string","pattern": "^[a-z0-9][a-z0-9_.-]*$","maxLength": 180},
+ "attemptId": {"type": "string","pattern": "^[a-z0-9][a-z0-9_.-]*$","maxLength": 180},
"trials": {
"type":"array",
"minItems":64,
diff --git a/experimental/CollectiveX/schemas/terminal-outcome-v1.schema.json b/experimental/CollectiveX/schemas/terminal-outcome-v1.schema.json
index 1ffa88fcae..2d4497e187 100644
--- a/experimental/CollectiveX/schemas/terminal-outcome-v1.schema.json
+++ b/experimental/CollectiveX/schemas/terminal-outcome-v1.schema.json
@@ -4,7 +4,7 @@
"title": "CollectiveX terminal outcome v1",
"type": "object",
"additionalProperties": false,
- "required": ["case","format","generated_at","identity","outcome","provenance","record_type","schema_version"],
+ "required": ["case","format","generated_at","identity","outcome","provenance","record_type","schema_version","version"],
"properties": {
"case": {"$ref": "#/$defs/case"},
"format": {"const": "collectivex.terminal.v1"},
@@ -12,10 +12,9 @@
"identity": {
"type": "object",
"additionalProperties": false,
- "required": ["allocation_factors","allocation_id","attempt_id","attempt_ordinal","case_factors","case_id"],
+ "required": ["allocation_factors","attempt_id","attempt_ordinal","case_factors","case_id"],
"properties": {
"allocation_factors": {"$ref": "#/$defs/allocationFactors"},
- "allocation_id": {"$ref": "#/$defs/allocationId"},
"attempt_id": {"$ref": "#/$defs/attemptId"},
"attempt_ordinal": {"type": "integer","minimum": 1},
"case_factors": {
@@ -60,7 +59,8 @@
}
},
"record_type": {"const": "terminal-outcome"},
- "schema_version": {"const": 1}
+ "schema_version": {"const": 1},
+ "version": {"type": "integer","minimum": 1}
},
"allOf": [
{
@@ -139,9 +139,8 @@
},
"sha256": {"type": "string","pattern": "^[0-9a-f]{64}$"},
"safeId": {"type": "string","pattern": "^[a-z0-9][a-z0-9_.-]*$","maxLength": 128},
- "caseId": {"type": "string","pattern": "^cxcase-v1-[0-9a-f]{64}$"},
- "allocationId": {"type": "string","pattern": "^cxallocation-v1-[0-9a-f]{64}$"},
- "attemptId": {"type": "string","pattern": "^cxattempt-v1-[0-9a-f]{64}$"},
+ "caseId": {"type": "string","pattern": "^[a-z0-9][a-z0-9_.-]*$","maxLength": 180},
+ "attemptId": {"type": "string","pattern": "^[a-z0-9][a-z0-9_.-]*$","maxLength": 180},
"nullableText": {"oneOf": [{"type": "null"},{"type": "string","minLength": 1}]},
"caseProfile": {
"type":"object",
@@ -170,12 +169,11 @@
}}
}
],
- "required":["activation_generator","activation_profile","combine_dtype","combine_quant_mode","combine_semantics","component_order_contract","conditioning_contract","contract","correctness_scope","dtype","eplb_planner","eplb_redundant_experts","eplb_reference_tokens_per_rank","mode","oracle_contract","oracle_tolerances","payload_unit","placement","percentile_method","rank_reduction","resource_mode","routing_generator","sampling_contract","seed","source_identity_contract"],
+ "required":["activation_generator","activation_profile","combine_dtype","combine_semantics","component_order_contract","conditioning_contract","contract","correctness_scope","dtype","eplb_planner","eplb_redundant_experts","eplb_reference_tokens_per_rank","mode","oracle_contract","payload_unit","placement","percentile_method","rank_reduction","resource_mode","routing_generator","sampling_contract","seed","source_identity_contract"],
"properties": {
"activation_generator":{"const":"collectivex-activation-counter-v4"},
"activation_profile":{"const":"canonical-counter-source-v4"},
"combine_dtype":{"const":"bf16"},
- "combine_quant_mode":{"const":"none"},
"combine_semantics":{"enum":["activation-only","gate-weighted"]},
"component_order_contract":{"const":"qualification-hash-rotated-components-v1"},
"conditioning_contract":{"const":"fixed-phase-ramp-8-roundtrips-v1"},
@@ -187,7 +185,6 @@
"eplb_reference_tokens_per_rank":{"const":2048},
"mode":{"enum":["normal","low-latency"]},
"oracle_contract":{"enum":["expert-specific-transform-v1","expert-assignment-transform-v1"]},
- "oracle_tolerances":{"const":"codec-specific-combine-v1"},
"payload_unit":{"enum":["token-rank","token-expert"]},
"placement":{"const":"packed"},
"percentile_method":{"const":"nearest-rank"},
diff --git a/experimental/CollectiveX/summarize.py b/experimental/CollectiveX/summarize.py
index df0e1eb53f..05c281235b 100644
--- a/experimental/CollectiveX/summarize.py
+++ b/experimental/CollectiveX/summarize.py
@@ -55,20 +55,20 @@ def render(documents: list[dict], markdown: bool) -> str:
if markdown:
lines = [
"## CollectiveX EP results", "",
- "| sku | backend | suite | phase | routing | ep | outcome | T* | p50 us | p99 us |",
- "|---|---|---|---|---|--:|---|--:|--:|--:|",
+ "| ver | sku | backend | suite | phase | routing | ep | outcome | T* | p50 us | p99 us |",
+ "|--:|---|---|---|---|---|--:|---|--:|--:|--:|",
]
for document in documents:
sku, suite, routing, phase, eplb, ep = _identity(document)
backend = document["case"]["backend"]
token, p50, p99 = _headline(document)
lines.append(
- f"| {sku} | `{backend}` | {suite} | {phase} | "
+ f"| {document['version']} | {sku} | `{backend}` | {suite} | {phase} | "
f"{routing}{'+eplb' if eplb else ''} | {ep} | "
f"{document['outcome']['status']} | {token} | {p50} | {p99} |"
)
if not documents:
- lines.append("\n> No valid native v1 outcome documents found.")
+ lines.append("\n> No valid native outcome documents found.")
return "\n".join(lines)
lines = ["CollectiveX EP results", "======================"]
for document in documents:
@@ -76,7 +76,7 @@ def render(documents: list[dict], markdown: bool) -> str:
backend = document["case"]["backend"]
token, _, p99 = _headline(document)
lines.append(
- f" {sku:<10} {backend:<16} {suite:<13} {phase:<7} "
+ f" v{document['version']} {sku:<10} {backend:<16} {suite:<13} {phase:<7} "
f"{routing}{'+eplb' if eplb else ''} ep{ep} "
f"{document['outcome']['status']} T={token} roundtrip_p99_us={p99}"
)
diff --git a/experimental/CollectiveX/sweep_matrix.py b/experimental/CollectiveX/sweep_matrix.py
index f36aef9061..cca8668d38 100644
--- a/experimental/CollectiveX/sweep_matrix.py
+++ b/experimental/CollectiveX/sweep_matrix.py
@@ -37,9 +37,9 @@
f"{ep_harness.TIMED_ITERS_PER_TRIAL}:{ep_harness.TRIALS_PER_POINT}:"
f"{ep_harness.WARMUP_ITERS_PER_TRIAL}"
)
-V1_WORKLOAD = ("deepseek-v3-v1", 7168, 8, 256)
+V1_WORKLOAD = ("deepseek-v3", 7168, 8, 256)
V1_SUITE_CONTRACTS = {
- "ep-core-v1": {
+ "ep-core": {
"mode": "normal",
"coordinates": {
("normal", "decode", "uniform", False),
@@ -50,7 +50,7 @@
"prefill": (256, 512),
},
},
- "ep-low-latency-v1": {
+ "ep-low-latency": {
"mode": "low-latency",
"backends": {"deepep", "uccl"},
"coordinates": {("low-latency", "decode", "uniform", False)},
@@ -141,7 +141,7 @@ def validate_config_documents(
"""Reject configuration that is ambiguous, unused, or outside the v1 grid."""
_fields(
suites_document, "configs/suites.yaml",
- {"schema_version", "suites"}, {"schema_version", "suites"},
+ {"schema_version", "version", "suites"}, {"schema_version", "version", "suites"},
)
_fields(
workloads, "configs/workloads.yaml",
@@ -149,6 +149,10 @@ def validate_config_documents(
)
if type(suites_document["schema_version"]) is not int or suites_document["schema_version"] != 1:
raise SystemExit("configs/suites.yaml schema_version must be integer 1")
+ # The iterable benchmark version is independent of the JSON schema_version: it
+ # is any positive integer the operator bumps to mark a new, incomparable grid.
+ if type(suites_document["version"]) is not int or suites_document["version"] < 1:
+ raise SystemExit("configs/suites.yaml version must be a positive integer")
if type(workloads["schema_version"]) is not int or workloads["schema_version"] != 1:
raise SystemExit("configs/workloads.yaml schema_version must be integer 1")
registry: dict[str, dict[str, Any]] = {}
@@ -524,6 +528,7 @@ def add_case(
return {
"format": "collectivex.matrix.v1",
"schema_version": 1,
+ "version": suites_document["version"],
"requested_cases": requested_cases,
"include": include,
}
@@ -590,7 +595,7 @@ def validate_shard_control(
if sku not in cap.PLATFORMS or backend not in cap.SWEEP_BACKENDS:
raise MatrixError("shard platform/backend is not registered")
top_fields = {
- "schema_version", "id", "sku", "backend", "nodes", "n", "cases",
+ "schema_version", "version", "id", "sku", "backend", "nodes", "n", "cases",
"execution_weight",
}
if (
@@ -599,6 +604,8 @@ def validate_shard_control(
or shard["schema_version"] != 1
):
raise MatrixError("shard fields or schema version differ from v1 contract")
+ if type(shard.get("version")) is not int or shard["version"] < 1:
+ raise MatrixError("shard benchmark version must be a positive integer")
if not isinstance(shard.get("id"), str) or not IDENTIFIER.fullmatch(shard["id"]):
raise MatrixError("shard has invalid id")
for field, expected in (("sku", sku), ("backend", backend)):
@@ -631,7 +638,7 @@ def validate_shard_control(
f"missing={sorted(required - fields)}, extra={sorted(fields - required)}"
)
case_id = case["case_id"]
- if not identity.is_typed_id(case_id, "case"):
+ if not identity.is_case_id(case_id):
raise MatrixError(f"case {index} has invalid case_id")
if case_id in seen:
raise MatrixError(f"duplicate case_id {case_id}")
@@ -715,7 +722,7 @@ def validate_shard_control(
def validate_matrix_document(document: Any) -> dict[str, Any]:
"""Validate the complete requested grid and its runnable shard partition."""
if not isinstance(document, dict) or set(document) != {
- "format", "schema_version", "requested_cases", "include"
+ "format", "schema_version", "version", "requested_cases", "include"
}:
raise MatrixError("matrix fields differ from the v1 contract")
if (
@@ -724,6 +731,8 @@ def validate_matrix_document(document: Any) -> dict[str, Any]:
or document["schema_version"] != 1
):
raise MatrixError("matrix format/schema differs from v1")
+ if type(document["version"]) is not int or document["version"] < 1:
+ raise MatrixError("matrix benchmark version must be a positive integer")
requested = document["requested_cases"]
include = document["include"]
if not isinstance(requested, list) or not requested:
@@ -768,6 +777,7 @@ def validate_matrix_document(document: Any) -> dict[str, Any]:
validate_shard_control(
{
"schema_version": 1,
+ "version": document["version"],
"id": "requested-case",
"sku": sku,
"backend": backend,
@@ -869,6 +879,7 @@ def extract_shard(
cases = [requested[case_id]["case"] for case_id in source["case_ids"]]
control = {
"schema_version": 1,
+ "version": document["version"],
"id": source.get("id"),
"sku": source.get("sku"),
"backend": source.get("backend"),
@@ -933,7 +944,7 @@ def emit_unsupported(
"profile": identity.profile_for_case(case),
"sku": wrapper["sku"],
}
- case_id = identity.digest("case", case_factors)
+ case_id = identity.case_id_from_factors(case_factors)
if case_id != scheduled["case_id"]:
raise MatrixError(f"unsupported case identity differs for {scheduled['case_id']}")
record = contracts.make_terminal_document(
@@ -949,6 +960,7 @@ def emit_unsupported(
return_code=5,
source="matrix-capability-resolver",
status="unsupported",
+ version=document["version"],
expected_case_id=case_id,
)
path = destination / f"unsupported_{case_id}.json"
@@ -966,59 +978,6 @@ def emit_unsupported(
return written
-def frontend_catalog(matrix: dict[str, Any]) -> dict[str, Any]:
- """Project the validated requested graph into a compact frontend test fixture."""
- document = validate_matrix_document(matrix)
- matrix_bytes = contracts.canonical_json_bytes(document) + b"\n"
- cases = []
- for wrapper in document["requested_cases"]:
- case = wrapper["case"]
- cases.append({
- "backend": case["backend"],
- "backend_generation": None,
- "case_id": case["case_id"],
- "disposition": wrapper["disposition"],
- "eplb": case["eplb"],
- "label": (
- f"{wrapper['sku'].upper()} / {case['backend']} / EP{case['ep']} / "
- f"{case['mode']} / {case['phase']} / {case['routing']}"
- ),
- "mode": case["mode"],
- "phase": case["phase"],
- "reason": wrapper["reason"],
- "required": True,
- "resource": {
- "mode": "fixed-profile",
- "profile": None,
- "comm_units_kind": None,
- "configured_units": None,
- },
- "routing": case["routing"],
- "sku": wrapper["sku"],
- "suite": case["suite"],
- "topology": {
- "ep_size": case["ep"],
- **{field: case[field] for field in TOPOLOGY_FIELDS},
- },
- "workload": case["workload"],
- "points": [
- {
- "global_tokens": int(token) * case["ep"],
- "tokens_per_rank": int(token),
- }
- for token in case["ladder"].split()
- ],
- })
- return {
- "case_count": len(cases),
- "format": "collectivex.frontend-catalog.v1",
- "matrix_sha256": hashlib.sha256(matrix_bytes).hexdigest(),
- "point_count": sum(len(case["points"]) for case in cases),
- "schema_version": 1,
- "cases": cases,
- }
-
-
def main() -> int:
parser = argparse.ArgumentParser(description="CollectiveX matrix resolver")
parser.add_argument("--suites", default="all", help="'all' or comma-list of suites")
@@ -1040,7 +999,6 @@ def main() -> int:
parser.add_argument("--validate-control", default="", metavar="SHARD")
parser.add_argument("--emit-unsupported-from", default="", metavar="MATRIX")
parser.add_argument("--out-dir", default="")
- parser.add_argument("--frontend-catalog", action="store_true")
parser.add_argument("--shard-id", default="")
parser.add_argument("--expect-sku", default="")
parser.add_argument("--expect-backend", default="")
@@ -1110,10 +1068,9 @@ def main() -> int:
validate_matrix_document(matrix)
except MatrixError as exc:
parser.error(str(exc))
- output_document = frontend_catalog(matrix) if args.frontend_catalog else matrix
if args.out:
with open(args.out, "w") as fh:
- json.dump(output_document, fh, sort_keys=True, separators=(",", ":"))
+ json.dump(matrix, fh, sort_keys=True, separators=(",", ":"))
fh.write("\n")
runnable = sum(
item["disposition"] == "runnable" for item in matrix["requested_cases"]
@@ -1124,7 +1081,7 @@ def main() -> int:
f"{runnable} runnable and {unsupported} unsupported cases",
file=sys.stderr,
)
- print(json.dumps(output_document))
+ print(json.dumps(matrix))
return 0
diff --git a/experimental/CollectiveX/tests/test_contracts.py b/experimental/CollectiveX/tests/test_contracts.py
new file mode 100644
index 0000000000..08b7f03c42
--- /dev/null
+++ b/experimental/CollectiveX/tests/test_contracts.py
@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+"""Focused checks for the neutral JSON schemas and strict JSON loader."""
+from __future__ import annotations
+
+import copy
+import json
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+import jsonschema
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT))
+
+import contracts # noqa: E402
+
+
+def schema(name):
+ return json.loads((ROOT / "schemas" / name).read_text())
+
+
+class ContractTests(unittest.TestCase):
+ def test_only_neutral_schemas_exist_and_are_valid(self):
+ paths = sorted((ROOT / "schemas").glob("*.schema.json"))
+ self.assertEqual(
+ {path.name for path in paths},
+ {"raw-case-v1.schema.json", "samples-v1.schema.json", "terminal-outcome-v1.schema.json"},
+ )
+ for path in paths:
+ document = json.loads(path.read_text())
+ jsonschema.Draft202012Validator.check_schema(document)
+ self.assertFalse(document["additionalProperties"])
+
+ def test_raw_and_terminal_share_the_scheduled_case_contract(self):
+ raw = schema("raw-case-v1.schema.json")["$defs"]["scheduled_case"]
+ terminal = schema("terminal-outcome-v1.schema.json")["$defs"]["case"]
+ self.assertEqual(set(raw["properties"]), set(terminal["properties"]))
+ self.assertEqual(set(raw["required"]), set(terminal["required"]))
+ self.assertFalse(raw["additionalProperties"])
+ self.assertFalse(terminal["additionalProperties"])
+ self.assertNotIn("precision_profile", raw["properties"])
+
+ def test_sample_component_is_exactly_512_or_unavailable(self):
+ samples = schema("samples-v1.schema.json")
+ validator = jsonschema.Draft202012Validator({
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$defs": samples["$defs"], "$ref": "#/$defs/component",
+ })
+ measured = {
+ "availability": "measured", "sample_count": 512,
+ "trials": [[1.0] * 8 for _ in range(64)],
+ }
+ validator.validate(measured)
+ for broken in (
+ {**measured, "sample_count": 511},
+ {**measured, "trials": measured["trials"][:-1]},
+ ):
+ with self.assertRaises(jsonschema.ValidationError):
+ validator.validate(broken)
+ validator.validate({"availability": "unavailable", "sample_count": 0, "trials": None})
+
+ def test_strict_loader_rejects_duplicate_keys_and_nonfinite_numbers(self):
+ with tempfile.TemporaryDirectory() as temporary:
+ path = Path(temporary) / "bad.json"
+ for payload in ('{"x":1,"x":2}', '{"x":NaN}'):
+ path.write_text(payload)
+ with self.assertRaises(contracts.ContractError):
+ contracts.strict_json_load(path)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/experimental/CollectiveX/tests/test_ep_backend.py b/experimental/CollectiveX/tests/test_ep_backend.py
index 5141f430a6..503ac3e1c7 100644
--- a/experimental/CollectiveX/tests/test_ep_backend.py
+++ b/experimental/CollectiveX/tests/test_ep_backend.py
@@ -1,17 +1,5 @@
#!/usr/bin/env python3
-"""Unit tests for the EPBackend abstract base class (torch-free).
-
-These exercise the shared lifecycle/plumbing the driver relies on — make_inputs
-sizing and early-returns, the timed_components matrix, the two Pass-2 timing
-branch rules, mode validation, and __init_subclass__ name enforcement — against a
-minimal in-process fake backend.
-
-torch is never installed in this environment, so the few base methods that
-``import torch`` locally are driven with a stub module injected into
-``sys.modules``; make_inputs is kept torch-free by overriding
-``_build_rank_inputs`` on the fake so no real routing/activation tensors are
-materialised.
-"""
+"""Small torch-free smoke tests for the shared EP backend lifecycle."""
from __future__ import annotations
import sys
@@ -21,282 +9,113 @@
from unittest import mock
ROOT = Path(__file__).resolve().parents[1]
-BENCH = ROOT / "bench"
-sys.path.insert(0, str(ROOT))
-sys.path.insert(0, str(BENCH))
+sys.path[:0] = [str(ROOT), str(ROOT / "bench")]
import ep_backend # noqa: E402
from ep_backend import EPBackend, RankInputs # noqa: E402
-def _args(**over):
- """A minimal args namespace with the fields make_inputs / __init__ read."""
- base = dict(
- experts=8,
- phase="decode",
- tokens_ladder="",
- routing="uniform",
- seed=0,
- hidden=16,
- topk=2,
- mode="normal",
- workload_dir="",
+def args(**updates):
+ values = dict(
+ experts=8, phase="decode", tokens_ladder="", routing="uniform", seed=0,
+ hidden=16, topk=2, mode="normal", workload_dir="",
)
- base.update(over)
- return types.SimpleNamespace(**base)
+ values.update(updates)
+ return types.SimpleNamespace(**values)
class FakeBackend(EPBackend):
- """Concrete no-op backend: records transport calls, never touches a device."""
-
name = "fake"
- def __init__(self, args, rank=0, world_size=1, local_rank=0, device="cpu", cap=None):
- super().__init__(args, rank, world_size, local_rank, device)
- self._cap = cap
- self.calls = []
- self.built = []
+ def __init__(self, options, *, cap=None, world_size=1):
+ super().__init__(options, 0, world_size, 0, "cpu")
+ self.cap = cap
+ self.calls: list[str] = []
- # ---- transport contract: recording no-ops -----------------------------------
def create_buffer(self, spec):
- self.calls.append("create_buffer")
return None
- def dispatch(self, p):
+ def dispatch(self, problem):
self.calls.append("dispatch")
- return types.SimpleNamespace()
+ return object()
- def stage(self, p, h):
+ def stage(self, problem, handle):
self.calls.append("stage")
- return None
- def combine(self, p, h):
+ def combine(self, problem, handle):
self.calls.append("combine")
- return None
- def recv_tokens(self, h):
- self.calls.append("recv_tokens")
+ def recv_tokens(self, handle):
return 0
- def inspect_dispatch(self, p, h):
- raise NotImplementedError
+ def inspect_dispatch(self, problem, handle):
+ return None
- def combine_transformed(self, p, h, transformed):
- raise NotImplementedError
+ def combine_transformed(self, problem, handle, transformed):
+ return None
- # ---- keep make_inputs torch-free --------------------------------------------
- def buffer_cap(self, args):
- return self._cap
+ def buffer_cap(self, options):
+ return self.cap
- def _build_rank_inputs(self, args, tokens_per_rank, *, canonical, retain_global):
- self.built.append((tokens_per_rank, canonical, retain_global))
- workload_id = f"wl-{tokens_per_rank}" if canonical else None
+ def _build_rank_inputs(self, options, tokens, *, canonical, retain_global):
return RankInputs(
- tokens_per_rank=tokens_per_rank,
- topk_idx=None,
- topk_weights=None,
- activations=None,
- workload_id=workload_id,
- checksums={"c": tokens_per_rank},
+ tokens_per_rank=tokens, topk_idx=None, topk_weights=None,
+ activations=None, workload_id=f"workload-{tokens}" if canonical else None,
+ checksums={"tokens": tokens},
)
-class MakeInputsTests(unittest.TestCase):
- def test_cap_below_conditioning_ladder_early_returns_rc2(self):
- # decode conditioning ladder tops out at 128; a cap of 64 cannot serve it.
- backend = FakeBackend(_args(phase="decode"), cap=64)
- spec = backend.make_inputs(backend.args)
- self.assertFalse(spec.ok)
- self.assertEqual(spec.rc, 2)
- self.assertIn("buffer cap 64", spec.message)
- self.assertEqual(backend.built, []) # no inputs materialised on the reject path
-
- def test_empty_ladder_early_returns_rc2(self):
- # "0" filters to no positive token counts -> empty ladder (cap is unbounded).
- backend = FakeBackend(_args(tokens_ladder="0"), cap=None)
- spec = backend.make_inputs(backend.args)
- self.assertFalse(spec.ok)
- self.assertEqual(spec.rc, 2)
- self.assertIn("empty token ladder", spec.message)
- self.assertEqual(backend.built, [])
-
- def test_success_spec_sizes_and_materialises(self):
- backend = FakeBackend(
- _args(tokens_ladder="8 16", workload_dir="/w"), world_size=2, cap=None
- )
+class BackendTests(unittest.TestCase):
+ def test_input_plan_sizes_for_measured_and_conditioning_ladders(self):
+ backend = FakeBackend(args(tokens_ladder="8 16", workload_dir="/tmp"), world_size=2)
spec = backend.make_inputs(backend.args)
self.assertTrue(spec.ok)
self.assertEqual(spec.ladder, [8, 16])
- self.assertEqual(spec.conditioning_ladder, [1, 2, 4, 8, 16, 32, 64, 128])
- # max tokens/rank spans measured + conditioning shapes (drives buffer sizing).
self.assertEqual(spec.max_tokens_per_rank, 128)
- self.assertEqual(spec.ep_size, 2)
- self.assertEqual(spec.experts_per_rank, 4)
+ self.assertEqual((spec.ep_size, spec.experts_per_rank), (2, 4))
self.assertEqual(sorted(spec.points), [8, 16])
- self.assertEqual(sorted(spec.conditioning_points), [1, 2, 4, 8, 16, 32, 64, 128])
- def test_success_registers_canonical_measured_workloads_only(self):
- backend = FakeBackend(
- _args(tokens_ladder="8 16", workload_dir="/w"), world_size=2, cap=None
- )
- spec = backend.make_inputs(backend.args)
- # canonical measured points register ids/checksums; conditioning shapes
- # (canonical=False -> workload_id None) never do.
- self.assertEqual(spec.loaded_workload_ids, ["wl-8", "wl-16"])
- self.assertEqual(spec.loaded_checksums["wl-8"], {"c": 8})
- # measured points are canonical + retain the global trace; conditioning is neither.
- self.assertIn((8, True, True), backend.built)
- self.assertIn((1, False, False), backend.built)
-
- def test_non_canonical_run_registers_no_workload_ids(self):
- backend = FakeBackend(_args(tokens_ladder="8 16"), world_size=2, cap=None)
- spec = backend.make_inputs(backend.args) # workload_dir "" -> non-canonical
- self.assertEqual(spec.loaded_workload_ids, [])
- self.assertIn((8, False, True), backend.built)
-
-
-class TimedComponentsTests(unittest.TestCase):
- def _backend(self, **flags):
- backend = FakeBackend(_args())
- for key, value in flags.items():
- setattr(backend, key, value)
- return backend
-
- def test_default_measures_roundtrip_dispatch_combine(self):
- self.assertEqual(
- self._backend().timed_components(), ["roundtrip", "dispatch", "combine"]
- )
-
- def test_stage_device_work_adds_stage(self):
+ def test_invalid_ladder_or_small_buffer_fails_before_execution(self):
+ for backend, message in (
+ (FakeBackend(args(tokens_ladder="0")), "empty token ladder"),
+ (FakeBackend(args(), cap=64), "buffer cap 64"),
+ ):
+ with self.subTest(message=message):
+ spec = backend.make_inputs(backend.args)
+ self.assertEqual(spec.rc, 2)
+ self.assertIn(message, spec.message)
+
+ def test_timed_components_follow_backend_contract(self):
+ backend = FakeBackend(args())
+ self.assertEqual(backend.timed_components(), ["roundtrip", "dispatch", "combine"])
+ backend.stage_device_work = True
self.assertEqual(
- self._backend(stage_device_work=True).timed_components(),
- ["roundtrip", "dispatch", "combine", "stage"],
+ backend.timed_components(), ["roundtrip", "dispatch", "combine", "stage"]
)
-
- def test_roundtrip_only_measures_roundtrip_alone(self):
- self.assertEqual(self._backend(roundtrip_only=True).timed_components(), ["roundtrip"])
-
- def test_roundtrip_only_suppresses_stage_flag(self):
- backend = self._backend(roundtrip_only=True, stage_device_work=True)
+ backend.roundtrip_only = True
self.assertEqual(backend.timed_components(), ["roundtrip"])
-
-class BenchmarkBranchTests(unittest.TestCase):
- """The two Pass-2 branch rules encoded in benchmark_dispatch/combine.
-
- time_us is replaced with a recorder that captures its pre/post kwargs without
- invoking the timed callables, so we observe branch *selection* rather than run
- a device timing loop.
- """
-
- def setUp(self):
- self.records = []
-
- def recorder(torch_module, fn, warmup, iters, pre=None, post=None):
- self.records.append(
- types.SimpleNamespace(fn=fn, warmup=warmup, iters=iters, pre=pre, post=post)
- )
- return [1.0]
-
- self._patch_time_us = mock.patch.object(ep_backend, "time_us", recorder)
- self._patch_time_us.start()
- self.addCleanup(self._patch_time_us.stop)
-
- fake_torch = types.SimpleNamespace(
- cuda=types.SimpleNamespace(synchronize=lambda: None)
- )
- self._patch_torch = mock.patch.dict(sys.modules, {"torch": fake_torch})
- self._patch_torch.start()
- self.addCleanup(self._patch_torch.stop)
-
- def test_dispatch_without_cleanup_passes_no_post(self):
- backend = FakeBackend(_args())
- backend.benchmark_dispatch(types.SimpleNamespace(), warmup=0, iters=3)
- self.assertEqual(len(self.records), 1)
- self.assertIsNone(self.records[0].post)
- self.assertEqual(self.records[0].iters, 3)
-
- def test_dispatch_with_cleanup_attaches_untimed_stage_combine(self):
- backend = FakeBackend(_args())
+ def test_dispatch_cleanup_is_outside_timed_call(self):
+ backend = FakeBackend(args())
backend.dispatch_needs_combine_cleanup = True
- backend.benchmark_dispatch(types.SimpleNamespace(), warmup=0, iters=3)
- self.assertTrue(callable(self.records[0].post))
- # the post hook runs stage then combine (untimed cleanup after a timed dispatch).
- backend.calls.clear()
- self.records[0].post(types.SimpleNamespace())
- self.assertEqual(backend.calls, ["stage", "combine"])
-
- def test_combine_redispatch_supplies_pre_and_defers_it(self):
- backend = FakeBackend(_args())
- backend.combine_needs_redispatch = True
- backend.calls.clear()
- backend.benchmark_combine(types.SimpleNamespace(), warmup=0, iters=3)
- # pre is handed to time_us but not executed here (recorder never calls it).
- self.assertTrue(callable(self.records[0].pre))
- self.assertEqual(backend.calls, [])
- # invoking pre performs exactly one untimed dispatch + stage per iteration.
- backend.calls.clear()
- self.records[0].pre()
- self.assertEqual(backend.calls, ["dispatch", "stage"])
+ captured = {}
- def test_combine_stateless_dispatches_once_and_reuses_handle(self):
- backend = FakeBackend(_args()) # combine_needs_redispatch stays False
- backend.calls.clear()
- backend.benchmark_combine(types.SimpleNamespace(), warmup=0, iters=3)
- self.assertIsNone(self.records[0].pre)
- # one dispatch+stage happens before timing; the handle is reused across iters.
- self.assertEqual(backend.calls, ["dispatch", "stage"])
+ def fake_time(_torch, operation, _warmup, _iters, **kwargs):
+ handle = operation()
+ kwargs["post"](handle)
+ captured.update(kwargs)
+ return [1.0]
+ with mock.patch.dict(sys.modules, {"torch": types.SimpleNamespace()}), mock.patch.object(
+ ep_backend, "time_us", side_effect=fake_time
+ ):
+ backend.benchmark_dispatch(object(), 0, 1)
+ self.assertIn("post", captured)
+ self.assertEqual(backend.calls, ["dispatch", "stage", "combine"])
-class ContractEnforcementTests(unittest.TestCase):
- def test_unsupported_mode_raises_valueerror(self):
+ def test_mode_is_fail_closed(self):
with self.assertRaises(ValueError):
- FakeBackend(_args(mode="low-latency"))
-
- def test_supported_mode_constructs(self):
- backend = FakeBackend(_args(mode="normal"))
- self.assertEqual(backend.mode, "normal")
-
- def test_multi_mode_backend_accepts_declared_mode(self):
- class MultiMode(FakeBackend):
- name = "multi"
- SUPPORTED_MODES = ("normal", "low-latency")
-
- backend = MultiMode(_args(mode="low-latency"))
- self.assertEqual(backend.mode, "low-latency")
-
- def test_missing_name_raises_at_class_definition(self):
- with self.assertRaises(TypeError):
-
- class NoName(EPBackend):
- pass
-
- def test_named_subclass_defines_cleanly(self):
- class Named(EPBackend):
- name = "named"
-
- def create_buffer(self, spec):
- return None
-
- def dispatch(self, p):
- return None
-
- def stage(self, p, h):
- return None
-
- def combine(self, p, h):
- return None
-
- def recv_tokens(self, h):
- return 0
-
- def inspect_dispatch(self, p, h):
- return None
-
- def combine_transformed(self, p, h, transformed):
- return None
-
- self.assertEqual(Named.name, "named")
+ FakeBackend(args(mode="low-latency"))
if __name__ == "__main__":
diff --git a/experimental/CollectiveX/tests/test_ep_precision_adapters.py b/experimental/CollectiveX/tests/test_ep_precision_adapters.py
deleted file mode 100644
index e5f4fd7868..0000000000
--- a/experimental/CollectiveX/tests/test_ep_precision_adapters.py
+++ /dev/null
@@ -1,80 +0,0 @@
-#!/usr/bin/env python3
-"""CPU-only contract tests for native EP precision adapter wiring."""
-from __future__ import annotations
-
-import sys
-import unittest
-from pathlib import Path
-from types import SimpleNamespace
-
-
-ROOT = Path(__file__).resolve().parents[1]
-BENCH = ROOT / "bench"
-sys.path.insert(0, str(ROOT))
-sys.path.insert(0, str(BENCH))
-
-import ep_precision # noqa: E402
-
-
-class PrecisionResolutionTests(unittest.TestCase):
- def test_blank_profile_resolves_to_bf16_control(self):
- profile_id, profile = ep_precision.resolve_precision(
- SimpleNamespace(precision_profile=""),
- backend="nccl-ep",
- mode="normal",
- supported_profiles={"d-bf16.c-bf16"},
- )
- self.assertEqual(profile_id, "d-bf16.c-bf16")
- self.assertEqual(profile["dispatch"]["communication_format"], "bf16")
- self.assertEqual(profile["combine"]["communication_format"], "bf16")
-
- def test_adapter_profile_rejection_is_fail_closed(self):
- with self.assertRaisesRegex(
- ep_precision.PrecisionError,
- "nccl-ep does not realize precision profile",
- ):
- ep_precision.resolve_precision(
- SimpleNamespace(
- precision_profile=(
- "d-fp8-e4m3fn-b128-f32-prequantized.c-bf16"
- )
- ),
- backend="nccl-ep",
- mode="normal",
- supported_profiles={"d-bf16.c-bf16"},
- )
-
- def test_profile_mode_mismatch_is_rejected(self):
- with self.assertRaisesRegex(ep_precision.PrecisionError, "not valid in mode"):
- ep_precision.resolve_precision(
- SimpleNamespace(
- precision_profile="d-fp8-e4m3fn-b128-f32-fused.c-bf16"
- ),
- backend="deepep",
- mode="normal",
- supported_profiles={"d-fp8-e4m3fn-b128-f32-fused.c-bf16"},
- )
-
- def test_required_native_keyword_is_checked(self):
- def native_api(*, use_fp8=False):
- return use_fp8
-
- ep_precision.require_keyword(native_api, "use_fp8", api="native_api")
- with self.assertRaisesRegex(ep_precision.PrecisionError, "omits 'use_logfmt'"):
- ep_precision.require_keyword(
- native_api, "use_logfmt", api="native_api"
- )
-
- def test_bf16_evidence_is_exact_and_has_no_scale_checks(self):
- evidence = ep_precision.exact_axis_evidence()
- self.assertTrue(evidence["passed"])
- self.assertEqual(evidence["max_abs_error"], 0.0)
- self.assertEqual(evidence["max_rel_error"], 0.0)
- self.assertEqual(evidence["saturation_count"], 0)
- self.assertEqual(evidence["saturation_rate"], 0.0)
- self.assertIsNone(evidence["scales_finite"])
- self.assertIsNone(evidence["scales_positive"])
-
-
-if __name__ == "__main__":
- unittest.main()
diff --git a/experimental/CollectiveX/tests/test_matrix.py b/experimental/CollectiveX/tests/test_matrix.py
new file mode 100644
index 0000000000..fb43f11e28
--- /dev/null
+++ b/experimental/CollectiveX/tests/test_matrix.py
@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+"""Matrix, subset, shard extraction, and unsupported-delivery tests."""
+from __future__ import annotations
+
+import json
+import os
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+from unittest import mock
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT))
+
+import contracts # noqa: E402
+import sweep_matrix # noqa: E402
+
+
+def matrix(**options):
+ return sweep_matrix.validate_matrix_document(
+ sweep_matrix.resolve_matrix(max_cases=128, **options)
+ )
+
+
+class MatrixTests(unittest.TestCase):
+ def test_shard_extraction_is_deterministic_and_preserves_cases(self):
+ document = matrix(backend="deepep", only_sku="h100-dgxc")
+ cell = document["include"][0]
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary)
+ source = root / "matrix.json"
+ source.write_text(json.dumps(document, sort_keys=True))
+ outputs = [
+ sweep_matrix.extract_shard(
+ source, cell["id"], root / f"shard-{index}.json",
+ sku=cell["sku"], backend=cell["backend"], nodes=cell["nodes"],
+ )
+ for index in range(2)
+ ]
+ self.assertEqual(outputs[0], outputs[1])
+ self.assertEqual([case["case_id"] for case in outputs[0]["cases"]], cell["case_ids"])
+
+ def test_sku_and_ep_filters_only_remove_cases(self):
+ full = matrix(suites="all", backends="all")
+ for options, keep in (
+ ({"exclude_skus": "b300"}, lambda item: item["sku"] != "b300"),
+ ({"ep_sizes": "8"}, lambda item: item["case"]["ep"] == 8),
+ ):
+ partial = matrix(suites="all", backends="all", **options)
+ expected = {
+ item["case"]["case_id"]: item for item in full["requested_cases"] if keep(item)
+ }
+ actual = {item["case"]["case_id"]: item for item in partial["requested_cases"]}
+ self.assertEqual(actual, expected)
+
+ def test_invalid_filters_fail_closed(self):
+ for options in (
+ {"exclude_skus": "unknown"},
+ {"only_sku": "b300", "exclude_skus": "b300"},
+ {"ep_sizes": "0"},
+ {"ep_sizes": "eight"},
+ ):
+ with self.subTest(options=options), self.assertRaises(SystemExit):
+ sweep_matrix.resolve_matrix(**options)
+
+ def test_unsupported_emission_is_complete_and_delivery_valid(self):
+ document = matrix(suites="all", backends="all", only_sku="h100-dgxc")
+ expected = [x for x in document["requested_cases"] if x["disposition"] == "unsupported"]
+ environment = {
+ "COLLECTIVEX_ARTIFACT_NAME": "cxunsupported-123-1",
+ "COLLECTIVEX_EXECUTION_ID": "123_1_unsupported",
+ "COLLECTIVEX_SOURCE_SHA": "a" * 40,
+ "GITHUB_JOB": "setup", "GITHUB_REF_NAME": "collectivex",
+ "GITHUB_REPOSITORY": "SemiAnalysisAI/InferenceX",
+ "GITHUB_RUN_ATTEMPT": "1", "GITHUB_RUN_ID": "123",
+ }
+ with tempfile.TemporaryDirectory() as temporary, mock.patch.dict(
+ os.environ, environment, clear=False
+ ):
+ root = Path(temporary)
+ source = root / "matrix.json"
+ source.write_text(json.dumps(document, sort_keys=True, separators=(",", ":")))
+ paths = sweep_matrix.emit_unsupported(source, root / "unsupported")
+ self.assertEqual(
+ contracts.validate_delivery(
+ [str(path) for path in paths], str(source), disposition="unsupported"
+ ),
+ len(expected),
+ )
+ emitted = [contracts.validate_result(path) for path in paths]
+ self.assertEqual(
+ {item["identity"]["case_id"] for item in emitted},
+ {item["case"]["case_id"] for item in expected},
+ )
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/experimental/CollectiveX/tests/test_qualification_planning.py b/experimental/CollectiveX/tests/test_qualification_planning.py
deleted file mode 100644
index 90b62c06af..0000000000
--- a/experimental/CollectiveX/tests/test_qualification_planning.py
+++ /dev/null
@@ -1,202 +0,0 @@
-#!/usr/bin/env python3
-"""CPU-only tests for matrix resolution, subset selection, and shard extraction."""
-from __future__ import annotations
-
-import json
-import os
-from pathlib import Path
-import sys
-import tempfile
-import unittest
-from unittest import mock
-
-
-HERE = Path(__file__).resolve().parent
-ROOT = HERE.parent
-sys.path[:0] = [str(ROOT), str(HERE)]
-
-import sweep_matrix # noqa: E402
-
-
-def _canonical(value: object) -> bytes:
- return json.dumps(
- value, ensure_ascii=True, sort_keys=True, separators=(",", ":")
- ).encode()
-
-
-class MatrixPlanningTest(unittest.TestCase):
- @classmethod
- def setUpClass(cls) -> None:
- cls.matrix = sweep_matrix.validate_matrix_document(
- sweep_matrix.resolve_matrix(
- backend="deepep", only_sku="h100-dgxc", max_cases=128
- )
- )
- cls.shard = next(item for item in cls.matrix["include"] if item["n"] >= 3)
-
- def test_extract_shard_is_deterministic_and_matrix_preserving(self) -> None:
- matrix_bytes = _canonical(self.matrix) + b"\n"
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary)
- matrix_path = root / "matrix.json"
- matrix_path.write_bytes(matrix_bytes)
- first = sweep_matrix.extract_shard(
- matrix_path,
- self.shard["id"],
- root / "first.json",
- sku=self.shard["sku"],
- backend=self.shard["backend"],
- nodes=self.shard["nodes"],
- )
- repeated = sweep_matrix.extract_shard(
- matrix_path,
- self.shard["id"],
- root / "repeated.json",
- sku=self.shard["sku"],
- backend=self.shard["backend"],
- nodes=self.shard["nodes"],
- )
- self.assertEqual(first, repeated)
- self.assertEqual(
- [case["case_id"] for case in first["cases"]], self.shard["case_ids"]
- )
- sweep_matrix.validate_shard_control(
- first,
- sku=self.shard["sku"],
- backend=self.shard["backend"],
- nodes=self.shard["nodes"],
- )
- # Extraction is read-only: the matrix on disk is untouched.
- self.assertEqual(matrix_path.read_bytes(), matrix_bytes)
-
- def test_full_v1_matrix_emits_every_unsupported_terminal_outcome(self) -> None:
- matrix = sweep_matrix.validate_matrix_document(
- sweep_matrix.resolve_matrix(suites="all", backends="all", max_cases=128)
- )
- expected = [
- item for item in matrix["requested_cases"]
- if item["disposition"] == "unsupported"
- ]
- environment = {
- "COLLECTIVEX_ARTIFACT_NAME": "cxunsupported-123-1",
- "COLLECTIVEX_EXECUTION_ID": "123_1_unsupported",
- "COLLECTIVEX_SOURCE_SHA": "a" * 40,
- "GITHUB_JOB": "setup",
- "GITHUB_REF_NAME": "collectivex",
- "GITHUB_REPOSITORY": "SemiAnalysisAI/InferenceX",
- "GITHUB_RUN_ATTEMPT": "1",
- "GITHUB_RUN_ID": "123",
- }
- with tempfile.TemporaryDirectory() as temporary:
- root = Path(temporary)
- matrix_path = root / "matrix.json"
- matrix_path.write_bytes(_canonical(matrix))
- with mock.patch.dict(os.environ, environment, clear=False):
- written = sweep_matrix.emit_unsupported(matrix_path, root / "unsupported")
- documents = [json.loads(path.read_text()) for path in written]
-
- self.assertEqual(len(documents), len(expected))
- self.assertTrue(documents)
- self.assertEqual(
- {document["outcome"]["reason"] for document in documents},
- {item["reason"] for item in expected},
- )
- self.assertEqual(
- {document["identity"]["case_id"] for document in documents},
- {item["case"]["case_id"] for item in expected},
- )
-
- def test_exclude_skus_narrows_matrix_to_a_subset(self) -> None:
- full = sweep_matrix.validate_matrix_document(
- sweep_matrix.resolve_matrix(suites="all", backends="all", max_cases=128)
- )
- partial = sweep_matrix.validate_matrix_document(
- sweep_matrix.resolve_matrix(
- suites="all", backends="all", max_cases=128, exclude_skus="b300"
- )
- )
- full_skus = {item["sku"] for item in full["include"]}
- partial_skus = {item["sku"] for item in partial["include"]}
- self.assertIn("b300", full_skus)
- self.assertNotIn("b300", partial_skus)
- self.assertEqual(full_skus - {"b300"}, partial_skus)
- # Excluding a pool only omits its cells; every surviving cell is byte-identical
- # to the full matrix (a partial run cannot invent or reclassify cases).
- def by_id(matrix: dict[str, object]) -> dict[str, object]:
- return {
- item["case"]["case_id"]: item
- for item in matrix["requested_cases"]
- if item.get("sku") != "b300"
- }
- self.assertEqual(by_id(partial), by_id(full))
- self.assertTrue(
- all(item.get("sku") != "b300" for item in partial["requested_cases"])
- )
-
- def test_exclude_skus_rejects_unknown_and_conflicting_pools(self) -> None:
- with self.assertRaisesRegex(SystemExit, "unknown --exclude-skus"):
- sweep_matrix.resolve_matrix(exclude_skus="nosuchsku")
- with self.assertRaisesRegex(SystemExit, "disjoint pools"):
- sweep_matrix.resolve_matrix(only_sku="b300", exclude_skus="b300")
-
- def test_ep_sizes_narrows_matrix_to_a_subset(self) -> None:
- full = sweep_matrix.validate_matrix_document(
- sweep_matrix.resolve_matrix(suites="all", backends="all", max_cases=128)
- )
- partial = sweep_matrix.validate_matrix_document(
- sweep_matrix.resolve_matrix(
- suites="all", backends="all", max_cases=128, ep_sizes="8"
- )
- )
- full_eps = {item["case"]["ep"] for item in full["requested_cases"]}
- partial_eps = {item["case"]["ep"] for item in partial["requested_cases"]}
- # The canonical matrix carries EP16; --ep-sizes=8 drops it entirely so a
- # comprehensive run can co-schedule EP8 across every SKU (GB EP8 is two nodes,
- # the 8-GPU SKUs' EP8 is one) with no EP16 leg to wall.
- self.assertIn(16, full_eps)
- self.assertEqual(partial_eps, {8})
- # Keeping a degree only omits the others; every surviving cell is byte-identical
- # to the full matrix (a partial run cannot invent or reclassify cases).
- def ep8_by_id(matrix: dict[str, object]) -> dict[str, object]:
- return {
- item["case"]["case_id"]: item
- for item in matrix["requested_cases"]
- if item["case"]["ep"] == 8
- }
- self.assertEqual(ep8_by_id(partial), ep8_by_id(full))
- self.assertTrue(
- all(item["case"]["ep"] == 8 for item in partial["requested_cases"])
- )
-
- def test_ep_sizes_rejects_non_positive_or_non_integer_degrees(self) -> None:
- with self.assertRaisesRegex(SystemExit, "invalid --ep-sizes"):
- sweep_matrix.resolve_matrix(ep_sizes="0")
- with self.assertRaisesRegex(SystemExit, "invalid --ep-sizes"):
- sweep_matrix.resolve_matrix(ep_sizes="eight")
-
- def test_frontend_catalog_covers_every_requested_case_and_point(self) -> None:
- catalog = sweep_matrix.frontend_catalog(self.matrix)
- self.assertEqual(catalog["format"], "collectivex.frontend-catalog.v1")
- self.assertEqual(catalog["case_count"], len(self.matrix["requested_cases"]))
- self.assertEqual(
- catalog["point_count"],
- sum(
- len(item["case"]["ladder"].split())
- for item in self.matrix["requested_cases"]
- ),
- )
- self.assertEqual(
- {item["case_id"] for item in catalog["cases"]},
- {item["case"]["case_id"] for item in self.matrix["requested_cases"]},
- )
- self.assertTrue(all(item["required"] for item in catalog["cases"]))
- self.assertTrue(all(item["backend_generation"] is None for item in catalog["cases"]))
- self.assertEqual(
- {item["resource"]["mode"] for item in catalog["cases"]},
- {"fixed-profile"},
- )
- self.assertLess(len(_canonical(catalog)) + 1, 1024 * 1024)
-
-
-if __name__ == "__main__":
- unittest.main()
diff --git a/experimental/CollectiveX/tests/test_raw_document_roundtrip.py b/experimental/CollectiveX/tests/test_raw_document_roundtrip.py
deleted file mode 100644
index f0523926f9..0000000000
--- a/experimental/CollectiveX/tests/test_raw_document_roundtrip.py
+++ /dev/null
@@ -1,494 +0,0 @@
-"""End-to-end raw-document contract test (no GPU, no torch).
-
-`test_ep_backend.py` exercises the mocked benchmark up to the measurement rows, and
-`test_schema_v1_contract.py` asserts facts about the schema files, but nothing drove a
-complete raw attempt document plus its detached samples through
-`contracts.validate_raw_document`. That left the emitter tail of `ep_harness.run_sweep`
-(the doc assembly that only runs on a real allocation) unguarded: a field the emitter
-writes but the frozen schema forbids, or vice versa, would only surface on a live sweep.
-
-This builds a document whose field sets mirror `ep_harness.run_sweep`'s assembly exactly,
-using the real `identity` module for every typed id and the real
-`identity.profile_for_case` for the case profile, then validates it. The negative cases
-pin the drift fields that were removed from the emitter so a re-introduction fails here
-instead of on the cluster.
-"""
-import copy
-import os
-import sys
-import unittest
-
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "bench"))
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
-
-import identity # noqa: E402
-import contracts # noqa: E402
-
-_HEX = "ab" * 32 # a syntactically valid lowercase sha256 digest (64 hex)
-_SHA40 = "abcdef0123456789abcdef0123456789abcdef01" # a git-style 40-hex sha
-_WORKLOAD_ID = identity.workload_id({"name": "deepseek-v3-v1", "trace": _HEX})
-
-# A representative EP8 H100 scheduled case, identical in shape to what sweep_matrix
-# materializes and ep_harness rebuilds from args (no case_id key: it is computed from
-# these factors).
-SCHEDULED_CASE = {
- "backend": "deepep",
- "canonical": True,
- "ep": 8,
- "eplb": False,
- "experts": 256,
- "gpus_per_node": 8,
- "hidden": 7168,
- "ladder": "1 2 4 8 16 32 64 128",
- "mode": "normal",
- "nodes": 1,
- "phase": "decode",
- "routing": "uniform",
- "samples_per_point": 512,
- "scale_out_transport": None,
- "scale_up_domain": 8,
- "scale_up_transport": "nvlink",
- "scope": "scale-up",
- "suite": "ep-core-v1",
- "timing": "8:64:32",
- "topk": 8,
- "topology_class": "h100-nvlink-island",
- "transport": "nvlink",
- "warmup_semantics": "full-roundtrip-before-each-component-trial-point-v1",
- "workload": "deepseek-v3-v1",
-}
-
-
-def _percentiles():
- return {"p50": 10.0, "p90": 20.0, "p95": 25.0, "p99": 30.0}
-
-
-def _component():
- return {
- "availability": "measured",
- "origin": "measured",
- "percentiles_us": _percentiles(),
- "sample_count": 512,
- }
-
-
-def _samples_component():
- # 64 trials x 8 iterations of non-negative numbers, per samples-v1 trials def.
- return {
- "availability": "measured",
- "sample_count": 512,
- "trials": [[1.0] * 8 for _ in range(64)],
- }
-
-
-def _byte_accounting():
- return {
- "accounting_contract": "activation-data-plus-scales-v1",
- "activation_data_bytes": 1024,
- "scale_bytes": 0,
- "total_logical_bytes": 1024,
- }
-
-
-def _histogram():
- return {"bins": 4, "counts": [1, 2, 3, 4], "max": 30.0, "min": 10.0, "n": 10}
-
-
-def _oracle():
- return {
- "atol": 0.02,
- "checks": {
- "combine_values": True,
- "counts": True,
- "metadata": True,
- "multiplicity": True,
- "payload": True,
- "source_set": True,
- "weights": True,
- },
- "combine_weight_semantics": "unweighted-rank-sum",
- "contract": "expert-specific-transform-v1",
- "dispatch_sha256": None,
- "max_absolute_error": 0.0,
- "max_elementwise_relative_error": 0.0,
- "max_relative_error": 0.0,
- "max_weight_error": None,
- "order_sha256": None,
- "ordering_contract": "roundtrip-first-then-components-v1",
- "passed": True,
- "receive_count": 8,
- "rtol": 0.05,
- }
-
-
-def _routing():
- return {
- "empty_expert_count": 0,
- "empty_rank_count": 0,
- "expert_assignment_rank_cv": 0.0,
- "expert_assignments_per_rank": [32, 32, 32, 32, 32, 32, 32, 32],
- "expert_load_cv": 0.0,
- "expert_load_max": 8,
- "expert_load_mean": 8.0,
- "expert_load_min": 8,
- "fanout_histogram": [0, 8],
- "fanout_max": 1,
- "fanout_mean": 1.0,
- "fanout_min": 1,
- "hash": _HEX,
- "hotspot_ratio": 1.0,
- "locality": None,
- "payload_copies_per_rank": [8, 8, 8, 8, 8, 8, 8, 8],
- "payload_rank_cv": 0.0,
- "routed_copies": 64,
- "source_token_stats": None,
- }
-
-
-def _correctness():
- return {
- "contract": "expert-specific-transform-v1",
- "max_relative_error": 0.0,
- "passed": True,
- "rank_evidence": [
- {
- "input_unchanged": True,
- "order_stable": True,
- "post_timing": _oracle(),
- "pre_timing": _oracle(),
- "rank": 0,
- }
- ],
- "scope": "dispatch-metadata-and-transformed-combine",
- }
-
-
-def _build_documents():
- """Return (raw_document, samples_document) with fully reconciled identities."""
- profile = identity.profile_for_case({"mode": SCHEDULED_CASE["mode"]})
- case_factors = {"case": SCHEDULED_CASE, "profile": profile, "sku": "h100-dgxc"}
- case_id = identity.digest("case", case_factors)
-
- allocation_factors = {
- "artifact": "collectivex-results-h100-dgxc-deepep-n1",
- "execution_id": "exec-0001",
- "job": "sweep",
- "repo": "SemiAnalysisAI/InferenceX",
- "run_attempt": "1",
- "run_id": "100200300",
- "runner": "h100-dgxc",
- "source_sha": _SHA40,
- }
- allocation_id = identity.allocation_id(allocation_factors)
- attempt_ordinal = 1
- attempt_id = identity.attempt_id(
- allocation=allocation_id, case=case_id, ordinal=attempt_ordinal
- )
-
- series_factors = {
- "backend": "deepep",
- "case_id": case_id,
- "image_digest": "sha256:" + _HEX,
- "source_sha": _SHA40,
- "squash_sha256": _HEX,
- "workload_id": _WORKLOAD_ID,
- }
- series_id = identity.series_id(series_factors)
-
- rows = []
- sample_points = []
- for token_count in (64, 128):
- sample_sha256 = identity.digest("point", {"t": token_count})[-64:]
- point_id = identity.point_id(series=series_id, tokens_per_rank=token_count)
- evidence_id = identity.evidence_id(
- point=point_id,
- allocation=allocation_id,
- attempt=attempt_id,
- sample_sha256=sample_sha256,
- )
- rows.append({
- "anomalies": [],
- "byte_provenance": {c: _byte_accounting() for c in
- ("combine", "dispatch", "roundtrip", "stage")},
- "components": {
- "combine": _component(),
- "dispatch": _component(),
- "isolated_sum": _component(),
- "roundtrip": _component(),
- "stage": _component(),
- },
- "correctness": _correctness(),
- "evidence_id": evidence_id,
- "global_tokens": token_count * 8,
- "point_id": point_id,
- "receive": {"max": 8, "mean": 8.0, "min": 8, "total": 64},
- "routing": _routing(),
- "sample_histograms": {
- "combine": None,
- "dispatch": None,
- "roundtrip": _histogram(),
- "stage": None,
- },
- "sample_sha256": sample_sha256,
- "token_rate_at_latency_percentile": _percentiles(),
- "tokens_per_rank": token_count,
- })
- sample_points.append({
- "components": {
- "combine": _samples_component(),
- "dispatch": _samples_component(),
- "roundtrip": _samples_component(),
- "stage": _samples_component(),
- },
- "evidence_id": evidence_id,
- "point_id": point_id,
- "sample_sha256": sample_sha256,
- "tokens_per_rank": token_count,
- })
-
- samples_document = {
- "allocation_id": allocation_id,
- "attempt_id": attempt_id,
- "case_id": case_id,
- "format": "collectivex.samples.v1",
- "points": sample_points,
- "sampling": {
- "iterations_per_trial": 8,
- "reduction": "cross-rank-max-per-iteration",
- "trials": 64,
- },
- "schema_version": 1,
- "series_id": series_id,
- }
-
- eplb_record = {
- "calibration_token_offset": None,
- "calibration_trace_sha256": None,
- "calibration_window": None,
- "calibration_workload_id": None,
- "enabled": False,
- "imbalance_after": None,
- "imbalance_before": None,
- "mapping_hash": None,
- "max_replicas": None,
- "num_logical_experts": 256,
- "num_physical_experts": 256,
- "num_redundant": 0,
- "planner": None,
- "reference_tokens_per_rank": None,
- "replicated_experts": 0,
- }
-
- document = {
- "format": "collectivex.ep.v1",
- "schema_version": 1,
- "record_type": "case-attempt",
- "generated_at": "2026-07-08T00:00:00Z",
- "identity": {
- "allocation_factors": allocation_factors,
- "allocation_id": allocation_id,
- "attempt_id": attempt_id,
- "attempt_ordinal": attempt_ordinal,
- "case_factors": case_factors,
- "case_id": case_id,
- "series_factors": series_factors,
- "series_id": series_id,
- },
- "case": {
- "attempt_ordinal": attempt_ordinal,
- "backend": "deepep",
- "eplb": eplb_record,
- "ep_size": 8,
- "mode": "normal",
- "phase": "decode",
- "resource_mode": "fixed-profile",
- "runner": "h100-dgxc",
- "shape": {
- "activation_profile": "canonical-counter-source-v4",
- "eplb": False,
- "experts": 256,
- "experts_per_rank": 32,
- "hidden": 7168,
- "kernel_gen": "image-pinned-v1",
- "num_logical_experts": 256,
- "routing": "uniform",
- "topk": 8,
- },
- "suite": "ep-core-v1",
- "workload_name": "deepseek-v3-v1",
- },
- "workload": {
- "activation_generator": "collectivex-activation-counter-v4",
- "activation_identity": _HEX,
- "activation_profile": "canonical-counter-source-v4",
- "cross_rank_consistent": True,
- "manifest_checksums": None,
- "members": None,
- "routing_generator": "collectivex-routing-counter-v3",
- "source": "seeded-runtime",
- "trace_hashes": [_HEX],
- "trace_signature": _HEX,
- "workload_id": _WORKLOAD_ID,
- },
- "measurement": {
- "component_order_contract": "qualification-hash-rotated-components-v1",
- "conditioning": {
- "contract": "fixed-phase-ramp-8-roundtrips-v1",
- "ladder": [1, 2, 4, 8, 16, 32, 64, 128],
- "roundtrips_per_shape": 8,
- },
- "contract": "layout-and-dispatch-v1",
- "rows": rows,
- "sampling": {
- "contract": "fixed-512-v1",
- "iterations_per_trial": 8,
- "percentile_method": "nearest-rank",
- "reduction": "cross-rank-max-per-iteration",
- "samples_per_component": 512,
- "trials": 64,
- "warmup_iterations": 32,
- "warmup_semantics": "full-roundtrip-before-each-component-trial-point-v1",
- },
- "source_allocation": "even",
- },
- "implementation": {
- "kernel_generation": "image-pinned-v1",
- "name": "deepep",
- "provenance": {},
- "resource_profile": {
- key: None for key in (
- "achieved_fraction", "comm_units_kind", "configured_units",
- "conformance_class", "device_units", "fixed_kernel",
- "nonconforming", "pareto_eligible", "persistent_bytes",
- "qps_per_rank", "requested_fraction", "resource_class",
- "target_achieved_within_tol", "tolerance", "tuned_source",
- "warps_combine", "warps_dispatch",
- )
- },
- },
- "topology": {
- "device_count": 8,
- "device_product": "NVIDIA H100 80GB HBM3",
- "gpus_per_node": 8,
- "nodes": 1,
- "placement": "packed",
- "realized_placement": {
- "gpus_per_node": 8,
- "nodes": 1,
- "ranks_per_node": 8,
- "unique_local_ranks": True,
- "valid": True,
- },
- "scale_up_domain": 8,
- "scale_up_transport": "nvlink",
- "scale_out_transport": None,
- "scope": "scale-up",
- "topology_class": "h100-nvlink-island",
- "transport": "nvlink",
- "world_size": 8,
- },
- "runtime_fingerprint": {
- "accelerator_runtime": {"kind": "cuda", "version": "12.4"},
- "collective_library": {"kind": "nccl", "version": "2.21.5"},
- "device": {
- "arch": "sm90",
- "compute_units": 132,
- "memory_bytes": 85899345920,
- "product": "NVIDIA H100 80GB HBM3",
- "warp_size": 32,
- },
- "driver_version": "550.54.15",
- "framework": {"kind": "torch", "version": "2.5.1"},
- "machine": "x86_64",
- "python_version": "3.11.9",
- "vendor": "nvidia",
- },
- "provenance": {
- "command": "run_ep.py ...",
- "distributed_launcher": None,
- "git_run": {
- "artifact": "collectivex-results-h100-dgxc-deepep-n1",
- "job": "sweep",
- "ref": "collectivex",
- "repo": "SemiAnalysisAI/InferenceX",
- "run_attempt": "1",
- "run_id": "100200300",
- "source_sha": _SHA40,
- },
- "image": {
- "arch": "x86_64",
- "digest": "sha256:" + _HEX,
- "digest_verified": True,
- "reference": "example/image:tag",
- "squash_sha256": _HEX,
- },
- "redaction": "sanitized-v1",
- },
- "sample_artifact": {
- "bytes": 1,
- "format": "collectivex.samples.v1",
- "path": "case.samples.json",
- "sha256": _HEX,
- },
- "outcome": {
- "reasons": [],
- "status": "success",
- "validity": {
- "anomaly_free": True,
- "execution_status": "complete",
- "measurement_conformance": "conformant",
- "provenance_complete": True,
- "resource_conformance": "conformant",
- "sampling_conformance": "conformant",
- "semantic_correctness": "pass",
- "workload_identity": "consistent-across-ranks",
- "workload_source": "seeded-runtime",
- },
- },
- }
- return document, samples_document
-
-
-class RawDocumentRoundTripTest(unittest.TestCase):
- def test_representative_raw_document_validates(self):
- document, samples = _build_documents()
- # Must not raise: schema shape + all cross-file identity checks pass.
- result = contracts.validate_raw_document(document, samples)
- self.assertEqual(result["identity"]["case_id"], document["identity"]["case_id"])
-
- def test_profile_is_bf16_fixed_fact(self):
- profile = identity.profile_for_case({"mode": "normal"})
- self.assertEqual(profile["combine_dtype"], "bf16")
- self.assertEqual(profile["combine_quant_mode"], "none")
-
- def test_removed_drift_fields_are_rejected(self):
- # Each of these keys was emitted by an earlier build but is forbidden by the
- # frozen schema. Re-introducing any of them must fail closed here.
- base_doc, base_samples = _build_documents()
- cases = [
- ("identity", "allocation_factors", "qualification_index"),
- ("identity", "series_factors", "runtime_fingerprint_sha256"),
- ("measurement", None, "execution_order_sha256"),
- ("measurement", None, "qualification_index"),
- ("provenance", None, "allocation_stratum_sha256"),
- ("provenance", "git_run", "qualification_index"),
- ("case", "shape", "precision_profile"),
- ("case", "shape", "dispatch_precision"),
- ]
- for top, sub, key in cases:
- with self.subTest(field=f"{top}.{sub or ''}.{key}"):
- doc = copy.deepcopy(base_doc)
- target = doc[top] if sub is None else doc[top][sub]
- target[key] = 1
- with self.assertRaises(contracts.ContractError):
- contracts.validate_raw_document(doc, copy.deepcopy(base_samples))
-
- def test_samples_qualification_index_is_rejected(self):
- doc, samples = _build_documents()
- samples["qualification_index"] = 1
- with self.assertRaises(contracts.ContractError):
- contracts.validate_raw_document(doc, samples)
-
-
-if __name__ == "__main__":
- unittest.main()
diff --git a/experimental/CollectiveX/tests/test_schema_v1_contract.py b/experimental/CollectiveX/tests/test_schema_v1_contract.py
deleted file mode 100644
index 15b8e552e9..0000000000
--- a/experimental/CollectiveX/tests/test_schema_v1_contract.py
+++ /dev/null
@@ -1,146 +0,0 @@
-#!/usr/bin/env python3
-"""Focused structural tests for the fail-closed CollectiveX neutral schemas."""
-from __future__ import annotations
-
-import copy
-import json
-import sys
-import unittest
-from pathlib import Path
-
-import jsonschema
-
-ROOT = Path(__file__).resolve().parents[1]
-if str(ROOT) not in sys.path:
- sys.path.insert(0, str(ROOT))
-
-
-# The scheduled case is the neutral contract every artifact agrees on. Both the
-# raw evidence schema and the terminal-outcome schema must enforce this exact set
-# of 24 fields with additionalProperties:false so a producer cannot invent,
-# rename, or drop a scheduling field on either side of a run.
-SCHEDULED_CASE_FIELDS = frozenset({
- "backend", "canonical", "ep", "eplb", "experts", "gpus_per_node", "hidden",
- "ladder", "mode", "nodes", "phase", "routing", "samples_per_point",
- "scale_out_transport", "scale_up_domain", "scale_up_transport", "scope",
- "suite", "timing", "topk", "topology_class", "transport", "warmup_semantics",
- "workload",
-})
-
-
-def _load(name: str) -> dict:
- return json.loads((ROOT / "schemas" / name).read_text())
-
-
-def _definition_validator(schema: dict, name: str) -> jsonschema.Validator:
- return jsonschema.Draft202012Validator(
- {
- "$schema": "https://json-schema.org/draft/2020-12/schema",
- "$defs": schema["$defs"],
- "$ref": f"#/$defs/{name}",
- }
- )
-
-
-class CollectiveXSchemaContractTest(unittest.TestCase):
- @classmethod
- def setUpClass(cls) -> None:
- cls.raw = _load("raw-case-v1.schema.json")
- cls.samples = _load("samples-v1.schema.json")
- cls.terminal = _load("terminal-outcome-v1.schema.json")
-
- def test_all_checked_in_schemas_are_draft_2020_12_valid(self) -> None:
- schemas = sorted((ROOT / "schemas").glob("*.schema.json"))
- # The publication schemas are gone; only the three neutral ones remain.
- self.assertEqual(
- {path.name for path in schemas},
- {
- "raw-case-v1.schema.json",
- "samples-v1.schema.json",
- "terminal-outcome-v1.schema.json",
- },
- )
- for path in schemas:
- with self.subTest(path=path.name):
- schema = json.loads(path.read_text())
- jsonschema.Draft202012Validator.check_schema(schema)
- self.assertFalse(schema["additionalProperties"])
-
- def test_scheduled_case_is_identical_across_raw_and_terminal(self) -> None:
- # Format consts pin the record type of each neutral artifact.
- self.assertEqual(
- self.raw["properties"]["format"]["const"], "collectivex.ep.v1"
- )
- self.assertEqual(
- self.terminal["properties"]["format"]["const"], "collectivex.terminal.v1"
- )
- self.assertEqual(
- self.samples["properties"]["format"]["const"], "collectivex.samples.v1"
- )
-
- raw_case = self.raw["$defs"]["scheduled_case"]
- terminal_case = self.terminal["$defs"]["case"]
- for descriptor in (raw_case, terminal_case):
- self.assertFalse(descriptor["additionalProperties"])
- self.assertEqual(set(descriptor["properties"]), SCHEDULED_CASE_FIELDS)
- self.assertEqual(set(descriptor["required"]), SCHEDULED_CASE_FIELDS)
- # No precision dimension survives on the scheduled case.
- self.assertNotIn("precision_profile", raw_case["properties"])
- self.assertNotIn("precision_profile", terminal_case["properties"])
-
- def test_stage_samples_are_absent_or_exactly_512(self) -> None:
- validator = _definition_validator(self.samples, "component")
- measured = {
- "availability": "measured",
- "sample_count": 512,
- "trials": [[1.0] * 8 for _ in range(64)],
- }
- validator.validate(measured)
- unavailable = {
- "availability": "unavailable",
- "sample_count": 0,
- "trials": None,
- }
- validator.validate(unavailable)
- for mutate in (
- lambda value: value.update(sample_count=511),
- lambda value: value["trials"].pop(),
- lambda value: value["trials"][0].pop(),
- ):
- broken = copy.deepcopy(measured)
- mutate(broken)
- with self.assertRaises(jsonschema.ValidationError):
- validator.validate(broken)
-
- raw_components = self.raw["properties"]["measurement"]["properties"]["rows"][
- "items"
- ]["properties"]["components"]
- self.assertIn("stage", raw_components["required"])
-
- def test_byte_accounting_is_exact_and_strict(self) -> None:
- expected = {
- "accounting_contract",
- "activation_data_bytes",
- "scale_bytes",
- "total_logical_bytes",
- }
- definition = self.raw["$defs"]["byte_accounting"]
- self.assertFalse(definition["additionalProperties"])
- self.assertEqual(set(definition["required"]), expected)
- self.assertEqual(set(definition["properties"]), expected)
-
- def test_eplb_calibration_provenance_is_explicit(self) -> None:
- fields = {
- "calibration_workload_id",
- "calibration_trace_sha256",
- "calibration_window",
- "calibration_token_offset",
- }
- descriptor = self.raw["properties"]["case"]["properties"]["eplb"]
- self.assertTrue(fields <= set(descriptor["required"]))
- self.assertTrue(fields <= set(descriptor["properties"]))
- self.assertFalse(descriptor["additionalProperties"])
-
-
-if __name__ == "__main__":
- unittest.main()
From b27ecf55a577f695933478fc5e9b1a403a58f4d8 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Wed, 8 Jul 2026 18:10:39 +0800
Subject: [PATCH 202/210] no more chinese
---
experimental/CollectiveX/README_zh.md | 154 ---------
.../CollectiveX/docs/methodology_zh.md | 297 ------------------
2 files changed, 451 deletions(-)
delete mode 100644 experimental/CollectiveX/README_zh.md
delete mode 100644 experimental/CollectiveX/docs/methodology_zh.md
diff --git a/experimental/CollectiveX/README_zh.md b/experimental/CollectiveX/README_zh.md
deleted file mode 100644
index fc8c6a459c..0000000000
--- a/experimental/CollectiveX/README_zh.md
+++ /dev/null
@@ -1,154 +0,0 @@
-# CollectiveX
-
-
-
-[English](./README.md) | **中文**
-
-
-
-CollectiveX 是实验性的 MoE 专家并行通信基准，用于测量不同 EP 库和加速器系统的
-dispatch、combine 及配对 roundtrip 延迟。
-
-> 发布暂停：历史 schema 3-5 数据仅供诊断。目前没有数据集获准用于排名、推荐或回归基线。
-
-## v1 执行配置
-
-每个调度用例均采用 BF16、后端调优资源和 packed placement。显式指定的 mode 选择以下两个
-契约之一：
-
-- Normal mode 使用 `layout-and-dispatch-v1`、按 rank 去重的 token payload 和 activation-only
- combine。核心覆盖使用 uniform routing，并保留一个 Zipf 敏感性场景；EPLB 只作为 Zipf
- 的修正方案测量。
-- Low-latency mode 使用 `expert-packed-weighted-combine-v1`、token-expert payload 和
- gate-weighted combine，并且只调用真正的 DeepEP V1 或 UCCL low-latency API。该模式仅覆盖
- 解码，绝不与 normal mode 共用排名 cohort。其他后端在此 suite 中均显式标为 unsupported。
-
-两种模式统一使用 `fixed-512-v1`：64 trials x 8 timed iterations；每个 trial/point 的每个被测
-组件前执行 32 次同步完整 roundtrip warmup。先测 roundtrip；每次 iteration 先取跨 rank 最大值，
-再按 nearest-rank 计算 p50/p90/p95/p99，主要延迟指标为 roundtrip p99。stdlib 整数计数器
-生成逐字节一致的 routing 和 gate weights。
-
-规范矩阵覆盖 H100、H200、B200、B300、GB200、GB300、MI325X 和 MI355X。矩阵请求
-608 个 cases / 1,600 个 token points：364 个可运行 cases / 940 个 points，并形成 58 个可执行
-workflow shards/allocation cells；另有 244 个显式 unsupported cases / 660 个 points。
-`sweep_matrix.py` 物化每个 token ladder，并拒绝缺失、过期、格式错误或被修改的 shard controls。
-分片按 SKU round-robin 发出，使受限的 GHA matrix 尽早使用所有 runner pools。
-
-| 系统 | EP8 | EP16 |
-|---|---|---|
-| H100/H200/B200/B300 | 1x8 NVLink，scale-up | 2x8 NVLink + RDMA，scale-out |
-| MI325X/MI355X | 1x8 XGMI，scale-up | 2x8 XGMI + RDMA，scale-out |
-| GB200/GB300 | 2x4 MNNVL，scale-up | 4x4 MNNVL，scale-up |
-
-物理主机数量不能决定通信范围：两种 GB 拓扑都位于同一个 72-GPU MNNVL scale-up domain 内。
-
-| 后端 | 当前范围 |
-|---|---|
-| DeepEP V1 | 镜像固定的 `deep_ep.Buffer`：提供 normal 和原生 low-latency API；x86 使用 upstream v1.2.1，arm64 使用镜像内 GB fork |
-| DeepEP V2 | PR #605 `ElasticBuffer` 加 #630：scale-up 使用 LSA，x86 EP16 scale-out 使用 GIN；JIT 可复现并绑定 source/SASS |
-| DeepEP Hybrid | 固定的 `HybridEPBuffer`：x86 EP16 使用 multi-domain RDMA/DOCA；GB EP8/EP16 位于同一个 MNNVL communication domain |
-| UCCL | Hopper 上固定的 0.1.1 wheel 和 wrapper，提供 normal 和原生 low-latency API；Blackwell 显式标为 unsupported |
-| NCCL/RCCL A2A | 可移植的 rank-deduplicated payload 加 expert/routing-metadata reference |
-| MoRI | EP8 使用 MI325X AsyncLL 或 MI355X IntraNode；EP16 固定使用 2x8 XGMI + RDMA 上的 InterNodeV1 |
-
-FlashInfer 不在 v1 范围内，因为已测试的 EP path 在运行时存在间歇性失败。该问题不会被误报为
-平台能力限制；在证明有稳定的固定实现后可重新加入。
-
-DeepEP V2 指 [DeepEP PR #605](https://github.com/deepseek-ai/DeepEP/pull/605) 引入的
-`ElasticBuffer` 实现，而不是更新的 legacy `Buffer` build。固定 source 使用最小化的 upstream
-[PR #630](https://github.com/deepseek-ai/DeepEP/pull/630) 后续修复：其 parent 是 #605 merge
-tree，唯一 source 变更是修复 GIN 不可用时的纯 scale-up 初始化。Scale-up cases 请求 NCCL
-Device API LSA；若实际建立的 LSA team 未覆盖整个 EP world，则直接失败。x86 EP16 scale-out
-cases 必须使用启用 GIN 的 hybrid path，其精确拓扑为两个逻辑 scale-out domains（由两个物理
-RDMA ranks 表示）、每个 domain 八个 scale-up ranks；GB EP16 仍是 MNNVL scale-up，因此继续
-使用 LSA。隔离构建会记录 API、source、loaded libraries、generated JIT source、executable
-SASS 与 raw CUBIN diagnostics。当前 H100 runner pool 被明确标记为 V2 unsupported，因为 NCCL
-2.30.4 报告其 EP8 communicator 不具备 Device API symmetric-memory 支持；只有该 pool 的
-runtime 支持全 rank CUDA P2P/LSA 后才能重新启用。其他 NVIDIA SKU 在 GPU outcome 通过 native
-correctness 和 publication gates 前仍为 unvalidated。
-
-v1 已移除的轴包括 cached-layout `[cl]`、runtime-visible `[rv]`、FP8、quantized combine、
-额外 routing distributions、activation profiles、uneven allocation、placement permutations、
-model envelopes 和 scaling studies。
-
-## Workflow 与产物
-
-`.github/workflows/collectivex-sweep.yml` 生成 public-SKU matrix，提取严格且被忽略的
-`.shards/.json` control，每个 shard 执行一次 allocation，对结果 JSON 做隐私检查并上传
-raw GitHub artifacts。运行默认使用 `release_tag=unversioned`，仅供诊断。V1 运行必须显式选择
-`release_tag=v1`；setup 随后要求固定的完整 matrix digest，并生成绑定 run、attempt 与 source 的
-`cxrelease-v1-*` marker。Partial 或 filtered 运行无法获得该 marker。
-
-`.github/workflows/collectivex-sweep.yml` 的 `publish-v1` 操作是显式的 V1 gate。它只接受一个位于
-qualification index 1、成功且带 V1 tag 的首次尝试 sweep run ID，重新校验 GitHub metadata 与
-release marker，并在 runner 本地可丢弃工作区中执行 `publisher.py`。只有完整通过 promotion、隐私检查和内容寻址的数据集才会以
-`cxpublication-v1-*` 上传；raw artifacts 与 publisher private workspace 永不暴露给前端。
-
-系统不需要 results server、attached store、Vercel storage、GCP、Neon、managed database 或
-managed object store。前端使用已有的 server-side `GITHUB_TOKEN`，即时发现最新成功且按版本隔离
-的 publication workflow，下载其 NDJSON artifact，校验 ZIP layout、UTF-8/NDJSON 结构、schema、
-promotion 状态与 SHA-256，随后提供带版本的 channel URL 和 immutable dataset URL。UI 保留显式
-benchmark-version selector；V2 及后续版本必须使用独立的 release tag 与 publication identity。
-完整 validation contract 见 [docs/methodology_zh.md](docs/methodology_zh.md)。
-
-## Runner 配置
-
-Runner 本地 Slurm 和 storage 值使用严格的 per-SKU JSON 文档，路径为
-`$XDG_CONFIG_HOME/inferencex/collectivex.json` 或 `COLLECTIVEX_OPERATOR_CONFIG`。该 mode-0600、
-同 owner、非 symlink 文件位于 checkout 外且永不上传。未知 runners、fields、duplicate keys、
-endpoint literals、unsafe paths 和非 JSON 输入均 fail closed；配置绝不作为 shell 执行。GHA
-仅将加密的 `COLLECTIVEX_OPERATOR_CONFIG_V1` 内容传给 launcher；launcher 验证后只导出所选
-SKU 的 allowlisted values，并在 allocation 前删除临时副本。必需 JSON fields 如下：
-
-| SKU | 变量 |
-|---|---|
-| `h100-dgxc`, `b200-dgxc` | `partition`, `account`, `squash_dir`, `stage_dir` |
-| `h200-dgxc` | `partition`, `squash_dir`, `stage_dir` |
-| `b300` | `partition`, `account`, `squash_dir`, `stage_dir` |
-| `gb200` | `partition`, `account`, 有序 `storage_roots` |
-| `gb300` | `partition`, `account`, `squash_dir`, `stage_dir`, `enroot_cache_path` |
-| `mi325x`, `mi355x` | `partition`, `squash_dir`, `stage_dir` |
-
-每个已选中的非 MNNVL EP16 placement 还必须提供 `socket_ifname` 和 `rdma_devices`，用来指定
-operator 审核过的 fabric；还可配置 allowlisted
-`ib_gid_index` 与 `rdma_service_level`。CollectiveX 不会通过启发式规则选择 management route 或
-HCA。Allocation 完成后，每个非 MNNVL scale-out 节点都必须证明所有已配置 interface 与 active
-HCA port 存在，之后才允许初始化 backend。Scale-up 和 MNNVL job 会清除这些 overrides。
-Scale-out NCCL/RCCL 固定使用 `IB` 与精确匹配的 HCA selectors；如果无法使用 RDMA，job 会失败，
-而不会回退到 socket 后仍被错误标记为 RDMA。
-
-`stage_dir` 必须是 checkout 与 workflow workspace 外预创建且由 runner owner 持有的 base，
-不能经过 symlink，group 和 world 都不能写入，并且 runner 与所有 allocation 节点必须以相同路径
-访问。Job 只创建带 marker 的 mode-0700 execution child，验证跨节点读写可见性，并在
-allocation teardown 后只删除该 child；不会挂载 runner checkout，也不会在 AMD image storage
-下创建 stage。
-
-导入前，每个 Docker Hub tag 都通过有界 registry requests 解析，并且必须匹配固定 digest；拒绝
-digest-qualified overrides。Enroot imports 使用固定 filesystem epoch 和带版本、绑定 registry
-digest 的 cache key。每个已挂载 squash 都重新计算 hash，同时记录 verified registry digest 和
-local squash hash。镜像提供的 DeepEP 会按精确 wheel 和 installed-file fingerprints 检查；
-source-built backends 使用固定 commits 和 runtime-verified GPU targets。DeepEP V2 的 mode-0700
-cluster-local build cache 由版本化 build recipe、verified image、architecture、upstream
-trees 和 dependency pins 共同寻址；container 只看到固定的 `/cx-cache` mount，且该 cache 永不
-进入 result artifacts。
-固定的 V2 与 Hybrid source 在每个 workflow 中只获取一次。每个 job 都会验证完整 archive，仅
-提取自身精确 backend root，只允许指向 archive 内 regular file 的受限相对 leaf symlink，并在
-staging 前重新核对 Git tree 与 submodule pins。
-Compute containers 仅接收显式 environment allowlist。Private host、address、device、NIC、
-credential、workspace 和 path 数据只保留在加密配置、忽略的 operator notes 或有界 mode-0600
-runner logs 中，永不上传。
-
-## 本地检查
-
-```bash
-uv run --with-requirements experimental/CollectiveX/requirements.txt \
- python -m unittest discover experimental/CollectiveX/tests -p 'test_*.py'
-uv run --with-requirements experimental/CollectiveX/requirements.txt \
- python experimental/CollectiveX/sweep_matrix.py --backends all --out /tmp/cx-matrix.json >/dev/null
-uv run --with-requirements experimental/CollectiveX/requirements.txt \
- python experimental/CollectiveX/publisher.py --store-root "$COLLECTIVEX_STORE_ROOT" verify
-bash -n experimental/CollectiveX/runtime/*.sh experimental/CollectiveX/launchers/*.sh
-```
-
-核心路径为 `capability.py`、`configs/`、`contracts.py`、`schemas/`、`sweep_matrix.py`、
-`publisher.py`、`runtime/`、`launchers/` 和 `tests/`。
diff --git a/experimental/CollectiveX/docs/methodology_zh.md b/experimental/CollectiveX/docs/methodology_zh.md
deleted file mode 100644
index 7f6dcb67af..0000000000
--- a/experimental/CollectiveX/docs/methodology_zh.md
+++ /dev/null
@@ -1,297 +0,0 @@
-# CollectiveX EP v1 契约
-
-
-
-[English](./methodology.md) | **中文**
-
-
-
-本文档定义新的 CollectiveX 结果。历史运行笔记是 evidence，不是 contract。
-
-## 产品边界
-
-CollectiveX 是通信 microbenchmark，用于：
-
-- 在同一 chip/topology 上比较 EP libraries；
-- 在相同 workload 下比较不同系统的 EP latency 和 logical payload bandwidth；
-- 展示 unsupported、failed、invalid 和 unstable evidence，同时避免污染决策。
-
-若没有单独的 correlation study，它不能预测 serving throughput。
-
-## 矩阵
-
-提升后的 workload 为 `deepseek-v3-v1`：hidden 7168、top-k 8、256 routed experts、BF16
-dispatch 和 combine、packed placement，以及 backend-tuned resources。每个 case 都显式选择
-normal `layout-and-dispatch-v1` 或 low-latency `expert-packed-weighted-combine-v1` 语义。
-
-- `ep-core-v1`：uniform routing；decode T=1..128 的 2 次幂；prefill T=256/512。
-- `ep-routing-v1`：Zipf，EPLB off/on；decode T=128；prefill T=512。
-- `ep-low-latency-v1`：使用 DeepEP V1/UCCL 原生 low-latency API；uniform decode T=1..128 的
- 2 次幂；capability contract 会拒绝其他后端，不会伪造 low-latency path。
-- 规范矩阵范围：请求 608 个 cases / 1,600 个 token points；364 个可运行 cases / 940 个
- points，分布在 58 个可执行 workflow shards/allocation cells；244 个 unsupported cases / 660 个
- points。
-
-| 系统 | EP8 | EP16 |
-|---|---|---|
-| H100/H200/B200/B300 | 1x8 NVLink，scale-up | 2x8 NVLink + RDMA，scale-out |
-| MI325X/MI355X | 1x8 XGMI，scale-up | 2x8 XGMI + RDMA，scale-out |
-| GB200/GB300 | 2x4 MNNVL，scale-up | 4x4 MNNVL，scale-up |
-
-物理主机数量不能定义通信范围。两个 GB 配置都位于同一个 72-GPU MNNVL scale-up domain 内。
-
-Unsupported combinations 是 terminal outcomes，不会被静默跳过。DeepEP V2 指 PR #605
-引入的 `ElasticBuffer`，并固定使用 upstream PR #630 的最小纯 scale-up 修复。V2 的 scale-up
-cases 请求 NCCL Device API LSA；若实际建立的 LSA team 未覆盖整个 EP world，则直接失败。x86
-EP16 scale-out 使用启用 GIN 的 hybrid path，并要求两个逻辑 scale-out domains（由两个物理 RDMA
-ranks 表示）、每个 domain 八个 scale-up ranks。GB EP16 仍是 MNNVL scale-up，因此使用 LSA。
-Source 中声明的 NVIDIA capabilities 在 GPU outcomes 通过 native oracle 和 publisher gates 前仍为
-unvalidated。当前 runner pool 上的 H100 V2 在 v1 中被声明为 unsupported，因为 NCCL 2.30.4
-报告其 EP8 communicator 不具备 Device API symmetric-memory 支持；只有该 pool 恢复全 rank
-CUDA P2P/LSA 支持后才能重新加入。已移除的轴包括 `[cl]`、`[rv]`、quantization、alternate
-activation/routing profiles、uneven allocation、placement
-permutations、model envelopes 和 scaling。
-FlashInfer 因可重复出现的间歇性执行失败而排除在 v1 外；这些失败不会转为 planned-unsupported
-coverage。
-MoRI EP8 在 normal mode 下使用 MI325X AsyncLL 或 MI355X IntraNode。EP16 固定使用 2x8 XGMI +
-RDMA 上的 InterNodeV1，配置为 96 blocks、64 RDMA blocks、8 warps、每个 PE 一个 QP，以及
-external input。MoRI 的 AsyncLL transport 不属于真正的 low-latency suite contract，也绝不会
-以该模式标注。
-
-## Workload 身份
-
-一个 canonical workload 在 global token batch 上生成，再按 source rank 切分。Expert indices
-和 gate weights 会序列化。Activations 使用带版本的整数计数器公式，其 BF16 值在不同 runtime
-中精确一致；完整身份绑定到 manifest。Manifest 还绑定 shape/EP coordinates 和 oracle version。
-SHA-256 覆盖 canonical bytes 和 parameters；重新生成 library RNG 不能证明身份一致。
-
-Routing traffic 区分：
-
-- token-expert assignments，决定 expert compute load；
-- rank-deduplicated token payload copies，决定 EP activation traffic。
-
-Adapters 不得生成 routing，也不得将两种量相互解释。
-
-## 测量
-
-Normal mode 使用 `layout-and-dispatch-v1`：dispatch timing 包括 layout 和 communication，combine
-通过 unweighted rank-sum path 返回 activation payload。Low-latency mode 使用
-`expert-packed-weighted-combine-v1`：DeepEP V1/UCCL 原生 API dispatch token-expert assignments，
-并执行 gate-weighted combine。Expert-output staging 不计入 isolated combine timing，但计入被测
-paired roundtrip。每个 component 声明 availability、origin、start/end states、stage scope 和 sample
-count。仅有 paired API 时，isolated components 报 null。`isolated_sum` 为派生值，不用于
-throughput 或 recommendations。Mode 属于 series identity；normal 和 low-latency evidence 不能
-共用排名 cohort。
-
-每个被测 component 均使用 `fixed-512-v1`：
-
-- 64 trials x 8 timed iterations = 512 observations；
-- 每个 trial/point 的每个可用被测 component 前，执行 32 次同步完整
- dispatch-stage-combine warmups；
-- 先测 roundtrip，再测 isolated dispatch 和 combine，并使用固定的 per-phase conditioning ladder；
-- 每次 iteration 先取跨 rank 最大 latency，再以 nearest-rank 计算 p50/p90/p95/p99。
-
-被测 roundtrip p99 是 headline latency。Retries 保持为独立 attempts；后续成功不会抹除早期失败。
-Decode 和 prefill 表示一个 MoE-layer collective 所代表的 serving regime；在其他 shape 相同时，
-它们不会改变 timed primitive。
-
-NCCL/RCCL reference 是 end-to-end Python adapter，而不是 bare fabric primitive。其 dispatch
-boundary 包含 layout、count exchange、device-to-host split synchronization、fresh receive
-allocation，以及四次 payload/metadata all-to-all；activation-only combine 还包含一次 all-to-all 和
-scatter/reduction。因此其 p99 测量完整 reference-adapter boundary，可能对 host/scheduler 敏感。
-它可作为 portable system control，但不得标记为 fabric、link、bus 或 single-collective latency。
-
-带版本的 conditioning 和 EPLB planner contracts（reference trace、redundant count 和
-placement/remap version）属于 scheduled 和 evidence identity。
-
-Logical payload bandwidth 为：
-
-`logical_payload_bytes / measured_latency_seconds`
-
-Normal-mode payload bytes 使用按 rank 去重的 token-rank activations；low-latency bytes 使用
-token-expert assignments。两种模式都在命名边界上加入必需 scale bytes，并排除 expert metadata、
-padding 和 backend buffer capacity。若没有定义 primitive model 或 transport counters，不发布
-algorithm bandwidth、bus bandwidth、wire utilization 或 physical-link utilization。Logical
-bandwidth 绝不能标为 physical bandwidth。已发布 payload 和 token rates 命名为
-`rate_at_latency_percentile`：bytes 或 tokens 除以对应 latency percentile。它们是 p99 latency
-下的 lower-tail service rates，不是 inverted rate distribution 的 p99 percentiles。
-
-## 正确性
-
-与实现无关的 oracle 使用 expert-specific deterministic transform，使错误 expert routing 无法
-通过 identity roundtrip。它对每个 rank 和 point 验证：
-
-1. destination rank/expert、source token、multiplicity、gate weight 和 receive counts；
-2. timing 前的 dispatched payload 和 metadata；
-3. timing 前的 combined output；
-4. 所有 timed samples 期间 semantic inputs 不变；
-5. timing 后再次验证 dispatched payload/metadata 和 combined output。
-
-Normal-mode adapters 使用 activation-only、unweighted rank-sum combine。Oracle 在 combine 前
-构造每个 rank 的 gate-weighted expert aggregate，独立计算 `sum(gate * expert(token))`，并检查
-dispatch metadata 和 transformed output。Low-latency adapters 单独验证 expert-packed
-source/expert assignment、原生 gate weights 和 gate-weighted combined output。两个契约都使用
-已记录的 `rtol=0.05` 和 `atol=0.02` 检查每个 element。任一 rank 或 point 失败都会使 case
-不合格。Pre/post dispatch evidence 按
-canonical source-token order 计算 hash。Native receive slots 可能非确定性分配，因此 physical
-receive order 不作为 correctness property。
-
-## Native 结果
-
-单个 raw case document 使用 `format: "collectivex.ep.v1"`，拒绝未知 fields，并包含：
-
-- `case`：稳定 case ID、suite、required tier 和 coordinate；
-- `workload`：canonical identity 和 logical MoE shape；
-- `measurement`：sampling、component states、timing 和 byte accounting；
-- `implementation`：实例化 class/API、固定 source、loaded libraries 和 resources；
-- `topology`：requested 和 realized SKU、devices、placement、scale-up domain 和 transport；
-- `provenance`：source SHA、image/squash hashes、allocation、run 和 attempt；
-- `rows`：point latency、byte accounting、token rate、correctness、load、fanout 和 anomaly evidence；
-- `outcome`：`success`、`failed`、`invalid`、`diagnostic` 或 `unsupported`，以及 reasons。
-
-Raw result documents 和 exact samples 会先经过临时 GitHub delivery artifacts，再由 publisher
-归档到 private bundle；它们不会进入 public tree。Private environment details 只保留在本地
-mode-0600 logs 和忽略的 operator notes 中；不会归档或发布。每个 expected case 有一个 terminal
-selected outcome，同时保留每次 attempt。
-
-## 身份与比较
-
-Canonical JSON 生成三个完整 SHA-256 IDs：
-
-- `series_id`：除 token coordinate 和 repeat allocation 外的所有 locked factors；
-- `point_id`：`series_id` 加 token coordinate；
-- `evidence_id`：`point_id` 加 allocation/run/attempt/sample checksum。
-
-Locked factors 包括 workload bytes、measurement 和 sampling contract、resources、realized
-topology、implementation/build、loaded libraries、image/squash、runtime 和 source SHA。
-Deferred code generation 会在 measurement 前捕获，并在之后再次捕获。DeepEP V2 使用固定的
-NVCC random seed，并绑定最终 cache keys、generated-source hashes 与 executable-SASS hashes；
-raw CUBIN bytes 仅保留为 private diagnostics。Hybrid 绑定实际自动调优配置与完整 kernel-key
-set，同时将各 rank 的 shared-object hashes 仅保留为 private diagnostics。本地构建的 extension
-hashes 属于 diagnostic；其固定 source trees、build recipe、runtime 与 dependencies 仍绑定到
-series。
-Series identity 包含 case ID；case ID 绑定完整 scheduled token ladder，以及固定的 percentile、
-rank-reduction、conditioning、warmup 和 correctness semantics。
-
-Controlled comparison 只声明一个 contrast：
-
-- `library`：backend implementation 及其 tuned resource profile 可以不同；realized system、
- workload、EP、resource policy、source 和 measurement 必须匹配；
-- `chip`：受控 platform contrast。完整 realized system/topology 和 tuned resource profile 可以不同，
- 但 workload、EP、placement class、resource policy、backend lineage、source 和 measurement 必须
- 匹配。它不是 silicon-only comparison；
-- `system`：保留所有 hardware/backend 差异，同时匹配 workload、EP 和 measurement；
-- `routing`：routing distribution/EPLB 可以不同，但 static implementation build/generator、system、
- model shape、resource profile 和 measurement 必须匹配。未启用 EPLB 的 Uniform 和 Zipf 复用
- 同一 generated implementation；EPLB 的 physical-expert/JIT configuration 是显式 treatment
- difference。
-
-任何未声明的 mismatch 都会拒绝 overlay。Chip/system results 描述 measured systems，而非仅描述
-silicon。
-
-## Evidence 策略
-
-Capability declarations 说明可以尝试什么；artifacts 决定 evidence status。Promotion 要求完整的
-expected coverage，不能有 missing、extra、duplicate、malformed 或 heterogeneous case。Public
-coverage 保留每个 matrix disposition；promotion 要求每个 runnable case 在所有 selected runs 中
-成功，且每个 planned-unsupported case 始终为 unsupported。只有固定 canonical full-v1 matrix，
-且具有 decision-grade library、chip、system 和 routing cohort，才能推进 `dev-latest`；partial
-matrices 仍为 diagnostic。Full-matrix digest 有意绑定精确 workflow shard grouping 和 requested
-cases，因此即使 case coverage 不变，修改 `--max-cases` 或 SKU round-robin scheduling order 也只
-会产生 diagnostic-only runs。Superseded retries、planned-unsupported outcomes 和 unstable
-comparison cohorts 可以用于诊断展示，但不能排名或推荐；promoted dataset 中每个成功的 required
-series 都必须保持 decision-grade。Runnable case 的任何 failed、invalid 或 diagnostic retry 都会
-阻止 promotion，即使后续 retry 成功。Routing cohorts 是 comparable-experimental sensitivities，
-不会产生 configuration recommendations；official library/platform/system cohorts 才能产生可执行
-recommendations。
-
-一个 point 只有在三个独立 workflow runs 和 allocation IDs 均通过 correctness、identity、
-provenance、tail gates、p50/p99 repeat-stability thresholds 和 stable ordering 后才成为
-decision-grade。Eligibility、controlled cohorts、sensitivity pairs 和 recommendations 由
-publisher 而非 frontend 计算。
-
-## 执行隔离
-
-每个非 MNNVL scale-out case 都使用 operator 固定的 socket 与 RDMA selectors。Launcher 会拒绝
-缺失或不完整的 profile，并在 backend 初始化前逐个 allocation 节点检查已配置 interface、active
-HCA port 与指定 GID。它不会改用 default route、继承的 runner environment 或 transport
-fallback。Scale-up 和 MNNVL case 会清除该 profile；scale-out NCCL/RCCL 强制设置
-`NCCL_NET=IB` 并精确匹配 HCA。Selector values 只保留在加密配置和 mode-0600 private logs 中。
-
-Repository staging 使用 checkout 与 workflow workspace 外预创建的 shared base；该 base 由
-runner owner 持有，group/world 均不可写。父进程在复制前解析精确 execution child，以
-runner-owned marker 声明所有权，并验证所有 allocation 节点读写的是同一份 bytes。Cleanup 会
-等待 allocation teardown 得到确认，并只删除该 child，包括可安全识别的未完成 claim。同一 run
-的 V2/Hybrid source archive 会在固定 member 数和解压大小上限内完整验证，并且只提取所选 fixed
-root；仅当相对 leaf symlink 指向同一 backend root 内的 regular member 时才允许创建，之后还要
-通过精确 Git tree/submodule 校验。
-
-## 产物验证与即时交付
-
-不使用 self-hosted service、Vercel storage、GCP、Neon、managed database 或 managed object
-store。Publication workflow 仅将 runner 本地临时存储用作可丢弃的 validation 与 promotion
-工作区：
-
-```text
-$COLLECTIVEX_STORE_ROOT/
- private/incoming/ # write-once downloaded GHA attempts
- private/bundles// # immutable source archives, native results/samples, matrix, checksums
- private/quarantine/ # rejected attempts plus machine-readable reasons
- public/datasets// # immutable sanitized frontend datasets
- public/channels/ # small atomic pointers: latest-attempt, dev-latest
- locks/
-```
-
-Private 和 public trees 使用不同 permissions。JSON manifests 和 checksums 是权威记录；可重建
-catalog 仅为 index。Raw sweep artifacts 只是 publisher 的临时输入；只有清理并完成 promotion
-的 NDJSON 会保留为前端 publication artifact。
-
-Container tags 会与固定 registry digests 核对。Enroot imports 使用固定
-`SOURCE_DATE_EPOCH` 和 versioned cache generation；每个 mounted squash 都重新计算 hash 并纳入
-series identity。Image-provided DeepEP 也按精确 per-architecture wheel 和 installed-file
-fingerprints 检查，因此 stale cache 不能继承固定 source identity。
-Source-built DeepEP V2 使用独立的 mode-0700 cluster-local cache，并且只以 `/cx-cache` 挂载。
-其 content key 绑定版本化 build recipe、verified image digest、CPU/GPU architecture、
-upstream source trees 和固定 build dependencies。该 cache 既不是 artifact，也不是 publisher
-input；每次执行的 source/results stage 仍然隔离且可丢弃，并在复用前以 marker 和 runtime probe
-fail closed。Runner UID 属于受信任的 cluster boundary：该 cache 用于防止 stale 或意外修改，
-不防御恶意的同 UID job。只有从未发布的 partial build 才能自动重置；已发布 cache 一旦未通过
-integrity 或 runtime 检查，将保持原样并被拒绝，避免并发 allocation 正在使用的文件被删除。
-
-Publication 采用 fail-closed：
-
-1. 获取 exclusive filesystem lock，并在 destination filesystem 上 stage；
-2. 解析前归档 source bytes；
-3. 要求精确 matrix-declared artifact set，并拒绝每个未消费 archive member；
-4. 验证 strict schemas、privacy、checksums、identities、timing 和精确 matrix outcomes；
-5. 写入 checksums 和 `COMPLETE`，fsync，然后原子 rename private bundle；
-6. 构建并验证 sanitized content-addressed dataset，fsync，然后原子 rename；
-7. 仅在全部 promotion gates 通过后原子替换 `dev-latest.json`。
-
-Rejected attempts 可以更新工作区中的 `latest-attempt`，但不能更新 `dev-latest`。工作区会随
-publication runner 销毁，且绝不连接到前端。只有三个选定 bundles 全部推进 `dev-latest` 后才会
-生成 artifact。
-
-`publisher.py ingest` 接受精确 matrix，并为每个 GitHub artifact 接受一个 `--artifact` directory
-或 ZIP。`promote` 接受显式 immutable bundle IDs。默认 `verify` 要求 `latest-attempt`；若存在
-`dev-latest` 也会验证，而显式 `--channel dev-latest` 则要求其存在。Workflow 只会将通过验证并
-清理后的 dataset 复制到单记录 `collectivex_public_v1_.ndjson` artifact。Raw artifacts 和
-private workspace 内容绝不打包进应用。
-
-Sweeps 默认使用 `release_tag=unversioned`。选择 `v1` 时必须匹配固定的完整 matrix digest，并
-生成绑定 run ID、attempt、source SHA 与 matrix SHA-256 的 marker。手动 publication workflow
-只接受三个唯一、成功、来自同一 source SHA 的 `CollectiveX Sweep` run IDs；它会重新校验
-metadata 与精确 markers，下载 immutable artifacts，并将相同 provenance assertions 传给
-`publisher.py ingest`。Partial、filtered、untagged、跨 source、失败或过期的输入都会 fail closed。
-
-前端使用 server-side GitHub read token，即时发现最新成功且按版本隔离的 publication run，并
-下载 publication artifact。它要求 ZIP 根目录只有一个 NDJSON entry，校验 UTF-8、schema、
-promotion 状态及 filename/body SHA-256，随后提供短期缓存的带版本 channel pointer 和 immutable
-带版本 dataset URL。Benchmark-version selector 当前只显示 V1；后续版本必须使用独立的 release
-与 publication identity。前端不会虚构 missing values、选择 retries，或重新计算 decision
-eligibility。
-
-## 历史数据
-
-Numeric schemas 3-5 不在 v1 publisher 和 frontend reader 范围内。它们仍是 historical
-diagnostic evidence，不能作为 `dev-latest` 初始数据或驱动 v1 decisions。
From e5f2f9ad91319eb6f2a4f639c1758f9adb5f968c Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Wed, 8 Jul 2026 18:57:38 +0800
Subject: [PATCH 203/210] refactor(collectivex): drop dead code and a redundant
profile override
- identity.py: remove no-op component_order_contract override in
V1_LOW_LATENCY_CASE_PROFILE (identical to the value inherited from
V1_NORMAL_CASE_PROFILE via spread).
- capability.py: remove unused DEEPEP_V1_IBGDA_NIC_HANDLERS constant; the
b200 CPU-proxied IBGDA handler is set live in runtime/common.sh.
- runtime/common.sh: remove uncalled cx_require_single_node and
cx_gha_workspace_stage_root helpers.
No behavior change; CPU battery green (13 passed, 6 subtests).
---
experimental/CollectiveX/capability.py | 4 ---
experimental/CollectiveX/identity.py | 1 -
experimental/CollectiveX/runtime/common.sh | 33 ----------------------
3 files changed, 38 deletions(-)
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
index bdc9b2cafb..22e2f3f0c6 100644
--- a/experimental/CollectiveX/capability.py
+++ b/experimental/CollectiveX/capability.py
@@ -23,10 +23,6 @@
"mi355x": {"schedulable": False, "basis": "nvidia-only"},
}
-# B200's publication pods cannot map NIC doorbells into GPU address space.
-# DeepEP V1 scale-out therefore uses NVSHMEM's CPU-proxied IBGDA handler.
-DEEPEP_V1_IBGDA_NIC_HANDLERS = {"b200-dgxc": "cpu"}
-
def _topologies(
product: str, *, gpus_per_node: int, scale_up_domain: int, scale_up_transport: str
diff --git a/experimental/CollectiveX/identity.py b/experimental/CollectiveX/identity.py
index 7d22fefd4c..2fa34bf8d4 100644
--- a/experimental/CollectiveX/identity.py
+++ b/experimental/CollectiveX/identity.py
@@ -55,7 +55,6 @@
V1_LOW_LATENCY_CASE_PROFILE = {
**V1_NORMAL_CASE_PROFILE,
- "component_order_contract": "qualification-hash-rotated-components-v1",
"combine_semantics": "gate-weighted",
"contract": "expert-packed-weighted-combine-v1",
"correctness_scope": "expert-assignment-and-weighted-combine",
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 0f91fe9d98..747cf556f8 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -760,10 +760,6 @@ cx_require_record_safe() {
done
}
-cx_require_single_node() {
- [ "${CX_NODES:-1}" = "1" ] || cx_die "$1 supports one-node EP only"
-}
-
cx_nccl_hca_device_name() {
local selector="${1#=}"
printf '%s' "${selector%%:*}"
@@ -1486,35 +1482,6 @@ cx_default_image_digest() {
esac
}
-# Canonical workflow runs must not inherit benchmark controls from a persistent
-# self-hosted runner service. Manual/SSH diagnostics retain their explicit
-# overrides by leaving COLLECTIVEX_CANONICAL_GHA unset.
-cx_gha_workspace_stage_root() {
- local workspace="${GITHUB_WORKSPACE:-}"
- python3 - "$workspace" <<'PY'
-import os
-import stat
-import sys
-
-workspace = sys.argv[1]
-try:
- if (
- not os.path.isabs(workspace)
- or os.path.realpath(workspace) != workspace
- or not os.path.isdir(workspace)
- ):
- raise OSError
- metadata = os.stat(workspace, follow_symlinks=False)
- # GitHub runner workspaces are runner-owned but commonly writable by the
- # trusted runner-service group. Keep the child mode 0700 and reject world write.
- if metadata.st_uid != os.getuid() or stat.S_IMODE(metadata.st_mode) & stat.S_IWOTH:
- raise OSError
-except OSError:
- raise SystemExit(1)
-print(workspace, end="")
-PY
-}
-
# Create a per-UID cache under validated cluster-local storage. Only the fixed
# /cx-cache mount enters the container; the operator host path does not.
cx_prepare_backend_cache() {
From d2c32ce902ead12cf7c4fc4d486a2081a45db17a Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Wed, 8 Jul 2026 20:02:48 +0800
Subject: [PATCH 204/210] refactor(collectivex): drop contract/schema layer;
success is return-code only
Remove contracts.py and schemas/*.json and rewire every dependent to run
without document validation, terminal-outcome emission, or delivery gating.
A case now counts as run on the benchmark's return code alone; failed or
unsupported cells emit no synthetic record.
- bench: drop `import contracts`; move ContractError/GIT_RUN_FIELDS into
ep_provenance.py; inline the frozen v1 conditioning ladders; remove the
validate_raw_document emit-time call.
- runtime: strip result-doc probes, terminal-outcome/setup-failure emission,
and delivery gating from common.sh and run_in_container.sh.
- summarize.py: drop the no-longer-emitted terminal document format.
- CI: remove both validate-delivery steps, emit-unsupported, and jsonschema;
re-gate summary/stage/upload/cleanup off delivery_contracts.
- requirements: drop jsonschema.
- docs/tests: README, methodology, and matrix tests updated to the
return-code-only reality; delete test_contracts.py.
---
.github/workflows/collectivex-sweep.yml | 67 +-
experimental/CollectiveX/README.md | 32 +-
experimental/CollectiveX/bench/ep_deepep.py | 3 +-
.../CollectiveX/bench/ep_deepep_hybrid.py | 1 -
experimental/CollectiveX/bench/ep_harness.py | 10 +-
experimental/CollectiveX/bench/ep_nccl.py | 1 -
.../CollectiveX/bench/ep_provenance.py | 15 +-
experimental/CollectiveX/contracts.py | 911 ------------
experimental/CollectiveX/docs/methodology.md | 26 +-
experimental/CollectiveX/requirements.txt | 3 -
experimental/CollectiveX/runtime/common.sh | 235 +---
.../CollectiveX/runtime/run_in_container.sh | 36 +-
.../schemas/raw-case-v1.schema.json | 1218 -----------------
.../schemas/samples-v1.schema.json | 86 --
.../schemas/terminal-outcome-v1.schema.json | 285 ----
experimental/CollectiveX/summarize.py | 29 +-
experimental/CollectiveX/sweep_matrix.py | 91 --
.../CollectiveX/tests/test_contracts.py | 74 -
experimental/CollectiveX/tests/test_matrix.py | 35 +-
19 files changed, 87 insertions(+), 3071 deletions(-)
delete mode 100644 experimental/CollectiveX/contracts.py
delete mode 100644 experimental/CollectiveX/schemas/raw-case-v1.schema.json
delete mode 100644 experimental/CollectiveX/schemas/samples-v1.schema.json
delete mode 100644 experimental/CollectiveX/schemas/terminal-outcome-v1.schema.json
delete mode 100644 experimental/CollectiveX/tests/test_contracts.py
diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml
index daec48c6a7..5c23e6b2e9 100644
--- a/.github/workflows/collectivex-sweep.yml
+++ b/.github/workflows/collectivex-sweep.yml
@@ -1,9 +1,9 @@
# CollectiveX Sweep — one structured run instead of thousands of dispatches.
#
# Shape: setup resolves the suites into a shard matrix, then one GPU cell runs per
-# shard, executes the benchmark, validates the emitted neutral artifacts, and uploads
-# them. This repo only schedules, executes, validates, and uploads — it does not
-# promote, rank, recommend, or decide what a frontend displays. This file is registered
+# shard, executes the benchmark, and uploads the emitted neutral artifacts as-is.
+# This repo only schedules, executes, and uploads — it does not validate, promote,
+# rank, recommend, or decide what a frontend displays. This file is registered
# on the default branch, so its collectivex branch revision can be dispatched with --ref.
name: CollectiveX Sweep
permissions:
@@ -49,10 +49,8 @@ jobs:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v5.0.0
with: { clean: true, persist-credentials: false }
- name: Install matrix dependencies
- # jsonschema is required by contracts.py (make_terminal_document via
- # --emit-unsupported-from, and validate-delivery) once it is exercised;
# PyYAML is used by sweep_matrix.py to read the suites config.
- run: python3 -m pip install --quiet PyYAML==6.0.2 jsonschema==4.25.1
+ run: python3 -m pip install --quiet PyYAML==6.0.2
- id: gen
working-directory: experimental/CollectiveX
env:
@@ -61,9 +59,6 @@ jobs:
INPUT_ONLY_SKU: ${{ inputs.only_sku }}
INPUT_EXCLUDE_SKUS: ${{ inputs.exclude_skus }}
INPUT_EP_SIZES: ${{ inputs.ep_sizes }}
- COLLECTIVEX_SOURCE_SHA: ${{ github.sha }}
- COLLECTIVEX_ARTIFACT_NAME: cxunsupported-${{ github.run_id }}-${{ github.run_attempt }}
- COLLECTIVEX_EXECUTION_ID: ${{ github.run_id }}_${{ github.run_attempt }}_unsupported
run: |
set -euo pipefail
args=(--suites "$INPUT_SUITES")
@@ -82,12 +77,6 @@ jobs:
echo "source_backends=$(python3 -c "import json;m=json.load(open('matrix_full.json'));print(' '.join(sorted({x['backend'] for x in m['include']} & {'deepep-v2','deepep-hybrid'})))")"
echo "max_parallel=$(python3 -c "import json;m=json.load(open('matrix_full.json'));w=max(x['execution_weight'] for x in m['include']);print(max(1,min(10,4096//w)))")"
} >> "$GITHUB_OUTPUT"
- unsupported_n=$(python3 -c "import json;m=json.load(open('matrix_full.json'));print(sum(x['disposition']=='unsupported' for x in m['requested_cases']))")
- echo "unsupported_n=$unsupported_n" >> "$GITHUB_OUTPUT"
- if [ "$unsupported_n" -gt 0 ]; then
- python3 sweep_matrix.py --emit-unsupported-from matrix_full.json \
- --out-dir unsupported
- fi
python3 -c "import json;m=json.load(open('matrix_full.json'));print('execution-cells:',len(m['include']))"
- name: Prepare pinned backend source archive
if: ${{ steps.gen.outputs.source_backends != '' }}
@@ -142,24 +131,6 @@ jobs:
if-no-files-found: error
overwrite: true
retention-days: 14
- - name: Validate unsupported outcomes
- id: unsupported_contracts
- if: ${{ always() && fromJSON(steps.gen.outputs.unsupported_n) > 0 }}
- env:
- COLLECTIVEX_ARTIFACT_NAME: cxunsupported-${{ github.run_id }}-${{ github.run_attempt }}
- COLLECTIVEX_EXECUTION_ID: ${{ github.run_id }}_${{ github.run_attempt }}_unsupported
- run: |
- python3 experimental/CollectiveX/contracts.py validate-delivery \
- --source experimental/CollectiveX/matrix_full.json \
- --disposition unsupported \
- experimental/CollectiveX/unsupported/*.json
- - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
- if: ${{ always() && fromJSON(steps.gen.outputs.unsupported_n) > 0 && steps.unsupported_contracts.outcome == 'success' }}
- with:
- name: cxunsupported-${{ github.run_id }}-${{ github.run_attempt }}
- path: experimental/CollectiveX/unsupported/*.json
- if-no-files-found: error
- retention-days: 14
# ---- sweep: ONE matrix cell per shard (the parent job with child jobs) ----
sweep:
@@ -191,13 +162,11 @@ jobs:
CX_SOURCE_ROOT: ${{ matrix.launcher == 'mi-amds' && format('/tmp/inferencex-collectivex-parent-{0}-{1}-{2}/inferencex-collectivex-{0}-{1}-{2}/source', github.run_id, github.run_attempt, matrix.id) || format('/tmp/inferencex-collectivex-{0}-{1}-{2}/source', github.run_id, github.run_attempt, matrix.id) }}
HOME: ${{ matrix.launcher == 'mi-amds' && format('/tmp/inferencex-collectivex-parent-{0}-{1}-{2}/inferencex-collectivex-{0}-{1}-{2}/home', github.run_id, github.run_attempt, matrix.id) || format('/tmp/inferencex-collectivex-{0}-{1}-{2}/home', github.run_id, github.run_attempt, matrix.id) }}
steps:
- - name: Provision host Python for contract validation
- # P2 moved contracts.py onto jsonschema (host-only publisher schemas that,
- # per requirements.txt, are never imported by GPU execution). The self-hosted
- # runners ship a bare python3 with no system pip, so build a venv and put it on
- # PATH for every later step: the launcher fail-safe (emit-terminal),
- # validate-delivery, and summarize.py all invoke bare `python3` and must resolve
- # jsonschema. venv + ensurepip are available even where system pip is not.
+ - name: Provision host Python
+ # The self-hosted runners ship a bare python3 with no system pip. The host
+ # launcher and summarize.py invoke bare `python3` and need PyYAML/numpy, so
+ # build a venv and put it on PATH for every later step. venv + ensurepip are
+ # available even where system pip is not.
env:
HOME: ${{ runner.temp }}
run: |
@@ -206,7 +175,7 @@ jobs:
python3 -m venv --clear "$venv"
"$venv/bin/python3" -m pip install --quiet --no-cache-dir --upgrade pip
"$venv/bin/python3" -m pip install --quiet --no-cache-dir \
- jsonschema==4.25.1 PyYAML==6.0.2 "numpy>=1.26,<3"
+ PyYAML==6.0.2 "numpy>=1.26,<3"
echo "$venv/bin" >> "$GITHUB_PATH"
- name: Prepare isolated source
id: source
@@ -486,17 +455,8 @@ jobs:
set -euo pipefail
[ -f "$CX_JOB_ROOT/cleanup-safe" ] && [ ! -e "$CX_JOB_ROOT/cleanup-unsafe" ] \
|| { echo "CollectiveX allocation cleanup was not confirmed" >&2; exit 1; }
- - name: Validate shard delivery completeness
- id: delivery_contracts
- if: ${{ always() && steps.allocation_cleanup.outcome == 'success' }}
- run: |
- cd "$CX_SOURCE_ROOT" 2>/dev/null \
- || { echo "CollectiveX source is unavailable" >&2; exit 1; }
- python3 experimental/CollectiveX/contracts.py validate-delivery \
- --source "experimental/CollectiveX/${CX_SHARD_FILE}" \
- experimental/CollectiveX/results/*.json
- name: Shard summary
- if: ${{ always() && steps.delivery_contracts.outcome == 'success' }}
+ if: ${{ always() && steps.allocation_cleanup.outcome == 'success' }}
run: |
cd "$CX_SOURCE_ROOT" 2>/dev/null \
|| { echo "CollectiveX source is unavailable" >&2; exit 1; }
@@ -504,7 +464,7 @@ jobs:
--results-dir experimental/CollectiveX/results --markdown >> "$GITHUB_STEP_SUMMARY" || true
- name: Stage shard artifact
id: stage_artifact
- if: ${{ always() && steps.delivery_contracts.outcome == 'success' }}
+ if: ${{ always() && steps.allocation_cleanup.outcome == 'success' }}
run: |
set -euo pipefail
cd "$CX_SOURCE_ROOT" 2>/dev/null \
@@ -512,7 +472,7 @@ jobs:
cp -- experimental/CollectiveX/results/*.json "$CX_JOB_ROOT/artifact/"
- name: Upload shard results
id: upload_artifact
- if: always() && steps.stage_artifact.outcome == 'success' && steps.delivery_contracts.outcome == 'success'
+ if: always() && steps.stage_artifact.outcome == 'success'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: ${{ format('cxshard-{0}-{1}-{2}', matrix.id, github.run_id, github.run_attempt) }}
@@ -536,7 +496,6 @@ jobs:
|| { echo "CollectiveX allocation cleanup was not confirmed; retaining isolated files" >&2; exit 1; }
if [ '${{ steps.sweep_shard.outcome }}' = success ] \
&& [ '${{ steps.allocation_cleanup.outcome }}' = success ] \
- && [ '${{ steps.delivery_contracts.outcome }}' = success ] \
&& [ '${{ steps.stage_artifact.outcome }}' = success ] \
&& [ '${{ steps.upload_artifact.outcome }}' = success ] \
&& [ -f "$CX_SOURCE_ROOT/experimental/CollectiveX/runtime/common.sh" ]; then
diff --git a/experimental/CollectiveX/README.md b/experimental/CollectiveX/README.md
index ef8d508909..6d937ecf5b 100644
--- a/experimental/CollectiveX/README.md
+++ b/experimental/CollectiveX/README.md
@@ -4,10 +4,10 @@ CollectiveX is an experimental MoE expert-parallel communication benchmark. It m
combine, and paired roundtrip latency across EP libraries and accelerator systems, then uploads
neutral result artifacts.
-CollectiveX schedules benchmarks, executes them on real allocations, validates what each run emits,
-and uploads neutral artifacts. It does not promote, rank, recommend, select, or decide what a
-consumer displays. Any downstream display or comparison is the consumer's responsibility. The full
-measurement and validation contract is in [docs/methodology.md](docs/methodology.md).
+CollectiveX schedules benchmarks, executes them on real allocations, and uploads the neutral
+artifacts each run emits. It does not validate those artifacts, promote, rank, recommend, select, or
+decide what a consumer displays. Any downstream display or comparison is the consumer's
+responsibility. The full measurement methodology is in [docs/methodology.md](docs/methodology.md).
## Execution Profile
@@ -28,8 +28,8 @@ iteration takes the cross-rank maximum before nearest-rank p50/p90/p95/p99, and
headline latency. A stdlib integer counter produces byte-identical routing and gate weights.
Correctness is checked against the reference activation. The combine gate is `rtol=0.05, atol=0.02`
-for the BF16 communication path. Any failed rank or point makes the case ineligible and is
-recorded as a terminal outcome.
+for the BF16 communication path. Any failed rank or point makes the case ineligible in the result
+it writes.
The matrix covers H100, H200, B200, B300, GB200, GB300, and MI355X. `sweep_matrix.py` materializes
the requested SKUs, backends, EP sizes, and token ladders, then extracts strict per-shard controls
@@ -68,7 +68,7 @@ hybrid path with GIN, two logical scale-out domains represented by two physical
scale-up ranks per domain; GB EP16 remains MNNVL scale-up and therefore uses LSA. The isolated build
records the API, source, loaded libraries, generated JIT source, and executable SASS; raw CUBIN bytes
stay private diagnostics. Whether a given SKU/backend/EP cell is attempted is a capability fact;
-whether it succeeded is decided only by the emitted artifact.
+whether it succeeded is decided by the benchmark's return code.
## Workflow And Artifacts
@@ -76,14 +76,14 @@ whether it succeeded is decided only by the emitted artifact.
(`backend`, `suites`, `only_sku`, `exclude_skus`, `ep_sizes` inputs), fetches the pinned backend
source archive, and uploads the matrix.
`sweep` extracts a strict ignored `.shards/.json` control per matrix entry, executes one
-allocation per shard, runs `contracts.py validate-delivery`, and uploads the result artifacts with
-`always()` so a red or partial run still uploads.
+allocation per shard, and uploads the result artifacts with `always()` so a red or partial run still
+uploads.
-Each shard emits per-case result and terminal JSON, detached sample JSON, and a small mechanical
-summary. `validate-delivery` asserts strict schemas, one terminal record per scheduled case,
-detached-sample path/SHA-256 consistency, and privacy before upload. No step promotes a run, builds a
-dataset, or advances a channel; the neutral artifacts are the output. A consumer downloads them and
-decides what to display.
+Each shard emits per-case result JSON, detached sample JSON, and a small mechanical summary. A case
+counts as successful on the benchmark's own return code; there is no schema, completeness, or privacy
+validation step, and failed or unsupported cells produce no synthetic record. No step promotes a run,
+builds a dataset, or advances a channel; the neutral artifacts are the output. A consumer downloads
+them and decides what to display.
Private host, address, device, NIC, credential, workspace, and path data stays in encrypted config,
ignored operator notes, or bounded mode-0600 runner logs; it is never uploaded.
@@ -159,5 +159,5 @@ uv run --with-requirements experimental/CollectiveX/requirements.txt \
bash -n experimental/CollectiveX/runtime/*.sh experimental/CollectiveX/launchers/*.sh
```
-Core paths are `capability.py`, `configs/`, `contracts.py`, `identity.py`, `schemas/`,
-`sweep_matrix.py`, `summarize.py`, `bench/`, `runtime/`, `launchers/`, and `tests/`.
+Core paths are `capability.py`, `configs/`, `identity.py`, `sweep_matrix.py`, `summarize.py`,
+`bench/`, `runtime/`, `launchers/`, and `tests/`.
diff --git a/experimental/CollectiveX/bench/ep_deepep.py b/experimental/CollectiveX/bench/ep_deepep.py
index 6346a421e1..28b0e23aab 100644
--- a/experimental/CollectiveX/bench/ep_deepep.py
+++ b/experimental/CollectiveX/bench/ep_deepep.py
@@ -8,7 +8,6 @@
import torch
import torch.distributed as dist
-import contracts
import ep_provenance
from ep_deepep_family import DeepEPFamilyBackend
@@ -49,7 +48,7 @@ def _mnnvl_buffer_configuration() -> tuple[dict[str, bool], str]:
requested=True, signature_parameters=parameters,
deepep_commit=os.environ.get("DEEPEP_COMMIT"),
)
- except contracts.ContractError as exc:
+ except ep_provenance.ContractError as exc:
raise RuntimeError(str(exc)) from exc
diff --git a/experimental/CollectiveX/bench/ep_deepep_hybrid.py b/experimental/CollectiveX/bench/ep_deepep_hybrid.py
index 3e7f3f0b1c..87737eace8 100644
--- a/experimental/CollectiveX/bench/ep_deepep_hybrid.py
+++ b/experimental/CollectiveX/bench/ep_deepep_hybrid.py
@@ -38,7 +38,6 @@
import torch
import torch.distributed as dist
-import contracts
import ep_provenance
from ep_backend import EPBackend
diff --git a/experimental/CollectiveX/bench/ep_harness.py b/experimental/CollectiveX/bench/ep_harness.py
index 4032c806c0..eab75d9c29 100644
--- a/experimental/CollectiveX/bench/ep_harness.py
+++ b/experimental/CollectiveX/bench/ep_harness.py
@@ -45,7 +45,6 @@
import os
import types
-import contracts
import ep_provenance
import identity
import workload as workload_contract
@@ -81,10 +80,14 @@
# clean log x-axis; clamped to the backend buffer ceiling (MoRI's registerable heap).
DECODE_LADDER = [1, 2, 4, 8, 16, 32, 64, 128]
PREFILL_LADDER = [128, 256, 512, 1024, 2048, 4096]
+# Conditioning replays a fixed phase ramp before each measured shape to settle
+# clocks and routing state; these rounds are never timed or emitted. The ladders
+# and round count are the frozen v1 measurement contract (see conditioning_contract).
CONDITIONING_LADDERS = {
- phase: list(ladder) for phase, ladder in contracts.V1_CONDITIONING_LADDERS.items()
+ "decode": [1, 2, 4, 8, 16, 32, 64, 128],
+ "prefill": [1, 2, 4, 8, 16, 32, 64, 128, 256, 512],
}
-CONDITIONING_ROUNDS_PER_SHAPE = contracts.V1_CONDITIONING_ROUNDS_PER_SHAPE
+CONDITIONING_ROUNDS_PER_SHAPE = 8
CONDITIONING_CONTRACT = identity.V1_CASE_PROFILE["conditioning_contract"]
ORACLE_CONTRACT = identity.V1_CASE_PROFILE["oracle_contract"]
# Dispatch and combine are fixed BF16, so the combine oracle uses one frozen gate.
@@ -1765,7 +1768,6 @@ def component(percentiles, count, *, derived=False):
"validity": validity,
},
}
- contracts.validate_raw_document(doc, samples_document)
if rank == 0:
_write_bytes_atomic(samples_path, samples_payload)
_write_json_atomic(args.out, doc)
diff --git a/experimental/CollectiveX/bench/ep_nccl.py b/experimental/CollectiveX/bench/ep_nccl.py
index 4dac5f4cc7..309d7125bf 100644
--- a/experimental/CollectiveX/bench/ep_nccl.py
+++ b/experimental/CollectiveX/bench/ep_nccl.py
@@ -19,7 +19,6 @@
import torch
import torch.distributed as dist
-import contracts
import ep_provenance
from ep_backend import EPBackend
diff --git a/experimental/CollectiveX/bench/ep_provenance.py b/experimental/CollectiveX/bench/ep_provenance.py
index 82a74c96fe..06c56259de 100644
--- a/experimental/CollectiveX/bench/ep_provenance.py
+++ b/experimental/CollectiveX/bench/ep_provenance.py
@@ -5,10 +5,8 @@
evidence it embeds in each raw attempt document: which backend libraries were
loaded, that a deepep-v2 attempt carries internally consistent JIT cubins, that
a hybrid attempt realized its kernels, and so on. Those are cross-field facts a
-JSON Schema cannot express, so they live in Python — but they are an emitter
-concern, not part of the neutral cross-file validation boundary. They are kept
-here, beside the executable bench modules that call them, so ``contracts.py``
-holds only the neutral document/delivery validation.
+JSON Schema cannot express, so they live in Python here, beside the executable
+bench modules that build and self-check them.
"""
from __future__ import annotations
@@ -21,7 +19,14 @@
from pathlib import Path, PurePosixPath
from typing import Any, Iterable
-from contracts import ContractError, GIT_RUN_FIELDS
+class ContractError(ValueError):
+ """A provenance payload differs from the CollectiveX emitter contract."""
+
+
+# Git run-identity fields an emitted attempt carries when produced under CI.
+GIT_RUN_FIELDS = {
+ "artifact", "job", "ref", "repo", "run_attempt", "run_id", "source_sha",
+}
def _finite_tree(value: Any, path: str = "$") -> None:
diff --git a/experimental/CollectiveX/contracts.py b/experimental/CollectiveX/contracts.py
deleted file mode 100644
index ea12270ccf..0000000000
--- a/experimental/CollectiveX/contracts.py
+++ /dev/null
@@ -1,911 +0,0 @@
-#!/usr/bin/env python3
-"""Result contracts for CollectiveX.
-
-JSON Schema (``schemas/*.schema.json``, Draft 2020-12) owns document shape and
-field constraints. The Python here covers only the facts JSON Schema cannot
-express: recomputing the readable identities from their factors, joining detached
-samples by point identity and digest, one terminal result per scheduled case,
-complete shard delivery, and the artifact privacy boundary. The public surface is
-intentionally small:
-
- strict_json_load() - duplicate/non-finite-rejecting JSON reader
- validate_result() - validate one emitted result file (raw+samples or terminal)
- make_terminal_result() - build and self-validate one non-success outcome
- validate_delivery() - reconcile a shard/matrix with its complete attempt set
-
-Backend implementation-provenance constants and canonical-JSON helpers now live in
-``bench/ep_provenance.py``, beside the executable modules that emit them.
-"""
-from __future__ import annotations
-
-import argparse
-import datetime as dt
-import hashlib
-import ipaddress
-import json
-import os
-from pathlib import Path
-import re
-import sys
-from typing import Any
-
-import identity
-
-RAW_FORMAT = "collectivex.ep.v1"
-SAMPLES_FORMAT = "collectivex.samples.v1"
-TERMINAL_FORMAT = "collectivex.terminal.v1"
-GIT_RUN_FIELDS = {
- "artifact", "job", "ref", "repo", "run_attempt", "run_id", "source_sha",
-}
-PRE_EXECUTION_FAILURE_REASONS = {
- "setup": "launcher-setup-failed",
- "repository-stage": "repository-staging-failed",
- "registry-verification": "container-registry-verification-failed",
- "scheduler-allocation": "scheduler-allocation-failed",
- "container-import": "container-image-preparation-failed",
- "container-hash": "container-image-identity-failed",
- "container-launch": "container-runtime-launch-failed",
- "backend-setup": "backend-setup-failed",
- "artifact-collection": "artifact-collection-failed",
-}
-RUNTIME_FAILURE_REASONS = {
- **PRE_EXECUTION_FAILURE_REASONS,
- "runtime-identity": "runtime-identity-mismatch",
- "timeout": "execution-timeout",
- "deadlock": "execution-deadlock",
- "execution": "distributed-command-failed",
-}
-POST_EMIT_FAILURE_REASONS = {
- mode: "post-emit-distributed-command-failed"
- for mode in ("runtime-identity", "timeout", "deadlock", "execution")
-}
-CAPABILITY_FAILURE_REASONS = frozenset({
- "backend-platform-unsupported",
- "backend-token-capacity",
-})
-RETURN_CODE_FAILURE_MODES = {
- 5: "runtime-identity",
- 124: "timeout",
- 137: "timeout",
-}
-V1_CONDITIONING_LADDERS = {
- "decode": (1, 2, 4, 8, 16, 32, 64, 128),
- "prefill": (1, 2, 4, 8, 16, 32, 64, 128, 256, 512),
-}
-V1_CONDITIONING_ROUNDS_PER_SHAPE = 8
-SCHEMA_DIR = Path(__file__).resolve().parent / "schemas"
-
-
-class ContractError(ValueError):
- """A document differs from the CollectiveX result contract."""
-
-
-# ---------------------------------------------------------------------------
-# Artifact privacy boundary (kept narrow; enforced in delivery + terminal).
-# ---------------------------------------------------------------------------
-SENSITIVE_FIELDS = frozenset({
- "environment", "env", "host", "hostname", "uuid", "gpu_uuid", "device_uuid",
- "pci_bus_id", "ip_address", "ip_addresses", "master_addr", "ssh", "ssh_target",
- "nodelist", "node_list", "nic_guid", "ib_guid", "topology_matrix", "rdma_devices",
- "user", "username", "password", "passwd", "secret", "token", "access_token",
- "api_token", "auth_token", "api_key", "private_key", "credential", "credentials",
- "address", "addresses", "ip", "ips",
-})
-SENSITIVE_FIELDS_COMPACT = frozenset(item.replace("_", "") for item in SENSITIVE_FIELDS)
-SENSITIVE_FIELD_SUFFIXES = (
- "_host", "_hostname", "_address", "_addresses", "_path", "_paths", "_ip", "_ips",
- "_password", "_passwd", "_secret", "_token", "_credential", "_credentials",
- "_uuid", "_guid", "_bus_id",
-)
-SENSITIVE_VALUE_PATTERNS = (
- ("private-path", re.compile(
- r"(? str:
- normalized = re.sub(r"([A-Z]+)([A-Z][a-z])", r"\1_\2", str(value).strip())
- normalized = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", normalized)
- return normalized.lower().replace("-", "_")
-
-
-def _sensitive_value_rule(value: str, *, contextual: bool = True) -> str | None:
- matched = next(
- (
- name for name, pattern in SENSITIVE_VALUE_PATTERNS
- if (contextual or name not in CONTEXTUAL_VALUE_RULES) and pattern.search(value)
- ),
- None,
- )
- if matched:
- return matched
- for candidate in IPV6_CANDIDATE.findall(value):
- try:
- address = candidate.split("%", 1)[0]
- if ipaddress.ip_address(address).version == 6:
- return "ipv6-address"
- except ValueError:
- continue
- return None
-
-
-def assert_publication_safe(docs: list[dict]) -> None:
- """Reject private infrastructure fields and value shapes at the artifact boundary."""
- def walk(value, doc_index: int, parent_field: str | None = None) -> None:
- if isinstance(value, dict):
- for key, child in value.items():
- field = _normalized_field(key)
- compact = field.replace("_", "")
- if (
- field in SENSITIVE_FIELDS
- or compact in SENSITIVE_FIELDS_COMPACT
- or field.endswith(SENSITIVE_FIELD_SUFFIXES)
- ):
- raise ArtifactSafetyError(
- f"artifact safety: doc[{doc_index}] contains forbidden private field"
- )
- key_rule = _sensitive_value_rule(str(key))
- if key_rule:
- raise ArtifactSafetyError(
- f"artifact safety: doc[{doc_index}] contains forbidden {key_rule} key"
- )
- walk(child, doc_index, field)
- elif isinstance(value, list):
- for child in value:
- walk(child, doc_index, parent_field)
- elif isinstance(value, str):
- rule = _sensitive_value_rule(value, contextual=parent_field != "ref")
- if rule:
- raise ArtifactSafetyError(
- f"artifact safety: doc[{doc_index}] contains forbidden {rule} value"
- )
-
- for index, doc in enumerate(docs):
- if not isinstance(doc, dict):
- raise ArtifactSafetyError(f"artifact safety: doc[{index}] is not a JSON object")
- walk(doc, index)
-
-
-# ---------------------------------------------------------------------------
-# Scheduled-case profile resolution (shared by the cross-file validation below
-# and by the bench emitter; backend implementation-provenance evidence now lives
-# in bench/ep_provenance.py).
-# ---------------------------------------------------------------------------
-def scheduled_case_profile(case: dict[str, Any], path: str = "case") -> dict[str, Any]:
- """Resolve an explicit scheduled mode to its immutable measurement profile."""
- try:
- return identity.profile_for_case(case)
- except identity.IdentityError as exc:
- raise ContractError(f"{path}: {exc}") from exc
-
-
-# ---------------------------------------------------------------------------
-# JSON IO helpers.
-# ---------------------------------------------------------------------------
-def strict_json_load(path: str | os.PathLike[str]) -> Any:
- """Load JSON while rejecting duplicate keys and non-finite constants."""
- def pairs(items):
- result = {}
- for key, value in items:
- if key in result:
- raise ContractError(f"duplicate JSON key {key!r}")
- result[key] = value
- return result
-
- def constant(value):
- raise ContractError(f"non-finite JSON number {value}")
-
- try:
- with open(path) as handle:
- return json.load(handle, object_pairs_hook=pairs, parse_constant=constant)
- except (OSError, json.JSONDecodeError) as exc:
- raise ContractError(f"invalid JSON {path}: {exc}") from exc
-
-
-# Compatibility alias for existing runtime/summary callers.
-strict_load = strict_json_load
-
-
-def _obj(value: Any, path: str) -> dict[str, Any]:
- if not isinstance(value, dict):
- raise ContractError(f"{path} must be an object")
- return value
-
-
-def _keys(value: Any, expected: set[str], path: str) -> dict[str, Any]:
- obj = _obj(value, path)
- actual = set(obj)
- if actual != expected:
- raise ContractError(
- f"{path} fields differ: missing={sorted(expected - actual)}, "
- f"extra={sorted(actual - expected)}"
- )
- return obj
-
-
-def _text(value: Any, path: str, *, nullable: bool = False) -> str | None:
- if nullable and value is None:
- return None
- if not isinstance(value, str) or not value:
- raise ContractError(f"{path} must be a non-empty string")
- return value
-
-
-# ---------------------------------------------------------------------------
-# Schema validation core (JSON Schema owns document shape).
-# ---------------------------------------------------------------------------
-_VALIDATOR_CACHE: dict[str, Draft202012Validator] = {}
-
-
-def _validator(schema_file: str) -> Draft202012Validator:
- # Imported lazily so that importing this module (e.g. for the stdlib-only
- # shard-control path in sweep_matrix.py) does not require jsonschema; it is
- # only needed once a document is actually validated.
- from jsonschema import Draft202012Validator
-
- validator = _VALIDATOR_CACHE.get(schema_file)
- if validator is None:
- try:
- schema = json.loads((SCHEMA_DIR / schema_file).read_text())
- except (OSError, json.JSONDecodeError) as exc:
- raise ContractError(f"schema {schema_file} is unavailable: {exc}") from exc
- Draft202012Validator.check_schema(schema)
- validator = Draft202012Validator(schema)
- _VALIDATOR_CACHE[schema_file] = validator
- return validator
-
-
-def _schema_validate(document: Any, schema_file: str, label: str) -> None:
- errors = sorted(
- _validator(schema_file).iter_errors(document),
- key=lambda error: list(error.absolute_path),
- )
- if errors:
- error = errors[0]
- location = "/".join(str(part) for part in error.absolute_path) or ""
- raise ContractError(f"{label} schema error at {location}: {error.message}")
-
-
-def validate_samples_document(document: Any) -> dict[str, Any]:
- """Validate one detached exact-samples document against its schema."""
- if not isinstance(document, dict):
- raise ContractError("samples document must be an object")
- _schema_validate(document, "samples-v1.schema.json", "samples")
- return document
-
-
-def validate_raw_document(document: Any, samples: Any) -> dict[str, Any]:
- """Validate a raw measurement document and its paired detached samples.
-
- JSON Schema owns both document shapes. The cross-file checks here recompute
- the typed identities from their factors and join the detached samples to the
- measurement rows by point identity and sample digest.
- """
- if not isinstance(document, dict):
- raise ContractError("raw document must be an object")
- _schema_validate(document, "raw-case-v1.schema.json", "raw")
- _schema_validate(samples, "samples-v1.schema.json", "samples")
- ids = document["identity"]
- case_id = identity.case_id_from_factors(ids["case_factors"])
- attempt_id = identity.attempt_id(case=case_id, ordinal=ids["attempt_ordinal"])
- if (case_id, attempt_id) != (ids["case_id"], ids["attempt_id"]):
- raise ContractError("raw case identity does not match its factors")
- for field in ("case_id", "attempt_id"):
- if samples.get(field) != ids[field]:
- raise ContractError(f"detached samples {field} differs from raw identity")
- rows = document["measurement"]["rows"]
- sample_points = {point["point_id"]: point for point in samples["points"]}
- if {row["point_id"] for row in rows} != set(sample_points):
- raise ContractError("raw rows and detached samples cover different points")
- success = document["outcome"]["status"] == "success"
- for row in rows:
- expected_point = identity.point_id(
- case=case_id, tokens_per_rank=row["tokens_per_rank"]
- )
- if row["point_id"] != expected_point:
- raise ContractError("raw point identity does not match its factors")
- point = sample_points[row["point_id"]]
- if (point["sample_sha256"], point["tokens_per_rank"]) != (
- row["sample_sha256"], row["tokens_per_rank"]
- ):
- raise ContractError("detached sample point differs from its raw row")
- if success and row["correctness"]["passed"] is not True:
- raise ContractError("successful raw case has a failed correctness check")
- return document
-
-
-def load_raw_attempt(path: str | os.PathLike[str]) -> dict[str, Any]:
- """Load a raw attempt, verify its detached sample bytes, and validate both."""
- document = strict_json_load(path)
- artifact = _obj(_obj(document, "raw").get("sample_artifact"), "raw.sample_artifact")
- sample_path = Path(path).with_name(_text(artifact.get("path"), "raw.sample_artifact.path"))
- payload = sample_path.read_bytes()
- if (
- len(payload) != artifact.get("bytes")
- or hashlib.sha256(payload).hexdigest() != artifact.get("sha256")
- ):
- raise ContractError("sample artifact bytes or digest differ")
- samples = strict_json_load(sample_path)
- return validate_raw_document(document, samples)
-
-
-def load_attempt(path: str | os.PathLike[str]) -> dict[str, Any]:
- """Fully validate and return one raw or terminal attempt."""
- document = strict_json_load(path)
- if isinstance(document, dict) and document.get("format") == RAW_FORMAT:
- return load_raw_attempt(path)
- if isinstance(document, dict) and document.get("format") == TERMINAL_FORMAT:
- return validate_terminal_document(document)
- raise ContractError("unknown result attempt format")
-
-
-def validate_result(path: str | os.PathLike[str]) -> dict[str, Any]:
- """Validate one emitted result file (raw+samples or terminal) and return it."""
- return load_attempt(path)
-
-
-# ---------------------------------------------------------------------------
-# Terminal outcome construction and validation.
-# ---------------------------------------------------------------------------
-_ALLOCATION_GIT_FIELDS = ("artifact", "job", "repo", "run_attempt", "run_id", "source_sha")
-
-
-def _allocation_factors(
- git_run: dict[str, Any] | None, runner: str, execution_id: str | None
-) -> dict[str, Any]:
- """Project the CI provenance into the eight canonical allocation factors."""
- factors: dict[str, Any] = {
- name: (git_run[name] if git_run is not None else None)
- for name in _ALLOCATION_GIT_FIELDS
- }
- factors["execution_id"] = execution_id
- factors["runner"] = runner
- return factors
-
-
-def make_terminal_result(
- *,
- allocation_factors: dict[str, Any],
- attempt_ordinal: int,
- case: dict[str, Any],
- case_factors: dict[str, Any],
- control_sha256: str | None,
- failure_mode: str,
- generated_at: str,
- git_run: dict[str, Any] | None,
- reason: str,
- return_code: int,
- source: str,
- status: str,
- version: int,
- expected_case_id: str | None = None,
-) -> dict[str, Any]:
- """Build and self-validate one attributable non-success attempt."""
- case_id = identity.case_id_from_factors(case_factors)
- if expected_case_id is not None and expected_case_id != case_id:
- raise ContractError(
- f"scheduled case ID differs from terminal factors: {expected_case_id} != {case_id}"
- )
- if type(version) is not int or version < 1:
- raise ContractError("terminal benchmark version must be a positive integer")
- attempt_id = identity.attempt_id(case=case_id, ordinal=attempt_ordinal)
- document = {
- "format": TERMINAL_FORMAT,
- "schema_version": 1,
- "version": version,
- "record_type": "terminal-outcome",
- "generated_at": generated_at,
- "identity": {
- "allocation_factors": allocation_factors,
- "attempt_id": attempt_id,
- "attempt_ordinal": attempt_ordinal,
- "case_factors": case_factors,
- "case_id": case_id,
- },
- "case": case,
- "provenance": {
- "git_run": git_run,
- "control_sha256": control_sha256,
- "redaction": "sanitized-v1",
- "source": source,
- },
- "outcome": {
- "status": status,
- "failure_mode": failure_mode,
- "reason": reason,
- "return_code": return_code,
- },
- }
- return validate_terminal_document(document)
-
-
-# Compatibility alias for existing matrix/runtime callers.
-make_terminal_document = make_terminal_result
-
-
-def validate_terminal_document(document: Any) -> dict[str, Any]:
- """Validate a terminal outcome against its schema and cross-field semantics."""
- if not isinstance(document, dict):
- raise ContractError("terminal document must be an object")
- _schema_validate(document, "terminal-outcome-v1.schema.json", "terminal")
- ids = document["identity"]
- case = document["case"]
- factors = ids["case_factors"]
- if factors["case"] != case or factors["profile"] != scheduled_case_profile(
- case, "terminal.case"
- ):
- raise ContractError("terminal case factors differ from the scheduled case/profile")
- allocation = ids["allocation_factors"]
- ordinal = ids["attempt_ordinal"]
- expected_case = identity.case_id_from_factors(factors)
- expected_attempt = identity.attempt_id(case=expected_case, ordinal=ordinal)
- if (ids["case_id"], ids["attempt_id"]) != (expected_case, expected_attempt):
- raise ContractError("terminal case identity does not match its factors")
- provenance = document["provenance"]
- git_run = provenance["git_run"]
- outcome = document["outcome"]
- failure_mode = outcome["failure_mode"]
- source = provenance["source"]
- if source == "runtime-emitter":
- expected_runner = factors["sku"]
- valid_outcome = (
- outcome["status"] == "failed"
- and outcome["reason"] == RUNTIME_FAILURE_REASONS.get(failure_mode)
- )
- elif source == "post-emit-command":
- expected_runner = factors["sku"]
- valid_outcome = (
- outcome["status"] == "failed"
- and outcome["reason"] == POST_EMIT_FAILURE_REASONS.get(failure_mode)
- )
- elif source == "matrix-capability-resolver":
- expected_runner = "capability-resolver"
- valid_outcome = (
- outcome["status"] == "unsupported"
- and failure_mode == "capability"
- and outcome["reason"] in CAPABILITY_FAILURE_REASONS
- )
- else:
- raise ContractError("terminal provenance source is not registered")
- if not valid_outcome:
- raise ContractError("terminal source and outcome are not registered")
- expected_allocation_factors = _allocation_factors(
- git_run, expected_runner, allocation["execution_id"]
- )
- if allocation != expected_allocation_factors:
- raise ContractError("terminal allocation factors differ from provenance or source")
- assert_publication_safe([document])
- return document
-
-
-def quarantine_invalid_attempt(path: str | os.PathLike[str]) -> bool:
- """Move an invalid attempt and its basename-safe sample outside JSON upload globs."""
- destination = Path(path)
- if not destination.is_file():
- return False
- try:
- load_attempt(destination)
- return False
- except (ContractError, OSError, ValueError):
- try:
- document = json.loads(destination.read_bytes())
- except (OSError, json.JSONDecodeError):
- document = {}
- artifact = document.get("sample_artifact") if isinstance(document, dict) else None
- sample_name = artifact.get("path") if isinstance(artifact, dict) else None
- if isinstance(sample_name, str) and Path(sample_name).name == sample_name:
- sample_path = destination.with_name(sample_name)
- if sample_path.is_file():
- os.replace(sample_path, sample_path.with_name(sample_path.name + ".quarantine"))
- os.replace(destination, destination.with_name(destination.name + ".quarantine"))
- return True
-
-
-# ---------------------------------------------------------------------------
-# Terminal construction from the runtime environment.
-# ---------------------------------------------------------------------------
-def _env_integer(name: str, default: int) -> int:
- try:
- return int(os.environ.get(name, str(default)))
- except ValueError:
- return default
-
-
-def _env_enabled(name: str) -> bool:
- return os.environ.get(name, "").lower() in {"1", "true", "yes"}
-
-
-def _terminal_case_from_environment(backend: str, phase: str) -> dict[str, Any]:
- ep = _env_integer("CX_EP", _env_integer("CX_NGPUS", 1))
- gpus_per_node = _env_integer("CX_GPUS_PER_NODE", ep)
- ladder = os.environ.get("CX_TOKENS_LADDER", "") or (
- "1 2 4 8 16 32 64 128"
- if phase == "decode"
- else "128 256 512 1024 2048 4096"
- )
- return {
- "suite": os.environ.get("CX_SUITE") or "manual",
- "workload": os.environ.get("CX_WORKLOAD_NAME") or "manual",
- "backend": backend,
- "mode": os.environ.get("CX_MODE", "normal"),
- "routing": os.environ.get("CX_ROUTING", "uniform"),
- "phase": phase,
- "ep": ep,
- "eplb": False,
- "hidden": _env_integer("CX_HIDDEN", 7168),
- "topk": _env_integer("CX_TOPK", 8),
- "experts": _env_integer("CX_EXPERTS", 256),
- "samples_per_point": _env_integer("CX_SAMPLES_PER_POINT", 512),
- "warmup_semantics": os.environ.get(
- "CX_WARMUP_SEMANTICS",
- "full-roundtrip-before-each-component-trial-point-v1",
- ),
- "ladder": ladder,
- "timing": (
- f'{_env_integer("CX_ITERS", 8)}:{_env_integer("CX_TRIALS", 64)}:'
- f'{_env_integer("CX_WARMUP", 32)}'
- ),
- "canonical": _env_enabled("CX_CANONICAL"),
- "nodes": _env_integer("CX_NODES", _env_integer("SLURM_NNODES", 1)),
- "gpus_per_node": gpus_per_node,
- "scale_up_domain": _env_integer("CX_SCALE_UP_DOMAIN", gpus_per_node),
- "scope": os.environ.get("CX_SCOPE", "scale-up"),
- "topology_class": os.environ.get("CX_TOPO", "manual"),
- "transport": os.environ.get("CX_TRANSPORT", "unknown"),
- "scale_up_transport": os.environ.get("CX_SCALE_UP_TRANSPORT", "unknown"),
- "scale_out_transport": os.environ.get("CX_SCALE_OUT_TRANSPORT") or None,
- }
-
-
-def _git_run_from_environment() -> dict[str, Any] | None:
- def value(name: str) -> str | None:
- return os.environ.get(name) or None
-
- git_run = {
- "run_id": value("GITHUB_RUN_ID"),
- "run_attempt": value("GITHUB_RUN_ATTEMPT"),
- "ref": value("GITHUB_REF_NAME") or value("GITHUB_REF"),
- "source_sha": value("COLLECTIVEX_SOURCE_SHA") or value("GITHUB_SHA"),
- "repo": value("GITHUB_REPOSITORY"),
- "job": value("GITHUB_JOB"),
- "artifact": value("COLLECTIVEX_ARTIFACT_NAME"),
- }
- if not any(item is not None for item in git_run.values()):
- return None
- return git_run
-
-
-def make_terminal_from_environment(
- *, backend: str, phase: str, return_code: int, failure_mode: str | None = None
-) -> dict[str, Any]:
- """Build a terminal document from the same exported case coordinates as run_ep."""
- mode = failure_mode or RETURN_CODE_FAILURE_MODES.get(return_code, "execution")
- reason = RUNTIME_FAILURE_REASONS.get(mode)
- if reason is None:
- raise ContractError("runtime failure mode is not registered")
- runner = os.environ.get("CX_RUNNER", "")
- case = _terminal_case_from_environment(backend, phase)
- case_factors = {
- "case": case,
- "profile": scheduled_case_profile(case, "runtime case"),
- "sku": runner,
- }
- git_run = _git_run_from_environment()
- control = os.environ.get("COLLECTIVEX_CONTROL_SHA256") or None
- return make_terminal_result(
- allocation_factors=_allocation_factors(
- git_run, runner, os.environ.get("COLLECTIVEX_EXECUTION_ID") or None
- ),
- attempt_ordinal=_env_integer("CX_ATTEMPT_ID", 1),
- case=case,
- case_factors=case_factors,
- control_sha256=control,
- failure_mode=mode,
- generated_at=dt.datetime.now(dt.timezone.utc).isoformat(),
- git_run=git_run,
- reason=reason,
- return_code=return_code,
- source="runtime-emitter",
- status="failed",
- version=_env_integer("CX_VERSION", 1),
- expected_case_id=os.environ.get("CX_CASE_ID") or None,
- )
-
-
-def _write_document(path: str | os.PathLike[str], document: dict[str, Any]) -> None:
- destination = Path(path)
- destination.parent.mkdir(parents=True, exist_ok=True)
- temporary = destination.with_name(destination.name + ".tmp")
- temporary.write_text(json.dumps(document, indent=2, sort_keys=True) + "\n")
- os.replace(temporary, destination)
-
-
-def demote_raw_attempt(path: str | os.PathLike[str], return_code: int) -> dict[str, Any]:
- """Replace a rank-zero raw result when the distributed command later fails."""
- destination = Path(path)
- raw = strict_json_load(destination)
- if not isinstance(raw, dict) or raw.get("format") != RAW_FORMAT:
- raise ContractError("only a raw attempt can be demoted")
- ids = _obj(raw.get("identity"), "raw.identity")
- required = {
- "allocation_factors", "attempt_id", "attempt_ordinal",
- "case_factors", "case_id",
- }
- if not required.issubset(ids):
- raise ContractError("raw identity lacks terminal factors")
- mode = RETURN_CODE_FAILURE_MODES.get(return_code, "execution")
- version = raw.get("version")
- if type(version) is not int or version < 1:
- raise ContractError("raw attempt lacks a valid benchmark version to demote")
- git_run = _obj(raw.get("provenance"), "raw.provenance").get("git_run")
- if git_run is not None:
- git_run = _keys(git_run, GIT_RUN_FIELDS, "raw.provenance.git_run")
- terminal = make_terminal_result(
- allocation_factors=ids["allocation_factors"],
- attempt_ordinal=ids["attempt_ordinal"],
- case=ids["case_factors"]["case"],
- case_factors=ids["case_factors"],
- control_sha256=os.environ.get("COLLECTIVEX_CONTROL_SHA256") or None,
- failure_mode=mode,
- generated_at=dt.datetime.now(dt.timezone.utc).isoformat(),
- git_run=git_run,
- reason=POST_EMIT_FAILURE_REASONS[mode],
- return_code=return_code,
- source="post-emit-command",
- status="failed",
- version=version,
- expected_case_id=ids["case_id"],
- )
- artifact = raw.get("sample_artifact") or {}
- sample_name = artifact.get("path")
- if isinstance(sample_name, str) and Path(sample_name).name == sample_name:
- destination.with_name(sample_name).unlink(missing_ok=True)
- _write_document(destination, terminal)
- return terminal
-
-
-# ---------------------------------------------------------------------------
-# Shard / matrix delivery reconciliation.
-# ---------------------------------------------------------------------------
-def _load_delivery_attempts(paths: list[str]) -> list[dict[str, Any]]:
- """Fully validate every result path once and return its raw/terminal attempts."""
- if not paths or len(paths) != len(set(paths)):
- raise ContractError("delivery requires unique result paths")
- attempts: list[dict[str, Any]] = []
- sample_paths: set[Path] = set()
- referenced_samples: set[Path] = set()
- for raw_path in paths:
- path = Path(raw_path).resolve()
- document = strict_json_load(path)
- kind = document.get("format") if isinstance(document, dict) else None
- if kind == RAW_FORMAT:
- document = load_raw_attempt(path)
- referenced_samples.add(path.with_name(document["sample_artifact"]["path"]))
- attempts.append(document)
- elif kind == TERMINAL_FORMAT:
- attempts.append(validate_terminal_document(document))
- elif kind == SAMPLES_FORMAT:
- validate_samples_document(document)
- sample_paths.add(path)
- else:
- raise ContractError(f"unknown result artifact {path.name}")
- if sample_paths != referenced_samples:
- raise ContractError("sample artifacts are missing, orphaned, or outside the validated set")
- if not attempts:
- raise ContractError("result set contains no attempts")
- return attempts
-
-
-def validate_delivery(
- paths: list[str], source_path: str, *, disposition: str | None = None
-) -> int:
- """Reconcile a shard or matrix disposition with its complete attempt set."""
- source_file = Path(source_path).resolve()
- source = strict_json_load(source_file)
- if isinstance(source, dict) and source.get("format") == "collectivex.matrix.v1":
- if disposition is None:
- raise ContractError("matrix delivery validation requires a disposition")
- wrappers = [
- item for item in source.get("requested_cases", [])
- if isinstance(item, dict) and item.get("disposition") == disposition
- ]
- expected = {
- item["case"]["case_id"]: (item["sku"], item["case"])
- for item in wrappers
- }
- expected_count = len(wrappers)
- require_one_allocation = disposition == "unsupported"
- elif isinstance(source, dict) and isinstance(source.get("cases"), list):
- expected = {
- case["case_id"]: (source.get("sku"), case)
- for case in source["cases"]
- }
- expected_count = len(source["cases"])
- require_one_allocation = True
- else:
- raise ContractError("delivery source is not a matrix or shard control")
- if not expected or len(expected) != expected_count:
- raise ContractError("delivery source has empty or duplicate case coverage")
- source_version = source.get("version")
- if type(source_version) is not int or source_version < 1:
- raise ContractError("delivery source lacks a valid benchmark version")
-
- attempts = _load_delivery_attempts(paths)
- assert_publication_safe(attempts)
- by_case: dict[str, list[dict[str, Any]]] = {}
- attempt_ids = set()
- allocation_keys = set()
- source_sha256 = hashlib.sha256(source_file.read_bytes()).hexdigest()
- for document in attempts:
- ids = document["identity"]
- case_id = ids["case_id"]
- if case_id not in expected or ids["attempt_id"] in attempt_ids:
- raise ContractError("delivery contains an extra case or duplicate attempt")
- if document.get("version") != source_version:
- raise ContractError("delivery attempt benchmark version differs from its source")
- attempt_ids.add(ids["attempt_id"])
- allocation_keys.add(
- json.dumps(ids["allocation_factors"], sort_keys=True, separators=(",", ":"))
- )
- sku, scheduled = expected[case_id]
- scheduled_case = {key: value for key, value in scheduled.items() if key != "case_id"}
- if ids["case_factors"] != {
- "case": scheduled_case,
- "profile": scheduled_case_profile(scheduled_case, "delivery case"),
- "sku": sku,
- }:
- raise ContractError("delivery attempt differs from its scheduled case")
- factors = ids["allocation_factors"]
- expected_environment = {
- "artifact": os.environ.get("COLLECTIVEX_ARTIFACT_NAME"),
- "execution_id": os.environ.get("COLLECTIVEX_EXECUTION_ID"),
- "job": os.environ.get("GITHUB_JOB"),
- "repo": os.environ.get("GITHUB_REPOSITORY"),
- "run_attempt": os.environ.get("GITHUB_RUN_ATTEMPT"),
- "run_id": os.environ.get("GITHUB_RUN_ID"),
- "source_sha": os.environ.get("COLLECTIVEX_SOURCE_SHA") or os.environ.get("GITHUB_SHA"),
- }
- expected_runner = (
- "capability-resolver"
- if document["format"] == TERMINAL_FORMAT
- and document["provenance"]["source"] == "matrix-capability-resolver"
- else sku
- )
- if any(
- value is not None and factors[field] != value
- for field, value in expected_environment.items()
- ) or factors["runner"] != expected_runner:
- raise ContractError("delivery allocation factors differ from the workflow")
- if document["format"] == TERMINAL_FORMAT:
- control = document["provenance"]["control_sha256"]
- if control != source_sha256:
- raise ContractError("terminal outcome does not reference its exact control document")
- by_case.setdefault(case_id, []).append(document)
- if set(by_case) != set(expected):
- raise ContractError("delivery case coverage is incomplete")
- for case_id, documents in by_case.items():
- ordinals = sorted(document["identity"]["attempt_ordinal"] for document in documents)
- if ordinals != list(range(1, len(ordinals) + 1)):
- raise ContractError(f"delivery attempt ordinals are not contiguous for {case_id}")
- if require_one_allocation and len(allocation_keys) != 1:
- raise ContractError("one shard must use exactly one allocation")
- return len(attempts)
-
-
-def main() -> int:
- parser = argparse.ArgumentParser(description="CollectiveX result contracts")
- subparsers = parser.add_subparsers(dest="command", required=True)
- probe = subparsers.add_parser("probe")
- probe.add_argument("path")
- probe.add_argument("--status", choices=("success", "invalid"))
- emit = subparsers.add_parser("emit-terminal")
- emit.add_argument("--out", required=True)
- emit.add_argument("--backend", required=True)
- emit.add_argument("--phase", required=True, choices=("decode", "prefill"))
- emit.add_argument("--return-code", required=True, type=int)
- emit.add_argument("--failure-mode")
- demote = subparsers.add_parser("demote")
- demote.add_argument("path")
- demote.add_argument("--return-code", required=True, type=int)
- quarantine = subparsers.add_parser("quarantine-invalid")
- quarantine.add_argument("path")
- delivery = subparsers.add_parser("validate-delivery")
- delivery.add_argument("--source", required=True)
- delivery.add_argument("--disposition")
- delivery.add_argument("paths", nargs="+")
- args = parser.parse_args()
- try:
- if args.command == "probe":
- document = load_attempt(args.path)
- if args.status is None:
- return 0
- if document.get("format") != RAW_FORMAT:
- return 1
- outcome = document["outcome"]
- validity = outcome.get("validity")
- return int(
- not (
- isinstance(validity, dict)
- and validity.get("execution_status") == "complete"
- and outcome.get("status") == args.status
- )
- )
- if args.command == "emit-terminal":
- document = make_terminal_from_environment(
- backend=args.backend,
- phase=args.phase,
- return_code=args.return_code,
- failure_mode=args.failure_mode,
- )
- _write_document(args.out, document)
- print(f"preserved terminal outcome ({document['outcome']['failure_mode']})")
- return 0
- if args.command == "quarantine-invalid":
- quarantine_invalid_attempt(args.path)
- return 0
- if args.command == "validate-delivery":
- print(
- f"validated {validate_delivery(args.paths, args.source, disposition=args.disposition)} "
- "delivery attempts"
- )
- return 0
- demote_raw_attempt(args.path, args.return_code)
- return 0
- except (ContractError, identity.IdentityError, OSError, ValueError) as exc:
- print(f"terminal contract error: {exc}", file=sys.stderr)
- return 1
-
-
-if __name__ == "__main__":
- raise SystemExit(main())
diff --git a/experimental/CollectiveX/docs/methodology.md b/experimental/CollectiveX/docs/methodology.md
index 33fa4c64e9..a4306fc0c8 100644
--- a/experimental/CollectiveX/docs/methodology.md
+++ b/experimental/CollectiveX/docs/methodology.md
@@ -1,8 +1,8 @@
# CollectiveX EP Benchmark Methodology
CollectiveX schedules expert-parallel (EP) communication benchmarks, executes them on real
-accelerator allocations, validates the results each run emits, and uploads neutral artifacts. It
-does **not** promote, rank, recommend, select, hide, or decide what any consumer displays. The
+accelerator allocations, and uploads the neutral artifacts each run emits. It does **not** validate
+those artifacts, promote, rank, recommend, select, hide, or decide what any consumer displays. The
frontend reads the neutral matrix, result, summary, and catalog artifacts and makes its own coverage
and display decisions. This document describes how a case is scheduled, measured, checked, and
recorded — not a publication or qualification contract.
@@ -13,7 +13,7 @@ CollectiveX is a communication microbenchmark for:
- comparing EP libraries on one chip/topology;
- comparing EP latency and logical payload bandwidth across systems under the same workload; and
-- recording unsupported, failed, invalid, and unstable cases as explicit terminal evidence.
+- surfacing unsupported, failed, invalid, and unstable cases rather than hiding them.
It does not predict serving throughput without a separate correlation study.
@@ -43,7 +43,7 @@ Physical host count does not define scope. Both GB cells remain inside one 72-GP
domain. The MI325X launcher/configuration path is retained for future versions but is not referenced
by any current suite or shard.
-Unsupported combinations are terminal outcomes, not silently skipped coverage. DeepEP V2 is the
+Unsupported combinations are explicitly classified in the matrix, not silently skipped coverage. DeepEP V2 is the
`ElasticBuffer` introduced by PR #605, pinned with upstream PR #630's minimal pure-scale-up fix and
the exact upstream PR #640 library matcher that excludes NCCL shared-memory mappings. Scale-up cases
request NCCL Device API LSA and fail closed unless the realized LSA team covers the full EP world.
@@ -133,7 +133,7 @@ checks the dispatch metadata and transformed output. Low-latency adapters separa
expert-packed source/expert assignment, native gate weights, and gate-weighted combined output. Both
contracts compare against the reference activation. The combine gate is `rtol=0.05, atol=0.02` for
the BF16 communication path. This threshold is a correctness gate, not an estimate of transport
-error. Any failed rank or point makes the case ineligible and is recorded as such.
+error. Any failed rank or point makes the case ineligible in the result it writes.
Pre/post dispatch evidence is hashed in canonical source-token order. Native receive slots may be
assigned nondeterministically, so physical receive order is not treated as a correctness property.
@@ -153,8 +153,8 @@ unknown fields, and contains:
- `outcome`: `success`, `failed`, `invalid`, `unsupported`, with `diagnostic` and reasons.
Exact per-point samples are emitted as detached `collectivex.samples.v1` documents referenced by path
-and SHA-256, so the raw document stays compact. Every scheduled case produces exactly one terminal
-record; every attempt is retained (a superseded retry does not overwrite an earlier failure). Private
+and SHA-256, so the raw document stays compact. Each dispatched case writes its raw result document;
+unsupported or never-run cells produce no synthetic record. Private
environment details (hosts, addresses, device selectors, credentials, workspace paths) remain in
local mode-0600 logs and ignored operator notes and never enter an emitted artifact.
@@ -229,14 +229,10 @@ files it is using.
## Neutral Artifact Delivery
There is no results server, attached store, or managed object store. Each shard runs one allocation,
-emits per-case result and terminal JSON plus detached sample JSON and a small mechanical summary, and
-uploads them as GitHub artifacts with `always()` so a red or partial run still uploads. Before upload,
-`contracts.py validate-delivery` checks, for the scheduled shard:
-
-- strict schema validity of every result, terminal, and samples document;
-- exactly one terminal record per scheduled case, with no missing, extra, or duplicate case;
-- detached-sample path and SHA-256 consistency (`point_id` / `sample_sha256`); and
-- privacy — no private host, address, selector, credential, or path leaks into any artifact.
+emits per-case result JSON plus detached sample JSON and a small mechanical summary, and uploads them
+as GitHub artifacts with `always()` so a red or partial run still uploads. A case counts as successful
+on the benchmark's own return code; there is no schema, completeness, detached-sample, or privacy
+validation step before upload, and failed or unsupported cells produce no synthetic record.
No step promotes a run, builds a dataset, or advances a channel; the artifacts are the output. Any
downstream display or comparison is the consumer's responsibility.
diff --git a/experimental/CollectiveX/requirements.txt b/experimental/CollectiveX/requirements.txt
index f68f97d83d..23ed3544c0 100644
--- a/experimental/CollectiveX/requirements.txt
+++ b/experimental/CollectiveX/requirements.txt
@@ -3,6 +3,3 @@ PyYAML==6.0.2
# Canonical workload serialization.
numpy>=1.26,<3
-
-# Host-only strict artifact publisher schemas (never imported by GPU execution).
-jsonschema==4.25.1
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 747cf556f8..fc8e1c6ccb 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -5,7 +5,6 @@
# model-serving. Logging goes to stderr so functions can `echo` a single
# result on stdout.
-_CX_COMMON_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
CX_SQUASH_FORMAT_VERSION="repro-v1"
CX_SQUASH_SOURCE_DATE_EPOCH=1
CX_DEEPEP_V2_COMMIT="fa8a9b16898204afd347c663b89e65ef87dc6ce6" # pragma: allowlist secret
@@ -2758,196 +2757,6 @@ PY
cx_log "removed generated per-execution stage directory"
}
-# Return success only when a benchmark output is a complete JSON result object.
-# Callers use this before synthesizing a terminal outcome so an emitted invalid result
-# is not shadowed by a second record for the same attempt.
-cx_has_result_doc() {
- local path="$1"
- python3 "$_CX_COMMON_ROOT/contracts.py" probe "$path" >/dev/null 2>&1
-}
-
-cx_result_doc_is() {
- local path="$1" expected="$2"
- python3 "$_CX_COMMON_ROOT/contracts.py" probe "$path" --status "$expected" \
- >/dev/null 2>&1
-}
-
-# A rank-zero result can be written before another rank or backend teardown fails. Preserve its
-# measurements, but make the distributed command's nonzero terminal status authoritative.
-cx_demote_result_doc() {
- local path="$1" rc="$2"
- python3 "$_CX_COMMON_ROOT/contracts.py" demote "$path" --return-code "$rc"
-}
-
-cx_quarantine_result_doc() {
- python3 "$_CX_COMMON_ROOT/contracts.py" quarantine-invalid "$1"
-}
-
-# cx_emit_ep_failed_case
-# Preserve failures from rack launchers that invoke run_ep.py directly and therefore cannot use
-# run_in_container.sh's emitter. Case identity is read from the exported CX_* variables.
-cx_emit_ep_failed_case() {
- local out="$1" backend="$2" phase="$3" rc="$4"
- local -a args=(emit-terminal --out "$out" --backend "$backend" --phase "$phase"
- --return-code "$rc")
- [ -z "${CX_FAILURE_MODE:-}" ] || args+=(--failure-mode "$CX_FAILURE_MODE")
- if ! python3 "$_CX_COMMON_ROOT/contracts.py" "${args[@]}"
- then
- cx_log "ERROR: could not preserve terminal outcome"
- return 1
- fi
-}
-
-cx_case_attempt_exists() {
- local out_dir="$1" case_id="$2"
- python3 - "$_CX_COMMON_ROOT" "$out_dir" "$case_id" <<'PY'
-import pathlib, sys
-
-sys.path.insert(0, sys.argv[1])
-import contracts
-
-sample_paths = set()
-referenced_samples = set()
-found = False
-
-def quarantine(path, document):
- sample = document.get("sample_artifact") if isinstance(document, dict) else None
- if (
- isinstance(sample, dict)
- and isinstance(sample.get("path"), str)
- and pathlib.Path(sample["path"]).name == sample["path"]
- ):
- sample_path = path.with_name(sample["path"])
- if sample_path.is_file():
- sample_path.replace(sample_path.with_name(sample_path.name + ".quarantine"))
- if path.is_file():
- path.replace(path.with_name(path.name + ".quarantine"))
-
-for path in pathlib.Path(sys.argv[2]).glob("*.json"):
- document = None
- try:
- document = contracts.strict_load(path)
- if not isinstance(document, dict):
- continue
- if document.get("format") == contracts.RAW_FORMAT:
- document = contracts.load_raw_attempt(path)
- referenced_samples.add(path.with_name(document["sample_artifact"]["path"]))
- elif document.get("format") == contracts.TERMINAL_FORMAT:
- document = contracts.validate_terminal_document(document)
- elif document.get("format") == contracts.SAMPLES_FORMAT:
- contracts.validate_samples_document(document)
- sample_paths.add(path)
- continue
- else:
- continue
- except (contracts.ContractError, OSError, ValueError):
- quarantine(path, document)
- continue
- if document["identity"]["case_id"] == sys.argv[3]:
- found = True
-for orphan in sample_paths - referenced_samples:
- quarantine(orphan, {})
-raise SystemExit(0 if found else 1)
-PY
-}
-
-# Emit one setup-failure record per requested case. Rack launchers call this when
-# backend preparation fails before rank processes can start.
-cx_emit_setup_failures() {
- local root="$1" out_dir="$2" backend="$3" rc="$4" shard="${CX_SHARD_FILE:-}" path
- local phase case_id suite workload required routing eplb ep hidden topk experts nodes
- local gpn domain ladder canonical timing mode scope scale_up_transport scale_out_transport
- local warmup_semantics
- local transport topology_class
- local cases_file expected emitted=0 covered=0
- mkdir -p "$out_dir" || return 1
- export CX_FAILURE_MODE="${CX_FAILSAFE_MODE:-setup}" CX_ATTEMPT_ID=1
- if [ -z "$shard" ]; then
- local phases="${CX_PHASE:-decode}"
- [ "$phases" = both ] && phases="decode prefill"
- for phase in $phases; do
- if [ -n "${CX_CASE_ID:-}" ] && cx_case_attempt_exists "$out_dir" "$CX_CASE_ID"; then
- continue
- fi
- cx_emit_ep_failed_case "$out_dir/failed_${backend}_${phase}_${CX_TS:-setup}-a01.json" \
- "$backend" "$phase" "$rc" || return 1
- done
- unset CX_FAILURE_MODE
- return 0
- fi
- path="$shard"
- [ -f "$path" ] || path="${root%/}/$shard"
- [ -f "$path" ] || {
- unset CX_FAILURE_MODE
- cx_log "ERROR: cannot emit setup failures without shard control"
- return 1
- }
- export COLLECTIVEX_CONTROL_SHA256
- COLLECTIVEX_CONTROL_SHA256="$(sha256sum "$path" | awk '{print $1}')"
- [[ "$COLLECTIVEX_CONTROL_SHA256" =~ ^[0-9a-f]{64}$ ]] || {
- unset CX_FAILURE_MODE COLLECTIVEX_CONTROL_SHA256
- cx_log "ERROR: cannot hash shard for setup-failure records"
- return 1
- }
- # Setup-failure terminal outcomes carry the shard's iterable benchmark version.
- CX_VERSION="$(python3 -c "import json,sys;print(json.load(open(sys.argv[1]))['version'])" "$path")"
- export CX_VERSION
- cases_file="$(mktemp)" || return 1
- if ! python3 - "$path" > "$cases_file" <<'PY'
-import json, sys
-
-with open(sys.argv[1]) as handle:
- cases = json.load(handle)["cases"]
-for case in cases:
- fields = (
- case["phase"], case["mode"], case["case_id"], case["suite"], case["workload"],
- case["routing"], "1" if case["eplb"] else "",
- case["ep"], case["hidden"], case["topk"], case["experts"], case["nodes"],
- case["gpus_per_node"], case["scale_up_domain"], case["scope"],
- case["scale_up_transport"], case.get("scale_out_transport") or "",
- case["transport"], case["topology_class"], case["ladder"],
- case["warmup_semantics"],
- "1" if case["canonical"] else "", case["timing"],
- )
- print("|".join(map(str, fields)))
-PY
- then
- rm -f "$cases_file"
- unset CX_FAILURE_MODE
- return 1
- fi
- expected="$(wc -l < "$cases_file" | tr -d ' ')"
- [ "$expected" -gt 0 ] || { rm -f "$cases_file"; unset CX_FAILURE_MODE; return 1; }
- while IFS='|' read -r phase mode case_id suite workload routing eplb ep hidden topk \
- experts nodes gpn domain scope scale_up_transport scale_out_transport transport \
- topology_class ladder warmup_semantics canonical timing; do
- export CX_CASE_ID="$case_id" CX_SUITE="$suite" CX_WORKLOAD_NAME="$workload"
- export CX_ROUTING="$routing" CX_EPLB="$eplb"
- export CX_EP="$ep" CX_NGPUS="$ep" CX_HIDDEN="$hidden" CX_TOPK="$topk" CX_EXPERTS="$experts"
- export CX_MODE="$mode" CX_NODES="$nodes" CX_GPUS_PER_NODE="$gpn"
- export CX_SCALE_UP_DOMAIN="$domain" CX_SCOPE="$scope"
- export CX_SCALE_UP_TRANSPORT="$scale_up_transport"
- export CX_SCALE_OUT_TRANSPORT="$scale_out_transport"
- export CX_TRANSPORT="$transport" CX_TOPO="$topology_class"
- export CX_TOKENS_LADDER="$ladder" CX_CANONICAL="$canonical"
- export CX_WARMUP_SEMANTICS="$warmup_semantics"
- IFS=: read -r CX_ITERS CX_TRIALS CX_WARMUP <<< "$timing"
- export CX_ITERS CX_TRIALS CX_WARMUP CX_SAMPLES_PER_POINT="$((CX_ITERS * CX_TRIALS))"
- if cx_case_attempt_exists "$out_dir" "$case_id"; then
- covered=$((covered + 1))
- continue
- fi
- cx_emit_ep_failed_case "$out_dir/failed_${case_id}-a01.json" "$backend" "$phase" "$rc" || return 1
- emitted=$((emitted + 1))
- done < "$cases_file"
- rm -f "$cases_file"
- unset CX_FAILURE_MODE
- [ "$((emitted + covered))" -eq "$expected" ] || {
- cx_log "ERROR: covered $((emitted + covered))/$expected terminal cases"
- return 1
- }
-}
-
# Run one validated shard with one Slurm task per GPU. Launchers provide only
# allocation/container policy through globals and CX_DISTRIBUTED_CONTAINER_ARGS.
# shellcheck disable=SC2153
@@ -2956,8 +2765,8 @@ cx_run_distributed_shard() {
local ph mode routing eplb hidden topk experts ladder suite workload
local canonical case_id ep timing case_iters case_trials case_warmup case_stem
local scope scale_up_transport scale_out_transport transport topology_class nodes gpn domain
- local workload_dir workload_ladder workload_log stage_rc attempt_tag out failure_out
- local runtime_log run_rc expected_out case_ok summary_log
+ local workload_dir workload_ladder workload_log stage_rc attempt_tag out
+ local runtime_log run_rc summary_log
local -a container_args workload_args ep_args
[ "${NODES:-0}" -gt 1 ] && [ "${NGPUS:-0}" = "$((NODES * GPN))" ] \
|| cx_die "invalid distributed launcher placement"
@@ -2996,8 +2805,6 @@ cx_run_distributed_shard() {
set -e
if [ "$build_rc" != 0 ]; then
cx_fail_stage backend-setup "$build_log" || true
- cx_emit_setup_failures "$CX_DIR" "$MOUNT_SRC/experimental/CollectiveX/results" \
- "$CX_BENCH" "$build_rc"
return "$build_rc"
fi
cx_set_failure_stage execution
@@ -3082,9 +2889,7 @@ PY
cx_log "EP${NGPUS}[$ci] id=${case_id:-manual} $mode/$ph $CX_BENCH"
if [ "$ep" != "$NGPUS" ] || [ "$nodes" != "$NODES" ] || [ "$gpn" != "$GPN" ] \
|| [ "$domain" != "$SCALE_UP_DOMAIN" ]; then
- export CX_ATTEMPT_ID=1
- failure_out="$MOUNT_SRC/experimental/CollectiveX/results/failed_${case_stem}-a01.json"
- cx_emit_ep_failed_case "$failure_out" "$CX_BENCH" "$ph" 5
+ cx_log "ERROR: EP${NGPUS}[$ci] topology mismatch (ep=$ep nodes=$nodes gpn=$gpn domain=$domain); skipping case"
failed_cases=$((failed_cases + 1))
continue
fi
@@ -3108,9 +2913,7 @@ PY
stage_rc=$?
set -e
if [ "$stage_rc" != 0 ]; then
- export CX_ATTEMPT_ID=1
- failure_out="$MOUNT_SRC/experimental/CollectiveX/results/failed_${case_stem}-a01.json"
- cx_emit_ep_failed_case "$failure_out" "$CX_BENCH" "$ph" "$stage_rc"
+ cx_log "ERROR: EP${NGPUS}[$ci] workload staging failed (rc=$stage_rc)"
failed_cases=$((failed_cases + 1))
continue
fi
@@ -3130,7 +2933,6 @@ PY
export CX_ATTEMPT_ID=1
attempt_tag=a01
out="results/${case_stem}_${attempt_tag}.json"
- failure_out="$MOUNT_SRC/experimental/CollectiveX/results/failed_${case_stem}-${attempt_tag}.json"
runtime_log="$(cx_private_log_path "runtime-c$(printf '%03d' "$ci")-$attempt_tag")"
set +e
timeout -k 30 "${CX_RUN_TIMEOUT:-900}" srun --jobid="$JOB_ID" --nodes="$NODES" \
@@ -3142,24 +2944,12 @@ PY
"$runtime_log" 2>&1
run_rc=$?
set -e
- expected_out="$MOUNT_SRC/experimental/CollectiveX/$out"
- case_ok=0
- if [ "$run_rc" = 0 ] && cx_result_doc_is "$expected_out" success; then
- case_ok=1
- elif [ "$run_rc" = 0 ] && cx_result_doc_is "$expected_out" invalid; then
- cx_log "ERROR: EP${NGPUS}[$ci] completed with invalid semantic evidence"
- else
- [ "$run_rc" != 0 ] || run_rc=1
- if cx_has_result_doc "$expected_out"; then
- cx_demote_result_doc "$expected_out" "$run_rc" \
- || { cx_quarantine_result_doc "$expected_out"; cx_emit_ep_failed_case "$failure_out" "$CX_BENCH" "$ph" "$run_rc"; }
- else
- cx_quarantine_result_doc "$expected_out"
- cx_emit_ep_failed_case "$failure_out" "$CX_BENCH" "$ph" "$run_rc"
- fi
- fi
- if [ "$case_ok" = 0 ]; then
- [ "$run_rc" = 0 ] || cx_fail_stage execution "$runtime_log" || true
+ # Terminal-outcome emission and result-document gating were removed with
+ # contracts.py; a case now counts as run purely on the distributed command's
+ # return code. The rank-zero result the harness wrote (if any) is left in place
+ # for the summary renderer, which validates nothing.
+ if [ "$run_rc" != 0 ]; then
+ cx_fail_stage execution "$runtime_log" || true
failed_cases=$((failed_cases + 1))
fi
done < "$cases_file"
@@ -3177,7 +2967,7 @@ PY
}
cx_launcher_cleanup() {
- local rc="$1" stage_root="${MOUNT_SRC:-}" source_root out_dir allocation_stopped=1
+ local rc="$1" stage_root="${MOUNT_SRC:-}" source_root allocation_stopped=1
source_root="${stage_root:-${REPO_ROOT:-}}"
trap - EXIT HUP INT TERM
if [ -n "${COLLECTIVEX_EPHEMERAL_CONFIG_PATH:-}" ]; then
@@ -3206,9 +2996,6 @@ cx_launcher_cleanup() {
&& [ -n "${REPO_ROOT:-}" ] && [ -n "${CX_BENCH:-}" ]; then
cx_log "ERROR: terminal-failure-class=${CX_FAILSAFE_MODE:-setup}"
[ -d "$source_root/experimental/CollectiveX" ] || source_root="$REPO_ROOT"
- out_dir="$source_root/experimental/CollectiveX/results"
- cx_emit_setup_failures \
- "$source_root/experimental/CollectiveX" "$out_dir" "$CX_BENCH" "$rc" || true
[ "$source_root" = "$REPO_ROOT" ] \
|| cx_collect_results "$source_root" "$REPO_ROOT" || true
fi
diff --git a/experimental/CollectiveX/runtime/run_in_container.sh b/experimental/CollectiveX/runtime/run_in_container.sh
index 16b06d0704..b2d3f01982 100644
--- a/experimental/CollectiveX/runtime/run_in_container.sh
+++ b/experimental/CollectiveX/runtime/run_in_container.sh
@@ -59,21 +59,13 @@ cx_stage_canonical() {
# run_ep_suite
# One bench/run_ep.py invocation per phase (decode/prefill/both); dispatch and
# combine are timed separately inside it. One JSON per (backend, phase).
-# Preserve a failed case with its full scheduled identity instead of letting it vanish.
-emit_failed_case() { # backend phase rc
- cx_emit_ep_failed_case \
- "results/failed_${CX_RUNNER}_${1}_${2}_${CX_TS}.json" "$1" "$2" "$3" || true
-}
-
run_ep_suite() {
local backend="$1" phase phases ladder failure_kind rc=0 rc_run
ladder="$(cx_ep_ladder)"
phases="${CX_PHASE:-decode}"
[ "$phases" = "both" ] && phases="decode prefill"
if ! cx_stage_canonical; then
- for phase in $phases; do
- emit_failed_case "$backend" "$phase" 2
- done
+ cx_log "ERROR: $backend canonical workload staging failed"
return 1
fi
for phase in $phases; do
@@ -99,14 +91,10 @@ run_ep_suite() {
else
rc_run=$?
fi
- if [ "$rc_run" = 0 ] && cx_result_doc_is "$out" invalid; then
- cx_log "WARN: $backend $phase completed with invalid semantic evidence"
- rc=1
- continue
- fi
- if [ "$rc_run" = 0 ] && ! cx_result_doc_is "$out" success; then
- rc_run=1
- fi
+ # Result-document gating and terminal-outcome emission were removed with
+ # contracts.py; success is now the run_ep.py return code alone. Any result
+ # document it wrote is left in place for the summary renderer, which validates
+ # nothing.
if [ "$rc_run" != 0 ]; then
failure_kind=failed
[ "$rc_run" != 124 ] && [ "$rc_run" != 137 ] || failure_kind="timed out"
@@ -115,14 +103,6 @@ run_ep_suite() {
else
cx_log "WARN: $backend $phase run failed rc=$rc_run"
fi
- if cx_has_result_doc "$out"; then
- cx_demote_result_doc "$out" "$rc_run" \
- || { cx_quarantine_result_doc "$out"; emit_failed_case "$backend" "$phase" "$rc_run"; }
- cx_log "preserved benchmark output as a failed attempt"
- else
- cx_quarantine_result_doc "$out"
- emit_failed_case "$backend" "$phase" "$rc_run"
- fi
rc=1
fi
done
@@ -963,16 +943,12 @@ cx_prepare_backend() {
}
prepare_backend_or_record() {
- local backend="$1" phases="${CX_PHASE:-decode}" phase
+ local backend="$1"
cx_write_runtime_stage backend-setup || return 1
if cx_prepare_backend "$backend"; then
return 0
fi
cx_log "WARN: $backend preparation failed"
- [ "$phases" = "both" ] && phases="decode prefill"
- for phase in $phases; do
- CX_FAILURE_MODE=backend-setup emit_failed_case "$backend" "$phase" 6
- done
return 1
}
diff --git a/experimental/CollectiveX/schemas/raw-case-v1.schema.json b/experimental/CollectiveX/schemas/raw-case-v1.schema.json
deleted file mode 100644
index 279c6da909..0000000000
--- a/experimental/CollectiveX/schemas/raw-case-v1.schema.json
+++ /dev/null
@@ -1,1218 +0,0 @@
-{
- "$id": "https://inferencex.com/schemas/collectivex/raw-case-v1.schema.json",
- "$schema": "https://json-schema.org/draft/2020-12/schema",
- "$defs": {
- "deepep_v2_jit_cubin": {
- "additionalProperties": false,
- "properties": {
- "cache_key": {
- "pattern":"^kernel\\.[A-Za-z0-9_+-]+\\.[0-9a-f]{32}$",
- "type":"string"
- },
- "cubin_sha256": {"pattern":"^[0-9a-f]{64}$","type":"string"},
- "sass_sha256": {"pattern":"^[0-9a-f]{64}$","type":"string"},
- "source_sha256": {"pattern":"^[0-9a-f]{64}$","type":"string"}
- },
- "required": ["cache_key","cubin_sha256","sass_sha256","source_sha256"],
- "type": "object"
- },
- "hybrid_jit_rank_artifact": {
- "additionalProperties": false,
- "properties": {
- "bytes": {"minimum":1,"type":"integer"},
- "rank": {"minimum":0,"type":"integer"},
- "sha256": {"pattern":"^[0-9a-f]{64}$","type":"string"}
- },
- "required": ["bytes","rank","sha256"],
- "type": "object"
- },
- "hybrid_realized_config": {
- "additionalProperties": false,
- "properties": {
- "backward_combine_api": {"type":"boolean"},
- "device_side_sync_combine_api": {"type":"boolean"},
- "device_side_sync_dispatch_api": {"type":"boolean"},
- "forward_dispatch_api": {"type":"boolean"},
- "hidden_dim": {"minimum":1,"type":"integer"},
- "max_num_of_tokens_per_rank": {"minimum":1,"type":"integer"},
- "num_of_additional_in_flight_s2g_combine_api": {"minimum":0,"type":"integer"},
- "num_of_additional_in_flight_s2g_dispatch_api": {"minimum":0,"type":"integer"},
- "num_of_blocks_combine_api": {"minimum":0,"type":"integer"},
- "num_of_blocks_dispatch_api": {"minimum":0,"type":"integer"},
- "num_of_blocks_permute": {"minimum":0,"type":"integer"},
- "num_of_blocks_preprocessing_api": {"minimum":0,"type":"integer"},
- "num_of_blocks_unpermute": {"minimum":0,"type":"integer"},
- "num_of_experts_per_rank": {"minimum":1,"type":"integer"},
- "num_of_in_flight_s2g_dispatch_api": {"minimum":0,"type":"integer"},
- "num_of_in_flight_s2g_permute_block_dispatch_api": {"minimum":0,"type":"integer"},
- "num_of_nodes": {"minimum":1,"type":"integer"},
- "num_of_ranks_per_node": {"minimum":1,"type":"integer"},
- "num_of_stages_dispatch_api": {"minimum":0,"type":"integer"},
- "num_of_stages_g2s_combine_api": {"minimum":0,"type":"integer"},
- "num_of_stages_permute_block_dispatch_api": {"minimum":0,"type":"integer"},
- "num_of_stages_s2g_combine_api": {"minimum":0,"type":"integer"},
- "num_of_threads_per_block_preprocessing_api": {"minimum":0,"type":"integer"},
- "num_of_tokens_per_chunk_combine_api": {"minimum":0,"type":"integer"},
- "num_of_tokens_per_chunk_dispatch_api": {"minimum":0,"type":"integer"},
- "num_of_tokens_per_chunk_preprocessing_api": {"minimum":0,"type":"integer"},
- "num_of_tokens_per_group_combine_api": {"minimum":0,"type":"integer"},
- "pad_multiple": {"minimum":0,"type":"integer"},
- "token_data_type": {"const":"UINT16"}
- },
- "required": [
- "backward_combine_api","device_side_sync_combine_api","device_side_sync_dispatch_api",
- "forward_dispatch_api","hidden_dim","max_num_of_tokens_per_rank",
- "num_of_additional_in_flight_s2g_combine_api",
- "num_of_additional_in_flight_s2g_dispatch_api","num_of_blocks_combine_api",
- "num_of_blocks_dispatch_api","num_of_blocks_permute","num_of_blocks_preprocessing_api",
- "num_of_blocks_unpermute","num_of_experts_per_rank",
- "num_of_in_flight_s2g_dispatch_api","num_of_in_flight_s2g_permute_block_dispatch_api",
- "num_of_nodes","num_of_ranks_per_node","num_of_stages_dispatch_api",
- "num_of_stages_g2s_combine_api","num_of_stages_permute_block_dispatch_api",
- "num_of_stages_s2g_combine_api","num_of_threads_per_block_preprocessing_api",
- "num_of_tokens_per_chunk_combine_api","num_of_tokens_per_chunk_dispatch_api",
- "num_of_tokens_per_chunk_preprocessing_api","num_of_tokens_per_group_combine_api",
- "pad_multiple","token_data_type"
- ],
- "type": "object"
- },
- "nullable_sha256": {"oneOf":[{"type":"null"},{"pattern":"^[0-9a-f]{64}$","type":"string"}]},
- "byte_accounting": {
- "additionalProperties": false,
- "properties": {
- "accounting_contract": {"const":"activation-data-plus-scales-v1"},
- "activation_data_bytes": {"minimum":0,"type":"integer"},
- "scale_bytes": {"minimum":0,"type":"integer"},
- "total_logical_bytes": {"minimum":0,"type":"integer"}
- },
- "required": [
- "accounting_contract","activation_data_bytes","scale_bytes","total_logical_bytes"
- ],
- "type": "object"
- },
- "case_profile": {
- "additionalProperties": false,
- "allOf": [
- {
- "if": {"properties":{"mode":{"const":"normal"}},"required":["mode"]},
- "then": {"properties": {
- "combine_semantics":{"const":"activation-only"},
- "component_order_contract":{"const":"qualification-hash-rotated-components-v1"},
- "contract":{"const":"layout-and-dispatch-v1"},
- "correctness_scope":{"const":"dispatch-metadata-and-transformed-combine"},
- "oracle_contract":{"const":"expert-specific-transform-v1"},
- "payload_unit":{"const":"token-rank"}
- }}
- },
- {
- "if": {"properties":{"mode":{"const":"low-latency"}},"required":["mode"]},
- "then": {"properties": {
- "combine_semantics":{"const":"gate-weighted"},
- "component_order_contract":{"const":"qualification-hash-rotated-components-v1"},
- "contract":{"const":"expert-packed-weighted-combine-v1"},
- "correctness_scope":{"const":"expert-assignment-and-weighted-combine"},
- "oracle_contract":{"const":"expert-assignment-transform-v1"},
- "payload_unit":{"const":"token-expert"}
- }}
- }
- ],
- "properties": {
- "activation_generator": {"const":"collectivex-activation-counter-v4"},
- "activation_profile": {"const":"canonical-counter-source-v4"},
- "combine_dtype": {"const":"bf16"},
- "combine_semantics": {"enum":["activation-only","gate-weighted"]},
- "component_order_contract": {"const":"qualification-hash-rotated-components-v1"},
- "conditioning_contract": {"const":"fixed-phase-ramp-8-roundtrips-v1"},
- "contract": {"enum":["layout-and-dispatch-v1","expert-packed-weighted-combine-v1"]},
- "correctness_scope": {"enum":["dispatch-metadata-and-transformed-combine","expert-assignment-and-weighted-combine"]},
- "dtype": {"const":"bf16"},
- "eplb_planner": {"const":"greedy-rank-major-v1"},
- "eplb_redundant_experts": {"const":32},
- "eplb_reference_tokens_per_rank": {"const":2048},
- "mode": {"enum":["normal","low-latency"]},
- "oracle_contract": {"enum":["expert-specific-transform-v1","expert-assignment-transform-v1"]},
- "payload_unit": {"enum":["token-rank","token-expert"]},
- "placement": {"const":"packed"},
- "percentile_method": {"const":"nearest-rank"},
- "rank_reduction": {"const":"cross-rank-max-per-iteration"},
- "resource_mode": {"const":"fixed-profile"},
- "routing_generator": {"const":"collectivex-routing-counter-v3"},
- "sampling_contract": {"const":"fixed-512-v1"},
- "seed": {"const":67},
- "source_identity_contract": {"const":"bounded-sign-bit-source-v1"}
- },
- "required": [
- "activation_generator","activation_profile","combine_dtype",
- "combine_semantics","component_order_contract","conditioning_contract","contract",
- "correctness_scope","dtype","eplb_planner","eplb_redundant_experts",
- "eplb_reference_tokens_per_rank","mode","oracle_contract",
- "payload_unit","placement","percentile_method","rank_reduction","resource_mode",
- "routing_generator","sampling_contract","seed","source_identity_contract"
- ],
- "type": "object"
- },
- "oracle": {
- "additionalProperties": false,
- "properties": {
- "checks": {
- "additionalProperties": false,
- "properties": {
- "combine_values": {"type":"boolean"},
- "counts": {"type":"boolean"},
- "metadata": {"type":"boolean"},
- "multiplicity": {"type":"boolean"},
- "payload": {"type":"boolean"},
- "source_set": {"type":"boolean"},
- "weights": {"type":"boolean"}
- },
- "required": ["combine_values","counts","metadata","multiplicity","payload","source_set","weights"],
- "type": "object"
- },
- "atol": {"const":0.02},
- "combine_weight_semantics": {"enum":["unweighted-rank-sum","gate-weighted-sum"]},
- "contract": {"enum":["expert-specific-transform-v1","expert-assignment-transform-v1"]},
- "dispatch_sha256": {"$ref":"#/$defs/nullable_sha256"},
- "max_absolute_error": {"oneOf":[{"type":"null"},{"minimum":0,"type":"number"}]},
- "max_elementwise_relative_error": {"oneOf":[{"type":"null"},{"minimum":0,"type":"number"}]},
- "max_relative_error": {"oneOf":[{"type":"null"},{"minimum":0,"type":"number"}]},
- "max_weight_error": {"oneOf":[{"type":"null"},{"minimum":0,"type":"number"}]},
- "order_sha256": {"$ref":"#/$defs/nullable_sha256"},
- "ordering_contract": {"minLength":1,"type":"string"},
- "passed": {"type":"boolean"},
- "receive_count": {"minimum":0,"type":"integer"},
- "rtol": {"const":0.05}
- },
- "required": [
- "atol",
- "checks",
- "combine_weight_semantics",
- "contract",
- "dispatch_sha256",
- "max_absolute_error",
- "max_elementwise_relative_error",
- "max_relative_error",
- "max_weight_error",
- "order_sha256",
- "ordering_contract",
- "passed",
- "receive_count",
- "rtol"
- ],
- "type": "object"
- },
- "percentiles": {
- "additionalProperties": false,
- "properties": {
- "p50": {"minimum":0,"type":"number"},
- "p90": {"minimum":0,"type":"number"},
- "p95": {"minimum":0,"type":"number"},
- "p99": {"minimum":0,"type":"number"}
- },
- "required": ["p50","p90","p95","p99"],
- "type": "object"
- },
- "component": {
- "additionalProperties": false,
- "allOf": [
- {
- "if": {"properties":{"availability":{"const":"measured"}},"required":["availability"]},
- "then": {
- "properties": {
- "origin": {"const":"measured"},
- "percentiles_us": {"$ref":"#/$defs/percentiles"},
- "sample_count": {"const":512}
- }
- }
- },
- {
- "if": {"properties":{"availability":{"const":"unavailable"}},"required":["availability"]},
- "then": {
- "properties": {
- "percentiles_us": {"type":"null"},
- "sample_count": {"const":0}
- }
- }
- }
- ],
- "properties": {
- "availability": {"enum":["measured","derived","unavailable"]},
- "origin": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]},
- "percentiles_us": {"oneOf":[{"type":"null"},{"$ref":"#/$defs/percentiles"}]},
- "sample_count": {"minimum":0,"type":"integer"}
- },
- "required": ["availability","origin","percentiles_us","sample_count"],
- "type": "object"
- },
- "histogram": {
- "additionalProperties": false,
- "properties": {
- "bins": {"minimum":1,"type":"integer"},
- "counts": {"items":{"minimum":0,"type":"integer"},"minItems":1,"type":"array"},
- "max": {"minimum":0,"type":"number"},
- "min": {"minimum":0,"type":"number"},
- "n": {"minimum":1,"type":"integer"}
- },
- "required": ["n","min","max","bins","counts"],
- "type": "object"
- },
- "scheduled_case": {
- "additionalProperties": false,
- "properties": {
- "backend": {"maxLength":128,"pattern":"^[a-z0-9][a-z0-9_.-]*$","type":"string"},
- "canonical": {"const":true},
- "ep": {"minimum":1,"type":"integer"},
- "eplb": {"type":"boolean"},
- "experts": {"minimum":1,"type":"integer"},
- "gpus_per_node": {"minimum":1,"type":"integer"},
- "hidden": {"minimum":1,"type":"integer"},
- "ladder": {"pattern":"^[1-9][0-9]*( [1-9][0-9]*)*$","type":"string"},
- "mode": {"enum":["normal","low-latency"]},
- "nodes": {"minimum":1,"type":"integer"},
- "phase": {"enum":["decode","prefill"]},
- "routing": {"enum":["uniform"]},
- "samples_per_point": {"const":512},
- "scale_out_transport": {"oneOf":[{"type":"null"},{"maxLength":128,"pattern":"^[a-z0-9][a-z0-9_.-]*$","type":"string"}]},
- "scale_up_domain": {"minimum":1,"type":"integer"},
- "scale_up_transport": {"maxLength":128,"pattern":"^[a-z0-9][a-z0-9_.-]*$","type":"string"},
- "scope": {"enum":["scale-up","scale-out"]},
- "suite": {"maxLength":128,"pattern":"^[a-z0-9][a-z0-9_.-]*$","type":"string"},
- "timing": {"const":"8:64:32"},
- "topk": {"minimum":1,"type":"integer"},
- "topology_class": {"maxLength":128,"pattern":"^[a-z0-9][a-z0-9_.-]*$","type":"string"},
- "transport": {"maxLength":128,"pattern":"^[a-z0-9][a-z0-9_.-]*$","type":"string"},
- "warmup_semantics": {"const":"full-roundtrip-before-each-component-trial-point-v1"},
- "workload": {"maxLength":128,"pattern":"^[a-z0-9][a-z0-9_.-]*$","type":"string"}
- },
- "required": [
- "backend",
- "canonical",
- "eplb",
- "ep",
- "experts",
- "gpus_per_node",
- "hidden",
- "ladder",
- "mode",
- "nodes",
- "phase",
- "routing",
- "samples_per_point",
- "scale_out_transport",
- "scale_up_domain",
- "scale_up_transport",
- "scope",
- "suite",
- "timing",
- "topk",
- "topology_class",
- "transport",
- "warmup_semantics",
- "workload"
- ],
- "type": "object"
- },
- "git_run": {
- "additionalProperties": false,
- "properties": {
- "artifact": {"minLength":1,"type":"string"},
- "job": {"minLength":1,"type":"string"},
- "ref": {"minLength":1,"type":"string"},
- "repo": {"pattern":"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$","type":"string"},
- "run_attempt": {"pattern":"^[1-9][0-9]*$","type":"string"},
- "run_id": {"pattern":"^[1-9][0-9]*$","type":"string"},
- "source_sha": {"pattern":"^[0-9a-f]{40}$","type":"string"}
- },
- "required": ["artifact","job","ref","repo","run_attempt","run_id","source_sha"],
- "type": "object"
- }
- },
- "additionalProperties": false,
- "properties": {
- "case": {
- "additionalProperties": false,
- "properties": {
- "attempt_ordinal": {"minimum":1,"type":"integer"},
- "backend": {"maxLength":128,"pattern":"^[a-z0-9][a-z0-9_.-]*$","type":"string"},
- "ep_size": {"minimum":1,"type":"integer"},
- "eplb": {
- "additionalProperties": false,
- "allOf": [
- {
- "if": {"properties":{"enabled":{"const":true}},"required":["enabled"]},
- "then": {
- "properties": {
- "calibration_token_offset": {"minimum":0,"type":"integer"},
- "calibration_trace_sha256": {"pattern":"^[0-9a-f]{64}$","type":"string"},
- "calibration_window": {"const":"collectivex-eplb-calibration-window-v1"},
- "calibration_workload_id": {"pattern":"^cxwork-v1-[0-9a-f]{64}$","type":"string"}
- }
- },
- "else": {
- "properties": {
- "calibration_token_offset": {"type":"null"},
- "calibration_trace_sha256": {"type":"null"},
- "calibration_window": {"type":"null"},
- "calibration_workload_id": {"type":"null"}
- }
- }
- }
- ],
- "properties": {
- "calibration_token_offset": {"oneOf":[{"type":"null"},{"minimum":0,"type":"integer"}]},
- "calibration_trace_sha256": {"$ref":"#/$defs/nullable_sha256"},
- "calibration_window": {"oneOf":[{"type":"null"},{"const":"collectivex-eplb-calibration-window-v1"}]},
- "calibration_workload_id": {"oneOf":[{"type":"null"},{"pattern":"^cxwork-v1-[0-9a-f]{64}$","type":"string"}]},
- "enabled": {"type":"boolean"},
- "imbalance_after": {"oneOf":[{"type":"null"},{"minimum":0,"type":"number"}]},
- "imbalance_before": {"oneOf":[{"type":"null"},{"minimum":0,"type":"number"}]},
- "mapping_hash": {"oneOf":[{"type":"null"},{"pattern":"^[0-9a-f]{64}$","type":"string"}]},
- "max_replicas": {"oneOf":[{"type":"null"},{"minimum":0,"type":"integer"}]},
- "num_logical_experts": {"minimum":1,"type":"integer"},
- "num_physical_experts": {"minimum":1,"type":"integer"},
- "num_redundant": {"minimum":0,"type":"integer"},
- "planner": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]},
- "reference_tokens_per_rank": {"oneOf":[{"type":"null"},{"minimum":1,"type":"integer"}]},
- "replicated_experts": {"minimum":0,"type":"integer"}
- },
- "required": [
- "calibration_token_offset",
- "calibration_trace_sha256",
- "calibration_window",
- "calibration_workload_id",
- "enabled",
- "imbalance_after",
- "imbalance_before",
- "mapping_hash",
- "max_replicas",
- "num_logical_experts",
- "num_physical_experts",
- "num_redundant",
- "planner",
- "reference_tokens_per_rank",
- "replicated_experts"
- ],
- "type": "object"
- },
- "mode": {"enum":["normal","low-latency"]},
- "phase": {"enum":["decode","prefill"]},
- "resource_mode": {"const":"fixed-profile"},
- "runner": {"maxLength":128,"pattern":"^[a-z0-9][a-z0-9_.-]*$","type":"string"},
- "shape": {
- "additionalProperties": false,
- "properties": {
- "activation_profile": {"const":"canonical-counter-source-v4"},
- "eplb": {"type":"boolean"},
- "experts": {"minimum":1,"type":"integer"},
- "experts_per_rank": {"minimum":1,"type":"integer"},
- "hidden": {"minimum":1,"type":"integer"},
- "kernel_gen": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]},
- "num_logical_experts": {"minimum":1,"type":"integer"},
- "routing": {"enum":["uniform"]},
- "topk": {"minimum":1,"type":"integer"}
- },
- "required": [
- "activation_profile",
- "eplb",
- "experts",
- "experts_per_rank",
- "hidden",
- "kernel_gen",
- "num_logical_experts",
- "routing",
- "topk"
- ],
- "type": "object"
- },
- "suite": {"maxLength":128,"pattern":"^[a-z0-9][a-z0-9_.-]*$","type":"string"},
- "workload_name": {"maxLength":128,"pattern":"^[a-z0-9][a-z0-9_.-]*$","type":"string"}
- },
- "required": [
- "attempt_ordinal",
- "backend",
- "eplb",
- "ep_size",
- "mode",
- "phase",
- "resource_mode",
- "runner",
- "shape",
- "suite",
- "workload_name"
- ],
- "type": "object"
- },
- "format": {"const":"collectivex.ep.v1"},
- "generated_at": {"format":"date-time","type":"string"},
- "identity": {
- "additionalProperties": false,
- "properties": {
- "allocation_factors": {
- "additionalProperties": false,
- "properties": {
- "artifact": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]},
- "execution_id": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]},
- "job": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]},
- "repo": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]},
- "run_attempt": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]},
- "run_id": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]},
- "runner": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]},
- "source_sha": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]}
- },
- "required": ["artifact","execution_id","job","repo","run_attempt","run_id","runner","source_sha"],
- "type": "object"
- },
- "attempt_id": {"maxLength":180,"pattern":"^[a-z0-9][a-z0-9_.-]*$","type":"string"},
- "attempt_ordinal": {"minimum":1,"type":"integer"},
- "case_factors": {
- "additionalProperties": false,
- "properties": {
- "case": {"$ref":"#/$defs/scheduled_case"},
- "profile": {"$ref":"#/$defs/case_profile"},
- "sku": {"maxLength":128,"pattern":"^[a-z0-9][a-z0-9_.-]*$","type":"string"}
- },
- "required": ["case","profile","sku"],
- "type": "object"
- },
- "case_id": {"maxLength":180,"pattern":"^[a-z0-9][a-z0-9_.-]*$","type":"string"}
- },
- "required": [
- "allocation_factors",
- "attempt_id",
- "attempt_ordinal",
- "case_factors",
- "case_id"
- ],
- "type": "object"
- },
- "implementation": {
- "additionalProperties": false,
- "properties": {
- "kernel_generation": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]},
- "name": {"maxLength":128,"pattern":"^[a-z0-9][a-z0-9_.-]*$","type":"string"},
- "provenance": {
- "properties": {
- "allow_hybrid_mode": {"type":"boolean"},
- "communication_backend": {"enum":["nccl-device-lsa","nccl-gin"]},
- "deepep_fix_pr": {"const":630},
- "deepep_nccl_check_commit": {"const":"93d0564188f7a0a6288c6e316484861b0efa042e"},
- "deepep_nccl_check_fix_pr": {"const":640},
- "deepep_pr": {"const":605},
- "deterministic": {"type": "boolean"},
- "gin_enabled": {"type":"boolean"},
- "jit_cubins": {
- "items": {"$ref":"#/$defs/deepep_v2_jit_cubin"},
- "maxItems": 5,
- "minItems": 5,
- "type": "array",
- "uniqueItems": true
- },
- "jit_kernel_keys": {
- "items": {"maxLength":512,"pattern":"^[A-Za-z0-9][A-Za-z0-9_.+-]*$","type":"string"},
- "maxItems": 3,
- "minItems": 3,
- "type": "array",
- "uniqueItems": true
- },
- "jit_random_seed": {"const":"collectivex-deepep-v2-fa8a9b1"},
- "jit_shared_objects": {
- "items": {
- "additionalProperties": false,
- "properties": {
- "kernel_key": {"maxLength":512,"pattern":"^[A-Za-z0-9][A-Za-z0-9_.+-]*$","type":"string"},
- "rank_artifacts": {
- "items": {"$ref":"#/$defs/hybrid_jit_rank_artifact"},
- "minItems": 1,
- "type": "array"
- }
- },
- "required": ["kernel_key","rank_artifacts"],
- "type": "object"
- },
- "maxItems": 3,
- "minItems": 3,
- "type": "array"
- },
- "num_experts": {"minimum": 1, "type": "integer"},
- "num_nvl_bytes": {"minimum": 0, "type": "integer"},
- "num_qps_per_rank": {"minimum": 1, "type": "integer"},
- "num_rdma_bytes": {"minimum": 0, "type": "integer"},
- "rdma_block_num": {"minimum": 0, "type": "integer"},
- "realized_config": {"$ref":"#/$defs/hybrid_realized_config"},
- "tuning_num_experts": {"minimum": 1, "type": "integer"},
- "uccl_dependency_versions": {
- "additionalProperties": false,
- "properties": {
- "intervaltree": {"const":"3.1.0"},
- "nvidia-cuda-runtime-cu12": {"const":"12.9.79"},
- "sortedcontainers": {"const":"2.4.0"}
- },
- "required": ["intervaltree","nvidia-cuda-runtime-cu12","sortedcontainers"],
- "type": "object"
- },
- "use_external_inp_buf": {"type": "boolean"}
- },
- "type": "object",
- "propertyNames": {
- "enum": [
- "allocated_qps",
- "allow_hybrid_mode",
- "allow_mnnvl",
- "allow_multiple_reduction",
- "api",
- "api_signature_sha256",
- "backend",
- "backend_lineage",
- "block_num",
- "block_num_floored",
- "block_num_target",
- "branch",
- "collective_library",
- "combine_dtype",
- "combine_warps",
- "communication_backend",
- "cuda_version",
- "deepep_commit",
- "deepep_distribution_version",
- "deepep_fix_pr",
- "deepep_nccl_check_commit",
- "deepep_nccl_check_fix_pr",
- "deepep_pr",
- "deepep_tree",
- "deepep_version",
- "deterministic",
- "device_cus",
- "device_sms",
- "dispatch_dtype",
- "dispatch_warps",
- "enable_sdma",
- "fmt_commit",
- "gpus_per_node",
- "gin_enabled",
- "heap_size",
- "impl",
- "jit_cache_key",
- "jit_cubins",
- "jit_kernel_keys",
- "jit_random_seed",
- "jit_shared_objects",
- "kernel_type",
- "loaded_libraries",
- "local_experts",
- "logical_scaleout_ranks",
- "logical_scaleup_ranks",
- "mapping_variant",
- "max_num_inp_token_per_rank",
- "max_num_tokens",
- "max_total_recv_tokens",
- "mnnvl_comm",
- "mode",
- "mori_commit",
- "nccl_communicator",
- "nccl_package_version",
- "nccl_version",
- "num_experts",
- "num_max_tokens_per_rank",
- "num_nvl_bytes",
- "num_qps",
- "num_qps_per_rank",
- "num_rdma_bytes",
- "num_sms",
- "nvshmem_ibgda_nic_handler",
- "nvshmem_package_version",
- "path",
- "physical_nvlink_ranks",
- "physical_rdma_ranks",
- "prefer_overlap_with_compute",
- "rdma_block_num",
- "reference_semantics",
- "realized_config",
- "requested_num_sms",
- "resource_mode",
- "routing_factor",
- "routing_metadata",
- "sm_fraction",
- "top_k",
- "torch_git_version",
- "torch_version",
- "transport",
- "trtllm",
- "tuned_source",
- "tuning_num_experts",
- "uccl_commit",
- "uccl_dependency_versions",
- "uccl_version",
- "uccl_wrapper_commit",
- "use_external_inp_buf",
- "workspace"
- ]
- }
- },
- "resource_profile": {
- "additionalProperties": false,
- "properties": {
- "achieved_fraction": {},
- "comm_units_kind": {},
- "configured_units": {},
- "conformance_class": {},
- "device_units": {},
- "fixed_kernel": {},
- "nonconforming": {},
- "pareto_eligible": {},
- "persistent_bytes": {},
- "qps_per_rank": {},
- "requested_fraction": {},
- "tuned_source": {},
- "target_achieved_within_tol": {},
- "tolerance": {},
- "resource_class": {},
- "warps_combine": {},
- "warps_dispatch": {}
- },
- "required": [
- "comm_units_kind",
- "requested_fraction",
- "configured_units",
- "device_units",
- "achieved_fraction",
- "warps_dispatch",
- "warps_combine",
- "qps_per_rank",
- "persistent_bytes",
- "tuned_source",
- "resource_class",
- "conformance_class",
- "tolerance",
- "target_achieved_within_tol",
- "nonconforming",
- "fixed_kernel",
- "pareto_eligible"
- ],
- "type": "object"
- }
- },
- "required": ["kernel_generation","name","provenance","resource_profile"],
- "type": "object"
- },
- "measurement": {
- "additionalProperties": false,
- "properties": {
- "component_order_contract": {"const":"qualification-hash-rotated-components-v1"},
- "conditioning": {
- "additionalProperties": false,
- "properties": {
- "contract": {"const":"fixed-phase-ramp-8-roundtrips-v1"},
- "ladder": {"items":{"minimum":1,"type":"integer"},"minItems":1,"type":"array"},
- "roundtrips_per_shape": {"const":8}
- },
- "required": ["contract","ladder","roundtrips_per_shape"],
- "type": "object"
- },
- "contract": {"enum":["layout-and-dispatch-v1","expert-packed-weighted-combine-v1"]},
- "rows": {
- "items": {
- "additionalProperties": false,
- "properties": {
- "anomalies": {
- "items": {
- "additionalProperties": false,
- "properties": {
- "T": {"minimum":1,"type":"integer"},
- "component_floor_p50": {"minimum":0,"type":"number"},
- "isolated_sum_p99": {"minimum":0,"type":"number"},
- "ratio": {"minimum":0,"type":"number"},
- "roundtrip_p50": {"minimum":0,"type":"number"},
- "roundtrip_p99": {"minimum":0,"type":"number"},
- "threshold": {"minimum":0,"type":"number"},
- "type": {"enum":["roundtrip_gt_isolated_sum","roundtrip_lt_component_floor"]}
- },
- "required": ["type","T"],
- "type": "object"
- },
- "type": "array"
- },
- "components": {
- "additionalProperties": false,
- "properties": {
- "combine": {"$ref":"#/$defs/component"},
- "dispatch": {"$ref":"#/$defs/component"},
- "isolated_sum": {"$ref":"#/$defs/component"},
- "roundtrip": {"$ref":"#/$defs/component"},
- "stage": {"$ref":"#/$defs/component"}
- },
- "required": ["combine","dispatch","isolated_sum","roundtrip","stage"],
- "type": "object"
- },
- "correctness": {
- "additionalProperties": false,
- "properties": {
- "contract": {"enum":["expert-specific-transform-v1","expert-assignment-transform-v1"]},
- "max_relative_error": {"minimum":0,"type":"number"},
- "passed": {"type":"boolean"},
- "rank_evidence": {
- "items": {
- "additionalProperties": false,
- "properties": {
- "input_unchanged": {"type":"boolean"},
- "order_stable": {"type":"boolean"},
- "post_timing": {"$ref":"#/$defs/oracle"},
- "pre_timing": {"$ref":"#/$defs/oracle"},
- "rank": {"minimum":0,"type":"integer"}
- },
- "required": ["input_unchanged","order_stable","post_timing","pre_timing","rank"],
- "type": "object"
- },
- "minItems": 1,
- "type": "array"
- },
- "scope": {"enum":["dispatch-metadata-and-transformed-combine","expert-assignment-and-weighted-combine"]}
- },
- "required": ["contract","max_relative_error","passed","rank_evidence","scope"],
- "type": "object"
- },
- "global_tokens": {"minimum":1,"type":"integer"},
- "byte_provenance": {
- "additionalProperties": false,
- "properties": {
- "combine": {"$ref":"#/$defs/byte_accounting"},
- "dispatch": {"$ref":"#/$defs/byte_accounting"},
- "roundtrip": {"$ref":"#/$defs/byte_accounting"},
- "stage": {"$ref":"#/$defs/byte_accounting"}
- },
- "required": ["combine","dispatch","roundtrip","stage"],
- "type": "object"
- },
- "point_id": {"maxLength":180,"pattern":"^[a-z0-9][a-z0-9_.-]*$","type":"string"},
- "receive": {
- "additionalProperties": false,
- "properties": {
- "max": {"minimum":0,"type":"integer"},
- "mean": {"minimum":0,"type":"number"},
- "min": {"minimum":0,"type":"integer"},
- "total": {"minimum":0,"type":"integer"}
- },
- "required": ["max","mean","min","total"],
- "type": "object"
- },
- "routing": {
- "additionalProperties": false,
- "properties": {
- "empty_expert_count": {"minimum":0,"type":"integer"},
- "empty_rank_count": {"minimum":0,"type":"integer"},
- "expert_assignment_rank_cv": {"minimum":0,"type":"number"},
- "expert_assignments_per_rank": {"items":{"minimum":0,"type":"integer"},"type":"array"},
- "expert_load_cv": {"minimum":0,"type":"number"},
- "expert_load_max": {"minimum":0,"type":"integer"},
- "expert_load_mean": {"minimum":0,"type":"number"},
- "expert_load_min": {"minimum":0,"type":"integer"},
- "fanout_histogram": {"items":{"minimum":0,"type":"integer"},"type":"array"},
- "fanout_max": {"minimum":1,"type":"integer"},
- "fanout_mean": {"minimum":0,"type":"number"},
- "fanout_min": {"minimum":1,"type":"integer"},
- "hash": {"pattern":"^[0-9a-f]{64}$","type":"string"},
- "hotspot_ratio": {"minimum":0,"type":"number"},
- "locality": {
- "oneOf": [
- {"type":"null"},
- {
- "additionalProperties": false,
- "properties": {
- "copies": {"minimum":0,"type":"integer"},
- "cross_domain_fraction": {"minimum":0,"type":"number"},
- "cross_node_fraction": {"minimum":0,"type":"number"},
- "gpus_per_node": {"minimum":1,"type":"integer"},
- "local_rank_fraction": {"minimum":0,"type":"number"},
- "placement": {"const":"packed"},
- "same_node_fraction": {"minimum":0,"type":"number"},
- "same_scaleup_domain_fraction": {"minimum":0,"type":"number"},
- "scale_up_domain": {"minimum":1,"type":"integer"}
- },
- "required": [
- "placement",
- "local_rank_fraction",
- "same_node_fraction",
- "same_scaleup_domain_fraction",
- "cross_node_fraction",
- "cross_domain_fraction",
- "gpus_per_node",
- "scale_up_domain",
- "copies"
- ],
- "type": "object"
- }
- ]
- },
- "payload_copies_per_rank": {"items":{"minimum":0,"type":"integer"},"type":"array"},
- "payload_rank_cv": {"minimum":0,"type":"number"},
- "routed_copies": {"minimum":1,"type":"integer"},
- "source_token_stats": {
- "oneOf": [
- {"type":"null"},
- {
- "additionalProperties": false,
- "properties": {
- "cv": {"minimum":0,"type":"number"},
- "empty_ranks": {"minimum":0,"type":"integer"},
- "max": {"minimum":0,"type":"integer"},
- "mean": {"minimum":0,"type":"number"},
- "min": {"minimum":0,"type":"integer"},
- "ranks": {"minimum":1,"type":"integer"},
- "total": {"minimum":0,"type":"integer"}
- },
- "required": ["min","mean","max","cv","empty_ranks","total","ranks"],
- "type": "object"
- }
- ]
- }
- },
- "required": [
- "empty_expert_count",
- "empty_rank_count",
- "expert_assignment_rank_cv",
- "expert_assignments_per_rank",
- "expert_load_cv",
- "expert_load_max",
- "expert_load_mean",
- "expert_load_min",
- "fanout_histogram",
- "fanout_max",
- "fanout_mean",
- "fanout_min",
- "hash",
- "hotspot_ratio",
- "locality",
- "payload_copies_per_rank",
- "payload_rank_cv",
- "routed_copies",
- "source_token_stats"
- ],
- "type": "object"
- },
- "sample_histograms": {
- "additionalProperties": false,
- "properties": {
- "combine": {"oneOf":[{"type":"null"},{"$ref":"#/$defs/histogram"}]},
- "dispatch": {"oneOf":[{"type":"null"},{"$ref":"#/$defs/histogram"}]},
- "roundtrip": {"$ref":"#/$defs/histogram"},
- "stage": {"oneOf":[{"type":"null"},{"$ref":"#/$defs/histogram"}]}
- },
- "required": ["dispatch","combine","roundtrip","stage"],
- "type": "object"
- },
- "sample_sha256": {"pattern":"^[0-9a-f]{64}$","type":"string"},
- "token_rate_at_latency_percentile": {"$ref":"#/$defs/percentiles"},
- "tokens_per_rank": {"minimum":1,"type":"integer"}
- },
- "required": [
- "anomalies",
- "components",
- "correctness",
- "byte_provenance",
- "global_tokens",
- "point_id",
- "receive",
- "routing",
- "sample_histograms",
- "sample_sha256",
- "token_rate_at_latency_percentile",
- "tokens_per_rank"
- ],
- "type": "object"
- },
- "minItems": 1,
- "type": "array"
- },
- "sampling": {
- "additionalProperties": false,
- "properties": {
- "contract": {"const":"fixed-512-v1"},
- "iterations_per_trial": {"const":8},
- "percentile_method": {"const":"nearest-rank"},
- "reduction": {"const":"cross-rank-max-per-iteration"},
- "samples_per_component": {"const":512},
- "trials": {"const":64},
- "warmup_iterations": {"const":32},
- "warmup_semantics": {"const":"full-roundtrip-before-each-component-trial-point-v1"}
- },
- "required": [
- "contract",
- "iterations_per_trial",
- "percentile_method",
- "reduction",
- "samples_per_component",
- "trials",
- "warmup_iterations",
- "warmup_semantics"
- ],
- "type": "object"
- },
- "source_allocation": {"const":"even"}
- },
- "required": [
- "component_order_contract",
- "conditioning",
- "contract",
- "rows",
- "sampling",
- "source_allocation"
- ],
- "type": "object"
- },
- "outcome": {
- "additionalProperties": false,
- "properties": {
- "reasons": {"items":{"type":"string"},"type":"array"},
- "status": {"enum":["success","invalid"]},
- "validity": {
- "additionalProperties": false,
- "properties": {
- "anomaly_free": {"type":"boolean"},
- "execution_status": {"enum":["complete","failed"]},
- "measurement_conformance": {"enum":["conformant","nonconformant"]},
- "provenance_complete": {"type":"boolean"},
- "resource_conformance": {"minLength":1,"type":"string"},
- "sampling_conformance": {"enum":["conformant","nonconformant"]},
- "semantic_correctness": {"enum":["pass","fail"]},
- "workload_identity": {"enum":["consistent-across-ranks","inconsistent"]},
- "workload_source": {"enum":["canonical-serialized","seeded-runtime"]}
- },
- "required": [
- "execution_status",
- "semantic_correctness",
- "workload_identity",
- "workload_source",
- "measurement_conformance",
- "sampling_conformance",
- "resource_conformance",
- "provenance_complete",
- "anomaly_free"
- ],
- "type": "object"
- }
- },
- "required": ["reasons","status","validity"],
- "type": "object"
- },
- "provenance": {
- "additionalProperties": false,
- "properties": {
- "command": {"minLength":1,"type":"string"},
- "distributed_launcher": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]},
- "git_run": {"oneOf":[{"type":"null"},{"$ref":"#/$defs/git_run"}]},
- "image": {
- "additionalProperties": false,
- "properties": {
- "arch": {"maxLength":128,"pattern":"^[a-z0-9][a-z0-9_.-]*$","type":"string"},
- "digest": {
- "oneOf": [{"type":"null"},{"pattern":"^sha256:[0-9a-f]{64}$","type":"string"}]
- },
- "digest_verified": {"type":"boolean"},
- "reference": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]},
- "squash_sha256": {"oneOf":[{"type":"null"},{"pattern":"^[0-9a-f]{64}$","type":"string"}]}
- },
- "required": ["arch","digest","digest_verified","reference","squash_sha256"],
- "type": "object"
- },
- "redaction": {"const":"sanitized-v1"}
- },
- "required": ["command","distributed_launcher","git_run","image","redaction"],
- "type": "object"
- },
- "record_type": {"const":"case-attempt"},
- "runtime_fingerprint": {
- "additionalProperties": false,
- "properties": {
- "accelerator_runtime": {
- "additionalProperties": false,
- "properties": {
- "kind": {"enum":["cuda","hip"]},
- "version": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]}
- },
- "required": ["kind","version"],
- "type": "object"
- },
- "collective_library": {
- "additionalProperties": false,
- "properties": {
- "kind": {"enum":["nccl","rccl"]},
- "version": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]}
- },
- "required": ["kind","version"],
- "type": "object"
- },
- "device": {
- "additionalProperties": false,
- "properties": {
- "arch": {"minLength":1,"type":"string"},
- "compute_units": {"minimum":1,"type":"integer"},
- "memory_bytes": {"minimum":1,"type":"integer"},
- "product": {"minLength":1,"type":"string"},
- "warp_size": {"minimum":1,"type":"integer"}
- },
- "required": ["arch","compute_units","memory_bytes","product","warp_size"],
- "type": "object"
- },
- "driver_version": {"oneOf":[{"type":"null"},{"minLength":1,"type":"string"}]},
- "framework": {
- "additionalProperties": false,
- "properties": {"kind":{"const":"torch"},"version":{"minLength":1,"type":"string"}},
- "required": ["kind","version"],
- "type": "object"
- },
- "machine": {"maxLength":128,"pattern":"^[a-z0-9][a-z0-9_.-]*$","type":"string"},
- "python_version": {"minLength":1,"type":"string"},
- "vendor": {"enum":["nvidia","amd"]}
- },
- "required": [
- "accelerator_runtime",
- "collective_library",
- "device",
- "driver_version",
- "framework",
- "machine",
- "python_version",
- "vendor"
- ],
- "type": "object"
- },
- "sample_artifact": {
- "additionalProperties": false,
- "properties": {
- "bytes": {"minimum":1,"type":"integer"},
- "format": {"const":"collectivex.samples.v1"},
- "path": {"pattern":"^[A-Za-z0-9_.-]+$","type":"string"},
- "sha256": {"pattern":"^[0-9a-f]{64}$","type":"string"}
- },
- "required": ["bytes","format","path","sha256"],
- "type": "object"
- },
- "schema_version": {"const":1},
- "topology": {
- "additionalProperties": false,
- "properties": {
- "device_count": {"minimum":1,"type":"integer"},
- "device_product": {"minLength":1,"type":"string"},
- "gpus_per_node": {"minimum":1,"type":"integer"},
- "nodes": {"minimum":1,"type":"integer"},
- "placement": {"const":"packed"},
- "realized_placement": {
- "additionalProperties": false,
- "properties": {
- "gpus_per_node": {"minimum":1,"type":"integer"},
- "nodes": {"minimum":1,"type":"integer"},
- "ranks_per_node": {"minimum":1,"type":"integer"},
- "unique_local_ranks": {"const":true},
- "valid": {"const":true}
- },
- "required": ["gpus_per_node","nodes","ranks_per_node","unique_local_ranks","valid"],
- "type": "object"
- },
- "scale_out_transport": {"oneOf":[{"type":"null"},{"maxLength":128,"pattern":"^[a-z0-9][a-z0-9_.-]*$","type":"string"}]},
- "scale_up_domain": {"minimum":1,"type":"integer"},
- "scale_up_transport": {"maxLength":128,"pattern":"^[a-z0-9][a-z0-9_.-]*$","type":"string"},
- "scope": {"enum":["scale-up","scale-out"]},
- "topology_class": {"maxLength":128,"pattern":"^[a-z0-9][a-z0-9_.-]*$","type":"string"},
- "transport": {"maxLength":128,"pattern":"^[a-z0-9][a-z0-9_.-]*$","type":"string"},
- "world_size": {"minimum":1,"type":"integer"}
- },
- "required": [
- "device_count",
- "device_product",
- "gpus_per_node",
- "nodes",
- "placement",
- "realized_placement",
- "scale_out_transport",
- "scale_up_domain",
- "scale_up_transport",
- "scope",
- "topology_class",
- "transport",
- "world_size"
- ],
- "type": "object"
- },
- "version": {"minimum":1,"type":"integer"},
- "workload": {
- "additionalProperties": false,
- "properties": {
- "activation_generator": {"const":"collectivex-activation-counter-v4"},
- "activation_identity": {"pattern":"^[0-9a-f]{64}$","type":"string"},
- "activation_profile": {"const":"canonical-counter-source-v4"},
- "cross_rank_consistent": {"const":true},
- "manifest_checksums": {
- "oneOf": [
- {"type":"null"},
- {
- "additionalProperties": {
- "additionalProperties": false,
- "properties": {
- "topk_idx": {"pattern":"^[0-9a-f]{64}$","type":"string"},
- "topk_weights": {"pattern":"^[0-9a-f]{64}$","type":"string"},
- "trace": {"pattern":"^[0-9a-f]{64}$","type":"string"}
- },
- "required": ["topk_idx", "topk_weights", "trace"],
- "type": "object"
- },
- "type": "object"
- }
- ]
- },
- "members": {
- "oneOf": [
- {"type":"null"},
- {
- "items": {"pattern":"^cxwork-v1-[0-9a-f]{64}$","type":"string"},
- "minItems": 1,
- "uniqueItems": true,
- "type": "array"
- }
- ]
- },
- "routing_generator": {"const":"collectivex-routing-counter-v3"},
- "source": {"enum":["canonical-serialized","seeded-runtime"]},
- "trace_hashes": {
- "items": {"pattern":"^[0-9a-f]{64}$","type":"string"},
- "minItems": 1,
- "type": "array"
- },
- "trace_signature": {"pattern":"^[0-9a-f]{64}$","type":"string"},
- "workload_id": {
- "oneOf": [{"type":"null"},{"pattern":"^cxwork-v1-[0-9a-f]{64}$","type":"string"}]
- }
- },
- "required": [
- "activation_generator",
- "activation_identity",
- "activation_profile",
- "cross_rank_consistent",
- "manifest_checksums",
- "members",
- "routing_generator",
- "source",
- "trace_hashes",
- "trace_signature",
- "workload_id"
- ],
- "type": "object"
- }
- },
- "required": [
- "case",
- "format",
- "generated_at",
- "identity",
- "implementation",
- "measurement",
- "outcome",
- "provenance",
- "record_type",
- "runtime_fingerprint",
- "sample_artifact",
- "schema_version",
- "topology",
- "version",
- "workload"
- ],
- "title": "CollectiveX raw case attempt v1",
- "type": "object"
-}
diff --git a/experimental/CollectiveX/schemas/samples-v1.schema.json b/experimental/CollectiveX/schemas/samples-v1.schema.json
deleted file mode 100644
index 496dcea998..0000000000
--- a/experimental/CollectiveX/schemas/samples-v1.schema.json
+++ /dev/null
@@ -1,86 +0,0 @@
-{
- "$schema": "https://json-schema.org/draft/2020-12/schema",
- "$id": "https://inferencex.com/schemas/collectivex/samples-v1.schema.json",
- "title": "CollectiveX exact private samples v1",
- "type": "object",
- "additionalProperties": false,
- "required": ["attempt_id","case_id","format","points","sampling","schema_version"],
- "properties": {
- "attempt_id": {"$ref": "#/$defs/attemptId"},
- "case_id": {"$ref": "#/$defs/caseId"},
- "format": {"const": "collectivex.samples.v1"},
- "points": {
- "type": "array",
- "minItems": 1,
- "items": {
- "type": "object",
- "additionalProperties": false,
- "required": ["components","point_id","sample_sha256","tokens_per_rank"],
- "properties": {
- "components": {
- "type": "object",
- "additionalProperties": false,
- "required": ["combine","dispatch","roundtrip","stage"],
- "properties": {
- "combine": {"$ref": "#/$defs/component"},
- "dispatch": {"$ref": "#/$defs/component"},
- "roundtrip": {"$ref": "#/$defs/component"},
- "stage": {"$ref":"#/$defs/component"}
- }
- },
- "point_id": {"$ref": "#/$defs/pointId"},
- "sample_sha256": {"$ref": "#/$defs/sha256"},
- "tokens_per_rank": {"type": "integer","minimum": 1}
- }
- }
- },
- "sampling": {
- "type": "object",
- "additionalProperties": false,
- "required": ["iterations_per_trial","reduction","trials"],
- "properties": {
- "iterations_per_trial": {"const": 8},
- "reduction": {"const": "cross-rank-max-per-iteration"},
- "trials": {"const": 64}
- }
- },
- "schema_version": {"const": 1}
- },
- "$defs": {
- "sha256": {"type": "string","pattern": "^[0-9a-f]{64}$"},
- "caseId": {"type": "string","pattern": "^[a-z0-9][a-z0-9_.-]*$","maxLength": 180},
- "pointId": {"type": "string","pattern": "^[a-z0-9][a-z0-9_.-]*$","maxLength": 180},
- "attemptId": {"type": "string","pattern": "^[a-z0-9][a-z0-9_.-]*$","maxLength": 180},
- "trials": {
- "type":"array",
- "minItems":64,
- "maxItems":64,
- "items":{"type":"array","minItems":8,"maxItems":8,"items":{"type":"number","minimum":0}}
- },
- "measuredComponent": {
- "type":"object",
- "additionalProperties":false,
- "required":["availability","sample_count","trials"],
- "properties": {
- "availability":{"const":"measured"},
- "sample_count":{"const":512},
- "trials":{"$ref":"#/$defs/trials"}
- }
- },
- "component": {
- "oneOf": [
- {"$ref":"#/$defs/measuredComponent"},
- {
- "type":"object",
- "additionalProperties":false,
- "required":["availability","sample_count","trials"],
- "properties": {
- "availability":{"const":"unavailable"},
- "sample_count":{"const":0},
- "trials":{"type":"null"}
- }
- }
- ]
- }
- }
-}
diff --git a/experimental/CollectiveX/schemas/terminal-outcome-v1.schema.json b/experimental/CollectiveX/schemas/terminal-outcome-v1.schema.json
deleted file mode 100644
index 2d4497e187..0000000000
--- a/experimental/CollectiveX/schemas/terminal-outcome-v1.schema.json
+++ /dev/null
@@ -1,285 +0,0 @@
-{
- "$schema": "https://json-schema.org/draft/2020-12/schema",
- "$id": "https://inferencex.com/schemas/collectivex/terminal-outcome-v1.schema.json",
- "title": "CollectiveX terminal outcome v1",
- "type": "object",
- "additionalProperties": false,
- "required": ["case","format","generated_at","identity","outcome","provenance","record_type","schema_version","version"],
- "properties": {
- "case": {"$ref": "#/$defs/case"},
- "format": {"const": "collectivex.terminal.v1"},
- "generated_at": {"type": "string","format": "date-time"},
- "identity": {
- "type": "object",
- "additionalProperties": false,
- "required": ["allocation_factors","attempt_id","attempt_ordinal","case_factors","case_id"],
- "properties": {
- "allocation_factors": {"$ref": "#/$defs/allocationFactors"},
- "attempt_id": {"$ref": "#/$defs/attemptId"},
- "attempt_ordinal": {"type": "integer","minimum": 1},
- "case_factors": {
- "type": "object",
- "additionalProperties": false,
- "required": ["case","profile","sku"],
- "properties": {
- "case": {"$ref": "#/$defs/case"},
- "profile": {"$ref":"#/$defs/caseProfile"},
- "sku": {"$ref": "#/$defs/safeId"}
- }
- },
- "case_id": {"$ref": "#/$defs/caseId"}
- }
- },
- "outcome": {
- "type": "object",
- "additionalProperties": false,
- "required": ["failure_mode","reason","return_code","status"],
- "properties": {
- "failure_mode": {"$ref": "#/$defs/safeId"},
- "reason": {"type": "string","minLength": 1,"maxLength": 240},
- "return_code": {"type": "integer","minimum": 0},
- "status": {"enum": ["failed","invalid","unsupported"]}
- }
- },
- "provenance": {
- "type": "object",
- "additionalProperties": false,
- "required": ["git_run","control_sha256","redaction","source"],
- "properties": {
- "git_run": {"oneOf": [{"type": "null"},{"$ref": "#/$defs/gitRun"}]},
- "control_sha256": {"oneOf": [{"type": "null"},{"$ref": "#/$defs/sha256"}]},
- "redaction": {"const": "sanitized-v1"},
- "source": {
- "enum": [
- "runtime-emitter",
- "post-emit-command",
- "matrix-capability-resolver"
- ]
- }
- }
- },
- "record_type": {"const": "terminal-outcome"},
- "schema_version": {"const": 1},
- "version": {"type": "integer","minimum": 1}
- },
- "allOf": [
- {
- "oneOf": [
- {
- "properties": {
- "provenance": {
- "properties": {"source": {"const": "runtime-emitter"}}
- },
- "outcome": {"$ref": "#/$defs/runtimeOutcome"}
- }
- },
- {
- "properties": {
- "provenance": {
- "properties": {"source": {"const": "post-emit-command"}}
- },
- "outcome": {"$ref": "#/$defs/postEmitOutcome"}
- }
- },
- {
- "properties": {
- "provenance": {
- "properties": {"source": {"const": "matrix-capability-resolver"}}
- },
- "outcome": {"$ref": "#/$defs/capabilityOutcome"}
- }
- }
- ]
- }
- ],
- "$defs": {
- "runtimeOutcome": {
- "type": "object",
- "properties": {"status": {"const": "failed"}},
- "allOf": [
- {
- "oneOf": [
- {"properties": {"failure_mode": {"const": "setup"}, "reason": {"const": "launcher-setup-failed"}}},
- {"properties": {"failure_mode": {"const": "repository-stage"}, "reason": {"const": "repository-staging-failed"}}},
- {"properties": {"failure_mode": {"const": "registry-verification"}, "reason": {"const": "container-registry-verification-failed"}}},
- {"properties": {"failure_mode": {"const": "scheduler-allocation"}, "reason": {"const": "scheduler-allocation-failed"}}},
- {"properties": {"failure_mode": {"const": "container-import"}, "reason": {"const": "container-image-preparation-failed"}}},
- {"properties": {"failure_mode": {"const": "container-hash"}, "reason": {"const": "container-image-identity-failed"}}},
- {"properties": {"failure_mode": {"const": "container-launch"}, "reason": {"const": "container-runtime-launch-failed"}}},
- {"properties": {"failure_mode": {"const": "backend-setup"}, "reason": {"const": "backend-setup-failed"}}},
- {"properties": {"failure_mode": {"const": "artifact-collection"}, "reason": {"const": "artifact-collection-failed"}}},
- {"properties": {"failure_mode": {"const": "runtime-identity"}, "reason": {"const": "runtime-identity-mismatch"}}},
- {"properties": {"failure_mode": {"const": "timeout"}, "reason": {"const": "execution-timeout"}}},
- {"properties": {"failure_mode": {"const": "deadlock"}, "reason": {"const": "execution-deadlock"}}},
- {"properties": {"failure_mode": {"const": "execution"}, "reason": {"const": "distributed-command-failed"}}}
- ]
- }
- ]
- },
- "postEmitOutcome": {
- "type": "object",
- "properties": {
- "status": {"const": "failed"},
- "failure_mode": {"enum": ["runtime-identity", "timeout", "deadlock", "execution"]},
- "reason": {"const": "post-emit-distributed-command-failed"}
- }
- },
- "capabilityOutcome": {
- "type": "object",
- "properties": {
- "status": {"const": "unsupported"},
- "failure_mode": {"const": "capability"},
- "reason": {
- "enum": [
- "backend-platform-unsupported",
- "backend-token-capacity"
- ]
- }
- }
- },
- "sha256": {"type": "string","pattern": "^[0-9a-f]{64}$"},
- "safeId": {"type": "string","pattern": "^[a-z0-9][a-z0-9_.-]*$","maxLength": 128},
- "caseId": {"type": "string","pattern": "^[a-z0-9][a-z0-9_.-]*$","maxLength": 180},
- "attemptId": {"type": "string","pattern": "^[a-z0-9][a-z0-9_.-]*$","maxLength": 180},
- "nullableText": {"oneOf": [{"type": "null"},{"type": "string","minLength": 1}]},
- "caseProfile": {
- "type":"object",
- "additionalProperties":false,
- "allOf":[
- {
- "if":{"properties":{"mode":{"const":"normal"}},"required":["mode"]},
- "then":{"properties":{
- "combine_semantics":{"const":"activation-only"},
- "component_order_contract":{"const":"qualification-hash-rotated-components-v1"},
- "contract":{"const":"layout-and-dispatch-v1"},
- "correctness_scope":{"const":"dispatch-metadata-and-transformed-combine"},
- "oracle_contract":{"const":"expert-specific-transform-v1"},
- "payload_unit":{"const":"token-rank"}
- }}
- },
- {
- "if":{"properties":{"mode":{"const":"low-latency"}},"required":["mode"]},
- "then":{"properties":{
- "combine_semantics":{"const":"gate-weighted"},
- "component_order_contract":{"const":"qualification-hash-rotated-components-v1"},
- "contract":{"const":"expert-packed-weighted-combine-v1"},
- "correctness_scope":{"const":"expert-assignment-and-weighted-combine"},
- "oracle_contract":{"const":"expert-assignment-transform-v1"},
- "payload_unit":{"const":"token-expert"}
- }}
- }
- ],
- "required":["activation_generator","activation_profile","combine_dtype","combine_semantics","component_order_contract","conditioning_contract","contract","correctness_scope","dtype","eplb_planner","eplb_redundant_experts","eplb_reference_tokens_per_rank","mode","oracle_contract","payload_unit","placement","percentile_method","rank_reduction","resource_mode","routing_generator","sampling_contract","seed","source_identity_contract"],
- "properties": {
- "activation_generator":{"const":"collectivex-activation-counter-v4"},
- "activation_profile":{"const":"canonical-counter-source-v4"},
- "combine_dtype":{"const":"bf16"},
- "combine_semantics":{"enum":["activation-only","gate-weighted"]},
- "component_order_contract":{"const":"qualification-hash-rotated-components-v1"},
- "conditioning_contract":{"const":"fixed-phase-ramp-8-roundtrips-v1"},
- "contract":{"enum":["layout-and-dispatch-v1","expert-packed-weighted-combine-v1"]},
- "correctness_scope":{"enum":["dispatch-metadata-and-transformed-combine","expert-assignment-and-weighted-combine"]},
- "dtype":{"const":"bf16"},
- "eplb_planner":{"const":"greedy-rank-major-v1"},
- "eplb_redundant_experts":{"const":32},
- "eplb_reference_tokens_per_rank":{"const":2048},
- "mode":{"enum":["normal","low-latency"]},
- "oracle_contract":{"enum":["expert-specific-transform-v1","expert-assignment-transform-v1"]},
- "payload_unit":{"enum":["token-rank","token-expert"]},
- "placement":{"const":"packed"},
- "percentile_method":{"const":"nearest-rank"},
- "rank_reduction":{"const":"cross-rank-max-per-iteration"},
- "resource_mode":{"const":"fixed-profile"},
- "routing_generator":{"const":"collectivex-routing-counter-v3"},
- "sampling_contract":{"const":"fixed-512-v1"},
- "seed":{"const":67},
- "source_identity_contract":{"const":"bounded-sign-bit-source-v1"}
- }
- },
- "allocationFactors": {
- "type": "object",
- "additionalProperties": false,
- "required": ["artifact","execution_id","job","repo","run_attempt","run_id","runner","source_sha"],
- "properties": {
- "artifact": {"$ref": "#/$defs/nullableText"},
- "execution_id": {"$ref": "#/$defs/nullableText"},
- "job": {"$ref": "#/$defs/nullableText"},
- "repo": {"$ref": "#/$defs/nullableText"},
- "run_attempt": {"$ref": "#/$defs/nullableText"},
- "run_id": {"$ref": "#/$defs/nullableText"},
- "runner": {"$ref": "#/$defs/nullableText"},
- "source_sha": {"$ref": "#/$defs/nullableText"}
- }
- },
- "gitRun": {
- "type": "object",
- "additionalProperties": false,
- "required": ["artifact","job","ref","repo","run_attempt","run_id","source_sha"],
- "properties": {
- "artifact": {"type": "string","minLength": 1},
- "job": {"type": "string","minLength": 1},
- "ref": {"type": "string","minLength": 1},
- "repo": {"type": "string","pattern": "^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$"},
- "run_attempt": {"type": "string","pattern": "^[1-9][0-9]*$"},
- "run_id": {"type": "string","pattern": "^[1-9][0-9]*$"},
- "source_sha": {"type": "string","pattern": "^[0-9a-f]{40}$"}
- }
- },
- "case": {
- "type": "object",
- "additionalProperties": false,
- "required": [
- "backend",
- "canonical",
- "eplb",
- "ep",
- "experts",
- "gpus_per_node",
- "hidden",
- "ladder",
- "mode",
- "nodes",
- "phase",
- "routing",
- "samples_per_point",
- "scale_out_transport",
- "scale_up_domain",
- "scale_up_transport",
- "scope",
- "suite",
- "timing",
- "topk",
- "topology_class",
- "transport",
- "warmup_semantics",
- "workload"
- ],
- "properties": {
- "backend": {"$ref": "#/$defs/safeId"},
- "canonical": {"type": "boolean"},
- "eplb": {"type": "boolean"},
- "ep": {"type": "integer","minimum": 1},
- "experts": {"type": "integer","minimum": 1},
- "gpus_per_node": {"type": "integer","minimum": 1},
- "hidden": {"type": "integer","minimum": 1},
- "ladder": {"type": "string","pattern": "^[1-9][0-9]*( [1-9][0-9]*)*$"},
- "mode": {"enum": ["normal","low-latency"]},
- "nodes": {"type": "integer","minimum": 1},
- "phase": {"enum": ["decode","prefill"]},
- "routing": {"enum": ["uniform"]},
- "samples_per_point": {"const": 512},
- "scale_out_transport": {"oneOf": [{"type": "null"},{"$ref": "#/$defs/safeId"}]},
- "scale_up_domain": {"type": "integer","minimum": 1},
- "scale_up_transport": {"$ref": "#/$defs/safeId"},
- "scope": {"enum": ["scale-up","scale-out"]},
- "suite": {"$ref": "#/$defs/safeId"},
- "timing": {"const": "8:64:32"},
- "topk": {"type": "integer","minimum": 1},
- "topology_class": {"$ref": "#/$defs/safeId"},
- "transport": {"$ref": "#/$defs/safeId"},
- "warmup_semantics": {"const": "full-roundtrip-before-each-component-trial-point-v1"},
- "workload": {"$ref": "#/$defs/safeId"}
- }
- }
- }
-}
diff --git a/experimental/CollectiveX/summarize.py b/experimental/CollectiveX/summarize.py
index 05c281235b..91197da20d 100644
--- a/experimental/CollectiveX/summarize.py
+++ b/experimental/CollectiveX/summarize.py
@@ -3,9 +3,12 @@
from __future__ import annotations
import argparse
+import json
from pathlib import Path
-import contracts
+# Emitted document format this summary reads. This is a best-effort renderer over
+# whatever raw attempts a shard produced; it validates nothing.
+RAW_FORMAT = "collectivex.ep.v1"
def load_results(directory: str, runner: str | None, timestamp: str | None) -> list[dict]:
@@ -16,24 +19,19 @@ def load_results(directory: str, runner: str | None, timestamp: str | None) -> l
if timestamp and timestamp not in path.name:
continue
try:
- document = contracts.strict_json_load(path)
- if document.get("format") == contracts.RAW_FORMAT:
- documents.append(contracts.load_raw_attempt(path))
- elif document.get("format") == contracts.TERMINAL_FORMAT:
- documents.append(contracts.validate_terminal_document(document))
- except (contracts.ContractError, OSError):
+ with path.open() as handle:
+ document = json.load(handle)
+ except (OSError, ValueError):
continue
+ if isinstance(document, dict) and document.get("format") == RAW_FORMAT:
+ documents.append(document)
return documents
def _identity(document: dict) -> tuple[str, str, str, str, bool, int]:
case = document["case"]
- if document["format"] == contracts.RAW_FORMAT:
- routing = case["shape"]["routing"]
- eplb = case["eplb"]["enabled"]
- else:
- routing = case["routing"]
- eplb = case["eplb"]
+ routing = case["shape"]["routing"]
+ eplb = case["eplb"]["enabled"]
sku = document["identity"]["case_factors"]["sku"]
return (
sku, case["suite"], routing, case["phase"], eplb,
@@ -42,8 +40,6 @@ def _identity(document: dict) -> tuple[str, str, str, str, bool, int]:
def _headline(document: dict) -> tuple[int | str, float | str, float | str]:
- if document["format"] != contracts.RAW_FORMAT:
- return "-", "-", "-"
rows = document["measurement"]["rows"]
row = next((item for item in rows if item["tokens_per_rank"] == 64), rows[len(rows) // 2])
latency = row["components"]["roundtrip"]["percentiles_us"]
@@ -95,8 +91,7 @@ def main() -> int:
if args.markdown:
return 0
return 0 if any(
- document["format"] == contracts.RAW_FORMAT
- and document["outcome"]["status"] == "success"
+ document["outcome"]["status"] == "success"
for document in documents
) else 1
diff --git a/experimental/CollectiveX/sweep_matrix.py b/experimental/CollectiveX/sweep_matrix.py
index cca8668d38..4a8ce4878d 100644
--- a/experimental/CollectiveX/sweep_matrix.py
+++ b/experimental/CollectiveX/sweep_matrix.py
@@ -8,8 +8,6 @@
from __future__ import annotations
import argparse
-import datetime as dt
-import hashlib
import itertools
import json
import os
@@ -28,7 +26,6 @@
yaml = None
import capability as cap # noqa: E402
-import contracts # noqa: E402
import ep_harness # noqa: E402
import identity # noqa: E402
@@ -902,82 +899,6 @@ def extract_shard(
return control
-def emit_unsupported(
- matrix_path: str | os.PathLike[str], output_dir: str | os.PathLike[str]
-) -> list[Path]:
- """Materialize one strict terminal outcome for each unsupported requested case."""
- source = Path(matrix_path)
- document = validate_matrix_document(_strict_json_load(source))
- control_sha256 = hashlib.sha256(source.read_bytes()).hexdigest()
- generated_at = dt.datetime.now(dt.timezone.utc).isoformat()
- git_run = {
- "run_id": os.environ.get("GITHUB_RUN_ID"),
- "run_attempt": os.environ.get("GITHUB_RUN_ATTEMPT"),
- "ref": os.environ.get("GITHUB_REF_NAME") or os.environ.get("GITHUB_REF"),
- "source_sha": os.environ.get("COLLECTIVEX_SOURCE_SHA") or os.environ.get("GITHUB_SHA"),
- "repo": os.environ.get("GITHUB_REPOSITORY"),
- "job": os.environ.get("GITHUB_JOB"),
- "artifact": os.environ.get("COLLECTIVEX_ARTIFACT_NAME"),
- }
- if not any(value is not None for value in git_run.values()):
- git_run = None
- allocation_factors = {
- "artifact": git_run["artifact"] if git_run is not None else None,
- "execution_id": os.environ.get("COLLECTIVEX_EXECUTION_ID"),
- "job": git_run["job"] if git_run is not None else None,
- "repo": git_run["repo"] if git_run is not None else None,
- "run_attempt": git_run["run_attempt"] if git_run is not None else None,
- "run_id": git_run["run_id"] if git_run is not None else None,
- "runner": "capability-resolver",
- "source_sha": git_run["source_sha"] if git_run is not None else None,
- }
- destination = Path(output_dir)
- destination.mkdir(parents=True, exist_ok=True)
- written: list[Path] = []
- for wrapper in document["requested_cases"]:
- if wrapper["disposition"] != "unsupported":
- continue
- scheduled = wrapper["case"]
- case = {key: value for key, value in scheduled.items() if key != "case_id"}
- case_factors = {
- "case": case,
- "profile": identity.profile_for_case(case),
- "sku": wrapper["sku"],
- }
- case_id = identity.case_id_from_factors(case_factors)
- if case_id != scheduled["case_id"]:
- raise MatrixError(f"unsupported case identity differs for {scheduled['case_id']}")
- record = contracts.make_terminal_document(
- allocation_factors=allocation_factors,
- attempt_ordinal=1,
- case=case,
- case_factors=case_factors,
- control_sha256=control_sha256,
- failure_mode="capability",
- generated_at=generated_at,
- git_run=git_run,
- reason=wrapper["reason"],
- return_code=5,
- source="matrix-capability-resolver",
- status="unsupported",
- version=document["version"],
- expected_case_id=case_id,
- )
- path = destination / f"unsupported_{case_id}.json"
- temporary = path.with_name(f".{path.name}.tmp-{os.getpid()}")
- try:
- with temporary.open("x") as handle:
- json.dump(record, handle, allow_nan=False, sort_keys=True, separators=(",", ":"))
- handle.write("\n")
- handle.flush()
- os.fsync(handle.fileno())
- os.replace(temporary, path)
- finally:
- temporary.unlink(missing_ok=True)
- written.append(path)
- return written
-
-
def main() -> int:
parser = argparse.ArgumentParser(description="CollectiveX matrix resolver")
parser.add_argument("--suites", default="all", help="'all' or comma-list of suites")
@@ -997,8 +918,6 @@ def main() -> int:
parser.add_argument("--max-cases", type=int, default=128)
parser.add_argument("--extract-from", default="", metavar="MATRIX")
parser.add_argument("--validate-control", default="", metavar="SHARD")
- parser.add_argument("--emit-unsupported-from", default="", metavar="MATRIX")
- parser.add_argument("--out-dir", default="")
parser.add_argument("--shard-id", default="")
parser.add_argument("--expect-sku", default="")
parser.add_argument("--expect-backend", default="")
@@ -1006,16 +925,6 @@ def main() -> int:
parser.add_argument("--out", default="")
args = parser.parse_args()
- if args.emit_unsupported_from:
- if not args.out_dir:
- parser.error("unsupported outcome emission requires --out-dir")
- try:
- written = emit_unsupported(args.emit_unsupported_from, args.out_dir)
- except MatrixError as exc:
- parser.error(str(exc))
- print(f"emitted {len(written)} unsupported terminal outcomes", file=sys.stderr)
- return 0
-
if args.validate_control:
if not all((args.expect_sku, args.expect_backend, args.expect_nodes)):
parser.error(
diff --git a/experimental/CollectiveX/tests/test_contracts.py b/experimental/CollectiveX/tests/test_contracts.py
deleted file mode 100644
index 08b7f03c42..0000000000
--- a/experimental/CollectiveX/tests/test_contracts.py
+++ /dev/null
@@ -1,74 +0,0 @@
-#!/usr/bin/env python3
-"""Focused checks for the neutral JSON schemas and strict JSON loader."""
-from __future__ import annotations
-
-import copy
-import json
-import sys
-import tempfile
-import unittest
-from pathlib import Path
-
-import jsonschema
-
-ROOT = Path(__file__).resolve().parents[1]
-sys.path.insert(0, str(ROOT))
-
-import contracts # noqa: E402
-
-
-def schema(name):
- return json.loads((ROOT / "schemas" / name).read_text())
-
-
-class ContractTests(unittest.TestCase):
- def test_only_neutral_schemas_exist_and_are_valid(self):
- paths = sorted((ROOT / "schemas").glob("*.schema.json"))
- self.assertEqual(
- {path.name for path in paths},
- {"raw-case-v1.schema.json", "samples-v1.schema.json", "terminal-outcome-v1.schema.json"},
- )
- for path in paths:
- document = json.loads(path.read_text())
- jsonschema.Draft202012Validator.check_schema(document)
- self.assertFalse(document["additionalProperties"])
-
- def test_raw_and_terminal_share_the_scheduled_case_contract(self):
- raw = schema("raw-case-v1.schema.json")["$defs"]["scheduled_case"]
- terminal = schema("terminal-outcome-v1.schema.json")["$defs"]["case"]
- self.assertEqual(set(raw["properties"]), set(terminal["properties"]))
- self.assertEqual(set(raw["required"]), set(terminal["required"]))
- self.assertFalse(raw["additionalProperties"])
- self.assertFalse(terminal["additionalProperties"])
- self.assertNotIn("precision_profile", raw["properties"])
-
- def test_sample_component_is_exactly_512_or_unavailable(self):
- samples = schema("samples-v1.schema.json")
- validator = jsonschema.Draft202012Validator({
- "$schema": "https://json-schema.org/draft/2020-12/schema",
- "$defs": samples["$defs"], "$ref": "#/$defs/component",
- })
- measured = {
- "availability": "measured", "sample_count": 512,
- "trials": [[1.0] * 8 for _ in range(64)],
- }
- validator.validate(measured)
- for broken in (
- {**measured, "sample_count": 511},
- {**measured, "trials": measured["trials"][:-1]},
- ):
- with self.assertRaises(jsonschema.ValidationError):
- validator.validate(broken)
- validator.validate({"availability": "unavailable", "sample_count": 0, "trials": None})
-
- def test_strict_loader_rejects_duplicate_keys_and_nonfinite_numbers(self):
- with tempfile.TemporaryDirectory() as temporary:
- path = Path(temporary) / "bad.json"
- for payload in ('{"x":1,"x":2}', '{"x":NaN}'):
- path.write_text(payload)
- with self.assertRaises(contracts.ContractError):
- contracts.strict_json_load(path)
-
-
-if __name__ == "__main__":
- unittest.main()
diff --git a/experimental/CollectiveX/tests/test_matrix.py b/experimental/CollectiveX/tests/test_matrix.py
index fb43f11e28..490d580ede 100644
--- a/experimental/CollectiveX/tests/test_matrix.py
+++ b/experimental/CollectiveX/tests/test_matrix.py
@@ -1,19 +1,16 @@
#!/usr/bin/env python3
-"""Matrix, subset, shard extraction, and unsupported-delivery tests."""
+"""Matrix, subset, and shard-extraction tests."""
from __future__ import annotations
import json
-import os
import sys
import tempfile
import unittest
from pathlib import Path
-from unittest import mock
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
-import contracts # noqa: E402
import sweep_matrix # noqa: E402
@@ -64,36 +61,6 @@ def test_invalid_filters_fail_closed(self):
with self.subTest(options=options), self.assertRaises(SystemExit):
sweep_matrix.resolve_matrix(**options)
- def test_unsupported_emission_is_complete_and_delivery_valid(self):
- document = matrix(suites="all", backends="all", only_sku="h100-dgxc")
- expected = [x for x in document["requested_cases"] if x["disposition"] == "unsupported"]
- environment = {
- "COLLECTIVEX_ARTIFACT_NAME": "cxunsupported-123-1",
- "COLLECTIVEX_EXECUTION_ID": "123_1_unsupported",
- "COLLECTIVEX_SOURCE_SHA": "a" * 40,
- "GITHUB_JOB": "setup", "GITHUB_REF_NAME": "collectivex",
- "GITHUB_REPOSITORY": "SemiAnalysisAI/InferenceX",
- "GITHUB_RUN_ATTEMPT": "1", "GITHUB_RUN_ID": "123",
- }
- with tempfile.TemporaryDirectory() as temporary, mock.patch.dict(
- os.environ, environment, clear=False
- ):
- root = Path(temporary)
- source = root / "matrix.json"
- source.write_text(json.dumps(document, sort_keys=True, separators=(",", ":")))
- paths = sweep_matrix.emit_unsupported(source, root / "unsupported")
- self.assertEqual(
- contracts.validate_delivery(
- [str(path) for path in paths], str(source), disposition="unsupported"
- ),
- len(expected),
- )
- emitted = [contracts.validate_result(path) for path in paths]
- self.assertEqual(
- {item["identity"]["case_id"] for item in emitted},
- {item["case"]["case_id"] for item in expected},
- )
-
if __name__ == "__main__":
unittest.main()
From ba157cb07deae9a67a07dcc5861e3205a29ec24f Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Wed, 8 Jul 2026 20:36:43 +0800
Subject: [PATCH 205/210] feat(collectivex): dump verbatim stage-log tail on
failure; tolerate empty results
cx_fail_stage now prints the failing stage's captured log tail (bounded by
CX_LOG_TAIL_LINES, default 100) verbatim to stdout after the diagnostic line,
so a failed leg surfaces the underlying tool output instead of just a
failure-class label. Stage shard artifact now no-ops cleanly when a leg
produced no result JSON (nullglob + staged output flag), so a failed leg shows
one red step instead of two.
---
.github/workflows/collectivex-sweep.yml | 12 ++++++++++--
experimental/CollectiveX/runtime/common.sh | 10 ++++++++++
2 files changed, 20 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml
index 5c23e6b2e9..49738a429b 100644
--- a/.github/workflows/collectivex-sweep.yml
+++ b/.github/workflows/collectivex-sweep.yml
@@ -469,10 +469,18 @@ jobs:
set -euo pipefail
cd "$CX_SOURCE_ROOT" 2>/dev/null \
|| { echo "CollectiveX source is unavailable" >&2; exit 1; }
- cp -- experimental/CollectiveX/results/*.json "$CX_JOB_ROOT/artifact/"
+ shopt -s nullglob
+ results=(experimental/CollectiveX/results/*.json)
+ if [ "${#results[@]}" -eq 0 ]; then
+ echo "staged=false" >> "$GITHUB_OUTPUT"
+ echo "No result JSON to stage; leg produced none."
+ exit 0
+ fi
+ cp -- "${results[@]}" "$CX_JOB_ROOT/artifact/"
+ echo "staged=true" >> "$GITHUB_OUTPUT"
- name: Upload shard results
id: upload_artifact
- if: always() && steps.stage_artifact.outcome == 'success'
+ if: always() && steps.stage_artifact.outputs.staged == 'true'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: ${{ format('cxshard-{0}-{1}-{2}', matrix.id, github.run_id, github.run_attempt) }}
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index fc8e1c6ccb..01ef7c448c 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -138,6 +138,16 @@ cx_fail_stage() {
fi
fi
cx_log "ERROR: failure-class=$stage diagnostic=$diagnostic"
+ # Surface the failing stage's captured output verbatim on stdout for debugging.
+ # Bounded by line count only (anti-flood, not redaction): this is public CI
+ # output, so the tail carries whatever the underlying tool emitted, unmasked.
+ if [ -n "$log_path" ] && [ -f "$log_path" ] && [ -s "$log_path" ]; then
+ local tail_lines="${CX_LOG_TAIL_LINES:-100}"
+ [[ "$tail_lines" =~ ^[0-9]+$ ]] || tail_lines=100
+ cx_log "--- $stage log tail (last $tail_lines lines, verbatim) ---"
+ tail -n "$tail_lines" "$log_path" >&2 || true
+ cx_log "--- end $stage log tail ---"
+ fi
return 1
}
From 4258e11104b66b33e35e0bac83e0e796de7f1e6d Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Wed, 8 Jul 2026 21:14:17 +0800
Subject: [PATCH 206/210] fix(collectivex): remove persistent pyxis container
on allocation teardown
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
The cluster runs pyxis with container_scope=global, so the distributed path's
named --container-writable container (cxep_) survives job teardown. Its
unpacked rootfs is ~60-70 GB per node, and it was never removed, so every EP16
leg left one behind on each allocated node's local image store
(/mnt/image-storage, 915 GB). After enough legs the store filled and the next
writable extraction failed at ~2 s with "No space left on device" — classified
as backend-setup/storage-capacity. That is exactly why every H200 2-node
(EP16) leg failed while every single-node (EP8) leg, which mounts the squash
read-only and creates no named container, succeeded.
cx_launcher_cleanup now runs `enroot remove -f pyxis_cxep_` across the
allocation's nodes before releasing it (best-effort, bounded, NODES>1 only).
Single-node legs use an unnamed ephemeral container that pyxis reclaims itself.
---
experimental/CollectiveX/runtime/common.sh | 18 ++++++++++++++++++
1 file changed, 18 insertions(+)
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 01ef7c448c..5679c10cb7 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -2976,6 +2976,23 @@ PY
return 0
}
+# Remove this allocation's persistent pyxis container before the allocation is
+# released. The cluster runs pyxis with container_scope=global, so a named
+# --container-writable container (the distributed path's cxep_) survives
+# job teardown and its unpacked rootfs — tens of GB per node — would otherwise
+# accumulate on every allocated node's local image store until it fills and the
+# next writable extraction fails with ENOSPC. Best-effort and bounded: teardown
+# must never hang or fail on this. Single-node legs use an unnamed, ephemeral
+# container that pyxis reclaims on its own, so only NODES>1 needs removal.
+cx_remove_distributed_container() {
+ local job_id="$1" nodes="${2:-1}"
+ [ -n "$job_id" ] || return 0
+ [ "$nodes" -gt 1 ] 2>/dev/null || return 0
+ timeout 120 srun --jobid="$job_id" --nodes="$nodes" --ntasks-per-node=1 \
+ --chdir=/tmp enroot remove -f "pyxis_cxep_${job_id}" \
+ /dev/null 2>&1 || true
+}
+
cx_launcher_cleanup() {
local rc="$1" stage_root="${MOUNT_SRC:-}" source_root allocation_stopped=1
source_root="${stage_root:-${REPO_ROOT:-}}"
@@ -2985,6 +3002,7 @@ cx_launcher_cleanup() {
unset COLLECTIVEX_EPHEMERAL_CONFIG_PATH
fi
if [ -n "${JOB_ID:-}" ]; then
+ cx_remove_distributed_container "$JOB_ID" "${NODES:-1}"
if ! cx_cancel_job "$JOB_ID"; then
allocation_stopped=0
[ "$rc" != 0 ] || rc=1
From 8c273a36b5b54e0a3788d3fcf952eb08e5dc012c Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Wed, 8 Jul 2026 23:03:55 +0800
Subject: [PATCH 207/210] refactor(collectivex): fetch pinned DeepEP source per
shard, drop seed archive
Replace the setup-job tar seed pipeline (build seed -> upload
cxbackend-sources -> download -> extract via source_archive.py) with a
per-shard git fetch of the pinned DeepEP-v2/hybrid source before
allocation, using the checkout path the shard already had. Removes
source_archive.py and the seed-copy branch in _cx_prepare_backend_source,
reduces the isolated-parent setup and operator-config overlay merge to
shell, and updates README/methodology to match.
---
.github/workflows/collectivex-sweep.yml | 233 ++-----------
experimental/CollectiveX/README.md | 7 +-
experimental/CollectiveX/docs/methodology.md | 6 +-
experimental/CollectiveX/runtime/common.sh | 23 --
experimental/CollectiveX/source_archive.py | 349 -------------------
5 files changed, 38 insertions(+), 580 deletions(-)
delete mode 100644 experimental/CollectiveX/source_archive.py
diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml
index 49738a429b..8e1fbf33d2 100644
--- a/.github/workflows/collectivex-sweep.yml
+++ b/.github/workflows/collectivex-sweep.yml
@@ -74,56 +74,9 @@ jobs:
{
echo "matrix=$SLIM"
echo "n=$(python3 -c "import json;print(len(json.load(open('matrix_full.json'))['include']))")"
- echo "source_backends=$(python3 -c "import json;m=json.load(open('matrix_full.json'));print(' '.join(sorted({x['backend'] for x in m['include']} & {'deepep-v2','deepep-hybrid'})))")"
echo "max_parallel=$(python3 -c "import json;m=json.load(open('matrix_full.json'));w=max(x['execution_weight'] for x in m['include']);print(max(1,min(10,4096//w)))")"
} >> "$GITHUB_OUTPUT"
python3 -c "import json;m=json.load(open('matrix_full.json'));print('execution-cells:',len(m['include']))"
- - name: Prepare pinned backend source archive
- if: ${{ steps.gen.outputs.source_backends != '' }}
- working-directory: experimental/CollectiveX
- env:
- SOURCE_BACKENDS: ${{ steps.gen.outputs.source_backends }}
- COLLECTIVEX_EXECUTION_ID: ${{ github.run_id }}_${{ github.run_attempt }}_sources
- run: |
- set -euo pipefail
- source runtime/common.sh
- work="$RUNNER_TEMP/collectivex-backend-sources"
- archive="$RUNNER_TEMP/collectivex-backend-sources.tar"
- rm -rf -- "$work" "$archive"
- umask 077
- mkdir -m 700 "$work"
- mkdir -p "$work/experimental/CollectiveX"
- read -r -a backends <<< "$SOURCE_BACKENDS"
- [ "${#backends[@]}" -gt 0 ]
- source_root="$work/experimental/CollectiveX/.cx_sources"
- members=()
- for backend in "${backends[@]}"; do
- cx_prepare_backend_source "$work" "$backend"
- source_path="$(cx_backend_source_path "$source_root" "$backend")"
- source_basename="${source_path#"$source_root/"}"
- [ -n "$source_basename" ] \
- && [ "$source_path" = "$source_root/$source_basename" ] \
- && [[ "$source_basename" != */* ]]
- members+=(".cx_sources/$source_basename")
- done
- cx_cleanup_private_logs 0
- tar --sort=name --mtime='@1' --owner=0 --group=0 --numeric-owner \
- --no-recursion -C "$work/experimental/CollectiveX" \
- -cf "$archive" .cx_sources
- for member in "${members[@]}"; do
- tar --sort=name --mtime='@1' --owner=0 --group=0 --numeric-owner \
- -C "$work/experimental/CollectiveX" -rf "$archive" "$member"
- done
- sha256sum "$archive"
- rm -rf -- "$work"
- - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
- if: ${{ steps.gen.outputs.source_backends != '' }}
- with:
- name: cxbackend-sources-${{ github.run_id }}
- path: ${{ runner.temp }}/collectivex-backend-sources.tar
- if-no-files-found: error
- overwrite: true
- retention-days: 14
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: cxsweep-matrix-${{ github.run_id }}
@@ -183,136 +136,47 @@ jobs:
COLLECTIVEX_REPOSITORY: ${{ github.repository }}
run: |
set -euo pipefail
- python3 - <<'PY'
- import os
- import re
- import shutil
- import stat
- import time
-
- pattern = re.compile(r"inferencex-collectivex-[0-9]+-[0-9]+-[A-Za-z0-9._-]+")
- link_pattern = re.compile(
- r"/tmp/inferencex-collectivex-parent-[0-9]+-[0-9]+-[A-Za-z0-9._-]+"
- )
- parent = os.environ["CX_JOB_PARENT"]
- scan_parent = parent
- if parent != "/tmp":
- if not link_pattern.fullmatch(parent):
- raise SystemExit("CollectiveX isolated parent is invalid")
- shared_parent = os.path.join(os.getcwd(), ".collectivex-jobs")
- try:
- shared_metadata = os.lstat(shared_parent)
- except FileNotFoundError:
- os.mkdir(shared_parent, mode=0o700)
- shared_metadata = os.lstat(shared_parent)
- if (
- not stat.S_ISDIR(shared_metadata.st_mode)
- or shared_metadata.st_uid != os.getuid()
- or stat.S_IMODE(shared_metadata.st_mode) != 0o700
- ):
- raise SystemExit("CollectiveX shared parent is unsafe")
- if os.path.lexists(parent):
- raise SystemExit("CollectiveX isolated parent already exists")
- os.symlink(shared_parent, parent)
- scan_parent = shared_parent
- cutoff = time.time() - 86400
- for entry in os.scandir(scan_parent):
- if not pattern.fullmatch(entry.name):
- continue
- try:
- metadata = entry.stat(follow_symlinks=False)
- except FileNotFoundError:
- continue
- if (
- not stat.S_ISDIR(metadata.st_mode)
- or metadata.st_uid != os.getuid()
- or stat.S_IMODE(metadata.st_mode) != 0o700
- or metadata.st_mtime >= cutoff
- ):
- continue
- marked = False
- for marker_name in ("cleanup-safe", "cleanup-unsafe"):
- try:
- marker = os.stat(
- os.path.join(entry.path, marker_name), follow_symlinks=False
- )
- except FileNotFoundError:
- continue
- marked = (
- stat.S_ISREG(marker.st_mode)
- and marker.st_uid == os.getuid()
- and stat.S_IMODE(marker.st_mode) == 0o600
- )
- if marked:
- break
- if marked:
- shutil.rmtree(entry.path)
- PY
[ "${CX_JOB_ROOT%/*}" = "$CX_JOB_PARENT" ] \
&& [[ "${CX_JOB_ROOT##*/}" =~ ^inferencex-collectivex-[0-9]+-[0-9]+-[A-Za-z0-9._-]+$ ]] \
|| { echo "CollectiveX isolated root is invalid" >&2; exit 1; }
[ "$CX_SOURCE_ROOT" = "$CX_JOB_ROOT/source" ] \
|| { echo "CollectiveX source root is invalid" >&2; exit 1; }
- if [ -e "$CX_JOB_ROOT" ] || [ -L "$CX_JOB_ROOT" ]; then
- echo "CollectiveX isolated root already exists" >&2
- exit 1
+ if [ "$CX_JOB_PARENT" != /tmp ]; then
+ [[ "$CX_JOB_PARENT" =~ ^/tmp/inferencex-collectivex-parent-[0-9]+-[0-9]+-[A-Za-z0-9._-]+$ ]] \
+ || { echo "CollectiveX isolated parent is invalid" >&2; exit 1; }
+ shared_parent="$GITHUB_WORKSPACE/.collectivex-jobs"
+ install -d -m 700 "$shared_parent"
+ [ "$(stat -c '%u:%a' "$shared_parent")" = "$(id -u):700" ] \
+ || { echo "CollectiveX shared parent is unsafe" >&2; exit 1; }
+ [ ! -e "$CX_JOB_PARENT" ] && [ ! -L "$CX_JOB_PARENT" ] \
+ || { echo "CollectiveX isolated parent already exists" >&2; exit 1; }
+ ln -s "$shared_parent" "$CX_JOB_PARENT"
fi
umask 077
- mkdir -m 700 -- "$CX_JOB_ROOT"
+ [ ! -e "$CX_JOB_ROOT" ] && [ ! -L "$CX_JOB_ROOT" ] \
+ || { echo "CollectiveX isolated root already exists" >&2; exit 1; }
+ mkdir -m 700 "$CX_JOB_ROOT"
trap 'rc=$?; if [ "$rc" != 0 ]; then rm -rf -- "$CX_JOB_ROOT"; [ "$CX_JOB_PARENT" = /tmp ] || rm -f -- "$CX_JOB_PARENT"; fi; exit "$rc"' EXIT
- mkdir -m 700 -- "$HOME" "$CX_JOB_ROOT/control" "$CX_JOB_ROOT/artifact" "$CX_SOURCE_ROOT"
+ mkdir -m 700 "$HOME" "$CX_JOB_ROOT/control" "$CX_JOB_ROOT/artifact" "$CX_SOURCE_ROOT"
: > "$CX_JOB_ROOT/cleanup-safe"
- if ! {
- GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null git init -q "$CX_SOURCE_ROOT"
- GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null \
- git -C "$CX_SOURCE_ROOT" remote add origin \
- "https://github.com/${COLLECTIVEX_REPOSITORY}.git"
- GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null \
- git -C "$CX_SOURCE_ROOT" -c credential.helper= -c protocol.version=2 \
- fetch -q --no-tags --depth=1 origin "$COLLECTIVEX_SOURCE_SHA"
- GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null \
- git -C "$CX_SOURCE_ROOT" -c advice.detachedHead=false \
- checkout -q --detach FETCH_HEAD
- [ "$(git -C "$CX_SOURCE_ROOT" rev-parse HEAD)" = "$COLLECTIVEX_SOURCE_SHA" ]
- } /dev/null 2>&1; then
+ export GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null
+ if ! git init -q "$CX_SOURCE_ROOT" \
+ || ! git -C "$CX_SOURCE_ROOT" remote add origin \
+ "https://github.com/${COLLECTIVEX_REPOSITORY}.git" \
+ || ! git -C "$CX_SOURCE_ROOT" -c credential.helper= -c protocol.version=2 \
+ fetch -q --no-tags --depth=1 origin "$COLLECTIVEX_SOURCE_SHA" \
+ || ! git -C "$CX_SOURCE_ROOT" -c advice.detachedHead=false \
+ checkout -q --detach FETCH_HEAD \
+ || [ "$(git -C "$CX_SOURCE_ROOT" rev-parse HEAD)" != "$COLLECTIVEX_SOURCE_SHA" ]; then
echo "CollectiveX source preparation failed" >&2
exit 1
fi
- [ "$(stat -c '%a' "$CX_JOB_ROOT")" = 700 ] \
- || { echo "CollectiveX isolated root has unsafe permissions" >&2; exit 1; }
echo 'prepared=true' >> "$GITHUB_OUTPUT"
trap - EXIT
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: cxsweep-matrix-${{ github.run_id }}
path: ${{ env.CX_JOB_ROOT }}/control
- - name: Download pinned backend source archive
- if: ${{ matrix.backend == 'deepep-v2' || matrix.backend == 'deepep-hybrid' }}
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
- with:
- name: cxbackend-sources-${{ github.run_id }}
- path: ${{ env.CX_JOB_ROOT }}/control
- - name: Install pinned backend source seed
- if: ${{ matrix.backend == 'deepep-v2' || matrix.backend == 'deepep-hybrid' }}
- env:
- EXPECTED_BACKEND: ${{ matrix.backend }}
- run: |
- set -euo pipefail
- archive="$CX_JOB_ROOT/control/collectivex-backend-sources.tar"
- destination="$CX_SOURCE_ROOT/experimental/CollectiveX"
- seed_root="$destination/.cx_sources"
- [ -f "$archive" ] && [ ! -e "$seed_root" ] && [ ! -L "$seed_root" ]
- chmod 600 -- "$archive"
- source "$destination/runtime/common.sh"
- source_path="$(cx_backend_source_path "$seed_root" "$EXPECTED_BACKEND")"
- source_basename="${source_path#"$seed_root/"}"
- [ -n "$source_basename" ] \
- && [ "$source_path" = "$seed_root/$source_basename" ] \
- && [[ "$source_basename" != */* ]]
- python3 "$destination/source_archive.py" \
- "$archive" "$destination" "$source_basename"
- cx_backend_source_is_valid "$EXPECTED_BACKEND" "$source_path"
- printf 'CX_BACKEND_SOURCE_SEED_ROOT=%s\n' "$seed_root" >> "$GITHUB_ENV"
- name: Extract and validate this execution control
run: |
set -euo pipefail
@@ -344,7 +208,6 @@ jobs:
python3 - "$operator_config" <<'PY'
import json
import os
- from pathlib import Path
import sys
def pairs(items):
@@ -362,6 +225,10 @@ jobs:
)
base = load("COLLECTIVEX_OPERATOR_CONFIG_CONTENT")
+ if set(base) != {"schema_version", "runners"} or base["schema_version"] != 1:
+ raise ValueError("invalid operator configuration")
+ if not isinstance(base["runners"], dict):
+ raise ValueError("invalid operator runners")
for overlay_name in (
"COLLECTIVEX_NETWORK_CONFIG_CONTENT", "COLLECTIVEX_H100_CONFIG_CONTENT",
"COLLECTIVEX_B300_CONFIG_CONTENT",
@@ -372,51 +239,18 @@ jobs:
if not overlay_raw:
continue
overlay = load(overlay_name)
- private_fields = {
- "socket_ifname", "rdma_devices", "ib_gid_index", "rdma_service_level",
- "rdma_traffic_class",
- }
if set(overlay) != {"schema_version", "runners"} or overlay["schema_version"] != 1:
- raise ValueError("invalid network overlay envelope")
- if not isinstance(overlay["runners"], dict) or not overlay["runners"]:
- raise ValueError("invalid network overlay runners")
- if overlay_name == "COLLECTIVEX_B200_CONFIG_CONTENT":
- if set(overlay["runners"]) != {"b200-dgxc"}:
- raise ValueError("invalid B200 overlay runner")
- private_fields.update({"nodelist", "qos"})
- elif overlay_name == "COLLECTIVEX_H100_CONFIG_CONTENT":
- if set(overlay["runners"]) != {"h100-dgxc"}:
- raise ValueError("invalid H100 overlay runner")
- private_fields.add("exclude_nodes")
- elif overlay_name == "COLLECTIVEX_B300_CONFIG_CONTENT":
- if set(overlay["runners"]) != {"b300"}:
- raise ValueError("invalid B300 overlay runner")
- private_fields.add("exclude_nodes")
- elif overlay_name == "COLLECTIVEX_MI355_CONFIG_CONTENT":
- if set(overlay["runners"]) != {"mi355x"}:
- raise ValueError("invalid MI355 overlay runner")
- elif overlay_name == "COLLECTIVEX_MI300_CONFIG_CONTENT":
- if set(overlay["runners"]) != {"mi300x"}:
- raise ValueError("invalid MI300 overlay runner")
- private_fields.update({
- "partition", "squash_dir", "stage_dir", "exclude_nodes",
- "nodelist", "lock_dir",
- })
+ raise ValueError("invalid operator overlay")
+ if not isinstance(overlay["runners"], dict):
+ raise ValueError("invalid overlay runners")
for runner, fields in overlay["runners"].items():
if not isinstance(fields, dict) or not fields:
- raise ValueError("invalid network overlay runner")
- if runner not in base.get("runners", {}):
- if overlay_name != "COLLECTIVEX_MI300_CONFIG_CONTENT":
- raise ValueError("invalid network overlay runner")
- base.setdefault("runners", {})[runner] = {}
- if set(fields) - private_fields:
- raise ValueError("private overlay contains a disallowed field")
- base["runners"][runner].update(fields)
+ raise ValueError("invalid overlay runner")
+ base["runners"].setdefault(runner, {}).update(fields)
payload = json.dumps(base, sort_keys=True, separators=(",", ":")) + "\n"
if len(payload.encode()) > 65536:
raise ValueError("merged operator configuration is too large")
- path = Path(sys.argv[1])
- descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
+ descriptor = os.open(sys.argv[1], os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
with os.fdopen(descriptor, "w", encoding="utf-8") as stream:
stream.write(payload)
PY
@@ -520,4 +354,3 @@ jobs:
fi
rm -rf -- "$CX_JOB_ROOT"
[ "$CX_JOB_PARENT" = /tmp ] || rm -f -- "$CX_JOB_PARENT"
-
diff --git a/experimental/CollectiveX/README.md b/experimental/CollectiveX/README.md
index 6d937ecf5b..d7b06bf367 100644
--- a/experimental/CollectiveX/README.md
+++ b/experimental/CollectiveX/README.md
@@ -73,11 +73,10 @@ whether it succeeded is decided by the benchmark's return code.
## Workflow And Artifacts
`.github/workflows/collectivex-sweep.yml` has two jobs. `setup` generates a public-SKU matrix
-(`backend`, `suites`, `only_sku`, `exclude_skus`, `ep_sizes` inputs), fetches the pinned backend
-source archive, and uploads the matrix.
+(`backend`, `suites`, `only_sku`, `exclude_skus`, `ep_sizes` inputs) and uploads the matrix.
`sweep` extracts a strict ignored `.shards/.json` control per matrix entry, executes one
-allocation per shard, and uploads the result artifacts with `always()` so a red or partial run still
-uploads.
+allocation per shard, fetches pinned DeepEP source before allocation when required, and uploads the
+result artifacts with `always()` so a red or partial run still uploads.
Each shard emits per-case result JSON, detached sample JSON, and a small mechanical summary. A case
counts as successful on the benchmark's own return code; there is no schema, completeness, or privacy
diff --git a/experimental/CollectiveX/docs/methodology.md b/experimental/CollectiveX/docs/methodology.md
index a4306fc0c8..5334e7a12e 100644
--- a/experimental/CollectiveX/docs/methodology.md
+++ b/experimental/CollectiveX/docs/methodology.md
@@ -194,10 +194,8 @@ Repository staging uses a pre-existing, runner-owned, group/world non-writable s
the checkout and workflow workspace. The parent process resolves the exact execution child before
copying, claims it with a runner-owned marker, and verifies that all allocated nodes can read and
write the same bytes. Cleanup waits for confirmed allocation teardown and removes only that child,
-including a safely identified partial claim. The same-run V2/Hybrid source archive is fully validated
-under fixed member and expanded-size bounds, and only the selected pinned root is extracted; a
-symlink is accepted only when it is a relative leaf pointing to a regular member inside the same
-backend root, followed by exact Git tree/submodule validation.
+including a safely identified partial claim. V2 and Hybrid source is fetched before allocation at an
+exact pinned revision, followed by exact Git tree, submodule, and local-patch validation.
H200, B200, and B300 may derive that private base beneath the validated operating-system account home
when it is compute-visible. H100 instead derives a sibling of its shared container directory, never a
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 5679c10cb7..619c6ef137 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -1824,7 +1824,6 @@ PY
_cx_prepare_backend_source() {
local mount_src="$1" backend="$2" root source temporary revision tree fmt nccl pin
local root_mode stage_mode root_owner stage_owner
- local seed_root="${CX_BACKEND_SOURCE_SEED_ROOT:-}" seed seed_mode
root="$mount_src/experimental/CollectiveX/.cx_sources"
CX_BACKEND_SOURCE_STEP="source mount creation"
if [ ! -e "$root" ] && [ ! -L "$root" ]; then
@@ -1860,28 +1859,6 @@ _cx_prepare_backend_source() {
cx_backend_source_is_valid "$backend" "$source"
return
fi
- if [ -n "$seed_root" ]; then
- CX_BACKEND_SOURCE_STEP="source seed validation"
- [[ "$seed_root" = /* ]] && [ -d "$seed_root" ] && [ ! -L "$seed_root" ] \
- || return 1
- seed_mode="$(stat -c '%a' "$seed_root" 2>/dev/null)" || return 1
- case "$seed_mode" in 700|[1-7]700) ;; *) return 1 ;; esac
- seed="$(cx_backend_source_path "$seed_root" "$backend")" || return 1
- cx_backend_source_is_valid "$backend" "$seed" || return 1
- CX_BACKEND_SOURCE_STEP="source seed copy"
- temporary="$(mktemp -d "$root/.${backend}.XXXXXX")" || return 1
- if ! cp -R -- "$seed/." "$temporary/" \
- || ! cx_backend_source_is_valid "$backend" "$temporary" \
- || ! mv -- "$temporary" "$source"; then
- rm -rf -- "$temporary"
- return 1
- fi
- return
- fi
- if [ "${COLLECTIVEX_CANONICAL_GHA:-0}" = 1 ]; then
- CX_BACKEND_SOURCE_STEP="source seed validation"
- return 1
- fi
CX_BACKEND_SOURCE_STEP="source checkout creation"
temporary="$(mktemp -d "$root/.${backend}.XXXXXX")" || return 1
CX_BACKEND_SOURCE_STEP="source pin resolution"
diff --git a/experimental/CollectiveX/source_archive.py b/experimental/CollectiveX/source_archive.py
deleted file mode 100644
index c027490a65..0000000000
--- a/experimental/CollectiveX/source_archive.py
+++ /dev/null
@@ -1,349 +0,0 @@
-#!/usr/bin/env python3
-"""Validate and extract one pinned backend from a shared source tar."""
-from __future__ import annotations
-
-import argparse
-import os
-from pathlib import Path, PurePosixPath
-import stat
-import tarfile
-from typing import Optional, Sequence
-
-
-PathParts = tuple[str, ...]
-_DIRECTORY_FLAGS = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC
-_FILE_FLAGS = os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC
-MAX_ARCHIVE_MEMBERS = 20_000
-MAX_MEMBER_BYTES = 512 * 1024 * 1024
-MAX_EXPANDED_BYTES = 2 * 1024 * 1024 * 1024
-MAX_ARCHIVE_BYTES = 4 * 1024 * 1024 * 1024
-MAX_ARCHIVE_HEADERS = 40_000
-MAX_EXTENSION_BYTES = 64 * 1024 * 1024
-MAX_EXTENSION_MEMBER_BYTES = 1024 * 1024
-MAX_EXTENSION_CHAIN = 8
-_TAR_BLOCK = 512
-_EXTENSION_TYPES = {b"L", b"K", b"x", b"g", b"X"}
-
-
-class SourceArchiveError(ValueError):
- """The backend source archive cannot be extracted safely."""
-
-
-def _tar_size(field: bytes) -> int:
- if field[0] in (0o200, 0o377):
- value = int.from_bytes(field[1:], "big")
- if field[0] == 0o377:
- value -= 256 ** (len(field) - 1)
- return value
- try:
- text = field.split(b"\0", 1)[0].decode("ascii").strip()
- return int(text or "0", 8)
- except (UnicodeDecodeError, ValueError) as exc:
- raise SourceArchiveError("archive contains an invalid size field") from exc
-
-
-def _preflight_archive(descriptor: int, archive_size: int) -> None:
- if archive_size <= 0 or archive_size > MAX_ARCHIVE_BYTES:
- raise SourceArchiveError("backend source archive exceeds the raw size limit")
- offset = headers = extension_bytes = extension_chain = 0
- while offset < archive_size:
- header = os.pread(descriptor, _TAR_BLOCK, offset)
- if len(header) != _TAR_BLOCK:
- raise SourceArchiveError("archive header is truncated")
- if not any(header):
- return
- headers += 1
- if headers > MAX_ARCHIVE_HEADERS:
- raise SourceArchiveError("archive has too many physical headers")
- size = _tar_size(header[124:136])
- if size < 0:
- raise SourceArchiveError("archive contains a negative payload size")
- type_flag = header[156:157]
- if type_flag in _EXTENSION_TYPES:
- extension_chain += 1
- extension_bytes += size
- if (
- extension_chain > MAX_EXTENSION_CHAIN
- or size > MAX_EXTENSION_MEMBER_BYTES
- or extension_bytes > MAX_EXTENSION_BYTES
- ):
- raise SourceArchiveError("archive extension metadata exceeds its limit")
- if type_flag in {b"x", b"g", b"X"}:
- payload = os.pread(descriptor, size, offset + _TAR_BLOCK)
- if len(payload) != size:
- raise SourceArchiveError("archive extension metadata is truncated")
- if b"GNU.sparse." in payload:
- raise SourceArchiveError("archive contains sparse extension metadata")
- else:
- extension_chain = 0
- if type_flag == b"S":
- raise SourceArchiveError("archive contains a sparse member")
- blocks = (size + _TAR_BLOCK - 1) // _TAR_BLOCK
- offset += _TAR_BLOCK + blocks * _TAR_BLOCK
- if offset > archive_size:
- raise SourceArchiveError("archive payload is truncated")
-
-
-def _member_parts(name: str) -> PathParts:
- if not name or "\\" in name or "\0" in name:
- raise SourceArchiveError("archive contains a noncanonical member path")
- path = PurePosixPath(name)
- if (
- path.is_absolute()
- or path.as_posix() != name
- or not path.parts
- or path.parts[0] != ".cx_sources"
- or any(part in {"", ".", ".."} for part in path.parts)
- ):
- raise SourceArchiveError("archive contains a noncanonical member path")
- return path.parts
-
-
-def _root_parts(root_basename: str) -> PathParts:
- path = PurePosixPath(root_basename)
- if (
- not root_basename
- or "\\" in root_basename
- or "\0" in root_basename
- or path.is_absolute()
- or path.as_posix() != root_basename
- or len(path.parts) != 1
- or path.parts[0] in {"", ".", ".."}
- ):
- raise SourceArchiveError("invalid backend source root")
- return (".cx_sources", root_basename)
-
-
-def _read_members(archive: tarfile.TarFile) -> list[tarfile.TarInfo]:
- members: list[tarfile.TarInfo] = []
- for member in archive:
- if len(members) >= MAX_ARCHIVE_MEMBERS:
- raise SourceArchiveError("archive has an invalid member count")
- members.append(member)
- return members
-
-
-def _validate_members(
- members: list[tarfile.TarInfo], selected_root: PathParts
-) -> dict[PathParts, tarfile.TarInfo]:
- if not members or len(members) > MAX_ARCHIVE_MEMBERS:
- raise SourceArchiveError("archive has an invalid member count")
- entries: dict[PathParts, tarfile.TarInfo] = {}
- expanded_bytes = 0
- for member in members:
- parts = _member_parts(member.name)
- if parts in entries:
- raise SourceArchiveError("archive contains duplicate member paths")
- if member.sparse is not None:
- raise SourceArchiveError("archive contains a sparse member")
- if member.isdir():
- if member.size != 0:
- raise SourceArchiveError("archive contains an invalid directory")
- elif member.isfile():
- if member.size < 0 or member.size > MAX_MEMBER_BYTES:
- raise SourceArchiveError("archive member exceeds the size limit")
- expanded_bytes += member.size
- if expanded_bytes > MAX_EXPANDED_BYTES:
- raise SourceArchiveError("archive exceeds the expanded size limit")
- elif member.issym():
- if member.size != 0:
- raise SourceArchiveError("archive contains an invalid symbolic link")
- else:
- raise SourceArchiveError("archive contains a non-file member")
- entries[parts] = member
-
- source_parent = entries.get((".cx_sources",))
- selected = entries.get(selected_root)
- if source_parent is None or not source_parent.isdir():
- raise SourceArchiveError("archive is missing its source directory")
- if selected is None or not selected.isdir():
- raise SourceArchiveError("archive is missing the selected backend source")
-
- for parts in entries:
- for depth in range(1, len(parts)):
- parent = entries.get(parts[:depth])
- if parent is None or not parent.isdir():
- raise SourceArchiveError("archive member has an unsafe parent")
-
- for parts, member in entries.items():
- if not member.issym():
- continue
- target_name = member.linkname
- target_path = PurePosixPath(target_name)
- if (
- not target_name
- or "\\" in target_name
- or "\0" in target_name
- or target_path.is_absolute()
- or target_path.as_posix() != target_name
- ):
- raise SourceArchiveError("archive contains an unsafe symbolic link")
- target = list(parts[:-1])
- for component in target_path.parts:
- if component == "..":
- if len(target) <= 2:
- raise SourceArchiveError("symbolic link escapes its backend source")
- target.pop()
- else:
- target.append(component)
- resolved = tuple(target)
- if resolved[:2] != parts[:2]:
- raise SourceArchiveError("symbolic link crosses backend sources")
- target_member = entries.get(resolved)
- if target_member is None or not target_member.isfile():
- raise SourceArchiveError("symbolic link target is not a regular archive file")
- return entries
-
-
-def _open_directory(root_fd: int, parts: PathParts) -> int:
- descriptor = os.dup(root_fd)
- try:
- for part in parts:
- child = os.open(part, _DIRECTORY_FLAGS, dir_fd=descriptor)
- os.close(descriptor)
- descriptor = child
- return descriptor
- except BaseException:
- os.close(descriptor)
- raise
-
-
-def _create_directory(root_fd: int, parts: PathParts) -> None:
- parent_fd = _open_directory(root_fd, parts[:-1])
- try:
- os.mkdir(parts[-1], mode=0o700, dir_fd=parent_fd)
- finally:
- os.close(parent_fd)
-
-
-def _extract_file(
- archive: tarfile.TarFile, root_fd: int, parts: PathParts, member: tarfile.TarInfo
-) -> None:
- parent_fd = _open_directory(root_fd, parts[:-1])
- descriptor = -1
- source = None
- try:
- mode = 0o700 if member.mode & 0o111 else 0o600
- descriptor = os.open(parts[-1], _FILE_FLAGS, mode, dir_fd=parent_fd)
- source = archive.extractfile(member)
- if source is None:
- raise SourceArchiveError("archive file has no readable payload")
- remaining = member.size
- while remaining:
- chunk = source.read(min(1024 * 1024, remaining))
- if not chunk:
- raise SourceArchiveError("archive file payload is truncated")
- view = memoryview(chunk)
- while view:
- written = os.write(descriptor, view)
- view = view[written:]
- remaining -= len(chunk)
- os.fchmod(descriptor, mode)
- finally:
- if source is not None:
- source.close()
- if descriptor >= 0:
- os.close(descriptor)
- os.close(parent_fd)
-
-
-def _extract_symlink(root_fd: int, parts: PathParts, member: tarfile.TarInfo) -> None:
- parent_fd = _open_directory(root_fd, parts[:-1])
- try:
- os.symlink(member.linkname, parts[-1], dir_fd=parent_fd)
- finally:
- os.close(parent_fd)
-
-
-def _extract_selected(
- archive: tarfile.TarFile,
- destination_fd: int,
- entries: dict[PathParts, tarfile.TarInfo],
- selected_root: PathParts,
-) -> None:
- try:
- os.stat(".cx_sources", dir_fd=destination_fd, follow_symlinks=False)
- except FileNotFoundError:
- pass
- else:
- raise SourceArchiveError("backend source output already exists")
-
- selected = {
- parts: member
- for parts, member in entries.items()
- if parts[: len(selected_root)] == selected_root
- }
- _create_directory(destination_fd, (".cx_sources",))
- directories = sorted(
- (parts for parts, member in selected.items() if member.isdir()),
- key=lambda parts: (len(parts), parts),
- )
- for parts in directories:
- _create_directory(destination_fd, parts)
- for parts, member in sorted(selected.items()):
- if member.isfile():
- _extract_file(archive, destination_fd, parts, member)
- for parts, member in sorted(selected.items()):
- if member.issym():
- _extract_symlink(destination_fd, parts, member)
-
-
-def extract_source_archive(
- archive_path: Path, destination: Path, root_basename: str
-) -> None:
- """Validate the complete tar, then safely extract one backend source root."""
- selected_root = _root_parts(root_basename)
- archive_fd = os.open(archive_path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC)
- try:
- metadata = os.fstat(archive_fd)
- if (
- not stat.S_ISREG(metadata.st_mode)
- or metadata.st_uid != os.getuid()
- or stat.S_IMODE(metadata.st_mode) & 0o022
- ):
- raise SourceArchiveError("backend source archive has unsafe metadata")
- _preflight_archive(archive_fd, metadata.st_size)
- with os.fdopen(os.dup(archive_fd), "rb") as stream:
- try:
- with tarfile.open(fileobj=stream, mode="r:") as archive:
- entries = _validate_members(_read_members(archive), selected_root)
- destination_fd = os.open(destination, _DIRECTORY_FLAGS)
- try:
- destination_metadata = os.fstat(destination_fd)
- if (
- destination_metadata.st_uid != os.getuid()
- or stat.S_IMODE(destination_metadata.st_mode) != 0o700
- ):
- raise SourceArchiveError("backend source destination is unsafe")
- previous_umask = os.umask(0o077)
- try:
- _extract_selected(
- archive, destination_fd, entries, selected_root
- )
- finally:
- os.umask(previous_umask)
- finally:
- os.close(destination_fd)
- except RecursionError as exc:
- raise SourceArchiveError("archive extension metadata is recursive") from exc
- finally:
- os.close(archive_fd)
-
-
-def main(argv: Optional[Sequence[str]] = None) -> int:
- parser = argparse.ArgumentParser(
- description="Safely install one pinned backend source archive"
- )
- parser.add_argument("archive", type=Path)
- parser.add_argument("destination", type=Path)
- parser.add_argument("root_basename")
- args = parser.parse_args(argv)
- try:
- extract_source_archive(args.archive, args.destination, args.root_basename)
- except (OSError, SourceArchiveError, tarfile.TarError) as exc:
- parser.error(f"backend source archive rejected: {exc}")
- return 0
-
-
-if __name__ == "__main__":
- raise SystemExit(main())
From 07b81abc3c55062e22a9dcd4dd73007bfb89339f Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Wed, 8 Jul 2026 23:58:45 +0800
Subject: [PATCH 208/210] cleanup sweep.yml
---
.github/workflows/collectivex-sweep.yml | 111 +++++++-----------------
1 file changed, 31 insertions(+), 80 deletions(-)
diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml
index 8e1fbf33d2..668d87d84e 100644
--- a/.github/workflows/collectivex-sweep.yml
+++ b/.github/workflows/collectivex-sweep.yml
@@ -1,10 +1,4 @@
-# CollectiveX Sweep — one structured run instead of thousands of dispatches.
-#
-# Shape: setup resolves the suites into a shard matrix, then one GPU cell runs per
-# shard, executes the benchmark, and uploads the emitted neutral artifacts as-is.
-# This repo only schedules, executes, and uploads — it does not validate, promote,
-# rank, recommend, or decide what a frontend displays. This file is registered
-# on the default branch, so its collectivex branch revision can be dispatched with --ref.
+# Generate a shard matrix, run each shard on its GPU pool, and upload raw results.
name: CollectiveX Sweep
permissions:
actions: read
@@ -49,7 +43,6 @@ jobs:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v5.0.0
with: { clean: true, persist-credentials: false }
- name: Install matrix dependencies
- # PyYAML is used by sweep_matrix.py to read the suites config.
run: python3 -m pip install --quiet PyYAML==6.0.2
- id: gen
working-directory: experimental/CollectiveX
@@ -70,13 +63,27 @@ jobs:
[ -n "$INPUT_EXCLUDE_SKUS" ] && args+=(--exclude-skus "$INPUT_EXCLUDE_SKUS")
[ -n "$INPUT_EP_SIZES" ] && args+=(--ep-sizes "$INPUT_EP_SIZES")
python3 sweep_matrix.py "${args[@]}" --out matrix_full.json >/dev/null
- SLIM=$(python3 -c "import json;m=json.load(open('matrix_full.json'));print(json.dumps({'include':[{k:v for k,v in x.items() if k!='case_ids'} for x in m['include']]}))")
- {
- echo "matrix=$SLIM"
- echo "n=$(python3 -c "import json;print(len(json.load(open('matrix_full.json'))['include']))")"
- echo "max_parallel=$(python3 -c "import json;m=json.load(open('matrix_full.json'));w=max(x['execution_weight'] for x in m['include']);print(max(1,min(10,4096//w)))")"
- } >> "$GITHUB_OUTPUT"
- python3 -c "import json;m=json.load(open('matrix_full.json'));print('execution-cells:',len(m['include']))"
+ python3 - "$GITHUB_OUTPUT" <<'PY'
+ import json
+ import pathlib
+ import sys
+
+ matrix = json.loads(pathlib.Path("matrix_full.json").read_text())
+ cells = matrix["include"]
+ slim = {"include": [
+ {key: value for key, value in cell.items() if key != "case_ids"}
+ for cell in cells
+ ]}
+ max_parallel = (
+ max(1, min(10, 4096 // max(cell["execution_weight"] for cell in cells)))
+ if cells else 1
+ )
+ with open(sys.argv[1], "a", encoding="utf-8") as output:
+ output.write(f"matrix={json.dumps(slim, separators=(',', ':'))}\n")
+ output.write(f"n={len(cells)}\n")
+ output.write(f"max_parallel={max_parallel}\n")
+ print(f"execution-cells: {len(cells)}")
+ PY
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: cxsweep-matrix-${{ github.run_id }}
@@ -115,21 +122,6 @@ jobs:
CX_SOURCE_ROOT: ${{ matrix.launcher == 'mi-amds' && format('/tmp/inferencex-collectivex-parent-{0}-{1}-{2}/inferencex-collectivex-{0}-{1}-{2}/source', github.run_id, github.run_attempt, matrix.id) || format('/tmp/inferencex-collectivex-{0}-{1}-{2}/source', github.run_id, github.run_attempt, matrix.id) }}
HOME: ${{ matrix.launcher == 'mi-amds' && format('/tmp/inferencex-collectivex-parent-{0}-{1}-{2}/inferencex-collectivex-{0}-{1}-{2}/home', github.run_id, github.run_attempt, matrix.id) || format('/tmp/inferencex-collectivex-{0}-{1}-{2}/home', github.run_id, github.run_attempt, matrix.id) }}
steps:
- - name: Provision host Python
- # The self-hosted runners ship a bare python3 with no system pip. The host
- # launcher and summarize.py invoke bare `python3` and need PyYAML/numpy, so
- # build a venv and put it on PATH for every later step. venv + ensurepip are
- # available even where system pip is not.
- env:
- HOME: ${{ runner.temp }}
- run: |
- set -euo pipefail
- venv="$RUNNER_TEMP/cx-hostpy"
- python3 -m venv --clear "$venv"
- "$venv/bin/python3" -m pip install --quiet --no-cache-dir --upgrade pip
- "$venv/bin/python3" -m pip install --quiet --no-cache-dir \
- PyYAML==6.0.2 "numpy>=1.26,<3"
- echo "$venv/bin" >> "$GITHUB_PATH"
- name: Prepare isolated source
id: source
env:
@@ -254,11 +246,6 @@ jobs:
with os.fdopen(descriptor, "w", encoding="utf-8") as stream:
stream.write(payload)
PY
- unset COLLECTIVEX_OPERATOR_CONFIG_CONTENT COLLECTIVEX_NETWORK_CONFIG_CONTENT
- unset COLLECTIVEX_H100_CONFIG_CONTENT COLLECTIVEX_B300_CONFIG_CONTENT
- unset COLLECTIVEX_B200_CONFIG_CONTENT
- unset COLLECTIVEX_MI300_CONFIG_CONTENT COLLECTIVEX_MI355_CONFIG_CONTENT
- unset COLLECTIVEX_OPERATOR_CONFIG_REQUIRED
export COLLECTIVEX_OPERATOR_CONFIG="$operator_config"
export COLLECTIVEX_OPERATOR_CONFIG_EPHEMERAL=1
: > "$CX_JOB_ROOT/cleanup-unsafe"
@@ -266,43 +253,18 @@ jobs:
cd "$CX_SOURCE_ROOT" 2>/dev/null \
|| { echo "CollectiveX source is unavailable" >&2; exit 1; }
bash "experimental/CollectiveX/launchers/launch_${{ matrix.launcher }}.sh"
- - name: Reconcile allocation cleanup
+ - name: Summarize and stage shard results
+ id: stage_artifact
if: ${{ always() && steps.source.outputs.prepared == 'true' }}
run: |
set -euo pipefail
cd "$CX_SOURCE_ROOT" 2>/dev/null \
|| { echo "CollectiveX source is unavailable" >&2; exit 1; }
source experimental/CollectiveX/runtime/common.sh
- cx_reconcile_recorded_allocation "$CX_JOB_ROOT"
- - name: Classify private scheduler failure
- if: ${{ always() && steps.source.outputs.prepared == 'true' && steps.sweep_shard.outcome == 'failure' }}
- run: |
- set -euo pipefail
- cd "$CX_SOURCE_ROOT" 2>/dev/null \
- || { echo "CollectiveX source is unavailable" >&2; exit 1; }
- source experimental/CollectiveX/runtime/common.sh
- cx_report_private_scheduler_failure
- - name: Confirm allocation cleanup
- id: allocation_cleanup
- if: ${{ always() && steps.source.outputs.prepared == 'true' }}
- run: |
- set -euo pipefail
- [ -f "$CX_JOB_ROOT/cleanup-safe" ] && [ ! -e "$CX_JOB_ROOT/cleanup-unsafe" ] \
- || { echo "CollectiveX allocation cleanup was not confirmed" >&2; exit 1; }
- - name: Shard summary
- if: ${{ always() && steps.allocation_cleanup.outcome == 'success' }}
- run: |
- cd "$CX_SOURCE_ROOT" 2>/dev/null \
- || { echo "CollectiveX source is unavailable" >&2; exit 1; }
+ cx_reconcile_recorded_allocation "$CX_JOB_ROOT" \
+ || { echo "CollectiveX allocation cleanup could not be reconciled; results may still be changing" >&2; exit 1; }
python3 experimental/CollectiveX/summarize.py \
--results-dir experimental/CollectiveX/results --markdown >> "$GITHUB_STEP_SUMMARY" || true
- - name: Stage shard artifact
- id: stage_artifact
- if: ${{ always() && steps.allocation_cleanup.outcome == 'success' }}
- run: |
- set -euo pipefail
- cd "$CX_SOURCE_ROOT" 2>/dev/null \
- || { echo "CollectiveX source is unavailable" >&2; exit 1; }
shopt -s nullglob
results=(experimental/CollectiveX/results/*.json)
if [ "${#results[@]}" -eq 0 ]; then
@@ -334,23 +296,12 @@ jobs:
|| { echo "CollectiveX cleanup root is invalid" >&2; exit 1; }
[ "$CX_SOURCE_ROOT" = "$CX_JOB_ROOT/source" ] \
|| { echo "CollectiveX cleanup source is invalid" >&2; exit 1; }
+ cd "$CX_SOURCE_ROOT" 2>/dev/null \
+ || { echo "CollectiveX source is unavailable" >&2; exit 1; }
+ source experimental/CollectiveX/runtime/common.sh
+ cx_reconcile_recorded_allocation "$CX_JOB_ROOT" \
+ || { echo "CollectiveX allocation cleanup could not be reconciled; retaining isolated files" >&2; exit 1; }
[ -f "$CX_JOB_ROOT/cleanup-safe" ] && [ ! -e "$CX_JOB_ROOT/cleanup-unsafe" ] \
|| { echo "CollectiveX allocation cleanup was not confirmed; retaining isolated files" >&2; exit 1; }
- if [ '${{ steps.sweep_shard.outcome }}' = success ] \
- && [ '${{ steps.allocation_cleanup.outcome }}' = success ] \
- && [ '${{ steps.stage_artifact.outcome }}' = success ] \
- && [ '${{ steps.upload_artifact.outcome }}' = success ] \
- && [ -f "$CX_SOURCE_ROOT/experimental/CollectiveX/runtime/common.sh" ]; then
- # shellcheck source=/dev/null
- if source "$CX_SOURCE_ROOT/experimental/CollectiveX/runtime/common.sh" \
- >/dev/null 2>&1; then
- cx_cleanup_private_logs 0
- fi
- fi
- if [ '${{ steps.sweep_shard.outcome }}' != success ] \
- && [ "$CX_JOB_PARENT" != /tmp ]; then
- rm -f -- "$CX_JOB_PARENT"
- exit 0
- fi
rm -rf -- "$CX_JOB_ROOT"
[ "$CX_JOB_PARENT" = /tmp ] || rm -f -- "$CX_JOB_PARENT"
From 6bd51e9de420d8ad3c7ea39c1e88a45046f51b37 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Thu, 9 Jul 2026 01:28:33 +0800
Subject: [PATCH 209/210] feat(collectivex): enable MoRI on MI355X EP8
Migrate mi355x onto the same mi35x-tagged 0701 image (sglang 0.5.14) that
mi300x/mi325x already run green, with the asyncll kernel and the XGMI/SDMA
env. The prior 0227 image (sglang 0.5.9) hung MoRI EpDispatchCombineOp
construction on gfx950; the 0227 image/commit constants are now dead and
removed. Drops the MI355X/mori/EP8 capability wall so the cell is runnable
(EP16 wall retained pending separate validation).
---
experimental/CollectiveX/capability.py | 3 ---
experimental/CollectiveX/runtime/common.sh | 31 ++++++----------------
2 files changed, 8 insertions(+), 26 deletions(-)
diff --git a/experimental/CollectiveX/capability.py b/experimental/CollectiveX/capability.py
index 22e2f3f0c6..5efb1dec1f 100644
--- a/experimental/CollectiveX/capability.py
+++ b/experimental/CollectiveX/capability.py
@@ -187,9 +187,6 @@ def _platform(
"DeepEP V2 EP16 requires GDRCopy /dev/gdrdrv for NVSHMEM-IBGDA, "
"unprovisioned on B300 hosts"
),
- ("mi355x", "mori", 8): (
- "Pinned MoRI backend construction does not complete on MI355X EP8"
- ),
("mi355x", "mori", 16): (
"Pinned MoRI backend construction does not complete on MI355X EP16"
),
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 619c6ef137..db4aa0536c 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -1419,17 +1419,14 @@ CX_IMAGE_MULTIARCH_DIGEST="sha256:061fb71f838e82000a1768c159654d526c2f17ebe751c2
# Runtime setup verifies the image-bundled DeepEP build for the detected GPU target.
CX_IMAGE_MULTIARCH="lmsysorg/sglang:v0.5.11-cu130"
-# AMD (ROCm/CDNA): separate single-arch images bundle MoRI.
-CX_IMAGE_AMD_MORI="rocm/sgl-dev:sglang-0.5.9-rocm720-mi35x-mori-0227-2"
-CX_IMAGE_AMD_MORI_DIGEST="sha256:24c3b30d64475937abbb6498e3b29528649adcb836dde7a468979f767809b0e8"
-CX_MORI_COMMIT_MI355="99bc0a3a6e7a70aacc6372cd9a4275ccfb4de567" # pragma: allowlist secret
+# AMD (ROCm/CDNA): single mi35x-tagged image bundles MoRI for all three CDNA
+# SKUs (gfx942 mi300x/mi325x + gfx950 mi355x).
CX_IMAGE_AMD_MORI_MI325="rocm/sgl-dev:sglang-0.5.14-rocm720-mi35x-mori-0701"
CX_IMAGE_AMD_MORI_MI325_DIGEST="sha256:ea42375343c2ef8f73b3bdb9e1b7b435556e3ca92aba5e3f74ada29ba217fabc"
CX_MORI_COMMIT_MI325="bf99bdf18fc69887a346913ca01c315c2aa9bd4c" # pragma: allowlist secret
cx_default_image() {
case "$1" in
- mi300x*|mi325x*) echo "$CX_IMAGE_AMD_MORI_MI325" ;;
- mi355x*) echo "$CX_IMAGE_AMD_MORI" ;;
+ mi300x*|mi325x*|mi355x*) echo "$CX_IMAGE_AMD_MORI_MI325" ;;
b200*|gb200*|b300*|gb300*|h100*|h200*) echo "$CX_IMAGE_MULTIARCH" ;;
*) cx_die "no default image for runner prefix: $1" ;;
esac
@@ -1486,7 +1483,6 @@ cx_verify_registry_image() {
cx_default_image_digest() {
case "$1" in
"$CX_IMAGE_MULTIARCH") printf '%s' "$CX_IMAGE_MULTIARCH_DIGEST" ;;
- "$CX_IMAGE_AMD_MORI") printf '%s' "$CX_IMAGE_AMD_MORI_DIGEST" ;;
"$CX_IMAGE_AMD_MORI_MI325") printf '%s' "$CX_IMAGE_AMD_MORI_MI325_DIGEST" ;;
esac
}
@@ -2131,10 +2127,13 @@ cx_lock_canonical_gha_env() {
CX_NCCL_HOME=/usr
CX_MASTER_PORT=29551
;;
- mi300x|mi325x)
+ mi300x|mi325x|mi355x)
expected_nodes="${CX_NODES:-}"; expected_gpn=8
[ "$expected_nodes" = 1 ] || [ "$expected_nodes" = 2 ] \
|| cx_die "canonical AMD execution requires one or two nodes"
+ # All three CDNA SKUs run the same MoRI-bundled image (mi35x tag covers
+ # gfx942 + gfx950); mi355x was migrated off the older 0227 image (sglang
+ # 0.5.9), whose MoRI build hung during EpDispatchCombineOp construction.
CX_IMAGE="$CX_IMAGE_AMD_MORI_MI325"
CX_IMAGE_DIGEST="$CX_IMAGE_AMD_MORI_MI325_DIGEST"
if [ "$expected_nodes" = 2 ]; then
@@ -2149,19 +2148,6 @@ cx_lock_canonical_gha_env() {
MORI_SHMEM_LOG_LEVEL=info
MORI_IO_LOG_LEVEL=info
;;
- mi355x)
- expected_nodes="${CX_NODES:-}"; expected_gpn=8
- [ "$expected_nodes" = 1 ] || [ "$expected_nodes" = 2 ] \
- || cx_die "canonical AMD execution requires one or two nodes"
- CX_IMAGE="$CX_IMAGE_AMD_MORI"
- CX_IMAGE_DIGEST="$CX_IMAGE_AMD_MORI_DIGEST"
- if [ "$expected_nodes" = 2 ]; then
- CX_MORI_KERNEL_TYPE=internode-v1
- else
- CX_MORI_KERNEL_TYPE=intranode
- fi
- MORI_COMMIT="$CX_MORI_COMMIT_MI355"
- ;;
*) cx_die "canonical CollectiveX runner is not registered" ;;
esac
case "$runner:$trusted_lock_dir" in
@@ -2195,11 +2181,10 @@ cx_lock_canonical_gha_env() {
case "$runner" in
h100-dgxc|h200-dgxc|b200-dgxc|b300) export CX_NCCL_HOME ;;
gb200|gb300) export CX_NCCL_HOME CX_MASTER_PORT ;;
- mi300x|mi325x)
+ mi300x|mi325x|mi355x)
export CX_MORI_KERNEL_TYPE MORI_COMMIT MORI_DISABLE_AUTO_XGMI MORI_ENABLE_SDMA
export MORI_APP_LOG_LEVEL MORI_SHMEM_LOG_LEVEL MORI_IO_LOG_LEVEL
;;
- mi355x) export CX_MORI_KERNEL_TYPE MORI_COMMIT ;;
esac
}
From 0de7df6eb587cb60be05a1bb52f798f5a9f5261a Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Thu, 9 Jul 2026 01:37:50 +0800
Subject: [PATCH 210/210] fix(collectivex): clear raw config secrets before
launcher in sweep
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Commit 07b81abc ("cleanup sweep.yml") dropped the block that unset the
COLLECTIVEX_*_CONFIG_CONTENT and _REQUIRED environment variables after the
merge step writes $CX_JOB_ROOT/operator-config.json. With those still set,
the launcher's cx_load_operator_config re-enters the ephemeral-config branch
and tries to create the same file with O_EXCL/noclobber, which the merge step
already created — failing every leg at setup with "cannot create ephemeral
runner configuration". Restore the unset so the launcher consumes the written
path instead.
---
.github/workflows/collectivex-sweep.yml | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git a/.github/workflows/collectivex-sweep.yml b/.github/workflows/collectivex-sweep.yml
index 668d87d84e..62b80b0e52 100644
--- a/.github/workflows/collectivex-sweep.yml
+++ b/.github/workflows/collectivex-sweep.yml
@@ -246,6 +246,16 @@ jobs:
with os.fdopen(descriptor, "w", encoding="utf-8") as stream:
stream.write(payload)
PY
+ # The merged config has been written to $operator_config; hand the
+ # launcher the path only. The raw secret *_CONTENT vars must be cleared
+ # here, otherwise cx_load_operator_config sees them still set and tries
+ # to re-create operator-config.json (O_EXCL) that this step already
+ # wrote, failing with "cannot create ephemeral runner configuration".
+ unset COLLECTIVEX_OPERATOR_CONFIG_CONTENT COLLECTIVEX_NETWORK_CONFIG_CONTENT
+ unset COLLECTIVEX_H100_CONFIG_CONTENT COLLECTIVEX_B300_CONFIG_CONTENT
+ unset COLLECTIVEX_B200_CONFIG_CONTENT
+ unset COLLECTIVEX_MI300_CONFIG_CONTENT COLLECTIVEX_MI355_CONFIG_CONTENT
+ unset COLLECTIVEX_OPERATOR_CONFIG_REQUIRED
export COLLECTIVEX_OPERATOR_CONFIG="$operator_config"
export COLLECTIVEX_OPERATOR_CONFIG_EPHEMERAL=1
: > "$CX_JOB_ROOT/cleanup-unsafe"