From 3cc775d6af5f49d866ab56a68af7b676cec22173 Mon Sep 17 00:00:00 2001 From: Dheeraj Date: Tue, 21 Jul 2026 17:33:23 +0530 Subject: [PATCH 1/6] feat(EC-1818): add stress benchmark CI workflow Add a report-only GitHub Actions workflow that runs the stress benchmark on PRs to main, surfacing peak memory and execution time in the job summary without blocking merges. Uses oras to pull pre-built benchmark data from Quay with upstream regeneration as fallback. Ref: EC-1818 Co-Authored-By: Claude Opus 4.6 --- .github/workflows/benchmark.yaml | 102 +++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 .github/workflows/benchmark.yaml diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml new file mode 100644 index 000000000..ca1287c95 --- /dev/null +++ b/.github/workflows/benchmark.yaml @@ -0,0 +1,102 @@ +# Copyright The Conforma Contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +--- +name: Benchmark + +"on": + pull_request: + branches: + - main + +permissions: + contents: read + +jobs: + + Stress: + runs-on: ubuntu-latest + continue-on-error: true + env: + EC_STRESS_COMPONENTS: "10" + EC_STRESS_WORKERS: "5" + steps: + - name: Harden Runner + uses: step-security/harden-runner@58077d3c7e43986b6b15fba718e8ea69e387dfcc # v2.15.1 + with: + egress-policy: audit + disable-telemetry: true + + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Setup Go environment + uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 + with: + go-version-file: go.mod + cache: false + + - name: Setup ORAS + uses: oras-project/setup-oras@1d808f7d7f6995cc68b7bf507bfe5c5446e1dc9d # v2.0.1 + + - name: Prepare benchmark data + run: | + cd benchmark/stress + ./prepare_data.sh + + - name: Run stress benchmark + run: | + cd benchmark/stress + go run . 2>/dev/null | tee benchmark-output.txt + + - name: Write job summary + if: always() + run: | + if [[ ! -f benchmark/stress/benchmark-output.txt ]]; then + echo "## Stress Benchmark" >> "$GITHUB_STEP_SUMMARY" + echo "Benchmark did not produce output." >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + + line=$(grep '^BenchmarkStress' benchmark/stress/benchmark-output.txt || true) + if [[ -z "$line" ]]; then + echo "## Stress Benchmark" >> "$GITHUB_STEP_SUMMARY" + echo "No benchmark results found in output." >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + + ns_op=$(echo "$line" | grep -oP '[\d.]+ ns/op' | awk '{print $1}') + peak_rss=$(echo "$line" | grep -oP '[\d.]+ peak-RSS-bytes' | awk '{print $1}') + alloc=$(echo "$line" | grep -oP '[\d.]+ allocated-bytes/op' | awk '{print $1}') + heap=$(echo "$line" | grep -oP '[\d.]+ heap-bytes-from-system' | awk '{print $1}') + + secs=$(awk "BEGIN {printf \"%.1f\", ${ns_op:-0} / 1000000000}") + rss_mb=$(awk "BEGIN {printf \"%.0f\", ${peak_rss:-0} / 1048576}") + alloc_mb=$(awk "BEGIN {printf \"%.0f\", ${alloc:-0} / 1048576}") + heap_mb=$(awk "BEGIN {printf \"%.0f\", ${heap:-0} / 1048576}") + + { + echo "## Stress Benchmark" + echo "" + echo "| Metric | Value |" + echo "|--------|-------|" + echo "| Components | ${EC_STRESS_COMPONENTS} |" + echo "| Workers | ${EC_STRESS_WORKERS} |" + echo "| Execution time | ${secs}s |" + echo "| Peak RSS | ${rss_mb} MB |" + echo "| Allocated memory | ${alloc_mb} MB |" + echo "| Heap from system | ${heap_mb} MB |" + } >> "$GITHUB_STEP_SUMMARY" From 26bde5cd6e4f423e9e616c6790e3a010ffe28505 Mon Sep 17 00:00:00 2001 From: Dheeraj Date: Tue, 21 Jul 2026 18:37:16 +0530 Subject: [PATCH 2/6] chore(EC-1818): bump stress benchmark to 40 components, 10 workers Initial CI run with 10/5 completed in under a minute. Increase to 40/10 to target 5-6 minute total job time on CI runners. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/benchmark.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index ca1287c95..c483c32af 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -31,8 +31,8 @@ jobs: runs-on: ubuntu-latest continue-on-error: true env: - EC_STRESS_COMPONENTS: "10" - EC_STRESS_WORKERS: "5" + EC_STRESS_COMPONENTS: "40" + EC_STRESS_WORKERS: "10" steps: - name: Harden Runner uses: step-security/harden-runner@58077d3c7e43986b6b15fba718e8ea69e387dfcc # v2.15.1 From 7b261b5d1a2ac7f7184427f65d96bd98bda6d7ed Mon Sep 17 00:00:00 2001 From: Dheeraj Date: Tue, 21 Jul 2026 19:37:04 +0530 Subject: [PATCH 3/6] fix(EC-1818): address review feedback on benchmark workflow Rename workflow to "Stress Benchmark", add workflow_dispatch trigger, separate build from execution to surface compilation errors, and document CI-specific env var overrides. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/benchmark.yaml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index c483c32af..072f26272 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -15,12 +15,13 @@ # SPDX-License-Identifier: Apache-2.0 --- -name: Benchmark +name: Stress Benchmark "on": pull_request: branches: - main + workflow_dispatch: permissions: contents: read @@ -31,6 +32,8 @@ jobs: runs-on: ubuntu-latest continue-on-error: true env: + # Tuned for 4 vCPU / 16 GB CI runners to complete within 6-8 minutes. + # Code defaults are 10 components / 35 workers. EC_STRESS_COMPONENTS: "40" EC_STRESS_WORKERS: "10" steps: @@ -57,10 +60,13 @@ jobs: cd benchmark/stress ./prepare_data.sh + - name: Build stress benchmark + run: go build -o benchmark/stress/stress ./benchmark/stress + - name: Run stress benchmark run: | cd benchmark/stress - go run . 2>/dev/null | tee benchmark-output.txt + ./stress 2>/dev/null | tee benchmark-output.txt - name: Write job summary if: always() From 4861700557a0b6b9fa3aa262d9118d3c76f20059 Mon Sep 17 00:00:00 2001 From: Dheeraj Date: Wed, 22 Jul 2026 17:27:48 +0530 Subject: [PATCH 4/6] chore(EC-1818): bump stress benchmark to 80 components Doubles component count from 40 to 80 to increase peak RSS (~4 GB), making memory regressions more visible in CI job summaries. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/benchmark.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index 072f26272..0d0eddc68 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -34,7 +34,7 @@ jobs: env: # Tuned for 4 vCPU / 16 GB CI runners to complete within 6-8 minutes. # Code defaults are 10 components / 35 workers. - EC_STRESS_COMPONENTS: "40" + EC_STRESS_COMPONENTS: "80" EC_STRESS_WORKERS: "10" steps: - name: Harden Runner From d3506318028bf7813b8b11fbd7f20e6d37d7b8ac Mon Sep 17 00:00:00 2001 From: Dheeraj Date: Wed, 22 Jul 2026 17:30:58 +0530 Subject: [PATCH 5/6] fix(EC-1818): capture stderr and use lowercase job ID Redirect stderr to a file instead of /dev/null so panics and errors are surfaced in the job summary. Rename job ID from Stress to stress to match the dominant lowercase convention in the repo. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/benchmark.yaml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index 0d0eddc68..8dede32ef 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -28,7 +28,7 @@ permissions: jobs: - Stress: + stress: runs-on: ubuntu-latest continue-on-error: true env: @@ -66,11 +66,20 @@ jobs: - name: Run stress benchmark run: | cd benchmark/stress - ./stress 2>/dev/null | tee benchmark-output.txt + ./stress 2>benchmark-stderr.txt | tee benchmark-output.txt - name: Write job summary if: always() run: | + if [[ -s benchmark/stress/benchmark-stderr.txt ]]; then + { + echo "### Stderr" + echo '```' + tail -50 benchmark/stress/benchmark-stderr.txt + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + fi + if [[ ! -f benchmark/stress/benchmark-output.txt ]]; then echo "## Stress Benchmark" >> "$GITHUB_STEP_SUMMARY" echo "Benchmark did not produce output." >> "$GITHUB_STEP_SUMMARY" From bb9a51653c63cd19480e7222049f56b3a9e00a3e Mon Sep 17 00:00:00 2001 From: Dheeraj Date: Wed, 22 Jul 2026 17:50:23 +0530 Subject: [PATCH 6/6] chore(EC-1818): revert stress components to default 10 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All components use the same image digest, so internal caches deduplicate after the first — higher counts don't add memory pressure. Use the code default to avoid misleading numbers. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/benchmark.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index 8dede32ef..61737b6ca 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -34,7 +34,7 @@ jobs: env: # Tuned for 4 vCPU / 16 GB CI runners to complete within 6-8 minutes. # Code defaults are 10 components / 35 workers. - EC_STRESS_COMPONENTS: "80" + EC_STRESS_COMPONENTS: "10" EC_STRESS_WORKERS: "10" steps: - name: Harden Runner