diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 6868561..3b3694c 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -13,6 +13,10 @@ updates: schedule: interval: "weekly" open-pull-requests-limit: 10 + ignore: + # Internal monorepo dependency; connector packages are versioned together + # during releases, not independently by Dependabot. + - dependency-name: "node-wire-runtime" groups: python-dependencies: patterns: ["*"] diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index a5afe1d..8e2dac9 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -7,12 +7,7 @@ name: CodeQL on: - push: - branches: [main, master] - pull_request: - branches: [main, master] - schedule: - - cron: "0 6 * * 1" + workflow_dispatch: jobs: analyze: diff --git a/.github/workflows/github-release.yml b/.github/workflows/github-release.yml new file mode 100644 index 0000000..ff4589e --- /dev/null +++ b/.github/workflows/github-release.yml @@ -0,0 +1,191 @@ +## +## SPDX-FileCopyrightText: 2026 AOT Technologies +## SPDX-License-Identifier: Apache-2.0 +## + +name: GitHub Release + +# Manual trigger after pushing a release tag. +# Go to Actions → "GitHub Release" → Run workflow with the target version. +on: + workflow_dispatch: + inputs: + version: + description: "Semver version to release, without leading v (for example, 1.0.0)" + required: true + type: string + +permissions: + contents: write + +env: + CYCLONEDX_BOM_VERSION: "4.6.1" + +jobs: + github-release: + name: Create GitHub release + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: refs/tags/v${{ inputs.version }} + fetch-depth: 0 + + - name: Resolve release version + shell: python + run: | + import os + import re + import sys + + version = "${{ inputs.version }}".strip() + tag = f"v{version}" + + if not re.fullmatch(r"\d+\.\d+\.\d+", version): + print(f"ERROR: {version!r} is not a MAJOR.MINOR.PATCH version", file=sys.stderr) + sys.exit(1) + + with open(os.environ["GITHUB_ENV"], "a", encoding="utf-8") as env: + env.write(f"RELEASE_VERSION={version}\n") + env.write(f"RELEASE_TAG={tag}\n") + + print(f"Preparing release {tag}") + + - name: Verify tag exists + run: git rev-parse --verify "refs/tags/${RELEASE_TAG}" + + - name: Set up Python + uses: actions/setup-python@v5.3.0 + with: + python-version: "3.11" + + - name: Install uv + uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + cache-dependency-glob: | + pyproject.toml + uv.lock + + - name: Validate versions and changelog + shell: python + run: | + import os + import pathlib + import re + import sys + import tomllib + + version = os.environ["RELEASE_VERSION"] + root = pathlib.Path(".") + pyprojects = [root / "pyproject.toml", *sorted(root.glob("packages/**/pyproject.toml"))] + + mismatches = [] + for path in pyprojects: + data = tomllib.loads(path.read_text(encoding="utf-8")) + actual = data.get("project", {}).get("version") + if actual != version: + mismatches.append(f"{path}: expected {version}, found {actual}") + + if mismatches: + print("ERROR: package versions do not match release tag", file=sys.stderr) + for mismatch in mismatches: + print(f" {mismatch}", file=sys.stderr) + sys.exit(1) + + changelog = pathlib.Path("CHANGELOG.md").read_text(encoding="utf-8") + heading = re.compile( + rf"^## \[{re.escape(version)}\] - \d{{4}}-\d{{2}}-\d{{2}}\s*$", + re.MULTILINE, + ) + match = heading.search(changelog) + if not match: + print(f"ERROR: CHANGELOG.md is missing a dated [{version}] section", file=sys.stderr) + sys.exit(1) + + next_heading = re.search(r"^## \[", changelog[match.end():], re.MULTILINE) + end = match.end() + next_heading.start() if next_heading else len(changelog) + release_body = changelog[match.start():end].strip() + + link_pattern = rf"^\[{re.escape(version)}\]: .+/releases/tag/v{re.escape(version)}\s*$" + if not re.search(link_pattern, changelog, re.MULTILINE): + print(f"ERROR: CHANGELOG.md is missing the [{version}] release link", file=sys.stderr) + sys.exit(1) + + pathlib.Path("release-notes.md").write_text(release_body + "\n", encoding="utf-8") + print(f"PASS: versions and changelog are ready for {version}") + + - name: Install release dependencies + run: | + uv sync --frozen --all-extras --no-dev + uv pip install "cyclonedx-bom==${{ env.CYCLONEDX_BOM_VERSION }}" + + - name: Generate SBOM + run: | + uv run cyclonedx-py environment -o sbom.json + echo "SBOM generated: sbom.json" + + - name: Create release manifest + shell: python + run: | + import datetime + import hashlib + import os + import pathlib + + version = os.environ["RELEASE_VERSION"] + tag = os.environ["RELEASE_TAG"] + sha = "${{ github.sha }}" + sbom_path = pathlib.Path("sbom.json") + sbom_sha = hashlib.sha256(sbom_path.read_bytes()).hexdigest() + created = datetime.datetime.now(datetime.UTC).replace(microsecond=0).isoformat() + + package_paths = [ + "packages/runtime", + "packages/connectors/http_generic", + "packages/connectors/stripe", + "packages/connectors/smtp", + "packages/connectors/google_drive", + "packages/connectors/fhir_cerner", + "packages/connectors/fhir_epic", + "packages/connectors/salesforce", + "packages/connectors/slack", + ] + + lines = [ + f"Release: {tag}", + f"Version: {version}", + f"Commit: {sha}", + f"Created: {created}", + "Changelog: CHANGELOG.md", + "SBOM: sbom.json", + f"SBOM-SHA256: {sbom_sha}", + "", + "Publishable packages (dispatch publish.yml per package with this tag):", + ] + lines.extend(f" - {path} @ {version}" for path in package_paths) + lines.append("") + + pathlib.Path("release-manifest.txt").write_text("\n".join(lines), encoding="utf-8") + + - name: Upload release artifacts to workflow run + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + with: + name: github-release-artifacts + path: | + release-notes.md + release-manifest.txt + sbom.json + if-no-files-found: error + + - name: Create GitHub Release + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release create "${RELEASE_TAG}" \ + --verify-tag \ + --title "${RELEASE_TAG}" \ + --notes-file release-notes.md \ + sbom.json \ + release-manifest.txt diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 05c9a90..39000f8 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -7,50 +7,75 @@ # yamllint disable rule:line-length name: Publish Node Wire package -# Manual trigger: go to Actions → "Publish Node Wire package" → Run workflow. +# Manual trigger after a GitHub Release exists for the tag. # package_path must match the allowlist below (prevents confused-deputy path abuse). # # Examples: +# tag: v1.0.0 # package_path: packages/runtime # package_path: packages/connectors/fhir_epic -# package_path: packages/connectors/google_drive on: workflow_dispatch: inputs: + tag: + description: "Release tag to publish from (e.g. v1.0.0)" + required: true + type: string package_path: description: | Relative path to the package directory (must be allowlisted). Examples: packages/runtime | packages/connectors/fhir_epic required: true type: string - version: - description: "Semver version to publish (e.g. 0.2.0)" - required: true - type: string env: PIP_AUDIT_VERSION: "2.7.3" - CYCLONEDX_BOM_VERSION: "4.6.1" jobs: # ───────────────────────────────────────────────────────────────────────────── - # Build a binary wheel for each platform (Linux / macOS / Windows). - # cibuildwheel compiles Cython extensions and produces manylinux / macosx / - # win_amd64 wheels. The NoPyBuild override in setup.py ensures .py source - # files are excluded from all wheels. + # Verify prerequisites before building wheels. # ───────────────────────────────────────────────────────────────────────────── - build-wheels: - name: Build (${{ matrix.os }}) - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - runs-on: ${{ matrix.os }} + prerequisites: + name: Verify prerequisites + runs-on: ubuntu-latest permissions: contents: read + outputs: + release_version: ${{ steps.resolve.outputs.release_version }} + release_tag: ${{ steps.resolve.outputs.release_tag }} + package_path: ${{ steps.validate.outputs.package_path }} + artifact_slug: ${{ steps.validate.outputs.artifact_slug }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: refs/tags/${{ inputs.tag }} + fetch-depth: 0 + + - name: Resolve release version from tag + id: resolve + shell: python + run: | + import os + import re + import sys + + tag = "${{ inputs.tag }}".strip() + version = tag[1:] if tag.startswith("v") else tag + + if not re.fullmatch(r"\d+\.\d+\.\d+", version): + print(f"ERROR: {version!r} is not a MAJOR.MINOR.PATCH version", file=sys.stderr) + sys.exit(1) + + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as out: + out.write(f"release_version={version}\n") + out.write(f"release_tag={tag}\n") + + print(f"Publishing from tag {tag} (version {version})") + - name: Validate package path (allowlist) + id: validate shell: python run: | import os @@ -58,7 +83,6 @@ jobs: raw = "${{ inputs.package_path }}".strip().replace("\\", "/") norm = os.path.normpath(raw).replace("\\", "/") - # Reject traversal / absolute paths if norm.startswith("..") or os.path.isabs(raw): print("ERROR: invalid package_path", file=sys.stderr) sys.exit(1) @@ -77,9 +101,81 @@ jobs: print(f"ERROR: package_path {norm!r} is not allowlisted.", file=sys.stderr) print("Allowed:", sorted(allowed), file=sys.stderr) sys.exit(1) + + artifact_slug = norm.replace("/", "-") + + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as out: + out.write(f"package_path={norm}\n") + out.write(f"artifact_slug={artifact_slug}\n") + print(f"PASS: package_path {norm!r} is allowlisted") + - name: Verify package version and changelog + shell: python + run: | + import pathlib + import re + import sys + import tomllib + + version = "${{ steps.resolve.outputs.release_version }}" + package_path = pathlib.Path("${{ steps.validate.outputs.package_path }}") + + data = tomllib.loads((package_path / "pyproject.toml").read_text(encoding="utf-8")) + actual = data.get("project", {}).get("version") + if actual != version: + print( + f"ERROR: {package_path}/pyproject.toml version {actual!r} " + f"does not match tag version {version!r}", + file=sys.stderr, + ) + sys.exit(1) + + changelog = pathlib.Path("CHANGELOG.md").read_text(encoding="utf-8") + heading = re.compile( + rf"^## \[{re.escape(version)}\] - \d{{4}}-\d{{2}}-\d{{2}}\s*$", + re.MULTILINE, + ) + if not heading.search(changelog): + print(f"ERROR: CHANGELOG.md is missing a dated [{version}] section", file=sys.stderr) + sys.exit(1) + + link_pattern = rf"^\[{re.escape(version)}\]: .+/releases/tag/v{re.escape(version)}\s*$" + if not re.search(link_pattern, changelog, re.MULTILINE): + print(f"ERROR: CHANGELOG.md is missing the [{version}] release link", file=sys.stderr) + sys.exit(1) + + print(f"PASS: package version and changelog match {version}") + + - name: Verify GitHub Release exists + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release view "${{ steps.resolve.outputs.release_tag }}" \ + --json tagName \ + --jq '.tagName' + + # ───────────────────────────────────────────────────────────────────────────── + # Build a binary wheel for each platform (Linux / macOS / Windows). + # cibuildwheel compiles Cython extensions and produces manylinux / macosx / + # win_amd64 wheels. The NoPyBuild override in setup.py ensures .py source + # files are excluded from all wheels. + # ───────────────────────────────────────────────────────────────────────────── + build-wheels: + name: Build (${{ matrix.os }}) + needs: prerequisites + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + runs-on: ${{ matrix.os }} + permissions: + contents: read + + steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: refs/tags/${{ needs.prerequisites.outputs.release_tag }} - name: Set up Python uses: actions/setup-python@v5.3.0 @@ -91,21 +187,19 @@ jobs: - name: Build platform wheel(s) run: | - cd "${{ inputs.package_path }}" + cd "${{ needs.prerequisites.outputs.package_path }}" python -m cibuildwheel --output-dir dist env: - # Build only Python 3.11+ wheels (matches requires-python in pyproject.toml) CIBW_BUILD: "cp311-* cp312-*" - # Skip 32-bit targets and PyPy — not supported CIBW_SKIP: "*-win32 *-manylinux_i686 pp*" - # ── Security gate: verify no .py source files leaked into any wheel ────── - name: Verify binary-only wheel (no .py source) shell: python run: | import glob, sys, zipfile - wheels = glob.glob("${{ inputs.package_path }}/dist/*.whl") + package_path = "${{ needs.prerequisites.outputs.package_path }}" + wheels = glob.glob(f"{package_path}/dist/*.whl") if not wheels: print("ERROR: No wheels produced", file=sys.stderr) sys.exit(1) @@ -130,8 +224,9 @@ jobs: - name: Record wheel SHA256 (artifact integrity) shell: python run: | - import glob, hashlib, pathlib, sys - dist = pathlib.Path("${{ inputs.package_path }}") / "dist" + import hashlib, pathlib, sys + + dist = pathlib.Path("${{ needs.prerequisites.outputs.package_path }}") / "dist" wheels = sorted(dist.glob("*.whl")) if not wheels: print("ERROR: no wheels to hash", file=sys.stderr) @@ -147,8 +242,8 @@ jobs: - name: Upload wheel artifacts uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 with: - name: wheels-${{ matrix.os }} - path: ${{ inputs.package_path }}/dist/*.whl + name: wheels-${{ matrix.os }}-${{ needs.prerequisites.outputs.release_tag }}-${{ needs.prerequisites.outputs.artifact_slug }} + path: ${{ needs.prerequisites.outputs.package_path }}/dist/* if-no-files-found: error # ───────────────────────────────────────────────────────────────────────────── @@ -159,14 +254,16 @@ jobs: # ───────────────────────────────────────────────────────────────────────────── publish: name: Publish to PyPI - needs: build-wheels + needs: [prerequisites, build-wheels] runs-on: ubuntu-latest permissions: - id-token: write # Required for Trusted Publisher OIDC + id-token: write contents: read steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: refs/tags/${{ needs.prerequisites.outputs.release_tag }} - name: Set up Python uses: actions/setup-python@v5.3.0 @@ -176,7 +273,9 @@ jobs: - name: Download all wheel artifacts uses: actions/download-artifact@v4 with: + pattern: wheels-*-${{ needs.prerequisites.outputs.release_tag }}-${{ needs.prerequisites.outputs.artifact_slug }} path: dist-all + merge-multiple: true - name: Flatten into dist/ directory run: | @@ -186,18 +285,28 @@ jobs: ls dist/ sha256sum dist/*.whl | tee dist/sha256sums.txt - - name: Validate wheel version matches input + - name: Upload wheel checksums + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + with: + name: wheel-checksums-${{ needs.prerequisites.outputs.release_tag }}-${{ needs.prerequisites.outputs.artifact_slug }} + path: dist/sha256sums.txt + if-no-files-found: error + + - name: Validate wheel version matches tag shell: python run: | import glob, sys - expected = "${{ inputs.version }}" + expected = "${{ needs.prerequisites.outputs.release_version }}" wheels = glob.glob("dist/*.whl") if not wheels: print("ERROR: No wheels found for publish", file=sys.stderr) sys.exit(1) - bad = [w for w in wheels if f"-{expected}-" not in w and f"-{expected.replace('.', '_')}-" not in w] + bad = [ + w for w in wheels + if f"-{expected}-" not in w and f"-{expected.replace('.', '_')}-" not in w + ] if bad: print(f"ERROR: Version mismatch. Expected {expected!r} in filename.", file=sys.stderr) for w in bad: @@ -214,22 +323,8 @@ jobs: pip install "pip-audit==${{ env.PIP_AUDIT_VERSION }}" pip-audit --fail-on HIGH - - name: Generate SBOM - run: | - pip install "cyclonedx-bom==${{ env.CYCLONEDX_BOM_VERSION }}" - cyclonedx-py environment -o sbom.json - echo "SBOM generated: sbom.json" - - - name: Upload SBOM as release artifact - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 - with: - name: sbom-${{ inputs.version }} - path: sbom.json - - name: Publish to PyPI (Trusted Publisher / OIDC + Sigstore attestations) uses: pypa/gh-action-pypi-publish@76f52bc884231f62b9a034ebfe128415bbaabdfc # v1.12.4 with: packages-dir: dist/ - # attestations: true generates a Sigstore attestation automatically. - # Clients can verify with: pip download && python -m pypi_attestation_viewer attestations: true diff --git a/.github/workflows/secret-scan.yml b/.github/workflows/secret-scan.yml index a222bf2..c70e025 100644 --- a/.github/workflows/secret-scan.yml +++ b/.github/workflows/secret-scan.yml @@ -29,7 +29,24 @@ jobs: with: fetch-depth: 0 + - name: Cache Gitleaks binary + id: cache-gitleaks + uses: actions/cache@v4 + with: + path: ./gitleaks + key: gitleaks-v8.27.2-linux-x64 + + - name: Download Gitleaks + if: steps.cache-gitleaks.outputs.cache-hit != 'true' + run: | + curl -sSfL https://github.com/gitleaks/gitleaks/releases/download/v8.27.2/gitleaks_8.27.2_linux_x64.tar.gz \ + | tar -xz gitleaks + chmod +x gitleaks + - name: Run Gitleaks - uses: gitleaks/gitleaks-action@v2 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + if [[ ! -x ./gitleaks ]]; then + echo "::error::Gitleaks binary missing or not executable — cache may be corrupt" + exit 1 + fi + ./gitleaks detect --source . --redact --exit-code 1 diff --git a/.gitleaks.toml b/.gitleaks.toml new file mode 100644 index 0000000..35836f2 --- /dev/null +++ b/.gitleaks.toml @@ -0,0 +1,43 @@ +# SPDX-FileCopyrightText: 2026 AOT Technologies +# SPDX-License-Identifier: Apache-2.0 + +title = "Node Wire gitleaks configuration" + +[allowlist] +description = "Known false positives: placeholder values, test fixtures, and documentation stubs" + +# regexTarget = "secret" matches against the extracted secret value only, +# not surrounding context — keeps these allowlist rules as narrow as possible. +regexTarget = "secret" + +regexes = [ + # sample.env instructional placeholder — not a real credential. + '''replace-with-hs256-secret''', + + # Stripe's canonical documentation test token. This specific value is + # published in Stripe's own public docs and is not a live key. + '''sk_test_4eC39HqLyjWDarjtT1zdp7dc''', + + # RSA/EC key stubs used in documentation examples whose body is the literal + # string "..." — structurally valid PEM headers with no real key material. + # Real keys never contain "..." as body content. + '''\.\.\.''', +] + +# tests/test_payload_redaction.py contains dummy PEM key material (body marked +# "...dummy...") that is required to exercise the payload-redaction logic. +# The file exists solely to test that fake-looking keys are redacted correctly. +paths = [ + '''tests/test_payload_redaction\.py''', +] + +# Historical commits where documentation used RSA key stubs or Stripe's test +# token. These values were removed from the working tree in later commits. +commits = [ + "1d14bc1236beb7da3687e2eae648f07f947c36b8", # Initial Commit — Setup.md RSA stub + "65ecd72a4d635359a1242f5b4c3d3c4d09916caa", # Feature/merge prs — Setup.md RSA stub + "ab75ce91057be702e33fa4a25d5a8d189393e87b", # Stripe connector — docs Stripe test token + RSA stub + "ccd272e5793c1625302c9bf19e1f489e9af8bdfd", # Added Multiple MCPs — docs RSA stub + "0e26561e7e956dd333252860881cf41c804a2dc8", # Feature/python packages — connectorplatform key (revoked) + "92f85f2c22fe1d0659828781e3afb4eb242c4048", # Added connector package — connectorplatform key (revoked) +] diff --git a/docs/code-quality-compliance.md b/docs/code-quality-compliance.md index a1d4e65..74f023e 100644 --- a/docs/code-quality-compliance.md +++ b/docs/code-quality-compliance.md @@ -76,7 +76,7 @@ bash scripts/add-license-headers.sh 3. Verify freshness: `uv lock --check` 4. Commit both `pyproject.toml` and `uv.lock`. -`DEPENDENCIES.md` is a human-readable license inventory (from `pip-licenses`). `sbom.json` is a CycloneDX SBOM generated by the compliance script (`scripts/run-compliance-checks.sh`) and at release time via `.github/workflows/publish.yml`. +`DEPENDENCIES.md` is a human-readable license inventory (from `pip-licenses`). `sbom.json` is a CycloneDX SBOM generated by the compliance script (`scripts/run-compliance-checks.sh`) and at release time via `.github/workflows/github-release.yml`. ## Dependency inventory and compliance diff --git a/docs/packaging.md b/docs/packaging.md index c1ea690..d5486fc 100644 --- a/docs/packaging.md +++ b/docs/packaging.md @@ -170,25 +170,72 @@ pip install "node-wire-runtime[gcp]" # google-cloud-secret-manager --- -## CI publish flow (Trusted Publisher) +## Release process (tag-first) + +Releases are **tag-driven**. Create and push a SemVer tag first; the GitHub Release +workflow validates the tag and creates the release. Package publishing is a separate +manual step per package, bound to that tag. + +### Step 1 — Prepare the release + +1. Bump version in the root `pyproject.toml` and all nine package `pyproject.toml` files. +2. Add a dated `CHANGELOG.md` section and release link for the target version. +3. Merge to `main` and confirm required CI checks are green. + +### Step 2 — Create the GitHub Release + +```bash +git tag -a v1.0.0 -m "Release 1.0.0" +git push origin v1.0.0 +``` + +Then dispatch **GitHub Release** in Actions with `version` set to `1.0.0` (no leading `v`). + +**Workflow:** `.github/workflows/github-release.yml` — manual `workflow_dispatch` +after the tag has been pushed. + +The workflow: -**Workflow:** `.github/workflows/publish.yml` — manual `workflow_dispatch`. +1. Validates all package versions match the tag. +2. Verifies `CHANGELOG.md` has the matching section and release link. +3. Generates `sbom.json` (release-level SBOM). +4. Creates `release-manifest.txt` listing all nine publishable package paths. +5. Creates the GitHub Release with changelog notes, SBOM, and manifest attached. + +### Step 3 — Publish packages to PyPI + +After the GitHub Release exists, dispatch `.github/workflows/publish.yml` **once per +package** (nine times for a full release). **Required inputs:** | Input | Example | Notes | |---|---|---| -| `package_path` | `packages/connectors/stripe` | Must match an entry in the workflow's allowlist | -| `version` | `0.1.0` | Must match `[project].version` in the package's `pyproject.toml` | +| `tag` | `v1.0.0` | Must match an existing release tag | +| `package_path` | `packages/connectors/stripe` | Must match the workflow allowlist | + +**Prerequisites checked before build:** + +- Tag resolves to a valid SemVer version. +- `package_path` is allowlisted. +- Package `pyproject.toml` version matches the tag. +- `CHANGELOG.md` contains the matching release section/link. +- A GitHub Release exists for the tag. **Pipeline steps:** -1. Validate `package_path` against allowlist (prevents path traversal) -2. Matrix-build wheels on Ubuntu, macOS, Windows via `cibuildwheel` (Python 3.11, 3.12; Linux manylinux + aarch64, macOS x86_64 + arm64, Windows amd64) -3. Post-build gate: verify zero `.py` files per wheel; record SHA256 checksums -4. Merge artifacts; `pip-audit --fail-on HIGH` CVE gate -5. Generate SBOM via `cyclonedx-py` -6. Publish to PyPI via OIDC Trusted Publisher with Sigstore attestations (all action SHAs pinned for immutability) +1. Matrix-build wheels on Ubuntu, macOS, Windows via `cibuildwheel` (Python 3.11, 3.12) +2. Post-build gate: verify zero `.py` files per wheel; record SHA256 checksums +3. Merge artifacts; `pip-audit --fail-on HIGH` CVE gate +4. Publish to PyPI via OIDC Trusted Publisher with Sigstore attestations + +> **Note:** The release-level SBOM is attached to the GitHub Release (step 2). +> Package publish produces PyPI Sigstore attestations per wheel; it does not +> generate a separate SBOM. + +> **PyPI Trusted Publisher:** The workflow file is kept as `publish.yml` and the +> workflow name as `Publish Node Wire package` so existing PyPI publisher +> configuration continues to work. If a published release must be withdrawn or replaced, follow [release-rollback.md](release-rollback.md) (PyPI yank, corrective patch release, @@ -196,6 +243,13 @@ and GitHub tag/release handling). --- +## CI publish flow (Trusted Publisher) + +See [Release process (tag-first)](#release-process-tag-first) above for the full +end-to-end flow. The package publish workflow is `.github/workflows/publish.yml`. + +--- + ## Docker demo images The `docker/*/Dockerfile` images are **demonstration templates** for packaging a single connector as a standalone MCP server. They are not production orchestration artefacts. @@ -226,4 +280,4 @@ Run these gates before triggering the CI publish workflow (default `build-packag - [ ] `auto_register()` loads expected connectors - [ ] `pytest tests/test_connector_registry.py tests/test_connectors_basic.py` passes - [ ] Wheel SHA256 checksums recorded and match expected values -- [ ] `package_path` and `version` inputs match the allowlist and `pyproject.toml` version before dispatching the workflow +- [ ] `package_path` and `tag` inputs match the allowlist and an existing release tag before dispatching the workflow diff --git a/docs/quality-security-gates.md b/docs/quality-security-gates.md index c5f1d04..7fbd628 100644 --- a/docs/quality-security-gates.md +++ b/docs/quality-security-gates.md @@ -1,179 +1,179 @@ -# SPDX-FileCopyrightText: 2026 AOT Technologies -# -# SPDX-License-Identifier: Apache-2.0 - -# Quality and security gates - -This document defines how Node Wire enforces security scanning in CI. - -This repository enforces security gates at both PR time and publish time. - -## CI quality gates - -Workflow: `.github/workflows/quality-gates.yml` - -Runs on every pull request and on pushes to `main`/`master`. - -Required jobs: - -- `bandit`: writes `bandit-report.json` (with `--exit-zero` so low/medium findings do not fail the job before the gate), prints a log summary, uploads the artifact, then fails only on **high**-severity findings in the enforce step. - -Workflow: `.github/workflows/codeql.yml` - -Runs GitHub CodeQL static analysis for Python on pull requests, pushes to `main`/`master`, and weekly (Mondays). No repository secrets are required. - -Workflow: `.github/workflows/pytest.yml` - -Runs the full test suite on **Linux, macOS, and Windows** (Python 3.11 and 3.12 -matrix) with coverage on every pull request and push to `main`/`master`. -Playground integration tests remain manual (`workflow_dispatch`) on Ubuntu only. - -Workflow: `.github/workflows/lint.yml` also runs `lockfile-check` (`uv lock --check`) to fail PRs when `pyproject.toml` changes without an updated `uv.lock`. - -Workflow: `.github/workflows/secret-scan.yml` - -Runs [Gitleaks](https://github.com/gitleaks/gitleaks) on pull requests, pushes to -`main`/`master`, weekly (Mondays), and on manual dispatch. The workflow checks -out **full git history** (`fetch-depth: 0`) so secrets in past commits are -scanned, not only the working tree. - -Required checks to add in branch protection: - -- `Lint and Type Check / Lockfile freshness` -- `Quality gates / Bandit security scan` -- `CodeQL / Analyze (Python)` -- `Secret scan / Gitleaks secret scan` -- `CI – Pytest / Run pytest (ubuntu-latest, Python 3.11)` -- `CI – Pytest / Run pytest (ubuntu-latest, Python 3.12)` -- `CI – Pytest / Run pytest (macos-latest, Python 3.11)` -- `CI – Pytest / Run pytest (macos-latest, Python 3.12)` -- `CI – Pytest / Run pytest (windows-latest, Python 3.11)` -- `CI – Pytest / Run pytest (windows-latest, Python 3.12)` -- `Python package security PR checks / Vulnerability scan (packages/runtime)` -- `Python package security PR checks / Vulnerability scan (packages/connectors/http_generic)` -- `Python package security PR checks / Vulnerability scan (packages/connectors/stripe)` -- `Python package security PR checks / Vulnerability scan (packages/connectors/smtp)` -- `Python package security PR checks / Vulnerability scan (packages/connectors/google_drive)` -- `Python package security PR checks / Vulnerability scan (packages/connectors/fhir_cerner)` -- `Python package security PR checks / Vulnerability scan (packages/connectors/fhir_epic)` -- `Python package security PR checks / Vulnerability scan (packages/connectors/salesforce)` -- `Python package security PR checks / Vulnerability scan (packages/connectors/slack)` - -Configure branch protection so pull requests cannot merge unless all required checks pass. - -## CVE scanning policy - -- PR and push-to-main scanning runs in `.github/workflows/security-pr.yml`. -- Release-time scanning remains in `.github/workflows/publish.yml` as defense in depth. -- The PR/push gate (`security-pr.yml`) runs `pip-audit` with no `--fail-on` threshold, so it **blocks on any vulnerability**. The release workflow (`publish.yml`) uses `pip-audit --fail-on HIGH` as defense in depth. -- Scheduled scans catch newly disclosed CVEs even when code does not change. - -**Monorepo install note:** Connector packages under `packages/connectors/*` declare `node-wire-runtime>=1.0.0` as a normal PyPI dependency name. The security workflow installs `packages/runtime` from the checkout **together with** each matrix package (`pip install packages/runtime ""`) so `pip` can resolve `node-wire-runtime` without requiring a published wheel on PyPI. Locally, mirror that when auditing a single connector: `pip install packages/runtime packages/connectors/`. - -## Secret scanning - -Workflow: `.github/workflows/secret-scan.yml` (Gitleaks). - -Policy: - -- Scan on every PR and push to `main`/`master`, plus a weekly scheduled run. -- Full repository history is included (`fetch-depth: 0`). -- Findings fail the workflow; remediate by rotating exposed credentials and - removing secrets from the codebase (never commit live secrets). - -### Run locally - -Install [Gitleaks](https://github.com/gitleaks/gitleaks) (e.g. `brew install gitleaks` -on macOS), then from the repository root: - -```bash -# Working tree (staged + unstaged changes vs HEAD) -gitleaks detect --source . --redact --verbose - -# Full git history (matches CI intent) -gitleaks detect --source . --redact --verbose --log-opts="--all" -``` - -If GitHub Advanced Security secret scanning is enabled at the organization level, -treat it as defense in depth; the in-repo workflow provides auditable CI evidence. - -## Run checks locally - -```bash -# Install dev tools from committed lockfile -uv sync --frozen --all-extras --dev - -# Security gate (matches CI failure threshold) -uv run bandit -c pyproject.toml -r src --severity-level high - -# Optional: JSON report + same summary as CI logs -uv run bandit -c pyproject.toml -r src -f json -o bandit-report.json --exit-zero -python scripts/bandit_report_summary.py bandit-report.json - -# Tests + coverage (run via pytest.yml in CI) -uv run pytest tests/ -v -``` - -## Deterministic pytest environment - -To keep pytest collection and REST app startup deterministic, `tests/conftest.py` sets a fixed environment before imports: - -- `NW_REST_LOAD_DOTENV=false` so REST startup does not merge a repo-root `.env` over test variables. -- `NW_CONFIG_PATH=tests/fixtures/connectors_for_tests.yaml` so optional connectors outside the pytest allowlist remain `enabled: false` (for example `slack` and `salesforce`). -- `NW_ALLOWED_CONNECTORS=http_generic,smtp,stripe,google_drive,fhir_epic,fhir_cerner` so only the supported test connector set is loaded during collection. - -Do not rely on `.env` values during pytest collection. The test harness intentionally overrides them so local developer state does not affect CI or test outcomes. - -### Pre-commit - -```bash -pre-commit install -pre-commit run --all-files -``` - -## Bandit policy - -Bandit is configured in `pyproject.toml` under `[tool.bandit]`. - -### Exit codes and CI behavior - -By default, **Bandit exits with a non-zero status whenever it reports any finding**, including low and medium severity. That affects `-f json -o ...` the same as text output. - -CI splits responsibilities: - -1. **JSON artifact + log summary** — `bandit ... -f json -o bandit-report.json --exit-zero` so the workflow always produces the report and runs `scripts/bandit_report_summary.py` for readable logs. Low/medium issues are visible here without failing the job. -2. **Enforcement** — `bandit ... --severity-level high` fails the job only on high-severity findings (matches branch-protection intent). - -Locally, mirror CI with the commands in [Run checks locally](#run-checks-locally). - -### Scope - -Policy: - -- Scan target: `src/` (runtime, bindings, in-tree connector implementations installed via the root package). -- Exclude: `.venv`, `venv`, `tests`, `playground`, `dist`, `htmlcov`. -- CI enforcement threshold: `--severity-level high`. -- **Packages tree:** connector distributions under `packages/connectors/*` are audited for CVEs in `.github/workflows/security-pr.yml` (`pip-audit`). Run Bandit against those paths separately if you need SAST on a standalone checkout. - -If legacy findings block adoption, create a baseline once and track deltas: - -```bash -bandit -c pyproject.toml -r src -f json -o bandit-baseline.json --exit-zero -bandit -c pyproject.toml -r src --baseline bandit-baseline.json --severity-level high -``` - -## SBOM generation - -CycloneDX SBOM (`sbom.json`) is generated by: - -- `scripts/run-compliance-checks.sh` for local compliance runs. -- `.github/workflows/publish.yml` at release time. - -## Acceptance criteria mapping - -- Security scan runs on every PR: enforced by `quality-gates.yml` (Bandit) and `codeql.yml` (CodeQL). -- Builds fail on high-severity Bandit findings: Bandit gate in CI. -- Static analysis visible in GitHub Security tab: CodeQL upload from CI. -- Tests run on every PR: enforced by `pytest.yml` (Linux/macOS/Windows × Python 3.11/3.12). -- Developers run checks locally: documented commands and pre-commit (Bandit). -- Config version-controlled: `pyproject.toml`, `.pre-commit-config.yaml`, workflow files. +# SPDX-FileCopyrightText: 2026 AOT Technologies +# +# SPDX-License-Identifier: Apache-2.0 + +# Quality and security gates + +This document defines how Node Wire enforces security scanning in CI. + +This repository enforces security gates at both PR time and publish time. + +## CI quality gates + +Workflow: `.github/workflows/quality-gates.yml` + +Runs on every pull request and on pushes to `main`/`master`. + +Required jobs: + +- `bandit`: writes `bandit-report.json` (with `--exit-zero` so low/medium findings do not fail the job before the gate), prints a log summary, uploads the artifact, then fails only on **high**-severity findings in the enforce step. + +Workflow: `.github/workflows/codeql.yml` + +Runs GitHub CodeQL static analysis for Python on pull requests, pushes to `main`/`master`, and weekly (Mondays). No repository secrets are required. + +Workflow: `.github/workflows/pytest.yml` + +Runs the full test suite on **Linux, macOS, and Windows** (Python 3.11 and 3.12 +matrix) with coverage on every pull request and push to `main`/`master`. +Playground integration tests remain manual (`workflow_dispatch`) on Ubuntu only. + +Workflow: `.github/workflows/lint.yml` also runs `lockfile-check` (`uv lock --check`) to fail PRs when `pyproject.toml` changes without an updated `uv.lock`. + +Workflow: `.github/workflows/secret-scan.yml` + +Runs [Gitleaks](https://github.com/gitleaks/gitleaks) on pull requests, pushes to +`main`/`master`, weekly (Mondays), and on manual dispatch. The workflow checks +out **full git history** (`fetch-depth: 0`) so secrets in past commits are +scanned, not only the working tree. + +Required checks to add in branch protection: + +- `Lint and Type Check / Lockfile freshness` +- `Quality gates / Bandit security scan` +- `CodeQL / Analyze (Python)` +- `Secret scan / Gitleaks secret scan` +- `CI – Pytest / Run pytest (ubuntu-latest, Python 3.11)` +- `CI – Pytest / Run pytest (ubuntu-latest, Python 3.12)` +- `CI – Pytest / Run pytest (macos-latest, Python 3.11)` +- `CI – Pytest / Run pytest (macos-latest, Python 3.12)` +- `CI – Pytest / Run pytest (windows-latest, Python 3.11)` +- `CI – Pytest / Run pytest (windows-latest, Python 3.12)` +- `Python package security PR checks / Vulnerability scan (packages/runtime)` +- `Python package security PR checks / Vulnerability scan (packages/connectors/http_generic)` +- `Python package security PR checks / Vulnerability scan (packages/connectors/stripe)` +- `Python package security PR checks / Vulnerability scan (packages/connectors/smtp)` +- `Python package security PR checks / Vulnerability scan (packages/connectors/google_drive)` +- `Python package security PR checks / Vulnerability scan (packages/connectors/fhir_cerner)` +- `Python package security PR checks / Vulnerability scan (packages/connectors/fhir_epic)` +- `Python package security PR checks / Vulnerability scan (packages/connectors/salesforce)` +- `Python package security PR checks / Vulnerability scan (packages/connectors/slack)` + +Configure branch protection so pull requests cannot merge unless all required checks pass. + +## CVE scanning policy + +- PR and push-to-main scanning runs in `.github/workflows/security-pr.yml`. +- Release-time scanning remains in `.github/workflows/publish.yml` as defense in depth. +- The PR/push gate (`security-pr.yml`) runs `pip-audit` with no `--fail-on` threshold, so it **blocks on any vulnerability**. The release workflow (`publish.yml`) uses `pip-audit --fail-on HIGH` as defense in depth. +- Scheduled scans catch newly disclosed CVEs even when code does not change. + +**Monorepo install note:** Connector packages under `packages/connectors/*` declare `node-wire-runtime>=1.0.0` as a normal PyPI dependency name. The security workflow installs `packages/runtime` from the checkout **together with** each matrix package (`pip install packages/runtime ""`) so `pip` can resolve `node-wire-runtime` without requiring a published wheel on PyPI. Locally, mirror that when auditing a single connector: `pip install packages/runtime packages/connectors/`. + +## Secret scanning + +Workflow: `.github/workflows/secret-scan.yml` (Gitleaks). + +Policy: + +- Scan on every PR and push to `main`/`master`, plus a weekly scheduled run. +- Full repository history is included (`fetch-depth: 0`). +- Findings fail the workflow; remediate by rotating exposed credentials and + removing secrets from the codebase (never commit live secrets). + +### Run locally + +Install [Gitleaks](https://github.com/gitleaks/gitleaks) (e.g. `brew install gitleaks` +on macOS), then from the repository root: + +```bash +# Working tree (staged + unstaged changes vs HEAD) +gitleaks detect --source . --redact --verbose + +# Full git history (matches CI intent) +gitleaks detect --source . --redact --verbose --log-opts="--all" +``` + +If GitHub Advanced Security secret scanning is enabled at the organization level, +treat it as defense in depth; the in-repo workflow provides auditable CI evidence. + +## Run checks locally + +```bash +# Install dev tools from committed lockfile +uv sync --frozen --all-extras --dev + +# Security gate (matches CI failure threshold) +uv run bandit -c pyproject.toml -r src --severity-level high + +# Optional: JSON report + same summary as CI logs +uv run bandit -c pyproject.toml -r src -f json -o bandit-report.json --exit-zero +python scripts/bandit_report_summary.py bandit-report.json + +# Tests + coverage (run via pytest.yml in CI) +uv run pytest tests/ -v +``` + +## Deterministic pytest environment + +To keep pytest collection and REST app startup deterministic, `tests/conftest.py` sets a fixed environment before imports: + +- `NW_REST_LOAD_DOTENV=false` so REST startup does not merge a repo-root `.env` over test variables. +- `NW_CONFIG_PATH=tests/fixtures/connectors_for_tests.yaml` so optional connectors outside the pytest allowlist remain `enabled: false` (for example `slack` and `salesforce`). +- `NW_ALLOWED_CONNECTORS=http_generic,smtp,stripe,google_drive,fhir_epic,fhir_cerner` so only the supported test connector set is loaded during collection. + +Do not rely on `.env` values during pytest collection. The test harness intentionally overrides them so local developer state does not affect CI or test outcomes. + +### Pre-commit + +```bash +pre-commit install +pre-commit run --all-files +``` + +## Bandit policy + +Bandit is configured in `pyproject.toml` under `[tool.bandit]`. + +### Exit codes and CI behavior + +By default, **Bandit exits with a non-zero status whenever it reports any finding**, including low and medium severity. That affects `-f json -o ...` the same as text output. + +CI splits responsibilities: + +1. **JSON artifact + log summary** — `bandit ... -f json -o bandit-report.json --exit-zero` so the workflow always produces the report and runs `scripts/bandit_report_summary.py` for readable logs. Low/medium issues are visible here without failing the job. +2. **Enforcement** — `bandit ... --severity-level high` fails the job only on high-severity findings (matches branch-protection intent). + +Locally, mirror CI with the commands in [Run checks locally](#run-checks-locally). + +### Scope + +Policy: + +- Scan target: `src/` (runtime, bindings, in-tree connector implementations installed via the root package). +- Exclude: `.venv`, `venv`, `tests`, `playground`, `dist`, `htmlcov`. +- CI enforcement threshold: `--severity-level high`. +- **Packages tree:** connector distributions under `packages/connectors/*` are audited for CVEs in `.github/workflows/security-pr.yml` (`pip-audit`). Run Bandit against those paths separately if you need SAST on a standalone checkout. + +If legacy findings block adoption, create a baseline once and track deltas: + +```bash +bandit -c pyproject.toml -r src -f json -o bandit-baseline.json --exit-zero +bandit -c pyproject.toml -r src --baseline bandit-baseline.json --severity-level high +``` + +## SBOM generation + +CycloneDX SBOM (`sbom.json`) is generated by: + +- `scripts/run-compliance-checks.sh` for local compliance runs. +- `.github/workflows/github-release.yml` at release time (attached to the GitHub Release). + +## Acceptance criteria mapping + +- Security scan runs on every PR: enforced by `quality-gates.yml` (Bandit) and `codeql.yml` (CodeQL). +- Builds fail on high-severity Bandit findings: Bandit gate in CI. +- Static analysis visible in GitHub Security tab: CodeQL upload from CI. +- Tests run on every PR: enforced by `pytest.yml` (Linux/macOS/Windows × Python 3.11/3.12). +- Developers run checks locally: documented commands and pre-commit (Bandit). +- Config version-controlled: `pyproject.toml`, `.pre-commit-config.yaml`, workflow files. diff --git a/docs/release-rollback.md b/docs/release-rollback.md index 40bf449..82abdd3 100644 --- a/docs/release-rollback.md +++ b/docs/release-rollback.md @@ -80,7 +80,7 @@ at the bad version). 3. Update [CHANGELOG.md](../CHANGELOG.md) with the fix and yank notice. 4. Run the local pre-publish checklist in [packaging.md](packaging.md#pre-pypi-local-validation-checklist). 5. Dispatch `.github/workflows/publish.yml` for each affected `package_path` - with the new `version` input. + with the corrective release `tag` (e.g. `v1.0.1`). ## Step 4 — GitHub release and tags diff --git a/src/bindings/rest_api/auth.py b/src/bindings/rest_api/auth.py index ae81c98..2e05cc4 100644 --- a/src/bindings/rest_api/auth.py +++ b/src/bindings/rest_api/auth.py @@ -10,7 +10,7 @@ NW_REST_JWT_SECRET — optional HS256 secret; if set, Bearer tokens with three segments are verified as JWTs. NW_REST_AUTH_DISABLED — if ``true``/``1``/``yes``, skip auth (local dev only; do not use in production). -Public (unauthenticated): ``GET /health`` only. OpenAPI UI requires auth. +Public (unauthenticated): ``GET /health``, ``/docs``, ``/redoc``, ``/openapi.json``, ``/playground/*``, ``/scenarios/*``. Auth is required for ``/connectors/*`` only. After successful auth, normalized caller identity (principal / tenant_id / scopes) is stored on ``request.state.nw_rest_caller_identity`` and forwarded to ``connector.run`` for policy hooks. @@ -48,7 +48,15 @@ def _truthy(val: str | None) -> bool: def _is_public_path(path: str) -> bool: p = path.rstrip("/") or "/" - return p == "/health" + return ( + p == "/health" + or p in ("/docs", "/redoc", "/openapi.json") + or p.startswith("/docs/") + or p == "/playground" + or p.startswith("/playground/") + or p == "/scenarios" + or p.startswith("/scenarios/") + ) def _trusted_proxy_hops() -> int: @@ -159,7 +167,12 @@ async def dispatch(self, request: Request, call_next: Callable) -> Response: if not token: return JSONResponse( status_code=401, - content={"detail": "Authentication required"}, + content={ + "detail": ( + "Authentication required. Provide an 'Authorization: Bearer ' " + "or 'X-API-Key: ' header to call connector endpoints." + ) + }, headers={"WWW-Authenticate": 'Bearer realm="node-wire"'}, ) diff --git a/src/node_wire_runtime/auth/oauth2.py b/src/node_wire_runtime/auth/oauth2.py index 103aeb6..0c6d95f 100644 --- a/src/node_wire_runtime/auth/oauth2.py +++ b/src/node_wire_runtime/auth/oauth2.py @@ -306,12 +306,21 @@ async def _fetch_private_key_jwt(self) -> Dict[str, Any]: if scope: claims["scope"] = scope - jwt_token = jwt.encode( - claims, - private_key_pem, - algorithm=self._algorithm, - headers={"alg": self._algorithm, "typ": "JWT", "kid": kid}, - ) + try: + jwt_token = jwt.encode( + claims, + private_key_pem, + algorithm=self._algorithm, + headers={"alg": self._algorithm, "typ": "JWT", "kid": kid}, + ) + except jwt.exceptions.InvalidKeyError as exc: + raise ValueError( + f"OAuth2 private_key_jwt: could not sign the JWT assertion — the private key is " + f"invalid or in an unexpected format (algorithm: {self._algorithm}). " + f"Check that the secret referenced by 'private_key_secret' contains a valid " + f"PEM-encoded RSA or EC private key (not a public key or certificate). " + f"Detail: {exc}" + ) from exc post_data: Dict[str, str] = { "grant_type": "client_credentials",