diff --git a/.github/workflows/docker-policy.yml b/.github/workflows/docker-policy.yml new file mode 100644 index 0000000..ecddd35 --- /dev/null +++ b/.github/workflows/docker-policy.yml @@ -0,0 +1,38 @@ +# Enforce baseline container hygiene: digest-pinned base image and non-root USER. +name: Docker policy + +on: + push: + branches: [main, master] + paths: + - "Dockerfile" + - "docker/**" + pull_request: + paths: + - "Dockerfile" + - "docker/**" + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Verify Dockerfiles use digest-pinned FROM and non-root USER + shell: bash + run: | + set -euo pipefail + shopt -s nullglob + files=(Dockerfile docker/*/Dockerfile) + for f in "${files[@]}"; do + echo "Checking $f" + if ! grep -qE '^FROM [^[:space:]]+@sha256:[a-f0-9]{64}' "$f"; then + echo "ERROR: $f must use FROM image@sha256:<64-hex-digest>" >&2 + exit 1 + fi + if ! grep -qE '^USER[[:space:]]' "$f"; then + echo "ERROR: $f must end with a non-root USER directive" >&2 + exit 1 + fi + done + echo "PASS: all Dockerfiles pinned and non-root" diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..749d23f --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,228 @@ +# Pinned action SHAs are immutable; update intentionally when upgrading. +# yamllint disable rule:line-length +name: Publish Node Wire package + +# Manual trigger: go to Actions → "Publish Node Wire package" → Run workflow. +# package_path must match the allowlist below (prevents confused-deputy path abuse). +# +# Examples: +# package_path: packages/runtime +# package_path: packages/connectors/fhir_epic +# package_path: packages/connectors/google_drive +on: + workflow_dispatch: + inputs: + package_path: + description: | + Relative path to the package directory (must be allowlisted). + Examples: packages/runtime | packages/connectors/fhir_epic + required: true + type: string + version: + description: "Semver version to publish (e.g. 0.2.0)" + required: true + type: string + +env: + PIP_AUDIT_VERSION: "2.7.3" + CYCLONEDX_BOM_VERSION: "4.6.1" + +jobs: + # ───────────────────────────────────────────────────────────────────────────── + # Build a binary wheel for each platform (Linux / macOS / Windows). + # cibuildwheel compiles Cython extensions and produces manylinux / macosx / + # win_amd64 wheels. The NoPyBuild override in setup.py ensures .py source + # files are excluded from all wheels. + # ───────────────────────────────────────────────────────────────────────────── + build-wheels: + name: Build (${{ matrix.os }}) + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + runs-on: ${{ matrix.os }} + permissions: + contents: read + + steps: + - name: Validate package path (allowlist) + shell: python + run: | + import os + import sys + + raw = "${{ inputs.package_path }}".strip().replace("\\", "/") + norm = os.path.normpath(raw).replace("\\", "/") + # Reject traversal / absolute paths + if norm.startswith("..") or os.path.isabs(raw): + print("ERROR: invalid package_path", file=sys.stderr) + sys.exit(1) + allowed = { + "packages/runtime", + "packages/connectors/http_generic", + "packages/connectors/stripe", + "packages/connectors/smtp", + "packages/connectors/google_drive", + "packages/connectors/fhir_cerner", + "packages/connectors/fhir_epic", + } + if norm not in allowed: + print(f"ERROR: package_path {norm!r} is not allowlisted.", file=sys.stderr) + print("Allowed:", sorted(allowed), file=sys.stderr) + sys.exit(1) + print(f"PASS: package_path {norm!r} is allowlisted") + + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Set up Python + uses: actions/setup-python@0b93645e9fea7318ecaed2b3599a311e5c0f207c # v5.3.0 + with: + python-version: "3.11" + + - name: Install build tools + run: python -m pip install --upgrade pip "cython>=3.0" "cibuildwheel>=2.16.0" + + - name: Build platform wheel(s) + run: | + cd "${{ inputs.package_path }}" + python -m cibuildwheel --output-dir dist + env: + # Build only Python 3.11+ wheels (matches requires-python in pyproject.toml) + CIBW_BUILD: "cp311-* cp312-*" + # Skip 32-bit targets and PyPy — not supported + CIBW_SKIP: "*-win32 *-manylinux_i686 pp*" + + # ── Security gate: verify no .py source files leaked into any wheel ────── + - name: Verify binary-only wheel (no .py source) + shell: python + run: | + import glob, sys, zipfile + + wheels = glob.glob("${{ inputs.package_path }}/dist/*.whl") + if not wheels: + print("ERROR: No wheels produced", file=sys.stderr) + sys.exit(1) + + leaked: dict[str, list[str]] = {} + for whl in wheels: + with zipfile.ZipFile(whl) as zf: + bad = [n for n in zf.namelist() if n.endswith(".py")] + if bad: + leaked[whl] = bad + + if leaked: + print("SECURITY FAIL: .py files found in wheel(s):", file=sys.stderr) + for whl, files in leaked.items(): + print(f" {whl}:", file=sys.stderr) + for f in files: + print(f" {f}", file=sys.stderr) + sys.exit(1) + + print(f"PASS: {len(wheels)} wheel(s) verified — no .py source files") + + - name: Record wheel SHA256 (artifact integrity) + shell: python + run: | + import glob, hashlib, pathlib, sys + dist = pathlib.Path("${{ inputs.package_path }}") / "dist" + wheels = sorted(dist.glob("*.whl")) + if not wheels: + print("ERROR: no wheels to hash", file=sys.stderr) + sys.exit(1) + lines = [] + for w in wheels: + h = hashlib.sha256(w.read_bytes()).hexdigest() + line = f"{h} {w.name}" + print(line) + lines.append(line) + (dist / "sha256sums.txt").write_text("\n".join(lines) + "\n", encoding="utf-8") + + - name: Upload wheel artifacts + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + with: + name: wheels-${{ matrix.os }} + path: ${{ inputs.package_path }}/dist/*.whl + if-no-files-found: error + + # ───────────────────────────────────────────────────────────────────────────── + # Collect all platform wheels, run supply-chain checks, publish to PyPI. + # Uses Trusted Publisher (OIDC) — no long-lived PyPI API tokens needed. + # Configure on PyPI: Settings → Publishing → Add Publisher (GitHub, this repo, + # workflow name = "Publish Node Wire package"). + # ───────────────────────────────────────────────────────────────────────────── + publish: + name: Publish to PyPI + needs: build-wheels + runs-on: ubuntu-latest + permissions: + id-token: write # Required for Trusted Publisher OIDC + contents: read + + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Set up Python + uses: actions/setup-python@0b93645e9fea7318ecaed2b3599a311e5c0f207c # v5.3.0 + with: + python-version: "3.11" + + - name: Download all wheel artifacts + uses: actions/download-artifact@fa0a91b85d4f404e442e0f254de474a18c628458 # v4.1.8 + with: + path: dist-all + + - name: Flatten into dist/ directory + run: | + mkdir -p dist + find dist-all -name "*.whl" -exec cp {} dist/ \; + echo "Wheels collected:" + ls dist/ + sha256sum dist/*.whl | tee dist/sha256sums.txt + + - name: Validate wheel version matches input + shell: python + run: | + import glob, sys + + expected = "${{ inputs.version }}" + wheels = glob.glob("dist/*.whl") + if not wheels: + print("ERROR: No wheels found for publish", file=sys.stderr) + sys.exit(1) + + bad = [w for w in wheels if f"-{expected}-" not in w and f"-{expected.replace('.', '_')}-" not in w] + if bad: + print(f"ERROR: Version mismatch. Expected {expected!r} in filename.", file=sys.stderr) + for w in bad: + print(f" {w}", file=sys.stderr) + sys.exit(1) + + print(f"PASS: {len(wheels)} wheel(s) match version {expected!r}") + + - name: Install built wheels for CVE scan + run: pip install dist/*.whl + + - name: Vulnerability scan (CVE gate — blocks publish on HIGH or higher) + run: | + pip install "pip-audit==${{ env.PIP_AUDIT_VERSION }}" + pip-audit --fail-on HIGH + + - name: Generate SBOM + run: | + pip install "cyclonedx-bom==${{ env.CYCLONEDX_BOM_VERSION }}" + cyclonedx-py environment -o sbom.json + echo "SBOM generated: sbom.json" + + - name: Upload SBOM as release artifact + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + with: + name: sbom-${{ inputs.version }} + path: sbom.json + + - name: Publish to PyPI (Trusted Publisher / OIDC + Sigstore attestations) + uses: pypa/gh-action-pypi-publish@76f52bc884231f62b9a034ebfe128415bbaabdfc # v1.12.4 + with: + packages-dir: dist/ + # attestations: true generates a Sigstore attestation automatically. + # Clients can verify with: pip download && python -m pypi_attestation_viewer + attestations: true diff --git a/.github/workflows/quality-gates.yml b/.github/workflows/quality-gates.yml new file mode 100644 index 0000000..31cde00 --- /dev/null +++ b/.github/workflows/quality-gates.yml @@ -0,0 +1,73 @@ +name: Quality gates + +on: + pull_request: + push: + branches: [main, master] + +# This workflow enforces Bandit, tests/coverage, and SonarQube. + +jobs: + bandit: + name: Bandit security scan + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Set up Python + uses: actions/setup-python@0b93645e9fea7318ecaed2b3599a311e5c0f207c # v5.3.0 + with: + python-version: "3.11" + - name: Install dependencies + run: python -m pip install --upgrade pip && pip install -e ".[dev,agents]" + - name: Generate Bandit JSON report + run: bandit -c pyproject.toml -r src -f json -o bandit-report.json + - name: Upload Bandit report artifact + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + with: + name: bandit-report + path: bandit-report.json + - name: Enforce high-severity Bandit gate + run: bandit -c pyproject.toml -r src --severity-level high + + test: + name: Tests and coverage + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Set up Python + uses: actions/setup-python@0b93645e9fea7318ecaed2b3599a311e5c0f207c # v5.3.0 + with: + python-version: "3.11" + - name: Install dependencies + run: python -m pip install --upgrade pip && pip install -e ".[dev,agents]" + - name: Run tests (coverage.xml generated via pyproject addopts) + run: pytest tests/ -v + - name: Upload coverage artifact + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + with: + name: coverage-xml + path: coverage.xml + if-no-files-found: error + + sonar: + name: SonarQube analysis + runs-on: ubuntu-latest + needs: [bandit, test] + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + - name: Download coverage artifact + uses: actions/download-artifact@fa0a91b85d4f404e442e0f254de474a18c628458 # v4.1.8 + with: + name: coverage-xml + path: . + - name: SonarQube scan (wait for quality gate) + uses: SonarSource/sonarqube-scan-action@v5.3.1 + env: + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }} + with: + args: > + -Dsonar.qualitygate.wait=true + -Dsonar.qualitygate.timeout=300 diff --git a/.gitignore b/.gitignore index 96bb6a5..f2f8975 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,23 @@ __pycache__/ dist/ .env .DS_Store -**/.DS_Store \ No newline at end of file +**/.DS_Store +.coverage +.coverage.* +htmlcov/ + +# GCP / cloud credentials +connectorplatform-*.json +*-service-account.json +*credentials*.json + +# Grafana exports (auto-generated) +grafana/*.json + +# Python lock files +uv.lock + +# Temporary test/upload files +upload-test.txt +*-test.txt +upload-*.txt \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..969ce00 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,9 @@ +# Bandit pre-commit hook. + +repos: + - repo: https://github.com/PyCQA/bandit + rev: 1.8.6 + hooks: + - id: bandit + args: ["-c", "pyproject.toml", "-r", "src"] + pass_filenames: false diff --git a/Dockerfile b/Dockerfile index 4afee60..ae02351 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # Node Wire — Docker Image # ======================== -# This image packages the connector platform as a FastMCP server. +# This image packages the connector platform as an MCP stdio server (manifest-driven). # ToolHive runs it as a container, injects secrets as env vars, # and proxies the stdio MCP transport to HTTP/SSE. # @@ -11,7 +11,8 @@ # thv run --name node-wire-connectors --transport stdio \ # --secret ... node-wire:latest -FROM python:3.12-slim +# Digest-pinned base (update when bumping tag). See .github/workflows/docker-policy.yml. +FROM python:3.12-slim@sha256:3d5ed973e45820f5ba5e46bd065bd88b3a504ff0724d85980dcd05eab361fcf4 # Install system deps needed by some connector libs RUN apt-get update && apt-get install -y --no-install-recommends \ @@ -21,19 +22,42 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ WORKDIR /app # Copy source (build context = repo root) -COPY pyproject.toml ./ COPY src/ ./src/ COPY config/ ./config/ - -# Install platform + agents extras -RUN pip install --no-cache-dir -e ".[agents]" +COPY packages/runtime/dist/*.whl /wheels/ +COPY packages/connectors/http_generic/dist/*.whl /wheels/ +COPY packages/connectors/stripe/dist/*.whl /wheels/ +COPY packages/connectors/smtp/dist/*.whl /wheels/ +COPY packages/connectors/google_drive/dist/*.whl /wheels/ +COPY packages/connectors/fhir_cerner/dist/*.whl /wheels/ +COPY packages/connectors/fhir_epic/dist/*.whl /wheels/ + +ENV PYTHONPATH=/app/src + +# Install runtime + connector packages using local wheel artifacts +RUN pip install --no-cache-dir --find-links=/wheels \ + node-wire-runtime \ + node-wire-http-generic \ + node-wire-stripe \ + node-wire-smtp \ + node-wire-google-drive \ + node-wire-fhir-cerner \ + node-wire-fhir-epic \ + "mcp>=1.6.0" \ + && rm -rf /wheels + +RUN groupadd --system --gid 1000 app \ + && useradd --system --uid 1000 --gid app --home /app app \ + && chown -R app:app /app + +USER app # Expose nothing — ToolHive manages the stdio proxy port internally # MCP_PORT / FASTMCP_PORT will be set by ToolHive if ever needed # Healthcheck: verify the package is importable HEALTHCHECK --interval=30s --timeout=5s --start-period=10s CMD \ - python -c "from agents.mcp_entrypoint import _make_server; print('ok')" || exit 1 + python -c "from agents.mcp_entrypoint import main; assert callable(main); print('ok')" || exit 1 -# Default entrypoint: run the FastMCP server on stdio +# Default entrypoint: run the MCP server on stdio CMD ["python", "-m", "agents.mcp_entrypoint"] diff --git a/README.md b/README.md index 66d68ab..19a07a3 100644 --- a/README.md +++ b/README.md @@ -10,24 +10,26 @@ For dependency management use any tool that understands `pyproject.toml` (e.g. ` Each connector can run as its own independent MCP server (Docker image). -| Image | Tool exposed | Docker image | -| ----------------------- | -------------------------- | -------------------------------- | -| `nw-google-drive` | `google_drive_upload_file` | `docker/google-drive/Dockerfile` | -| `nw-smartonfhir-epic` | `fhir_epic_read_patient` | `docker/fhir-epic/Dockerfile` | -| `nw-smartonfhir-cerner` | `fhir_cerner_read_patient` | `docker/fhir-cerner/Dockerfile` | -| `nw-smtp` | `smtp_send_email` | `docker/smtp/Dockerfile` | +| Image | MCP tools (manifest) | Docker image | +| ----------------------- | -------------------- | -------------------------------- | +| `nw-google-drive` | All `google_drive.` (e.g. `google_drive.files.upload`) | `docker/google-drive/Dockerfile` | +| `nw-smartonfhir-epic` | All `fhir_epic.` (e.g. `fhir_epic.read_patient`) | `docker/fhir-epic/Dockerfile` | +| `nw-smartonfhir-cerner` | All `fhir_cerner.` (e.g. `fhir_cerner.read_patient`) | `docker/fhir-cerner/Dockerfile` | +| `nw-smtp` | `smtp.send_email` | `docker/smtp/Dockerfile` | See [docs/mcp-servers.md](docs/mcp-servers.md) for build, env config, docker-compose, and ToolHive registration. +**Packaging & Publishing (PyPI wheels, CI publish flow, secrets config):** [docs/packaging.md](docs/packaging.md). + --- ## High-level architecture The platform is split into three layers: -- **Layer A – Runtime** (`runtime`): The engine that every connector runs inside. It defines the execution contract, a standard error taxonomy, retries and circuit breaking, and telemetry. -- **Layer B – Connectors** (`connectors`): Adapters that implement that contract and call external systems (HTTP Generic, SMTP, Stripe, Google Drive, FHIR Epic, FHIR Cerner). Each connector has its own input/output schema and business logic. -- **Layer C – Bindings** (`bindings`): How the platform is exposed to the outside world—REST API, gRPC server, MCP server—and how connectors are loaded from configuration (ConnectorFactory + `config/connectors.yaml`). +- **Layer A – Runtime** (`src/node_wire_runtime/`): The engine that every connector runs inside. It defines the execution contract, a standard error taxonomy, retries and circuit breaking, and telemetry. +- **Layer B – Connectors** (`src/node_wire_/`): Adapters that implement that contract and call external systems (HTTP Generic, SMTP, Stripe, Google Drive, FHIR Epic, FHIR Cerner). Each connector has its own input/output schema and business logic. +- **Layer C – Bindings** (`src/bindings/`): How the platform is exposed to the outside world—REST API, gRPC server, MCP server—and how connectors are loaded from configuration (ConnectorFactory + `config/connectors.yaml`). **Data flow (simplified):** A request arrives via REST, gRPC, or MCP → the factory resolves the right connector → the runtime runs it (validate input → optional policy check → retry/circuit-breaker wrapper → execute) → the response is returned in a standard shape (`ConnectorResponse`). @@ -37,7 +39,7 @@ The platform is split into three layers: **Purpose:** Provide shared execution and reliability so every connector behaves in a consistent way (validation, errors, retries, telemetry) without each connector reimplementing the same plumbing. -**Location:** `src/runtime/` (base.py, models.py, errors.py, resilience.py, secrets.py, policy.py). +**Location:** `src/node_wire_runtime/` (base_connector.py, models.py, errors.py, resilience.py, policy.py, observability.py, connector_registry.py, manifest.py). ### Main pieces @@ -74,7 +76,7 @@ The platform is split into three layers: **Purpose:** System adapters that talk to external services. Each connector defines input/output models and implements `internal_execute` (and optionally registers its own exceptions with the ErrorMapper). -**Location:** `src/connectors/`. Each connector lives in its own subpackage (e.g. `google_drive/`, `smtp/`, `stripe/`, `http_generic/`). +**Location:** `src/node_wire_/` (e.g. `src/node_wire_google_drive/`, `src/node_wire_smtp/`, `src/node_wire_stripe/`, `src/node_wire_http_generic/`). ### Common structure per connector @@ -91,14 +93,16 @@ The platform is split into three layers: | **smtp** | Send email via SMTP | `send_email` | rest, grpc, mcp | | **stripe** | Stripe charge | `charge` | grpc, mcp (no rest in config)| | **google_drive**| Google Drive (list, create, get, update, upload, delete, permissions) | `execute` (payload discriminator) | rest, grpc, mcp | -| **fhir_epic** | FHIR R4 integration for Epic (multi-action) | `read_patient`, `search_encounter`, `create_document_reference`, `search_document_reference` | rest, grpc, mcp | -| **fhir_cerner** | FHIR R4 integration for Cerner (multi-action) | `read_patient`, `search_encounter`, `create_document_reference`, `search_document_reference` | rest, grpc, mcp | +| **fhir_epic** | FHIR R4 integration for Epic (multi-action) | `read_patient`, `search_patients`, `search_encounter`, `create_document_reference`, `search_document_reference` | rest, grpc, mcp | +| **fhir_cerner** | FHIR R4 integration for Cerner (multi-action) | `read_patient`, `search_patients`, `search_encounter`, `create_document_reference`, `search_document_reference` | rest, grpc, mcp | ### Connector-specific documentation +**Connectors guide (`BaseConnector`, factory, manifest):** [docs/connectors.md](docs/connectors.md). + **Details for each connector**—operations, request/response bodies, examples, and error handling—**are documented in that connector’s folder.** -Examples: Google Drive has a full doc at `src/connectors/google_drive/README.md`; FHIR connectors are documented at `src/connectors/fhir_epic/README.md` and `src/connectors/fhir_cerner/README.md`. Other connectors may have a similar `.md` in their folder or document behavior in code and docstrings; always check the connector’s folder for up-to-date details. +Examples: Google Drive has a full doc at `src/node_wire_google_drive/README.md`; FHIR connectors are documented at `src/node_wire_fhir_epic/README.md` and `src/node_wire_fhir_cerner/README.md`. Other connectors may have a similar `.md` in their folder or document behavior in code and docstrings; always check the connector’s folder for up-to-date details. --- @@ -106,7 +110,7 @@ Examples: Google Drive has a full doc at `src/connectors/google_drive/README.md` **Purpose:** Expose connectors over different protocols and load them from configuration. No business logic lives here—only routing, config, and protocol translation. -**Location:** `src/bindings/` (factory.py, rest_api/app.py, grpc_server/, mcp_server/), and the entrypoint `bindings_entrypoint.py` at the package root. +**Location:** `src/bindings/` (factory.py, rest_api/app.py, grpc_server/, mcp_server/). The CLI entrypoint is the `node-wire` script, which maps to module **`bindings_entrypoint`** in `src/bindings_entrypoint.py` (run as `python -m bindings_entrypoint`). ### ConnectorFactory @@ -123,14 +127,14 @@ Examples: Google Drive has a full doc at `src/connectors/google_drive/README.md` ### gRPC / MCP - **gRPC:** Started when `MODE=GRPC`; server listens on port 50051. -- **MCP:** Started when `MODE=MCP`; server exposes tools for discovery and invocation. +- **MCP:** `MODE=MCP` starts a minimal MCP-style placeholder server (sufficient for local, manual inspection), but it is not the full stdio MCP server used for ToolHive and the agent layer. ### Entrypoint - Run with `python -m bindings_entrypoint` (or the `node-wire` script after install). The **MODE** environment variable selects: - **API** (default) – REST API on port 8000. - **GRPC** – gRPC server on port 50051. - - **MCP** – MCP server. + - **MCP** – minimal MCP-style placeholder server (see note above). --- @@ -142,7 +146,7 @@ Examples: Google Drive has a full doc at `src/connectors/google_drive/README.md` - `exposed_via`: list of protocols (`rest`, `grpc`, `mcp`). Only listed protocols expose that connector. - **Secrets** - Supplied via environment variables. The factory uses `EnvSecretProvider`; keys are connector-specific (e.g. Google Drive expects a variable documented in `src/connectors/google_drive/README.md`). + Supplied via environment variables. The factory uses `EnvSecretProvider`; keys are connector-specific (e.g. Google Drive expects a variable documented in `src/node_wire_google_drive/README.md`). ### Google Drive service account setup (quick) @@ -156,8 +160,7 @@ Examples: Google Drive has a full doc at `src/connectors/google_drive/README.md` Set credential secret used by this platform (`GOOGLE_DRIVE_SA_JSON`): -- **Option A (recommended for local):** set it to the absolute path of the JSON file. -- **Option B:** set it to the full JSON content as a string. +The connector expects the **full service account JSON as a string** (not a filesystem path). PowerShell example (load JSON content into env var for current shell): @@ -174,6 +177,7 @@ $env:GOOGLE_DRIVE_SA_JSON = Get-Content -Path $saPath -Raw ```bash pip install . ``` + For agents, ToolHive, or the stdio MCP server (`python -m agents.mcp_entrypoint`), install with optional dependencies, e.g. `pip install -e ".[agents]"`, or follow **[Setup.md](Setup.md)** for the full install matrix. 2. **Start the REST API** (default): - **Windows (cmd):** `set MODE=API && python -m bindings_entrypoint` @@ -193,3 +197,12 @@ $env:GOOGLE_DRIVE_SA_JSON = Get-Content -Path $saPath -Raw ## Dependencies All dependencies are declared in `pyproject.toml` (Python >=3.11). They include: pydantic, FastAPI, uvicorn, tenacity, pybreaker, OpenTelemetry, grpcio, and connector-specific libraries (httpx, aiosmtplib, stripe, google-auth, google-api-python-client, etc.). See `pyproject.toml` for the full list and versions. + +--- + +## Setup and development docs + +- Platform setup (REST/gRPC/agents MCP): [Setup.md](Setup.md) +- Individual connector MCP servers (ToolHive): [docs/mcp-servers.md](docs/mcp-servers.md) +- Creating a new connector: [docs/connectors.md](docs/connectors.md) +- Quality/security gates (Bandit, SonarQube): [docs/quality-security-gates.md](docs/quality-security-gates.md) diff --git a/Setup.md b/Setup.md index 7be559f..6441d8d 100644 --- a/Setup.md +++ b/Setup.md @@ -23,10 +23,11 @@ Node Wire is a Python framework that runs connector adapters (Google Drive, SMTP | Requirement | Version | Notes | | ----------- | ------- | --------------------------------------- | -| Python | 3.12+ | `python --version` to check | +| Python | 3.11+ | `python --version` to check | | pip or uv | Latest | `pip install --upgrade pip` | | Git | Any | To clone the repo | | Docker | Latest | Only needed for ToolHive MCP deployment | +| Node.js | Any LTS | Only needed for `npx @modelcontextprotocol/inspector` | --- @@ -36,7 +37,7 @@ Node Wire is a Python framework that runs connector adapters (Google Drive, SMTP ```bash # 1. Clone the repository git clone -cd connector-platform +cd # the folder git creates (rename if you like) # 2. Install dependencies (recommended: uv) uv sync --extra agents @@ -45,6 +46,8 @@ uv sync --extra agents uv run node-wire --help ``` +> **Install uv:** See the official installer docs at `https://docs.astral.sh/uv/`. +> > **REST/gRPC only** (no AI agent features): `uv sync` without the extra is sufficient. > > **Alternative (pip):** If you’re not using `uv`, install editable deps with pip: @@ -52,6 +55,8 @@ uv run node-wire --help > - `pip install -e ".[agents]"` (includes MCP/LLM agent dependencies) > - `pip install -e .` (REST/gRPC only, no agent dependencies) +> **Installing from PyPI wheels instead of source?** See [docs/packaging.md](docs/packaging.md) for the wheel build lifecycle, client install model, and pre-publish validation checklist. + --- ## Configuration @@ -67,6 +72,8 @@ cp sample.env .env You only need to fill in the sections for the connectors you plan to use. The platform starts successfully even if some credentials are missing — those connectors will simply return an error when called. +> **Doc convention:** Environment variable names in the docs follow `sample.env`. Some legacy keys (like `stripe_api_key`) are intentionally lower-case because that is what the connector reads. + ### Environment Variable Sections @@ -74,7 +81,7 @@ You only need to fill in the sections for the connectors you plan to use. The pl | ---------------- | ------------------------------------------------------------------------------------------------------------------- | ---------------------- | | **FHIR Epic** | `EPIC_FHIR_BASE_URL`, `EPIC_TOKEN_URL`, `EPIC_CLIENT_ID`, `EPIC_KID`, `EPIC_PRIVATE_KEY` | Epic EHR integration | | **FHIR Cerner** | `CERNER_FHIR_BASE_URL`, `CERNER_TOKEN_URL`, `CERNER_CLIENT_ID`, `CERNER_KID`, `CERNER_PRIVATE_KEY`, `CERNER_SCOPES` | Cerner EHR integration | -| **Google Drive** | `google_drive_sa_json`, `GOOGLE_DRIVE_FOLDER_ID` | Google Drive connector | +| **Google Drive** | `GOOGLE_DRIVE_SA_JSON`, `GOOGLE_DRIVE_FOLDER_ID` | Google Drive connector | | **SMTP** | `SMTP_HOST`, `SMTP_PORT`, `SMTP_USERNAME`, `SMTP_PASSWORD` | Sending emails | | **LLM / Agent** | `LLM_PROVIDER`, `GROQ_API_KEY` (or other provider key) | AI agent / ToolHive | | **ToolHive** | `TOOLHIVE_MCP_URL` (single) or `TOOLHIVE_MCP_URLS` (comma-separated, multi-server) | ToolHive MCP proxy | @@ -95,10 +102,26 @@ The platform supports three modes. Set the `MODE` environment variable to switch | **gRPC** | `MODE=GRPC uv run node-wire` | `50051` | gRPC clients | | **MCP (stdio)** | `python -m agents.mcp_entrypoint` | stdio | AI agents, ToolHive, Claude Desktop | +> **Important:** `MODE=MCP` for `node-wire` / `python -m bindings_entrypoint` starts a minimal MCP-style placeholder server, not the full stdio MCP server used with ToolHive and the agent layer. For ToolHive/Inspector/agents, use `python -m agents.mcp_entrypoint` (or the per-connector MCP servers in `docs/mcp-servers.md`). + +### Configuration file (`config/connectors.yaml`) + +Connectors are loaded from `config/connectors.yaml`. Each connector has: + +- `enabled`: whether the connector is instantiated at startup +- `exposed_via`: which protocols can access it (`rest`, `grpc`, `mcp`) + +If a connector is disabled (or not exposed for a protocol), requests to it will fail with “not configured / not available” even if your `.env` is correct. + +For details on adding a new connector to the runtime, see [docs/connectors.md](docs/connectors.md). + ### REST API Quick Start ```bash +# Local development: disable REST auth (do not use in production) +export NW_REST_AUTH_DISABLED=true + # Default port 8000 uv run node-wire @@ -106,16 +129,25 @@ uv run node-wire PORT=8001 uv run node-wire ``` +**Production / secured REST:** set `NW_REST_API_KEY` and send `Authorization: Bearer ` or `X-API-Key: ` on every route except `GET /health`. Set `NW_REST_LOAD_DOTENV=false` so secrets are not loaded from a `.env` file. See [docs/connectors.md](docs/connectors.md) (Security section). + +Equivalent entrypoint (without `uv`): + +```bash +MODE=API python -m bindings_entrypoint +``` + Once running: -- **Health check:** `GET http://localhost:8000/health` -- **Interactive docs (Swagger UI):** `http://localhost:8000/docs` +- **Health check (no auth):** `GET http://localhost:8000/health` +- **Interactive docs (Swagger UI):** `http://localhost:8000/docs` (requires API key when auth is enabled) - **Call a connector:** `POST http://localhost:8000/connectors/{connector_id}/{action}` -Example — send an HTTP request via the generic connector: +Example — send an HTTP request via the generic connector (with auth enabled): ```bash curl -X POST http://localhost:8000/connectors/http_generic/request \ + -H "Authorization: Bearer $NW_REST_API_KEY" \ -H "Content-Type: application/json" \ -d '{"url": "https://httpbin.org/get", "method": "GET"}' ``` @@ -137,6 +169,9 @@ All responses use the same standard shape: ## Connectors Overview +**Developer guide (`BaseConnector`, config, factory):** [docs/connectors.md](docs/connectors.md). + + | Connector | What It Does | Credentials Needed | Setup Guide | | ---------------- | ------------------------------------------ | -------------------------------------- | --------------------------------------------------------------------------------------------- | @@ -156,6 +191,11 @@ All responses use the same standard shape: No credentials required. Works out of the box. +Security defaults: +- Allowed methods: `GET`, `POST`, `PUT`, `PATCH`, `DELETE` (input is normalized to uppercase). +- Internal targets are blocked: `localhost`, loopback, private/link-local IPs, and metadata endpoints. +- Connector logs omit URL query strings and fragments (scheme/host/path only). + ```bash curl -X POST http://localhost:8000/connectors/http_generic/request \ -H "Content-Type: application/json" \ @@ -194,7 +234,7 @@ Supported configurations: Add to your `.env`: ```env -stripe_api_key=sk_test_your_key_here +STRIPE_API_KEY=sk_test_your_key_here ``` Use a **test key** (`sk_test_...`) during development. Switch to a live key (`sk_live_...`) for production. @@ -215,8 +255,10 @@ Quick summary of what you'll need: Add to your `.env`: +`GOOGLE_DRIVE_SA_JSON` must be the **full service account JSON as a string** (not a filesystem path). For local development you can paste minified JSON on one line, or load the file into the variable in your shell (see [docs/google_drive_connector.md](docs/google_drive_connector.md)). + ```env -google_drive_sa_json=/absolute/path/to/service-account.json +GOOGLE_DRIVE_SA_JSON={"type":"service_account",...} GOOGLE_DRIVE_FOLDER_ID=your-folder-id-from-drive-url ``` @@ -238,7 +280,7 @@ EPIC_PRIVATE_KEY="-----BEGIN RSA PRIVATE KEY-----\n...\n-----END RSA PRIVATE KEY You obtain these credentials by registering a backend application in the [Epic App Orchard](https://appmarket.epic.com/) (or your organization's Epic sandbox). -**Available actions:** `read_patient`, `search_encounter`, `create_document_reference`, `search_document_reference` +**Available actions:** `read_patient`, `search_patients`, `search_encounter`, `create_document_reference`, `search_document_reference` --- @@ -259,7 +301,7 @@ CERNER_SCOPES="system/Patient.read system/Encounter.read system/DocumentReferenc Register your application in the [Cerner Developer Portal](https://code.cerner.com/) to obtain these credentials. -**Available actions:** `read_patient`, `search_encounter`, `create_document_reference`, `search_document_reference` +**Available actions:** `read_patient`, `search_patients`, `search_encounter`, `create_document_reference`, `search_document_reference` --- @@ -272,23 +314,32 @@ The platform exposes connector tools for AI agents via the MCP (Model Context Pr Each connector runs as its own independent MCP server. This is the preferred approach for modular, scalable deployments. -| Image | Tool exposed | Docker image | -| ----------------------- | -------------------------- | -------------------------------- | -| `nw-google-drive` | `google_drive_upload_file` | `docker/google-drive/Dockerfile` | -| `nw-smartonfhir-epic` | `fhir_epic_read_patient` | `docker/fhir-epic/Dockerfile` | -| `nw-smartonfhir-cerner` | `fhir_cerner_read_patient` | `docker/fhir-cerner/Dockerfile` | -| `nw-smtp` | `smtp_send_email` | `docker/smtp/Dockerfile` | +| Image | MCP tools (manifest) | Docker image | +| ----------------------- | -------------------- | -------------------------------- | +| `nw-google-drive` | All `google_drive.` (e.g. `google_drive.files.upload`) | `docker/google-drive/Dockerfile` | +| `nw-smartonfhir-epic` | All `fhir_epic.` (e.g. `fhir_epic.read_patient`) | `docker/fhir-epic/Dockerfile` | +| `nw-smartonfhir-cerner` | All `fhir_cerner.` (e.g. `fhir_cerner.read_patient`) | `docker/fhir-cerner/Dockerfile` | +| `nw-smtp` | `smtp.send_email` | `docker/smtp/Dockerfile` | **Full guide (build, env config, ToolHive registration, multi-server agent usage):** [docs/mcp-servers.md](docs/mcp-servers.md) +**FHIR tool arguments (Cerner / Epic)** — tool names are `fhir_cerner.` and `fhir_epic.`. Use field names from `tools/list` / the connector manifest. Typical payloads: + +| Action | When to use | Example arguments | +| ------ | ----------- | ------------------- | +| `read_patient` | You have a Patient id | `{"resource_id": "12724066"}` (Epic ids often start with `e`) | +| `search_patients` | No id, or name-based search | `{"resource_ids": ["id1"]}` or `{"given_name": "...", "family_name": "..."}` or `{"search_params": {"identifier": "...", "family": "..."}}` (FHIR search param names) | + +The MCP server normalizes common LLM/legacy aliases (`patientId` / `patient_id` → `resource_id`; `patientId` inside `search_params` → `identifier`) before validation. Prefer canonical fields above when authoring prompts or clients. + Quick start: ```bash -# Build all three images +# Build all four images ./scripts/build-mcp-images.sh -# Start all three locally +# Start all four locally docker compose -f docker-compose.mcp.yml up ``` @@ -315,6 +366,12 @@ npx @modelcontextprotocol/inspector python -m agents.google_drive_mcp npx @modelcontextprotocol/inspector python -m agents.mcp_entrypoint ``` +### Troubleshooting quick hits + +- **Port 8000 in use**: set `PORT=8001` (or any free port) when starting the REST API. +- **Connector “not configured”**: confirm it is `enabled: true` (and exposed for your protocol) in `config/connectors.yaml`. +- **Google Drive auth failure**: `GOOGLE_DRIVE_SA_JSON` must be valid **inline JSON** (not a path). Use PowerShell `Get-Content -Raw` or equivalent to load a key file into the variable (see [docs/google_drive_connector.md](docs/google_drive_connector.md)). + --- ## Running Tests @@ -336,6 +393,57 @@ Most tests are unit tests that run without real credentials. Integration tests t --- +## Code quality and security gates + +Node Wire enforces security and coverage-backed analysis in CI for pull requests and pushes to `main`/`master`: + +- Bandit (`bandit -c pyproject.toml -r src --severity-level high`) for Python SAST. +- SonarQube Community Edition scan with `sonar.qualitygate.wait=true` so PRs fail when the quality gate fails. + +### Run checks locally + +```bash +# Install dev tools +pip install -e ".[dev,agents]" + +# Security (matches CI) +bandit -c pyproject.toml -r src --severity-level high + +# Tests + coverage.xml (required by SonarQube) +pytest tests/ -v +``` + +### Pre-commit + +```bash +pre-commit install +pre-commit run --all-files +``` + +### Run SonarQube scan locally (Docker) + +```bash +# from repository root, after coverage.xml is generated +docker run --rm \ + -e SONAR_TOKEN=YOUR_TOKEN \ + -v "G:\SPACE\node-wire:/usr/src" \ + -w /usr/src \ + sonarsource/sonar-scanner-cli \ + -Dsonar.host.url=http://host.docker.internal:9000 \ + -Dsonar.token=YOUR_TOKEN +``` + +### SonarQube configuration + +The repository includes [`sonar-project.properties`](sonar-project.properties) and CI expects these GitHub secrets: + +- `SONAR_HOST_URL` (example: `https://sonarqube.company.internal`) +- `SONAR_TOKEN` (project analysis token) + +For server setup and quality gate policy details, see [docs/quality-security-gates.md](docs/quality-security-gates.md). + +--- + ## Playground UI The repository includes an interactive web playground that showcases 5 orchestration scenarios: diff --git a/config/connectors.yaml b/config/connectors.yaml index 0c1682c..fae37f0 100644 --- a/config/connectors.yaml +++ b/config/connectors.yaml @@ -1,3 +1,17 @@ +# Connector enable/list — see docs/connectors.md +# +# REST API auth (not stored here; set in environment): +# NW_REST_API_KEY — required for /connectors, /playground, /scenarios unless NW_REST_AUTH_DISABLED=true +# NW_REST_LOAD_DOTENV=false — recommended in production (no .env file) +# +# Plugin allowlist (optional): +# NW_ALLOWED_CONNECTORS=fhir_epic,http_generic,... +# NW_CONNECTOR_MODULE_PREFIX=node_wire_ +# +# Secrets backend (optional): +# NW_SECRET_BACKEND=env | aws_env +# NW_AWS_SECRETS_MANAGER_SECRET_ID=... (when aws_env) +# connectors: http_generic: enabled: true diff --git a/docker/fhir-cerner/Dockerfile b/docker/fhir-cerner/Dockerfile index f53bb53..e0bf8f3 100644 --- a/docker/fhir-cerner/Dockerfile +++ b/docker/fhir-cerner/Dockerfile @@ -1,8 +1,8 @@ -FROM python:3.12-slim +FROM python:3.12-slim@sha256:3d5ed973e45820f5ba5e46bd065bd88b3a504ff0724d85980dcd05eab361fcf4 LABEL org.opencontainers.image.title="nw-smartonfhir-cerner" \ org.opencontainers.image.description="Node Wire — SMART on FHIR Cerner MCP server" \ - org.opencontainers.image.source="https://github.com/AOT-Technologies/connector-platform" + org.opencontainers.image.source="https://github.com/AOT-Technologies/node-wire" RUN apt-get update && apt-get install -y --no-install-recommends \ curl ca-certificates \ @@ -10,14 +10,26 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ WORKDIR /app -COPY pyproject.toml ./ COPY src/ ./src/ COPY config/ ./config/ +COPY packages/runtime/dist/*.whl /wheels/ +COPY packages/connectors/fhir_cerner/dist/*.whl /wheels/ -RUN pip install --no-cache-dir -e ".[agents]" +ENV PYTHONPATH=/app/src \ + NW_ALLOWED_CONNECTORS=fhir_cerner + +RUN pip install --no-cache-dir --find-links=/wheels \ + node-wire-runtime node-wire-fhir-cerner "mcp>=1.6.0" \ + && rm -rf /wheels + +RUN groupadd --system --gid 1000 app \ + && useradd --system --uid 1000 --gid app --home /app app \ + && chown -R app:app /app + +USER app HEALTHCHECK --interval=30s --timeout=5s --start-period=10s CMD \ - python -c "from agents.fhir_cerner_mcp import _make_server; print('ok')" || exit 1 + python -c "from agents.fhir_cerner_mcp import main; assert callable(main); print('ok')" || exit 1 CMD ["python", "-m", "agents.fhir_cerner_mcp"] diff --git a/docker/fhir-epic/Dockerfile b/docker/fhir-epic/Dockerfile index 633f031..5d7aa08 100644 --- a/docker/fhir-epic/Dockerfile +++ b/docker/fhir-epic/Dockerfile @@ -1,8 +1,8 @@ -FROM python:3.12-slim +FROM python:3.12-slim@sha256:3d5ed973e45820f5ba5e46bd065bd88b3a504ff0724d85980dcd05eab361fcf4 LABEL org.opencontainers.image.title="nw-smartonfhir-epic" \ org.opencontainers.image.description="Node Wire — SMART on FHIR Epic MCP server" \ - org.opencontainers.image.source="https://github.com/AOT-Technologies/connector-platform" + org.opencontainers.image.source="https://github.com/AOT-Technologies/node-wire" RUN apt-get update && apt-get install -y --no-install-recommends \ curl ca-certificates \ @@ -10,14 +10,26 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ WORKDIR /app -COPY pyproject.toml ./ COPY src/ ./src/ COPY config/ ./config/ +COPY packages/runtime/dist/*.whl /wheels/ +COPY packages/connectors/fhir_epic/dist/*.whl /wheels/ -RUN pip install --no-cache-dir -e ".[agents]" +ENV PYTHONPATH=/app/src \ + NW_ALLOWED_CONNECTORS=fhir_epic + +RUN pip install --no-cache-dir --find-links=/wheels \ + node-wire-runtime node-wire-fhir-epic "mcp>=1.6.0" \ + && rm -rf /wheels + +RUN groupadd --system --gid 1000 app \ + && useradd --system --uid 1000 --gid app --home /app app \ + && chown -R app:app /app + +USER app HEALTHCHECK --interval=30s --timeout=5s --start-period=10s CMD \ - python -c "from agents.fhir_epic_mcp import _make_server; print('ok')" || exit 1 + python -c "from agents.fhir_epic_mcp import main; assert callable(main); print('ok')" || exit 1 CMD ["python", "-m", "agents.fhir_epic_mcp"] diff --git a/docker/google-drive/Dockerfile b/docker/google-drive/Dockerfile index 43cbe2b..b73c983 100644 --- a/docker/google-drive/Dockerfile +++ b/docker/google-drive/Dockerfile @@ -1,8 +1,8 @@ -FROM python:3.12-slim +FROM python:3.12-slim@sha256:3d5ed973e45820f5ba5e46bd065bd88b3a504ff0724d85980dcd05eab361fcf4 LABEL org.opencontainers.image.title="nw-google-drive" \ org.opencontainers.image.description="Node Wire — Google Drive MCP server" \ - org.opencontainers.image.source="https://github.com/AOT-Technologies/connector-platform" + org.opencontainers.image.source="https://github.com/AOT-Technologies/node-wire" RUN apt-get update && apt-get install -y --no-install-recommends \ curl ca-certificates \ @@ -10,14 +10,26 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ WORKDIR /app -COPY pyproject.toml ./ COPY src/ ./src/ COPY config/ ./config/ +COPY packages/runtime/dist/*.whl /wheels/ +COPY packages/connectors/google_drive/dist/*.whl /wheels/ -RUN pip install --no-cache-dir -e ".[agents]" +ENV PYTHONPATH=/app/src \ + NW_ALLOWED_CONNECTORS=google_drive + +RUN pip install --no-cache-dir --find-links=/wheels \ + node-wire-runtime node-wire-google-drive "mcp>=1.6.0" \ + && rm -rf /wheels + +RUN groupadd --system --gid 1000 app \ + && useradd --system --uid 1000 --gid app --home /app app \ + && chown -R app:app /app + +USER app HEALTHCHECK --interval=30s --timeout=5s --start-period=10s CMD \ - python -c "from agents.google_drive_mcp import _make_server; print('ok')" || exit 1 + python -c "from agents.google_drive_mcp import main; assert callable(main); print('ok')" || exit 1 CMD ["python", "-m", "agents.google_drive_mcp"] diff --git a/docker/smtp/Dockerfile b/docker/smtp/Dockerfile index c4d725b..276d094 100644 --- a/docker/smtp/Dockerfile +++ b/docker/smtp/Dockerfile @@ -1,8 +1,8 @@ -FROM python:3.12-slim +FROM python:3.12-slim@sha256:3d5ed973e45820f5ba5e46bd065bd88b3a504ff0724d85980dcd05eab361fcf4 LABEL org.opencontainers.image.title="nw-smtp" \ org.opencontainers.image.description="Node Wire — SMTP MCP server" \ - org.opencontainers.image.source="https://github.com/AOT-Technologies/connector-platform" + org.opencontainers.image.source="https://github.com/AOT-Technologies/node-wire" RUN apt-get update && apt-get install -y --no-install-recommends \ curl ca-certificates \ @@ -10,14 +10,26 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ WORKDIR /app -COPY pyproject.toml ./ COPY src/ ./src/ COPY config/ ./config/ +COPY packages/runtime/dist/*.whl /wheels/ +COPY packages/connectors/smtp/dist/*.whl /wheels/ -RUN pip install --no-cache-dir -e ".[agents]" +ENV PYTHONPATH=/app/src \ + NW_ALLOWED_CONNECTORS=smtp + +RUN pip install --no-cache-dir --find-links=/wheels \ + node-wire-runtime node-wire-smtp "mcp>=1.6.0" \ + && rm -rf /wheels + +RUN groupadd --system --gid 1000 app \ + && useradd --system --uid 1000 --gid app --home /app app \ + && chown -R app:app /app + +USER app HEALTHCHECK --interval=30s --timeout=5s --start-period=10s CMD \ - python -c "from agents.smtp_mcp import _make_server; print('ok')" || exit 1 + python -c "from agents.smtp_mcp import main; assert callable(main); print('ok')" || exit 1 CMD ["python", "-m", "agents.smtp_mcp"] diff --git a/docs/connectors.md b/docs/connectors.md new file mode 100644 index 0000000..7a3c9ea --- /dev/null +++ b/docs/connectors.md @@ -0,0 +1,481 @@ +# Connectors guide (`src/node_wire_*`) + +This guide explains how **connectors** fit into Node Wire, how to build your own connector, and how the runtime and bindings wire everything together. Connector implementations live under `src/node_wire_/` (e.g. `src/node_wire_google_drive/`); the shared base class lives at **`src/node_wire_runtime/base_connector.py`**. + +## How connectors fit into the platform + +- **Layer B — Connectors** (`src/node_wire_/`): adapter packages (schemas, logic, optional `registration.py`). +- **Layer C — Bindings** (`src/bindings/`): REST, gRPC, and MCP servers plus `ConnectorFactory` loading from `config/connectors.yaml`. + +At startup, bindings call **`node_wire_runtime.connector_registry.auto_register()`**, which loads connector entry points, imports each connector’s `logic` module (registering the class), then imports optional `registration.py` for `ErrorMapper` side effects. **`ConnectorFactory`** resolves connectors from the registry — **do not add per-connector branches in `src/bindings/factory.py`.** + +--- + +## Package layout and registration + +Each connector is a **top-level package** under `src/` (e.g. `node_wire_fhir_epic`): + +| File | Role | +|------|------| +| `schema.py` | Pydantic input/output models. Each input model has an `action: Literal[...]` discriminator field (often combined into a discriminated union). | +| `logic.py` | Connector class: `BaseConnector` subclass — either explicit `@nw_action` methods, or **`action_specs`** plus an optional `_execute_action_spec` override for SDK dispatch. | +| `action_spec.py` (optional) | Declarative `SdkActionSpec` entries mapping validated models to vendor SDK calls (see Google Drive). | +| `registration.py` | Optional: registers connector-specific exceptions with `ErrorMapper`. | +| `exceptions.py` | Optional: custom exception types. | + +At startup, call **`node_wire_runtime.connector_registry.auto_register()`**: it loads entry points in group `node_wire.connectors`, imports each connector's `logic` module (triggering `BaseConnector.__init_subclass__` and `_CONNECTOR_REGISTRY`), then imports optional `registration.py` for `ErrorMapper` side effects. + +--- + +## The unified `BaseConnector` + +There is one base class for all connectors: **`BaseConnector`** (`src/node_wire_runtime/base_connector.py`). It handles: + +- Input validation via a Pydantic **discriminated union** (the `action` field selects the right model) +- Optional **policy hook** enforcement +- **Retries and circuit breaking** via `with_resilience` +- **Error mapping** via `ErrorMapper` +- OpenTelemetry **tracing** +- A standard **`ConnectorResponse`** envelope + +Actions are declared either with the **`@nw_action("name")`** decorator on async methods, or by listing them in **`action_specs`** (the runtime generates equivalent handlers). A connector can have **one or many** actions — there is no separate "single-action" type. + +``` +flowchart LR + yaml[connectors.yaml] + factory[ConnectorFactory.load] + inst[BaseConnector subclass] + run[connector.run] + exec[internal_execute → @nw_action dispatch] + resp[ConnectorResponse] + yaml --> factory --> inst --> run --> exec --> resp +``` + +--- + +## Building a connector (Google Drive SDK example) + +The production **Google Drive** connector (`src/node_wire_google_drive/`) is a good template for wrapping a **vendor Python SDK** (here `googleapiclient` / Drive API v3): service-account auth in `build_client()`, a discriminated union of operations in `schema.py`, and **`action_specs`** so each API surface becomes a manifest action without duplicating boilerplate. + +### Step 1 — Define your schemas (`schema.py`) + +Each operation is a Pydantic model with an **`action`** field whose type is a `Literal["…"]` unique to that operation. Those models are combined into a **discriminated union** (and often wrapped in `RootModel` for a single top-level validator), which the runtime uses to pick the correct handler. + +```python +# src/node_wire_google_drive/schema.py (conceptual excerpt) +from __future__ import annotations + +from typing import Annotated, Literal, Optional, Union + +from pydantic import BaseModel, ConfigDict, Field, RootModel + + +class BaseDriveOperation(BaseModel): + model_config = ConfigDict(extra="forbid") + + +class FilesListOperation(BaseDriveOperation): + action: Literal["files.list"] + page_size: int = Field(10, ge=1, le=100) + query: Optional[str] = None + fields: Optional[str] = None + page_token: Optional[str] = None + + +class FilesUploadOperation(BaseDriveOperation): + action: Literal["files.upload"] + name: str + mime_type: str + parents: Optional[list[str]] = None + content: Optional[str] = None + content_base64: Optional[str] = None + + +# …other operations (files.create, files.get, …) — see the repo. + +_GoogleDriveOperationUnion = Annotated[ + Union[ + FilesListOperation, + FilesUploadOperation, + # … FilesCreateOperation, FilesGetOperation, … + ], + Field(discriminator="action"), +] + +GoogleDriveOperationInput = RootModel[_GoogleDriveOperationUnion] + + +class GoogleDriveOperationOutput(BaseModel): + raw: dict + description: str +``` + +When a connector only has **one** action, the `action` field is still required — the runtime always validates through the discriminated union. + +### Step 2 — Map operations to the SDK (`action_spec.py`) + +**`SdkActionSpec`** describes how to turn a validated model into a single SDK call: resource path (`resource_segments`), HTTP-style method name (`method_name`), and how to build `body` / keyword arguments from the model. The full Drive registry lives in [`src/node_wire_google_drive/action_spec.py`](../src/node_wire_google_drive/action_spec.py). + +```python +# src/node_wire_google_drive/action_spec.py (illustrative) +from node_wire_runtime.sdk_action_spec import SdkActionSpec + +from .schema import FilesCreateOperation, FilesListOperation + +# def _build_files_list_kwargs(drive, model): ... + +# Real module builds this dict via register helpers — see repo for uploads, permissions, etc. + +GOOGLE_DRIVE_ACTION_SPECS: dict[str, SdkActionSpec] = { + "files.list": SdkActionSpec( + resource_segments=("files",), + method_name="list", + build_kwargs=_build_files_list_kwargs, # optional: defaults, shared drives flags + input_model=FilesListOperation, + ), + "files.create": SdkActionSpec( + resource_segments=("files",), + method_name="create", + body_from_model={"name": "name", "mime_type": "mimeType", "parents": "parents"}, + constant_kwargs={"fields": "id, name, webViewLink", "supportsAllDrives": True}, + input_model=FilesCreateOperation, + ), +} +``` + +`googleapiclient` is **synchronous**. The shared helper **`execute_spec_in_thread`** runs the generated `.execute()` call in a thread pool so the connector’s public API stays async. + +### Step 3 — Implement the connector class (`logic.py`) + +Subclass `BaseConnector`, set **`connector_id`**, **`output_model`**, and **`action_specs`**. The base class **generates** one async `@nw_action` handler per spec. Override **`_execute_action_spec`** to add logging, thread offload, and translation of vendor exceptions (e.g. `HttpError` → your `error_map` types). + +```python +# src/node_wire_google_drive/logic.py (conceptual excerpt) +from __future__ import annotations + +import json +from typing import Any + +from google.oauth2 import service_account +from googleapiclient.discovery import build +from googleapiclient.errors import HttpError + +from node_wire_runtime import BaseConnector +from node_wire_runtime.models import ErrorCategory +from node_wire_runtime.sdk_action_spec import execute_spec_in_thread + +from .action_spec import GOOGLE_DRIVE_ACTION_SPECS +from .exceptions import GoogleDriveAuthError, GoogleDriveRateLimitError # + other mapped types +from .schema import GoogleDriveOperationOutput + + +class GoogleDriveConnector(BaseConnector): + connector_id = "google_drive" + output_model = GoogleDriveOperationOutput + action_specs = GOOGLE_DRIVE_ACTION_SPECS + + error_map = { + GoogleDriveAuthError: (ErrorCategory.AUTH, "GDRIVE_AUTH"), + GoogleDriveRateLimitError: (ErrorCategory.RETRYABLE, "GDRIVE_RATE_LIMIT"), + # … + } + + def build_client(self) -> Any: + raw_sa = self.secret_provider.get_secret("GOOGLE_DRIVE_SA_JSON") + info = json.loads(raw_sa) # production code: inline JSON only; see node_wire_google_drive.logic + creds = service_account.Credentials.from_service_account_info( + info, + scopes=["https://www.googleapis.com/auth/drive"], + ) + return build("drive", "v3", credentials=creds) + + async def _execute_action_spec( + self, + action_name: str, + params: Any, + *, + trace_id: str, + log_extra: dict[str, Any] | None = None, + ) -> GoogleDriveOperationOutput: + spec = GOOGLE_DRIVE_ACTION_SPECS[action_name] + drive = self.get_client() + try: + raw = await execute_spec_in_thread(drive, spec, params) + except HttpError as exc: + self._translate_and_raise_http_error(exc) + return GoogleDriveOperationOutput( + raw=raw, + description=f"Successfully executed {action_name}", + ) +``` + +Key points: +- **`connector_id`** — unique string; used for routing, config, and registry lookup. +- **`output_model`** — the Pydantic class returned by every action (Drive uses one shared envelope with `raw` + `description`). +- **`error_map`** — maps exception types to `(ErrorCategory, error_code)`. Entries are registered with `ErrorMapper` automatically at class definition time. +- **`build_client()`** — override to create the Google API client. `get_client()` caches the result in `self._client`. +- **`action_specs`** — each key becomes a manifest action (e.g. `files.list`). Do **not** also add a manual `@nw_action` with the same name. +- **`_execute_action_spec`** — **required** when using **`action_specs`**: each generated handler delegates here. Typically call **`execute_spec_in_thread`** for blocking SDKs (such as `googleapiclient`). Connectors that only use hand-written `@nw_action` methods do not implement this hook. + +**Adding a new Drive operation:** add a Pydantic variant and extend the union in `schema.py`, register a new `SdkActionSpec` in `action_spec.py`, and rely on auto-generated handlers (see [`src/node_wire_google_drive/README.md`](../src/node_wire_google_drive/README.md)). + +### Step 4 — Register in `config/connectors.yaml` + +```yaml +connectors: + google_drive: + enabled: true + exposed_via: + - rest + - grpc + - mcp +``` + +`exposed_via` controls which bindings surface the connector. Use any subset of **`rest`**, **`grpc`**, and **`mcp`** (omit protocols you do not need). + +### Step 5 — Auto-registration (nothing extra needed) + +`BaseConnector.__init_subclass__` adds your class to `_CONNECTOR_REGISTRY[connector_id]` as soon as `logic.py` is imported. **`node_wire_runtime.connector_registry.auto_register()`** performs those imports at startup. **No manual factory branch is required.** + +--- + +## Single-action connector example + +A connector with one action is identical in structure — just add one `@nw_action` method: + +```python +# src/node_wire_sms/schema.py +from __future__ import annotations +from typing import Literal +from pydantic import BaseModel + +class SmsSendInput(BaseModel): + action: Literal["send"] = "send" + to: str + message: str + +class SmsSendOutput(BaseModel): + message_sid: str + status: str +``` + +```python +# src/node_wire_sms/logic.py +from __future__ import annotations + +from node_wire_runtime import BaseConnector, nw_action +from .schema import SmsSendInput, SmsSendOutput + + +class SmsConnector(BaseConnector): + connector_id = "sms" + output_model = SmsSendOutput + + @nw_action("send") + async def send(self, params: SmsSendInput, *, trace_id: str) -> SmsSendOutput: + api_key = self.secret_provider.get_secret("sms_api_key") + # ... call SMS vendor API ... + return SmsSendOutput(message_sid="SM123", status="queued") +``` + +--- + +## Calling a connector directly (in-process) + +Use `connector.run(dict)` for the full pipeline (validation, policy, retries, error mapping): + +```python +from node_wire_runtime.connector_registry import auto_register +from bindings.factory import ConnectorFactory + +auto_register() +factory = ConnectorFactory() +factory.load() + +connector = factory.get_for_protocol("google_drive", "rest", action="files.list") +response = await connector.run( + {"action": "files.list", "page_size": 10, "query": "mimeType = 'application/vnd.google-apps.folder'"} +) + +if response.success: + print(response.data) # {"raw": {"files": [...], ...}, "description": "Successfully executed files.list"} +else: + print(response.error_code, response.message) +``` + +For composing actions within a connector, use **`self.call_action`** (returns the action’s output model, not `ConnectorResponse`): + +```python +from node_wire_runtime import BaseConnector, nw_action + +@nw_action("upload_then_describe") +async def upload_then_describe( + self, params: MyInput, *, trace_id: str +) -> GoogleDriveOperationOutput: + created = await self.call_action( + "files.create", + {"action": "files.create", "name": params.name, "mime_type": params.mime_type}, + ) + file_id = created.raw["id"] + return await self.call_action( + "files.get", + {"action": "files.get", "file_id": file_id}, + ) +``` + +--- + +## Integrating with binding layers + +The factory and manifest drive all bindings. Once a connector is registered and `load()` is called, REST, gRPC, and MCP discover enabled connectors according to `exposed_via`. + +### Optional: MCP under `src/agents/` (ToolHive / stdio) + +The repo also ships **stdio MCP servers** for agents and ToolHive under `src/agents/` (e.g. `python -m agents.mcp_entrypoint`, per-connector modules). Those are separate from `MODE=MCP` on `node-wire`; see **[mcp-servers.md](mcp-servers.md)** for images, env, and registration. Wiring a connector in `config/connectors.yaml` does not by itself add a ToolHive image — follow **mcp-servers.md** when you need a dedicated MCP deployment. + +### REST binding + +`src/bindings/rest_api/app.py` calls `build_manifest(connectors)` and registers a `POST /connectors/{connector_id}/{action}` route for every manifest entry: + +``` +POST /connectors/google_drive/files.list +Content-Type: application/json + +{ "page_size": 10, "query": "name contains 'report'" } +``` + +The `action` field in the body is optional for REST — the binding injects it from the URL path (see `src/node_wire_runtime/ingress.py`). Per-action **argument normalizers** (`mcp_normalize` on each action) run on the JSON body the same way as MCP, so LLM-friendly aliases work for REST as well. If the body includes an `action` field, it **must** match the path segment; otherwise the API returns **400**. + +The runtime then performs full Pydantic validation and returns a `ConnectorResponse`. + +**Response envelope:** + +```json +{ + "success": true, + "data": { + "raw": { "files": [{ "id": "...", "name": "...", "mimeType": "..." }], "nextPageToken": null }, + "description": "Successfully executed files.list" + }, + "trace_id": "4f3a...", + "error_code": null, + "error_category": null, + "message": null +} +``` + +HTTP status codes are mapped from `ErrorCategory`: + +| `ErrorCategory` | HTTP status | +|-----------------|-------------| +| `BUSINESS` | 400 | +| `AUTH` | 401 | +| `RETRYABLE` | 503 | +| `FATAL` / other | 500 | + +### MCP binding + +`src/bindings/mcp_server/server.py` registers one **MCP tool** per manifest entry. Tool names follow the pattern `{connector_id}.{action}` (e.g. `google_drive.files.list`, `google_drive.files.upload`). + +The MCP server calls `connector.run(args_dict)` and serialises the `ConnectorResponse` as the tool result. + +The **tool name** (`.`) is authoritative: after normalizers run, the binding sets `action` from the tool name. A conflicting `action` in the payload is rejected (see `enforce_authoritative_action` in `src/node_wire_runtime/ingress.py`). + +Optional per-action **argument normalizers** (`mcp_normalize` on `@sdk_action` / `SdkActionSpec`) run before `connector.run` to map LLM aliases to canonical fields. Actions default to **strict** JSON Schema (`additionalProperties: false`); set `alias_tolerant=True` only where extra keys must pass MCP SDK validation before normalization. + +Published **`input_schema` omits the `action` property** (manifest contract v2+): clients must not rely on sending `action` inside tool arguments; the MCP tool name (or REST path) is authoritative. + +**FHIR `search_encounter` (Epic/Cerner):** normalizers map root-level `patient` / `patientId` to `patient_id`, and `sort` → `_sort` (via `search_params`). Encounter search **requires** a patient filter (`patient_id` or `patient` in `search_params`) before any outbound FHIR call. + +### Manifest + +`build_manifest(connectors)` is the single source of truth for both bindings (by default it strips `action` from each entry’s `input_schema`). It returns one entry per `@sdk_action`: + +```python +[ + { + "connector_id": "weather", + "action": "current_weather", + "input_schema": { ... }, # JSON Schema from CurrentWeatherInput (action not required) + "output_schema": { ... }, # ConnectorResponse envelope; data typed to the action output model (nullable on errors) + }, + { + "connector_id": "google_drive", + "action": "files.upload", + ... + } +] +``` + +--- + +## Connector inventory + +| Connector | Primary actions | +|-----------|-----------------| +| `http_generic` | `request` | +| `smtp` | `send_email` | +| `stripe` | `charge` | +| `google_drive` | `files.list`, `files.upload`, … (see `action_specs`) | +| `fhir_epic` | `read_patient`, `search_patients`, `search_encounter`, `create_document_reference`, `search_document_reference` | +| `fhir_cerner` | Same family as Epic with Cerner-specific schemas | + +MCP tool names: **`.`** (e.g. `fhir_epic.read_patient`). See [`docs/mcp-servers.md`](mcp-servers.md). + +--- + +## Adding a new connector (checklist) + +1. Create `src/node_wire_/` with `schema.py` and `logic.py`. +2. In `schema.py`: define one Pydantic input model per action, each with `action: Literal[""]`, and one or more output models (union + `RootModel` if you validate a single envelope). +3. In `logic.py`: subclass `BaseConnector`, set `connector_id` and `output_model`, then either add `@nw_action` methods with full type annotations or wire **`action_specs`** (and optionally `_execute_action_spec`) like Google Drive. +4. For SDK-style connectors, add an `action_spec.py` (or similar) with `SdkActionSpec` entries and use **`execute_spec_in_thread`** when the vendor client is blocking. +5. Optionally add `error_map` and/or `registration.py` for custom exception handling. +6. Add the connector to **`config/connectors.yaml`** with `enabled: true` and the desired `exposed_via` protocols. +7. That's it — `auto_register()` handles the rest. No factory branch required. + +--- + +## Configuration reference + +### `config/connectors.yaml` + +```yaml +connectors: + : + enabled: true # false → connector not instantiated + exposed_via: # controls which bindings surface this connector + - rest + - grpc + - mcp + # connector-specific keys passed via SecretProvider or connector __init__ +``` + +### `ConnectorFactory` API + +| Method | Description | +|--------|-------------| +| `load()` | Reads YAML, instantiates all enabled connectors from `_CONNECTOR_REGISTRY`. | +| `get_for_protocol(id, protocol, action=None)` | Returns connector if enabled and exposed for that protocol; `None` otherwise. | +| `list_for_protocol(protocol)` | All connectors exposed for a given protocol. | + +--- + +## Security (REST, plugins, secrets) + +**REST API (`bindings.rest_api`)** — `GET /health` is unauthenticated. All other routes (`/connectors/...`, `/playground/...`, `/scenarios/...`, OpenAPI) require **`NW_REST_API_KEY`** via `Authorization: Bearer ` or `X-API-Key: `, optional **`NW_REST_JWT_SECRET`** for HS256 JWTs. Set **`NW_REST_AUTH_DISABLED=true`** only for local development. Production: set **`NW_REST_LOAD_DOTENV=false`** so secrets are not read from a `.env` file on disk. + +**HTTP Generic outbound policy** — `http_generic.request` allows only `GET`, `POST`, `PUT`, `PATCH`, `DELETE`. URLs targeting internal destinations are rejected (`localhost`, loopback, private/link-local IP ranges, metadata endpoints). Connector logs sanitize URL fields by dropping query strings and fragments. + +**Connector entry points** — Any installed distribution may register `node_wire.connectors`. For production, set **`NW_ALLOWED_CONNECTORS`** to a comma-separated list of entry point names (e.g. `fhir_epic,http_generic`). **`NW_CONNECTOR_MODULE_PREFIX`** defaults to `node_wire_`; modules not under that prefix are skipped. + +**Secrets** — `EnvSecretProvider` looks up the key **as given**, then **`key.upper()`** (e.g. `my_key` then `MY_KEY`). It raises **`SecretNotFoundError`** when a variable is missing (fail-closed). Set **`NW_ENV_SECRET_LEGACY_EMPTY=true`** only if you need legacy empty-string behaviour. **`NW_SECRET_BACKEND=aws_env`** with **`NW_AWS_SECRETS_MANAGER_SECRET_ID`** composes AWS Secrets Manager JSON + env fallback via `ChainedSecretProvider` (see `bindings.factory._build_secret_provider`). + +--- + +## Related documentation + +- [packaging.md](packaging.md) — Wheel build lifecycle, PyPI publish flow, client install model, secrets config, and pre-publish checklist. +- [mcp-servers.md](mcp-servers.md) — MCP images, ToolHive, env vars. +- [google_drive_connector.md](google_drive_connector.md) — Drive REST API and setup. +- Per-connector READMEs under `src/node_wire_*/README.md` where present. diff --git a/docs/google_drive_connector.md b/docs/google_drive_connector.md index a66f116..30bc63b 100644 --- a/docs/google_drive_connector.md +++ b/docs/google_drive_connector.md @@ -5,7 +5,7 @@ This document covers the Google Drive connector under `connectors/google_drive` 1. **[Google Drive service account setup](#google-drive-service-account-setup)** — Create a GCP service account, enable the Drive API, configure `.env`, share a folder, and verify connectivity. 2. **[REST API reference](#rest-api-reference)** — The `execute` action, all seven operations, request/response shapes, and the platform error taxonomy. -For **MCP** (e.g. ToolHive), the connector is exposed as the `google_drive_upload_file` tool. End-to-end agent setup is documented in [docs/toolhive_agent_scenario.md](toolhive_agent_scenario.md). +For **MCP** (e.g. ToolHive), tools are named `google_drive.` from the connector manifest (e.g. `google_drive.files.upload`). End-to-end agent setup is documented in [docs/toolhive_agent_scenario.md](toolhive_agent_scenario.md). --- @@ -90,24 +90,24 @@ The file looks like this: Move the downloaded JSON key file to a safe location on your machine (e.g., `~/.secrets/google-drive-sa.json`). -Add the following to your `.env` file: +The connector reads **`GOOGLE_DRIVE_SA_JSON` as inline JSON only** (not a filesystem path). Paste minified JSON into `.env`, or load a key file into the variable in your shell. + +**Example `.env` (paste your real JSON on one line):** ```env -# Absolute path to your downloaded service account JSON key file -GOOGLE_DRIVE_SA_JSON=/Users/you/.secrets/google-drive-sa.json +GOOGLE_DRIVE_SA_JSON={"type":"service_account","project_id":"your-project",...} ``` -**For Windows (PowerShell) — alternative to editing `.env` directly:** +**Windows (PowerShell) — load from a key file into the current session:** ```powershell -# Read the service account JSON and set it as an environment variable $saPath = "C:\path\to\service_account.json" # Replace with your actual path $env:GOOGLE_DRIVE_SA_JSON = Get-Content -Path $saPath -Raw ``` -> This sets the variable for the current PowerShell session. To persist it across sessions, use `[System.Environment]::SetEnvironmentVariable("GOOGLE_DRIVE_SA_JSON", (Get-Content -Path $saPath -Raw), "User")` or add the path to your `.env` file manually. +> To persist across sessions, use `[System.Environment]::SetEnvironmentVariable("GOOGLE_DRIVE_SA_JSON", (Get-Content -Path $saPath -Raw), "User")` or put the JSON string in `.env` as above. -> **For ToolHive deployment:** Instead of a file path, paste the entire JSON content as a single-line string into the ToolHive secret named `GOOGLE_DRIVE_SA_JSON`. See [docs/toolhive_agent_scenario.md](toolhive_agent_scenario.md). +> **ToolHive:** Paste the entire JSON as the secret value. See [docs/toolhive_agent_scenario.md](toolhive_agent_scenario.md). ### Step 6: Share a Google Drive Folder with the Service Account @@ -165,13 +165,13 @@ To upload a test file, use the request body documented under [files.upload](#fil | Error | Likely Cause | Fix | |---|---|---| -| `GDRIVE_AUTH` / `401` or `403` | Service account key file path is wrong, or the JSON is invalid | Double-check `GOOGLE_DRIVE_SA_JSON` points to the correct absolute path | +| `GDRIVE_AUTH` / `401` or `403` | `GOOGLE_DRIVE_SA_JSON` is missing, not valid JSON, or credentials lack Drive access | Ensure the env var is the full JSON string; verify API enablement and IAM | | `GDRIVE_AUTH` / `403` on a specific file | Service account doesn't have permission to that file/folder | Share the folder with the service account email | | `GDRIVE_BUSINESS_RULE` / `404` | Folder ID is wrong | Check the URL in Drive and re-copy the folder ID | -| `FileNotFoundError` | `GOOGLE_DRIVE_SA_JSON` path doesn't exist | Use an absolute path, not a relative one | ### Security Notes +- **`files.list` query** is validated in the connector (max length, no control characters); Google Drive still enforces query semantics server-side. - **Never commit the JSON key file to version control.** Add it to `.gitignore`: ``` *.json @@ -339,7 +339,7 @@ The service account must have edit permission on the file. #### files.upload -Create a new file with text content. +Create a new file with content (text or binary). Request body: @@ -353,14 +353,21 @@ Request body: } ``` +For **MCP** (`google_drive.files.upload`), omit `action` in the tool arguments object; the server injects `files.upload` from the tool name. The published `inputSchema` does not include an `action` property. + Fields: - `name` (string, required). - `mime_type` (string, required). - `parents` (array of string, optional). -- `content` (string, required): UTF-8 text content that will be uploaded. +- `content` (string, optional): UTF-8 text content that will be uploaded. +- `content_base64` (string, optional): base64-encoded binary content (e.g. PDFs, images). + +Exactly one of `content` or `content_base64` must be provided (enforced at validation). + +Content is uploaded using `MediaInMemoryUpload`; this is suitable for small payloads. -Content is uploaded using `MediaInMemoryUpload`; this is suitable for small text payloads. +> For MCP callers (e.g. ToolHive): use canonical fields (`content` / `content_base64`). Legacy `media` / `media_body` shapes are normalized when possible but are not part of the public schema. Legacy `action: "upload"` in the payload is deprecated; set `NODE_WIRE_LEGACY_GDRIVE_ACTION_UPLOAD=reject` to hard-fail during rollout. #### files.delete diff --git a/docs/local-packages-to-images.md b/docs/local-packages-to-images.md new file mode 100644 index 0000000..c57defa --- /dev/null +++ b/docs/local-packages-to-images.md @@ -0,0 +1,141 @@ +# Local package -> Docker image workflow + +This guide walks through building Node Wire packages locally (as wheels) and using those wheels to build the Docker images in `docker/`. + +The Dockerfiles in this repo install local wheel artifacts from `packages/**/dist/*.whl`, so **you must build wheels first**. + +--- + +## Prerequisites + +- Python 3.12 available in your shell +- Docker installed and running +- Build tooling installed: + +```bash +python -m pip install --upgrade build cython wheel +``` + +Run all commands from the repository root: + +```bash +cd /path/to/vinaayakh-node-wire +``` + +--- + +## 1) Build wheel packages locally + +Build all runtime + connector wheels: + +```bash +bash scripts/build-packages.sh +``` + +Build only specific packages (faster when iterating): + +```bash +bash scripts/build-packages.sh \ + packages/runtime \ + packages/connectors/smtp +``` + +The script (`scripts/build-packages.sh` in default mode, not `--all`): +- builds host wheels and Linux-compatible wheels (via Docker), +- writes artifacts under each package's `dist/` folder, +- fails if any `.py` source files leak into a wheel. + +For optional local `cibuildwheel` builds (broader wheel matrix on your host), see **Optional: broader wheels** in [docs/packaging.md](packaging.md). + +--- + +## 2) Confirm wheel artifacts exist + +Quick check (example for SMTP): + +```bash +ls packages/runtime/dist/*.whl +ls packages/connectors/smtp/dist/*.whl +``` + +If `ls` fails, rebuild that package before continuing. + +--- + +## 3) Build Docker images from local wheels + +### Build all MCP connector images + +```bash +./scripts/build-mcp-images.sh +``` + +With explicit version tag: + +```bash +./scripts/build-mcp-images.sh --version 0.1.0 +``` + +This builds: +- `nw-google-drive` +- `nw-smartonfhir-epic` +- `nw-smartonfhir-cerner` +- `nw-smtp` + +### Build one image manually + +```bash +docker build -f docker/smtp/Dockerfile -t nw-smtp:local . +``` + +--- + +## Wheel requirements by image + +Each Dockerfile expects specific wheel files to exist in `dist/`: + +| Image | Required wheels | +|---|---| +| `docker/smtp/Dockerfile` | `packages/runtime/dist/*.whl`, `packages/connectors/smtp/dist/*.whl` | +| `docker/google-drive/Dockerfile` | `packages/runtime/dist/*.whl`, `packages/connectors/google_drive/dist/*.whl` | +| `docker/fhir-epic/Dockerfile` | `packages/runtime/dist/*.whl`, `packages/connectors/fhir_epic/dist/*.whl` | +| `docker/fhir-cerner/Dockerfile` | `packages/runtime/dist/*.whl`, `packages/connectors/fhir_cerner/dist/*.whl` | +| `Dockerfile` (unified MCP server) | runtime + all connector wheels (`http_generic`, `stripe`, `smtp`, `google_drive`, `fhir_epic`, `fhir_cerner`) | + +--- + +## Common failures and fixes + +### `COPY ... dist/*.whl` failed: no source files were specified + +A required wheel is missing. Re-run `scripts/build-packages.sh` for the missing package(s), then rebuild the image. + +### Docker build cannot find `src/` or `config/` + +Use repo root as build context (`.`): + +```bash +docker build -f docker/smtp/Dockerfile -t nw-smtp:local . +``` + +Do not run `docker build` from inside `docker//`. + +### Docker daemon not running + +Start Docker Desktop (or daemon) and retry package/image builds. + +--- + +## Recommended local loop + +```bash +# 1) Rebuild changed packages +bash scripts/build-packages.sh packages/runtime packages/connectors/smtp + +# 2) Build image(s) +docker build -f docker/smtp/Dockerfile -t nw-smtp:local . + +# 3) Verify image exists +docker images --filter reference=nw-smtp +``` + diff --git a/docs/mcp-servers.md b/docs/mcp-servers.md index 1dfe8de..0d66cd1 100644 --- a/docs/mcp-servers.md +++ b/docs/mcp-servers.md @@ -41,12 +41,26 @@ flowchart TD ## Naming conventions -| Connector | Python entrypoint | Docker image | ToolHive name | MCP tool(s) exposed | +| Connector | Python entrypoint | Docker image | ToolHive name | MCP tools exposed | |---|---|---|---|---| -| Google Drive | `python -m agents.google_drive_mcp` | `nw-google-drive` | `nw-google-drive` | `google_drive_upload_file` | -| SMART on FHIR (Epic) | `python -m agents.fhir_epic_mcp` | `nw-smartonfhir-epic` | `nw-smartonfhir-epic` | `fhir_epic_read_patient` | -| SMART on FHIR (Cerner) | `python -m agents.fhir_cerner_mcp` | `nw-smartonfhir-cerner` | `nw-smartonfhir-cerner` | `fhir_cerner_read_patient` | -| SMTP | `python -m agents.smtp_mcp` | `nw-smtp` | `nw-smtp` | `smtp_send_email` | +| Google Drive | `python -m agents.google_drive_mcp` | `nw-google-drive` | `nw-google-drive` | All manifest actions for `google_drive` (names `google_drive.`, e.g. `google_drive.files.upload`) | +| SMART on FHIR (Epic) | `python -m agents.fhir_epic_mcp` | `nw-smartonfhir-epic` | `nw-smartonfhir-epic` | All manifest actions for `fhir_epic` (e.g. `fhir_epic.read_patient`) | +| SMART on FHIR (Cerner) | `python -m agents.fhir_cerner_mcp` | `nw-smartonfhir-cerner` | `nw-smartonfhir-cerner` | All manifest actions for `fhir_cerner` (e.g. `fhir_cerner.read_patient`) | +| SMTP | `python -m agents.smtp_mcp` | `nw-smtp` | `nw-smtp` | `smtp.send_email` | + +The unified server (`python -m agents.mcp_entrypoint`) exposes **every** connector enabled for MCP in `config/connectors.yaml` (e.g. `http_generic.request`, `stripe.charge`, plus the rows above). + +### Tool arguments and security + +- Tool name (`.`) determines the routed action; do not rely on a separate `action` field in the JSON body to select a different operation. +- Per-action normalizers in `src/node_wire_runtime/mcp_normalizers.py` map common LLM mistakes to canonical schema fields; see also `src/node_wire_runtime/ingress.py` for shared MCP/REST behavior. +- **`tools/list` input schemas** omit the `action` field (manifest contract v2+). Pass only the fields shown in `inputSchema`; the server injects `action` from the tool name. + +**Legacy rollout (Google Drive `google_drive.files.upload` only):** + +| Variable | Values | Purpose | +|----------|--------|---------| +| `NODE_WIRE_LEGACY_GDRIVE_ACTION_UPLOAD` | `warn` (default), `allow` (same mapping, no deprecation log), `reject` | Legacy payload `action: "upload"` for `google_drive.files.upload`. Use `reject` once all clients omit `action` or use canonical `files.upload` only in pre-invoke validation paths. | --- @@ -64,15 +78,15 @@ cp sample.env .env | Variable | Description | |---|---| -| `GOOGLE_DRIVE_SA_JSON` | Absolute path to the service account JSON key file **or** the full JSON content as a string | +| `GOOGLE_DRIVE_SA_JSON` | Full service account JSON as a string (not a filesystem path) | | `GOOGLE_DRIVE_FOLDER_ID` | Drive folder ID (from the URL: `.../folders/`) | ```env -GOOGLE_DRIVE_SA_JSON=/absolute/path/to/service-account.json +GOOGLE_DRIVE_SA_JSON={"type":"service_account",...} GOOGLE_DRIVE_FOLDER_ID=your-google-drive-folder-id ``` -> **ToolHive note:** When running inside ToolHive, set `GOOGLE_DRIVE_SA_JSON` to the JSON *contents* (not a file path) because ToolHive injects secrets as string values, not files. +> Load a key file into the variable in your shell if needed (see [docs/google_drive_connector.md](google_drive_connector.md)). ToolHive injects string secrets — paste JSON contents, not a path. #### `nw-smartonfhir-epic` @@ -118,7 +132,7 @@ Register your application at the [Cerner Developer Portal](https://code.cerner.c #### `nw-smtp` -The SMTP MCP server exposes one tool: `smtp_send_email`. When running under ToolHive, inject these as secrets: +The SMTP MCP server exposes one tool: `smtp.send_email`. When running under ToolHive, inject these as secrets: | Variable | Description | |---|---| @@ -165,6 +179,8 @@ GROQ_API_KEY=your-groq-api-key ## Build images +Before building images, build local wheels first. See [docs/local-packages-to-images.md](local-packages-to-images.md) for the full package -> image workflow and required wheel artifacts per image. + All four images are built from the repository root using the automation script: ```bash @@ -208,7 +224,7 @@ docker build -f docker/smtp/Dockerfile -t nw-smtp:latest . ## Run with docker-compose -`docker-compose.mcp.yml` starts all three MCP servers as stdio containers in one command. This is useful for local validation before configuring ToolHive. +`docker-compose.mcp.yml` starts all four MCP servers as stdio containers in one command. This is useful for local validation before configuring ToolHive. ```bash # Ensure your .env is populated, then: @@ -264,7 +280,7 @@ thv run --name nw-smtp --transport stdio \ nw-smtp:latest ``` -> **Google Drive + ToolHive:** Set `GOOGLE_DRIVE_SA_JSON` to the JSON *contents* (not a file path) when storing in ToolHive secrets, because ToolHive injects secrets as string values. +> **Google Drive + ToolHive:** Set `GOOGLE_DRIVE_SA_JSON` to the full JSON string (the connector does not accept a filesystem path). ToolHive injects string secrets, not files. After registration, copy the proxy URLs from the ToolHive UI and set them in your `.env`: @@ -337,7 +353,7 @@ python -m agents.toolhive --local --patient-id 12724066 --recipient-email you@ex |---|---|---| | `TOOLHIVE_MCP_URL(S) is not set` | Missing env var | Set `TOOLHIVE_MCP_URL` (single) or `TOOLHIVE_MCP_URLS` (multi) in `.env` | | `Failed to list MCP tools: Connection refused` | ToolHive proxy stopped | Re-register with `thv run`; confirm the proxy URL matches what ToolHive UI shows | -| Google Drive auth failures | Secret injected as a file path | For ToolHive, set `GOOGLE_DRIVE_SA_JSON` to JSON *contents* (not a path) | +| Google Drive auth failures | `GOOGLE_DRIVE_SA_JSON` is empty, invalid JSON, or not the full key | Paste JSON contents into ToolHive secrets or `.env`; see [Google Drive setup](google_drive_connector.md) | | `fhir_epic connector not configured` | Missing Epic env vars | Ensure all `EPIC_*` variables are set and non-empty | | `fhir_cerner connector not configured` | Missing Cerner env vars | Ensure all `CERNER_*` variables are set and non-empty | | Docker build fails with `COPY src/ not found` | Wrong build context | Always run `docker build` from the **repository root**, not from `docker//` | diff --git a/docs/packaging.md b/docs/packaging.md new file mode 100644 index 0000000..68c81b2 --- /dev/null +++ b/docs/packaging.md @@ -0,0 +1,213 @@ +# Packaging & Publishing + +Node Wire ships as **seven independent PyPI packages** built from a single monorepo. All wheels are binary-only (Cython-compiled `.so`/`.pyd` files) — no `.py` source is included in any published wheel. + +--- + +## Package inventory + +| PyPI name | Source path | Entry-point key | +|---|---|---| +| `node-wire-runtime` | `src/node_wire_runtime/` | — (no entry point; this is the runtime) | +| `node-wire-fhir-cerner` | `src/node_wire_fhir_cerner/` | `fhir_cerner` | +| `node-wire-fhir-epic` | `src/node_wire_fhir_epic/` | `fhir_epic` | +| `node-wire-google-drive` | `src/node_wire_google_drive/` | `google_drive` | +| `node-wire-http-generic` | `src/node_wire_http_generic/` | `http_generic` | +| `node-wire-smtp` | `src/node_wire_smtp/` | `smtp` | +| `node-wire-stripe` | `src/node_wire_stripe/` | `stripe` | + +Each connector's `pyproject.toml` lives at `packages/connectors//pyproject.toml`; the runtime's is at `packages/runtime/pyproject.toml`. + +--- + +## Python package build lifecycle + +### Build all packages (default) + +```bash +bash scripts/build-packages.sh +``` + +Default mode builds each of the **seven** known package paths (see inventory above): `python -m build --wheel` on the **host**, then again inside **Docker** (`python:3.12-slim`) so you get Linux-tagged wheels suitable for containers. **Docker must be installed and the daemon running.** After each package, the script scans every produced wheel and fails if any `.py` file appears inside the archive. + +Prerequisites: `pip install build cython wheel` (and a usable `python` on the host). Run `bash scripts/build-packages.sh --help` for usage. + +### Artifact layout and safe command usage + +`scripts/build-packages.sh` writes wheels per package under `packages/**/dist/` (there is no single repo-root `dist/` output). + +Before using wildcard wheel commands, clear old wheel artifacts so commands do not accidentally match stale versions: + +```bash +rm -f packages/runtime/dist/*.whl +rm -f packages/connectors/stripe/dist/*.whl +``` + +### Build a single package + +```bash +bash scripts/build-packages.sh packages/connectors/stripe +``` + +### Optional: broader wheels with cibuildwheel (`--all`) + +For additional platform wheels from your **current machine** (whatever `cibuildwheel` can target there), install it and use the same script: + +```bash +python -m pip install 'cibuildwheel>=2.16.0' +bash scripts/build-packages.sh --all +bash scripts/build-packages.sh --all packages/runtime +``` + +`CIBW_BUILD` / `CIBW_SKIP` default to the same patterns as `.github/workflows/publish.yml` unless you override them in the environment. Full Linux + macOS + Windows coverage is still best done in CI, not guaranteed from one laptop. + +### Inspect wheel contents + +After building, confirm no source leaks: + +```bash +unzip -l packages/connectors/stripe/dist/node_wire_stripe-*.whl +# Must show .so/.pyd files only — no .py files +``` + +### Install from wheels and verify entry points + +```bash +# Install into an active (clean) virtual env +pip install \ + packages/runtime/dist/node_wire_runtime-*.whl \ + packages/connectors/stripe/dist/node_wire_stripe-*.whl + +# Confirm entry points registered +python -c " +from importlib.metadata import entry_points +print(list(entry_points(group='node_wire.connectors'))) +" +``` + +### Verify connector loading + +```bash +python -c " +from node_wire_runtime.connector_registry import auto_register +loaded = auto_register() +print('Loaded:', loaded) +" +``` + +--- + +## Client consumption model + +A downstream client installs only what it needs: + +```bash +pip install node-wire-runtime node-wire-stripe node-wire-fhir-epic +``` + +At startup, `auto_register()` discovers all installed connectors via the `node_wire.connectors` [entry-point group](https://packaging.python.org/en/latest/specifications/entry-points/) — no explicit import list required. + +### Runtime loading knobs + +| Env var | Default | Purpose | +|---|---|---| +| `NW_ALLOWED_CONNECTORS` | _(all discovered)_ | Comma-separated allowlist of entry-point names (e.g. `stripe,fhir_epic`). In development, leave unset to load everything. In production, set explicitly. | +| `NW_CONNECTOR_MODULE_PREFIX` | `node_wire_` | Connectors whose target module doesn't start with this prefix are skipped with a warning. Set to `""` to disable the check. | + +--- + +## `connectors.yaml` and secrets + +### Minimal `connectors.yaml` + +```yaml +connectors: + stripe: + enabled: true + exposed_via: ["mcp"] + fhir_epic: + enabled: false + exposed_via: [] +``` + +`enabled` gates whether the connector is instantiated. `exposed_via` controls which protocols (`rest`, `grpc`, `mcp`) surface it. A connector that is installed but `enabled: false` will not run. + +See `config/connectors.yaml` for the full working example and `src/node_wire_runtime/connectors.yaml.sample` for a commented template with all supported fields. + +For per-connector detail (operations, env vars, request/response shapes) see `docs/connectors.md` and each connector's `README.md` under `src/node_wire_/`. + +### Secret backend (`NW_SECRET_BACKEND`) + +| Value | Behavior | +|---|---| +| `env` _(default)_ | Reads from process environment. Raises `SecretNotFoundError` for absent keys (fail-closed). | +| `aws_env` | Tries AWS Secrets Manager JSON bundle first; falls back to env on `SecretNotFoundError`. Propagates `SecretProviderError` immediately (broken provider is never silently swallowed). | + +Required env vars for `aws_env`: + +- `NW_AWS_SECRETS_MANAGER_SECRET_ID` — secret name or ARN (required) +- `AWS_REGION` — defaults to `us-east-1` + +**Legacy flag:** `NW_ENV_SECRET_LEGACY_EMPTY=true` returns `""` for missing keys instead of raising. This exists for backwards compatibility only — do not use in production. + +Additional cloud backends (`vault`, `azure`, `gcp`) ship as optional extras in `node-wire-runtime` but are not currently wired into the factory: + +```bash +pip install "node-wire-runtime[aws]" # boto3 +pip install "node-wire-runtime[vault]" # hvac +pip install "node-wire-runtime[azure]" # azure-keyvault-secrets +pip install "node-wire-runtime[gcp]" # google-cloud-secret-manager +``` + +--- + +## CI publish flow (Trusted Publisher) + +**Workflow:** `.github/workflows/publish.yml` — manual `workflow_dispatch`. + +**Required inputs:** + +| Input | Example | Notes | +|---|---|---| +| `package_path` | `packages/connectors/stripe` | Must match an entry in the workflow's allowlist | +| `version` | `0.1.0` | Must match `[project].version` in the package's `pyproject.toml` | + +**Pipeline steps:** + +1. Validate `package_path` against allowlist (prevents path traversal) +2. Matrix-build wheels on Ubuntu, macOS, Windows via `cibuildwheel` (Python 3.11, 3.12; Linux manylinux + aarch64, macOS x86_64 + arm64, Windows amd64) +3. Post-build gate: verify zero `.py` files per wheel; record SHA256 checksums +4. Merge artifacts; `pip-audit --fail-on HIGH` CVE gate +5. Generate SBOM via `cyclonedx-py` +6. Publish to PyPI via OIDC Trusted Publisher with Sigstore attestations (all action SHAs pinned for immutability) + +--- + +## Docker demo images + +The `docker/*/Dockerfile` images are **demonstration templates** for packaging a single connector as a standalone MCP server. They are not production orchestration artefacts. + +For a local end-to-end walkthrough (build wheels first, then build Docker images that consume those wheels), see [docs/local-packages-to-images.md](local-packages-to-images.md). + +```bash +docker build -f docker/smtp/Dockerfile -t nw-smtp . +docker build -f docker/google-drive/Dockerfile -t nw-google-drive . +docker build -f docker/fhir-epic/Dockerfile -t nw-smartonfhir-epic . +docker build -f docker/fhir-cerner/Dockerfile -t nw-smartonfhir-cerner . +``` + +For compose and ToolHive registration see `docs/mcp-servers.md`. + +--- + +## Pre-PyPI local validation checklist + +Run these gates before triggering the CI publish workflow (default `build-packages.sh` is enough; `--all` is optional for broader local wheels): + +- [ ] `bash scripts/build-packages.sh` exits 0 +- [ ] `unzip -l packages//dist/*.whl` shows no `.py` files +- [ ] Install wheels into a clean venv; confirm entry points resolve +- [ ] `auto_register()` loads expected connectors +- [ ] `pytest tests/test_connector_registry.py tests/test_connectors_basic.py` passes +- [ ] Wheel SHA256 checksums recorded and match expected values +- [ ] `package_path` and `version` inputs match the allowlist and `pyproject.toml` version before dispatching the workflow diff --git a/docs/quality-security-gates.md b/docs/quality-security-gates.md new file mode 100644 index 0000000..1143e72 --- /dev/null +++ b/docs/quality-security-gates.md @@ -0,0 +1,112 @@ +# Quality and security gates + +This document defines how Node Wire enforces security scanning and SonarQube analysis in CI, plus the SonarQube Community Edition setup required for centralized reporting. + +## CI quality gates + +Workflow: `.github/workflows/quality-gates.yml` + +Runs on every pull request and on pushes to `main`/`master`. + +Required jobs: + +- `bandit`: publishes `bandit-report.json` and fails on high-severity findings. +- `test`: runs `pytest` and produces `coverage.xml`. +- `sonar`: runs SonarQube scan and waits for quality gate result (runs after `bandit` and `test`). + +## Local commands + +```bash +pip install -e ".[dev,agents]" +bandit -c pyproject.toml -r src --severity-level high +pytest tests/ -v +pre-commit install +pre-commit run --all-files +``` + +## Connector hardening (Google Drive) + +- **`GOOGLE_DRIVE_SA_JSON`**: the Google Drive connector accepts **inline service account JSON only** (no file-path fallback). +- **`files.list` `query`**: validated for basic hygiene (length cap, no ASCII control characters); Google Drive still validates query syntax. + +## Local Sonar scan with Docker + +After generating `coverage.xml`, run scanner from the repository root: + +```bash +docker run --rm \ + -e SONAR_TOKEN=YOUR_TOKEN \ + -v "G:\SPACE\node-wire:/usr/src" \ + -w /usr/src \ + sonarsource/sonar-scanner-cli \ + -Dsonar.host.url=http://host.docker.internal:9000 \ + -Dsonar.token=YOUR_TOKEN +``` + +## Bandit policy + +Bandit is configured in `pyproject.toml` under `[tool.bandit]`. + +Policy: + +- Scan target: `src/`. +- Exclude: `.venv`, `venv`, `tests`, `playground`, `dist`, `htmlcov`. +- CI enforcement threshold: `--severity-level high`. + +If legacy findings block adoption, create a baseline once and track deltas: + +```bash +bandit -c pyproject.toml -r src -f json -o bandit-baseline.json +bandit -c pyproject.toml -r src --baseline bandit-baseline.json --severity-level high +``` + +## SonarQube Community Edition setup + +### 1) Run SonarQube CE (example Docker) + +```bash +docker volume create sonarqube_data +docker volume create sonarqube_logs +docker volume create sonarqube_extensions + +docker run -d --name sonarqube \ + -p 9000:9000 \ + -v sonarqube_data:/opt/sonarqube/data \ + -v sonarqube_logs:/opt/sonarqube/logs \ + -v sonarqube_extensions:/opt/sonarqube/extensions \ + sonarqube:lts-community +``` + +For production, place SonarQube behind HTTPS/reverse proxy and persistent backup strategy. + +### 2) Create project and token + +1. Open SonarQube UI (`http://:9000`). +2. Create project key `node-wire` (or update `sonar-project.properties` if using a different key). +3. Generate project analysis token. + +### 3) Configure GitHub secrets + +In repository settings, add: + +- `SONAR_HOST_URL` +- `SONAR_TOKEN` + +### 4) Configure quality gate + +Create or update a quality gate to enforce at minimum: + +- No new blocker issues. +- No new critical vulnerabilities. +- Coverage on new code >= 80%. + +Attach the gate to the Node Wire project. + +## Acceptance criteria mapping + +- Security scan runs on every PR: enforced by `quality-gates.yml` (Bandit). +- Builds fail on high-severity Bandit findings: Bandit gate in CI. +- SonarQube dashboard visible: SonarQube CE project + scanner upload from CI. +- Coverage visible in SonarQube: `pytest-cov` generates `coverage.xml`, scanner consumes it via `sonar.python.coverage.reportPaths`. +- Developers run checks locally: documented commands and pre-commit (Bandit). +- Config version-controlled: `pyproject.toml`, `.pre-commit-config.yaml`, `sonar-project.properties`, workflow file. diff --git a/docs/security-gap-report.md b/docs/security-gap-report.md new file mode 100644 index 0000000..50c23c5 --- /dev/null +++ b/docs/security-gap-report.md @@ -0,0 +1,361 @@ +# Security & Architecture Gap Report — Node Wire MCP Platform + +> **Perspective:** Secure MCP server platform integrator reviewing the runtime and connectors for production readiness. +> **Date:** 2026-04-01 +> **Branch:** `feature/python-packages` + +--- + +## Executive Summary + +The platform has a solid foundation: clean layered architecture (runtime → connectors → bindings), Pydantic-enforced input validation, OpenTelemetry observability, and resilience patterns (circuit breaker, retry). However, **critical security gaps must be addressed before production use**, particularly around authentication, credential management, PHI/PII logging, and network security. + +**Finding counts:** 5 Critical · 10 High · 7 Medium · 5 Low + +--- + +## Severity Definitions + +| Severity | Meaning | +|----------|---------| +| **CRITICAL** | Exploit path exists now; immediate remediation required | +| **HIGH** | Significant attack surface; address before production | +| **MEDIUM** | Increases risk; address in next sprint | +| **LOW** | Best-practice gap; backlog item | + +--- + +## CRITICAL Findings + +### C1 — Credentials in `.env` Committed to Repository + +- **Location:** `.env` (repository root) +- **What's exposed:** `EPIC_PRIVATE_KEY` (RSA private key), `CERNER_PRIVATE_KEY`, `EPIC_CLIENT_ID`, `GROQ_API_KEY`, `SMTP_PASSWORD`, path to `connectorplatform-*.json` service account file +- **Impact:** Anyone with read access to the repo can impersonate the Epic/Cerner OAuth client, read Google Drive, send email as the platform, and call Groq +- **Fix:** + 1. Revoke all exposed credentials immediately and rotate + 2. Add `.env` and `connectorplatform-*.json` to `.gitignore` + 3. Move secrets to a secrets manager (HashiCorp Vault, AWS Secrets Manager, K8s Secrets) + +--- + +### C2 — PHI Logged in Error Paths (HIPAA Violation) + +- **Location:** `src/connectors/fhir_epic/logic.py` (~line 485), `src/connectors/fhir_cerner/logic.py` (~line 592) +- **What's logged:** Full FHIR `DocumentReference` payload on failure — contains patient names, birthdates, MRNs, diagnoses +- **Code pattern:** + ```python + logger.error("... sent_payload=%s", json.dumps(doc_ref)) + ``` +- **Impact:** Violates HIPAA § 164.312(b) audit controls; PHI written to log aggregation systems in plaintext +- **Fix:** Log only resource type, resource ID, and HTTP status code. Implement a `PHIScrubber` log filter for all healthcare connectors + +--- + +### C3 — No Authentication on REST API or gRPC Binding + +- **Location:** `src/bindings/rest_api/app.py`, `src/bindings/grpc_server/server.py` +- **What's missing:** Zero authentication or authorization on any endpoint +- **gRPC uses an insecure port:** + ```python + server.add_insecure_port(f"[::]:{port}") # no TLS, no mTLS + ``` +- **Impact:** Any network-adjacent caller can invoke any connector action with no audit trail +- **Fix:** + - REST: Add API key or OAuth2 bearer token middleware to FastAPI + - gRPC: Switch to `add_secure_port` with TLS credentials; enforce mTLS for service-to-service + +--- + +### C4 — SSRF via HTTP Generic Connector + +- **Location:** `src/connectors/http_generic/schema.py`, `src/connectors/http_generic/logic.py` +- **What's missing:** `HttpUrl` validates URL format but not destination host +- **Attack path:** + ```json + { "url": "http://169.254.169.254/latest/meta-data", "method": "GET" } + ``` + → Returns AWS instance metadata including IAM credentials +- **Fix:** Block RFC-1918, loopback (`127.0.0.0/8`), and link-local (`169.254.0.0/16`) address ranges at the schema validator level; optionally implement an egress allowlist + +--- + +### C5 — Configurable Secret Key Names in SMTP Connector + +- **Location:** `src/connectors/smtp/schema.py` (fields `username_secret_key`, `password_secret_key`) +- **Attack path:** Caller provides `"username_secret_key": "STRIPE_API_KEY"` → connector fetches the Stripe key and uses it as an SMTP credential; SMTP auth error may reveal whether the key value exists or its format +- **Impact:** Secret enumeration and partial exfiltration via error side-channel +- **Fix:** Hardcode secret key names inside the connector; remove `username_secret_key` and `password_secret_key` from the public input schema entirely + +--- + +## HIGH Findings + +### H1 — OAuth Error Response Body Logged + +- **Location:** `src/connectors/fhir_epic/logic.py` (~line 130), `src/connectors/fhir_cerner/logic.py` +- **What's logged:** Full `token_response.text` on OAuth failure — may include client credential reflections, token hints, or infrastructure error details +- **Fix:** Log only the `error` and `error_description` fields from the JSON response + +--- + +### H2 — Unvalidated HTTP Method in Generic Connector + +- **Location:** `src/connectors/http_generic/schema.py` +- **Current:** `method: str` — accepts any string value +- **Risk:** Arbitrary or non-standard HTTP methods forwarded to target servers; undefined server behavior +- **Fix:** + ```python + method: Literal["GET", "POST", "PUT", "DELETE", "PATCH", "HEAD", "OPTIONS"] + ``` + +--- + +### H3 — Stripe Global API Key (Race Condition) + +- **Location:** `src/connectors/stripe/logic.py` (~line 48) +- **Pattern:** `stripe.api_key = api_key` mutates global module state +- **Risk:** Concurrent requests clobber each other's API key; Stripe exceptions may include the key value in tracebacks +- **Fix:** Pass `api_key=` explicitly to each Stripe API call rather than setting global state + +--- + +### H4 — Auto-Discovery Loads All Connector Modules Without Allowlist + +- **Location:** `src/connectors/__init__.py` (`auto_register()`) +- **Risk:** Any file placed at `src/connectors/*/logic.py` is imported and executed automatically on startup — no explicit allowlist +- **Fix:** Validate each discovered connector ID against an explicit allowlist from `connectors.yaml`; skip unknown modules with a logged warning + +--- + +### H5 — No Rate Limiting on Any Binding + +- **Location:** All binding layers (REST, gRPC, MCP) +- **Risk:** Unlimited invocation rate; no protection against DoS, credential-stuffing, or API quota exhaustion at upstream services +- **Fix:** Add `slowapi` (or equivalent) rate-limiting middleware; define per-tenant quotas in connector configuration + +--- + +### H6 — Circuit Breaker Shared Across Tenants + +- **Location:** `src/runtime/base_connector.py` (constructor — `self._breaker = CircuitBreaker(...)`) +- **Risk:** One tenant triggering repeated failures opens the circuit breaker for all tenants — classic noisy-neighbor DoS +- **Fix:** Key the circuit breaker on `(connector_id, tenant_id)` rather than connector class alone + +--- + +### H7 — Unvalidated Base64 Content in FHIR Connectors + +- **Location:** `src/connectors/fhir_cerner/logic.py`, `src/connectors/fhir_epic/logic.py` +- **Pattern:** + ```python + attachment["data"] = params.data # no format check, no size limit + ``` +- **Risk:** Malformed base64 forwarded to EHR; unbounded payload size enables memory exhaustion +- **Fix:** Add a Pydantic `field_validator` that calls `base64.b64decode(v, validate=True)` and enforces a max size (e.g., 10 MB) + +--- + +### H8 — No Dependency Vulnerability Scanning + +- **Location:** `pyproject.toml`, `uv.lock` +- **Gap:** No `pip-audit`, `safety`, or Dependabot configured; several deps use unbounded `>=VERSION` (e.g., `tenacity>=8.2.0`, `fastapi>=0.111.0`, `uvicorn>=0.30.0`) +- **Fix:** Add `pip-audit` to CI pipeline; add upper bounds to all runtime dependencies; add `detect-secrets` as a pre-commit hook + +--- + +### H9 — No Structured Audit Trail for Policy Hook Decisions + +- **Location:** `src/runtime/base_connector.py` (policy hook execution block) +- **Gap:** Policy denials are logged at `WARNING` level but not emitted as structured security events queryable by a SIEM +- **Fix:** Emit a structured `POLICY_DENIED` event containing `principal`, `tenant_id`, `connector_id`, `action`, and `reason` to a dedicated audit log sink + +--- + +### H10 — Zero Security Test Coverage + +- **Location:** `tests/` (all files) +- **Gaps:** No tests exist for authentication failures, SSRF attempts, malformed payloads, credential leakage in error messages, multi-tenant isolation, rate limit enforcement, or concurrent state safety +- **Fix:** Add a `tests/security/` suite covering at minimum: + - SSRF via HTTP Generic (`127.0.0.1`, `169.254.169.254`) + - Secret enumeration via SMTP `username_secret_key` + - PHI absence in FHIR error log output + - Circuit breaker isolation across tenants + +--- + +## MEDIUM Findings + +### M1 — Stripe Input Has No Validation Bounds + +- **Location:** `src/connectors/stripe/schema.py` +- `amount: int` — no minimum (1 cent) or maximum cap +- `currency: str` — no ISO 4217 pattern check +- **Fix:** + ```python + amount: int = Field(..., ge=1, le=999_999_999) + currency: str = Field(..., pattern=r'^[A-Z]{3}$') + ``` + +--- + +### M2 — Config Variable Substitution (`${VAR:default}`) Not Implemented + +- **Location:** `src/bindings/factory.py`; `config/connectors.yaml` uses `${VAR:default}` syntax in comments and values +- **Current behavior:** Variables are loaded as literal strings — the `${...}` is never expanded +- **Fix:** Implement regex-based substitution in the YAML loader; raise at startup if a required variable is unset + +--- + +### M3 — Factory Returns `None` on Missing Connector (Silent Failure) + +- **Location:** `src/bindings/factory.py` (`get_for_protocol()`) +- **Risk:** A misconfigured connector silently returns `None`; failures surface at request time rather than startup +- **Fix:** Validate all enabled connectors during factory initialization and raise immediately on any misconfiguration + +--- + +### M4 — Hardcoded Timeouts and Circuit Breaker Parameters + +- **Location:** `src/connectors/http_generic/logic.py` (`timeout=30.0`), `src/runtime/base_connector.py` (`fail_max=5, reset_timeout=30`) +- **Risk:** A 30-second timeout is inappropriate for large FHIR document uploads; a 5-failure threshold may be too sensitive for high-traffic deployments +- **Fix:** Expose these as per-connector configuration keys in `connectors.yaml` + +--- + +### M5 — OpenTelemetry Trace Data May Export PHI + +- **Location:** `src/runtime/observability.py` +- **Risk:** Span attributes populated by FHIR connectors may include patient identifiers that flow unfiltered to the OTLP collector +- **Fix:** Add a `SpanSanitizer` processor that removes or hashes known PHI field names before export + +--- + +### M6 — Google Drive `query` Parameter Accepts Arbitrary String + +- **Location:** `src/connectors/google_drive/schema.py` (`query: Optional[str]`) +- **Risk:** No client-side validation; the platform relies entirely on Google's server-side handling of malformed or adversarial query strings +- **Fix:** Document and enforce the allowed query syntax subset; reject queries that don't match a safe pattern + +--- + +### M7 — Service Account File Path Not Sandboxed + +- **Location:** `src/connectors/google_drive/logic.py` +- **Pattern:** `Credentials.from_service_account_file(raw_sa.strip())` — path is fully controlled by the env var +- **Risk:** Path traversal if the environment variable is tampered with +- **Fix:** Resolve the path with `Path.resolve()` and assert it falls within the application directory before opening + +--- + +## LOW Findings + +### L1 — SMTP Connector Logs Recipient Email Addresses + +- **Location:** `src/connectors/smtp/logic.py` +- `"from_email": str(params.from_email)` written to structured log output +- **Risk:** Email addresses are PII; log aggregators retain them indefinitely, creating a compliance liability +- **Fix:** Log only recipient count and domain (e.g., `example.com`), never full addresses + +--- + +### L2 — MCP Manifest Lacks Security Metadata + +- **Location:** `src/connectors/manifest.py` +- **Missing:** Required OAuth scopes, auth requirements, per-action rate limits, deprecation status +- **Impact:** LLM clients have no way to determine required permissions before invoking a tool +- **Fix:** Add an optional `security` block to each manifest entry describing required scopes and auth type + +--- + +### L3 — No MCP Prompt Templates Defined + +- **Location:** `src/bindings/mcp_server/server.py` +- **Gap:** The MCP spec supports pre-built prompt templates to guide safe, correct tool use +- **Risk:** Without templates, LLM clients must independently discover correct multi-step usage patterns (e.g., FHIR patient lookup → document create) +- **Fix:** Define prompt templates for common connector flows + +--- + +### L4 — No Sampling or Pagination Limits in MCP Binding + +- **Location:** `src/bindings/mcp_server/server.py` +- **Gap:** A single `files.list` or FHIR search with a large page size could return megabytes of data in one tool response +- **Fix:** Enforce maximum page sizes at the MCP binding layer; add streaming for large result sets + +--- + +### L5 — PEM Key Reconstruction Is Brittle + +- **Location:** `src/connectors/fhir_cerner/logic.py`, `src/connectors/fhir_epic/logic.py` +- **Pattern:** `private_key_str.replace("\\n", "\n")` to reconstruct a PEM key from env var +- **Risk:** Silently produces an invalid key if the env var format is wrong; error only surfaces at JWT signing time +- **Fix:** Parse and validate the key with the `cryptography` library at connector startup; reject the connector if the key is unparseable + +--- + +## Summary Table + +| ID | Category | Issue | Severity | +|----|----------|-------|----------| +| C1 | Credentials | `.env` with real secrets committed to repo | CRITICAL | +| C2 | Privacy | PHI logged in FHIR error paths | CRITICAL | +| C3 | AuthN/AuthZ | No authentication on REST or gRPC bindings | CRITICAL | +| C4 | Network | SSRF via HTTP Generic connector | CRITICAL | +| C5 | AuthN | Configurable secret key names in SMTP | CRITICAL | +| H1 | Privacy | OAuth error response body logged | HIGH | +| H2 | Validation | Unvalidated HTTP method in generic connector | HIGH | +| H3 | Concurrency | Stripe global API key mutation (race condition) | HIGH | +| H4 | Supply Chain | All connector modules auto-loaded without allowlist | HIGH | +| H5 | DoS | No rate limiting on any binding | HIGH | +| H6 | Isolation | Circuit breaker shared across all tenants | HIGH | +| H7 | Validation | Unvalidated base64 content in FHIR connectors | HIGH | +| H8 | Dependencies | No CVE scanning; unbounded version ranges | HIGH | +| H9 | Audit | Policy hook denials not structured/auditable | HIGH | +| H10 | Testing | Zero security test coverage | HIGH | +| M1 | Validation | Stripe amount/currency unbounded | MEDIUM | +| M2 | Config | `${VAR}` substitution not implemented | MEDIUM | +| M3 | Reliability | Silent `None` on missing connector config | MEDIUM | +| M4 | Config | Hardcoded timeouts and circuit breaker params | MEDIUM | +| M5 | Privacy | OTel traces may export PHI to collector | MEDIUM | +| M6 | Validation | Drive query accepts arbitrary string | MEDIUM | +| M7 | Path Safety | Service account file path not sandboxed | MEDIUM | +| L1 | Privacy | SMTP logs full recipient email addresses | LOW | + +**Google Drive mitigations (M6/M7):** `GOOGLE_DRIVE_SA_JSON` is **inline JSON only** (no file-path reads). `files.list` `query` is bounded and rejects ASCII control characters; Google Drive still validates query syntax server-side. +| L2 | MCP | Manifest lacks security metadata (scopes, auth) | LOW | +| L3 | MCP | No MCP prompt templates defined | LOW | +| L4 | MCP | No sampling/pagination limits in MCP binding | LOW | +| L5 | Reliability | Brittle PEM key reconstruction from env var | LOW | + +--- + +## Recommended Remediation Order + +### Immediate — before any external network access + +1. Revoke all credentials in `.env`; rotate FHIR private keys, Groq key, SMTP password (C1) +2. Remove PHI from FHIR error log lines (C2) +3. Add API key middleware to REST binding (C3) +4. Block RFC-1918 / loopback hosts in HTTP Generic URL validator (C4) +5. Hardcode SMTP secret key names; remove from input schema (C5) + +### Before production + +6. Add TLS + mTLS to gRPC server (C3 continuation) +7. Add connector allowlist validation in `auto_register()` (H4) +8. Add rate limiting middleware to all bindings (H5) +9. Add `pip-audit` to CI and pin dependency upper bounds (H8) +10. Write `tests/security/` suite (H10) + +### Next sprint + +11. Implement per-tenant circuit breakers (H6) +12. Add OTLP `SpanSanitizer` for PHI fields (M5) +13. Implement `${VAR:default}` config substitution (M2) +14. Add Stripe `amount`/`currency` field validators (M1) +15. Add manifest `security` metadata block (L2) + +--- + +*Generated: 2026-04-01 | Branch: feature/python-packages* diff --git a/docs/toolhive_agent_scenario.md b/docs/toolhive_agent_scenario.md index 666c39b..1d137c2 100644 --- a/docs/toolhive_agent_scenario.md +++ b/docs/toolhive_agent_scenario.md @@ -36,10 +36,10 @@ This guide walks you through running the platform as an MCP server using ToolHiv ``` ToolHive UI ────────────────────────────────────────────────────── │ MCP Server (Docker): node-wire │ -│ ├── Tool: fhir_cerner_read_patient ← fetch patient from Cerner │ -│ ├── Tool: fhir_epic_read_patient ← fetch patient from Epic │ -│ ├── Tool: google_drive_upload_file ← write file to Drive │ -│ └── Tool: smtp_send_email ← email the summary │ +│ ├── Tool: fhir_cerner.read_patient ← fetch patient from Cerner │ +│ ├── Tool: fhir_epic.read_patient ← fetch patient from Epic │ +│ ├── Tool: google_drive.files.upload ← write file to Drive │ +│ └── Tool: smtp.send_email ← email the summary │ │ ↕ stdio → HTTP proxy │ ────────────────────────────────────────────────────────────────── ↕ MCP JSON-RPC over HTTP @@ -62,6 +62,7 @@ For modular deployments, each connector can be run as an independent MCP server - `nw-google-drive` (Google Drive) - `nw-smartonfhir-epic` (Epic SMART on FHIR) - `nw-smartonfhir-cerner` (Cerner SMART on FHIR) +- `nw-smtp` (SMTP email) When running multiple MCP servers, configure the agent with **`TOOLHIVE_MCP_URLS`** (comma-separated list of ToolHive proxy URLs). The agent will merge tools across servers. @@ -82,14 +83,14 @@ You can think of it as a local "MCP server manager" — you register your server ## What does the Node Wire MCP server expose? -When running as an MCP server, the platform exposes 4 tools that AI agents can discover and call: +When running **this scenario’s** minimal multi-connector stack (one MCP server per connector image registered in ToolHive), agents typically see **four** tools (Cerner read patient, Epic read patient, Drive upload, SMTP send). The **unified** MCP server (`python -m agents.mcp_entrypoint`) exposes **all** manifest actions for every connector enabled for MCP in `config/connectors.yaml` (often 18+ tools). This section describes the **four-tool** happy path; see [mcp-servers.md](mcp-servers.md) for the full surface. | Tool | Description | |---|---| -| `fhir_cerner_read_patient` | Fetch a patient's record from a Cerner FHIR R4 endpoint | -| `fhir_epic_read_patient` | Fetch a patient's record from an Epic FHIR R4 endpoint | -| `google_drive_upload_file` | Create and upload a text file to Google Drive | -| `smtp_send_email` | Send an email via SMTP | +| `fhir_cerner.read_patient` | Fetch a patient's record from a Cerner FHIR R4 endpoint | +| `fhir_epic.read_patient` | Fetch a patient's record from an Epic FHIR R4 endpoint | +| `google_drive.files.upload` | Create and upload a text file to Google Drive | +| `smtp.send_email` | Send an email via SMTP | The agent uses an LLM's tool-calling capability to decide which tools to call, in what order, and with what parameters. @@ -109,7 +110,7 @@ The MCP server automatically masks sensitive health information before any data | [ToolHive UI](https://stacklok.com/toolhive) installed | Download for macOS / Linux / Windows | | Docker running | Only needed to build the image | | Cerner FHIR credentials | `client_id`, `kid`, private key, tenant URL | -| Google Drive service account JSON | `service_account.json` or file path | +| Google Drive service account JSON | Paste full JSON (not a filesystem path) | | SMTP credentials | Gmail App Password recommended | | Groq API key (free) | [console.groq.com](https://console.groq.com) | @@ -135,13 +136,13 @@ Below is the full set of environment variables used by the connector platform an | `GROQ_API_KEY` | LLM (Groq) | Your Groq API key | | `GROQ_MODEL` | LLM | Example: `openai/gpt-oss-120b` | | `MCP_TRANSPORT` | ToolHive / local | `stdio` when running in ToolHive container | -| `PYTHONPATH` | Runtime | e.g. `/app/src` for container; `d:\connector-platform\src` locally | +| `PYTHONPATH` | Runtime | e.g. `/app/src` for container; `**/node-wire/src` locally | | `SMTP_HOST` | SMTP connector | Example: `sandbox.smtp.mailtrap.io` | | `SMTP_PORT` | SMTP connector | Example: `2525` | | `SMTP_USERNAME` | SMTP connector | Mailtrap / SMTP user | | `SMTP_PASSWORD` | SMTP connector | Mailtrap / SMTP password | | `SMTP_USE_TLS` | SMTP connector | `true` or `false` | -| `GOOGLE_DRIVE_SA_JSON` | Google Drive | Either paste full JSON into ToolHive secret or provide absolute file path to the service account JSON | +| `GOOGLE_DRIVE_SA_JSON` | Google Drive | Full service account JSON string (see [Google Drive setup](google_drive_connector.md#step-5-configure-the-connector)) | --- @@ -160,7 +161,7 @@ Option A — Recommended: ToolHive UI (no code) Option B — Local quick run (Windows PowerShell) -Prerequisite: Install Python 3.10+ and Git. If you cannot install, ask an administrator to run Option A. +Prerequisite: Install Python 3.11+ and Git. If you cannot install, ask an administrator to run Option A. 1. Open PowerShell and clone or navigate to the project folder. 2. Create a simple `.env` file in the project root (replace placeholder values): @@ -204,8 +205,6 @@ Notes for non-developers: From the root of the repository: ```bash -cd connector-platform - docker build -t node-wire:latest . ``` @@ -317,7 +316,7 @@ In the ToolHive UI under **Installed**, you should see: |---|---| | Name | `node-wire-connectors` | | Status | `Running` | -| Tools | `fhir_cerner_read_patient`, `fhir_epic_read_patient`, `google_drive_upload_file`, `smtp_send_email` | +| Tools | Manifest-driven `.` (e.g. `fhir_cerner.read_patient`, `fhir_epic.read_patient`, `google_drive.files.upload`, `smtp.send_email`; unified server also lists Stripe, HTTP generic, and other MCP-enabled connectors) | | Endpoint | `http://localhost:/sse` | --- @@ -404,11 +403,11 @@ I have completed all three steps: 3. Sent a summary email to your-email@example.com with a link to the file. Steps executed (3): - ✓ Step 1: fhir_cerner_read_patient + ✓ Step 1: fhir_cerner.read_patient result : {"patient_id": "123*****", "full_name": "Nancy Smart", ...} - ✓ Step 2: google_drive_upload_file + ✓ Step 2: google_drive.files.upload result : {"file_id": "1XYZ...", "web_view_link": "https://docs.google.com/..."} - ✓ Step 3: smtp_send_email + ✓ Step 3: smtp.send_email result : {"sent": true} ``` @@ -501,10 +500,10 @@ In Cursor's MCP settings, add the same endpoint URL. The tools will appear in th | `TOOLHIVE_MCP_URL is not set` | Missing env var | Copy the endpoint URL from ToolHive UI → Installed → `node-wire-connectors` and add to `.env` | | `Failed to list MCP tools: Connection refused` | ToolHive server stopped | Re-start via ToolHive UI, or run `thv run ...` again; check `thv list` to see running servers | | `Secret 'CERNER_PRIVATE_KEY' is not configured` | Secret not stored in ToolHive | Run `thv secret set CERNER_PRIVATE_KEY` or add it via the ToolHive UI | -| `google_drive connector: authentication failed` | `GOOGLE_DRIVE_SA_JSON` is a file path, not JSON content | For ToolHive, paste the actual JSON *contents* of the file (not the file path) as the secret value; for local `.env`, use an absolute path to the JSON file per [Google Drive service account setup](google_drive_connector.md#google-drive-service-account-setup) | +| `google_drive connector: authentication failed` | `GOOGLE_DRIVE_SA_JSON` is not valid JSON or is empty | Paste the full JSON string (or load it into the env var from a file per [Google Drive service account setup](google_drive_connector.md#step-5-configure-the-connector)) | | `SMTP authentication failed` | Wrong username or password | For Gmail, use an App Password not your regular password; confirm `SMTP_USERNAME` includes `@` | | `groq SDK not installed` | Missing optional dependency | `pip install -e ".[agents]"` | -| Agent loops forever without completing | LLM reasoning issue | Try increasing `--max-steps`; try a different LLM provider; check that all four tools are visible in ToolHive | +| Agent loops forever without completing | LLM reasoning issue | Try increasing `--max-steps`; try a different LLM provider; check that the expected tools are visible in ToolHive (`tools/list`); refresh after MCP image upgrades | | `docker: Cannot connect to the Docker daemon` | Docker not running | Start Docker Desktop | | Container starts but shows 0 tools | MCP server failed to start | Check container logs: `docker logs `; verify the image built successfully | @@ -519,33 +518,21 @@ pip install -e ".[dev,agents]" pytest tests/test_toolhive_agent.py -v ``` -Expected output (all 9 tests passing): - -``` -tests/test_toolhive_agent.py::test_llm_factory_groq_created PASSED -tests/test_toolhive_agent.py::test_llm_factory_openai_created PASSED -tests/test_toolhive_agent.py::test_llm_factory_unknown_raises PASSED -tests/test_toolhive_agent.py::test_llm_factory_case_insensitive PASSED -tests/test_toolhive_agent.py::test_agent_runs_three_tool_sequence PASSED -tests/test_toolhive_agent.py::test_agent_respects_max_steps PASSED -tests/test_toolhive_agent.py::test_agent_handles_tool_error_graceful PASSED -tests/test_toolhive_agent.py::test_agent_fails_when_mcp_unreachable PASSED -tests/test_toolhive_agent.py::test_mcp_entrypoint_registers_three_to PASSED -``` +Expect every test collected from `tests/test_toolhive_agent.py` to pass (names and count change as the suite evolves). If a test fails, re-run with `-v` and compare against the current definitions in that file. --- ## File layout (`agents`) ``` -connector-platform/ +node-wire/ ├── Dockerfile ← Docker image for ToolHive ├── pyproject.toml ← [agents] extras added ├── sample.env ← env var reference └── src/ └── agents/ ├── __init__.py - ├── mcp_entrypoint.py ← FastMCP server (4 tools) + ├── mcp_entrypoint.py ← MCP stdio server (manifest; all MCP connectors) ├── toolhive.py ← ReAct agent + CLI ├── llm_factory.py ← Provider factory └── providers/ diff --git a/grafana/Connector Logs & Status - Updated-1773917850709.json b/grafana/Connector Logs & Status - Updated-1773917850709.json deleted file mode 100644 index 08b2a3a..0000000 --- a/grafana/Connector Logs & Status - Updated-1773917850709.json +++ /dev/null @@ -1,304 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "description": "Real-time log monitoring with success rate and status distribution", - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "links": [], - "panels": [ - { - "datasource": { - "type": "loki", - "uid": "loki" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": 0 - }, - { - "color": "orange", - "value": 80 - }, - { - "color": "green", - "value": 95 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 1, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "12.4.1", - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "loki" - }, - "direction": "backward", - "editorMode": "code", - "expr": "sum(count_over_time({service_name=\"aot-connector-platform\"} | logfmt | connector_id =~ \"$connector_type.*\" |= \"completed successfully\" [$__range])) / sum(count_over_time({service_name=\"aot-connector-platform\"} | logfmt | connector_id =~ \"$connector_type.*\" |= \"Starting connector execution\" [$__range])) * 100", - "queryType": "range", - "refId": "A" - } - ], - "title": "Success Rate", - "type": "stat" - }, - { - "datasource": { - "type": "loki", - "uid": "loki" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - } - }, - "mappings": [] - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Success" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Error" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 2, - "options": { - "displayLabels": [ - "percent" - ], - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": true, - "values": [ - "value", - "percent" - ] - }, - "pieType": "donut", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "sort": "desc", - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.4.1", - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "loki" - }, - "direction": "backward", - "editorMode": "code", - "expr": "sum(count_over_time({service_name=\"aot-connector-platform\"} | logfmt | connector_id =~ \"$connector_type.*\" |~ \"(?i)completed successfully\" [$__range]))", - "legendFormat": "Success", - "queryType": "range", - "refId": "A" - }, - { - "datasource": { - "type": "loki", - "uid": "loki" - }, - "direction": "backward", - "editorMode": "code", - "expr": "sum(count_over_time({service_name=\"aot-connector-platform\"} |~ \"(?i)error\" [$__range]))", - "legendFormat": "Error", - "queryType": "range", - "refId": "B" - } - ], - "title": "Success vs Error Rate", - "type": "piechart" - }, - { - "datasource": { - "type": "loki", - "uid": "loki" - }, - "description": "Search logs using Ctrl+F or click on log lines for details", - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "gridPos": { - "h": 12, - "w": 24, - "x": 0, - "y": 6 - }, - "id": 3, - "options": { - "dedupStrategy": "none", - "enableInfiniteScrolling": true, - "enableLogDetails": true, - "prettifyLogMessage": true, - "showControls": true, - "showLabels": true, - "showTime": true, - "sortOrder": "Descending", - "syntaxHighlighting": true, - "unwrappedColumns": false, - "wrapLogMessage": true - }, - "pluginVersion": "12.4.1", - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "loki" - }, - "direction": "backward", - "editorMode": "code", - "expr": "{service_name=\"aot-connector-platform\"} | logfmt | connector_id =~ \"$connector_type.*\"", - "queryType": "range", - "refId": "A" - } - ], - "title": "All Connector Logs", - "type": "logs" - } - ], - "preload": false, - "refresh": "30s", - "schemaVersion": 42, - "tags": [ - "connectors", - "logs", - "monitoring" - ], - "templating": { - "list": [ - { - "allValue": ".*", - "allowCustomValue": false, - "current": { - "text": [ - "$__all" - ], - "value": [ - "$__all" - ] - }, - "includeAll": true, - "label": "Connector Type", - "multi": true, - "name": "connector_type", - "options": [], - "query": "fhir,google_drive", - "type": "custom", - "valuesFormat": "csv" - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": {}, - "timezone": "browser", - "title": "Connector Logs & Status - Updated", - "uid": "connector_logs_fixed", - "version": 17, - "weekStart": "" -} \ No newline at end of file diff --git a/packages/connectors/fhir_cerner/pyproject.toml b/packages/connectors/fhir_cerner/pyproject.toml new file mode 100644 index 0000000..1e4c6e3 --- /dev/null +++ b/packages/connectors/fhir_cerner/pyproject.toml @@ -0,0 +1,23 @@ +[project] +name = "node-wire-fhir-cerner" +version = "0.1.0" +description = "Node Wire connector — Cerner FHIR R4 (read/search patients and encounters)" +requires-python = ">=3.11" +authors = [{ name = "AOT", email = "dev@aot.local" }] + +dependencies = [ + "node-wire-runtime>=0.1.0", + "httpx[http2]>=0.27.0,<0.28.0", + "PyJWT[crypto]>=2.8.0", +] + +[project.entry-points."node_wire.connectors"] +fhir_cerner = "node_wire_fhir_cerner.logic" + +[build-system] +requires = ["setuptools>=69.0.0", "cython>=3.0", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +where = ["../../../src"] +include = ["node_wire_fhir_cerner*"] diff --git a/packages/connectors/fhir_cerner/setup.py b/packages/connectors/fhir_cerner/setup.py new file mode 100644 index 0000000..5a2537c --- /dev/null +++ b/packages/connectors/fhir_cerner/setup.py @@ -0,0 +1,16 @@ +import glob, os +from Cython.Build import cythonize +from setuptools import setup +from setuptools.command.build_py import build_py as _BuildPy + +class NoPyBuild(_BuildPy): + def find_package_modules(self, package, package_dir): + return [] + +src_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../src/node_wire_fhir_cerner")) +py_files = glob.glob(os.path.join(src_root, "**", "*.py"), recursive=True) + +setup( + cmdclass={"build_py": NoPyBuild}, + ext_modules=cythonize(py_files, compiler_directives={"language_level": "3"}, build_dir="build"), +) diff --git a/packages/connectors/fhir_epic/pyproject.toml b/packages/connectors/fhir_epic/pyproject.toml new file mode 100644 index 0000000..f8800c6 --- /dev/null +++ b/packages/connectors/fhir_epic/pyproject.toml @@ -0,0 +1,23 @@ +[project] +name = "node-wire-fhir-epic" +version = "0.1.0" +description = "Node Wire connector — Epic FHIR R4 (read/search patients and encounters)" +requires-python = ">=3.11" +authors = [{ name = "AOT", email = "dev@aot.local" }] + +dependencies = [ + "node-wire-runtime>=0.1.0", + "httpx[http2]>=0.27.0,<0.28.0", + "PyJWT[crypto]>=2.8.0", +] + +[project.entry-points."node_wire.connectors"] +fhir_epic = "node_wire_fhir_epic.logic" + +[build-system] +requires = ["setuptools>=69.0.0", "cython>=3.0", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +where = ["../../../src"] +include = ["node_wire_fhir_epic*"] diff --git a/packages/connectors/fhir_epic/setup.py b/packages/connectors/fhir_epic/setup.py new file mode 100644 index 0000000..19169aa --- /dev/null +++ b/packages/connectors/fhir_epic/setup.py @@ -0,0 +1,16 @@ +import glob, os +from Cython.Build import cythonize +from setuptools import setup +from setuptools.command.build_py import build_py as _BuildPy + +class NoPyBuild(_BuildPy): + def find_package_modules(self, package, package_dir): + return [] + +src_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../src/node_wire_fhir_epic")) +py_files = glob.glob(os.path.join(src_root, "**", "*.py"), recursive=True) + +setup( + cmdclass={"build_py": NoPyBuild}, + ext_modules=cythonize(py_files, compiler_directives={"language_level": "3"}, build_dir="build"), +) diff --git a/packages/connectors/google_drive/pyproject.toml b/packages/connectors/google_drive/pyproject.toml new file mode 100644 index 0000000..c809367 --- /dev/null +++ b/packages/connectors/google_drive/pyproject.toml @@ -0,0 +1,23 @@ +[project] +name = "node-wire-google-drive" +version = "0.1.0" +description = "Node Wire connector — Google Drive API v3 (files and permissions)" +requires-python = ">=3.11" +authors = [{ name = "AOT", email = "dev@aot.local" }] + +dependencies = [ + "node-wire-runtime>=0.1.0", + "google-auth>=2.0.0", + "google-api-python-client>=2.100.0", +] + +[project.entry-points."node_wire.connectors"] +google_drive = "node_wire_google_drive.logic" + +[build-system] +requires = ["setuptools>=69.0.0", "cython>=3.0", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +where = ["../../../src"] +include = ["node_wire_google_drive*"] diff --git a/packages/connectors/google_drive/setup.py b/packages/connectors/google_drive/setup.py new file mode 100644 index 0000000..21d7a2e --- /dev/null +++ b/packages/connectors/google_drive/setup.py @@ -0,0 +1,16 @@ +import glob, os +from Cython.Build import cythonize +from setuptools import setup +from setuptools.command.build_py import build_py as _BuildPy + +class NoPyBuild(_BuildPy): + def find_package_modules(self, package, package_dir): + return [] + +src_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../src/node_wire_google_drive")) +py_files = glob.glob(os.path.join(src_root, "**", "*.py"), recursive=True) + +setup( + cmdclass={"build_py": NoPyBuild}, + ext_modules=cythonize(py_files, compiler_directives={"language_level": "3"}, build_dir="build"), +) diff --git a/packages/connectors/http_generic/pyproject.toml b/packages/connectors/http_generic/pyproject.toml new file mode 100644 index 0000000..e37dfed --- /dev/null +++ b/packages/connectors/http_generic/pyproject.toml @@ -0,0 +1,22 @@ +[project] +name = "node-wire-http-generic" +version = "0.1.0" +description = "Node Wire connector — generic HTTP REST client (GET/POST/PUT/DELETE/PATCH)" +requires-python = ">=3.11" +authors = [{ name = "AOT", email = "dev@aot.local" }] + +dependencies = [ + "node-wire-runtime>=0.1.0", + "httpx[http2]>=0.27.0,<0.28.0", +] + +[project.entry-points."node_wire.connectors"] +http_generic = "node_wire_http_generic.logic" + +[build-system] +requires = ["setuptools>=69.0.0", "cython>=3.0", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +where = ["../../../src"] +include = ["node_wire_http_generic*"] diff --git a/packages/connectors/http_generic/setup.py b/packages/connectors/http_generic/setup.py new file mode 100644 index 0000000..1945677 --- /dev/null +++ b/packages/connectors/http_generic/setup.py @@ -0,0 +1,16 @@ +import glob, os +from Cython.Build import cythonize +from setuptools import setup +from setuptools.command.build_py import build_py as _BuildPy + +class NoPyBuild(_BuildPy): + def find_package_modules(self, package, package_dir): + return [] + +src_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../src/node_wire_http_generic")) +py_files = glob.glob(os.path.join(src_root, "**", "*.py"), recursive=True) + +setup( + cmdclass={"build_py": NoPyBuild}, + ext_modules=cythonize(py_files, compiler_directives={"language_level": "3"}, build_dir="build"), +) diff --git a/packages/connectors/smtp/pyproject.toml b/packages/connectors/smtp/pyproject.toml new file mode 100644 index 0000000..a956c96 --- /dev/null +++ b/packages/connectors/smtp/pyproject.toml @@ -0,0 +1,23 @@ +[project] +name = "node-wire-smtp" +version = "0.1.0" +description = "Node Wire connector — SMTP email sending (async)" +requires-python = ">=3.11" +authors = [{ name = "AOT", email = "dev@aot.local" }] + +dependencies = [ + "node-wire-runtime>=0.1.0", + "aiosmtplib>=3.0.1", + "email-validator>=2.0.0", +] + +[project.entry-points."node_wire.connectors"] +smtp = "node_wire_smtp.logic" + +[build-system] +requires = ["setuptools>=69.0.0", "cython>=3.0", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +where = ["../../../src"] +include = ["node_wire_smtp*"] diff --git a/packages/connectors/smtp/setup.py b/packages/connectors/smtp/setup.py new file mode 100644 index 0000000..f8867c5 --- /dev/null +++ b/packages/connectors/smtp/setup.py @@ -0,0 +1,16 @@ +import glob, os +from Cython.Build import cythonize +from setuptools import setup +from setuptools.command.build_py import build_py as _BuildPy + +class NoPyBuild(_BuildPy): + def find_package_modules(self, package, package_dir): + return [] + +src_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../src/node_wire_smtp")) +py_files = glob.glob(os.path.join(src_root, "**", "*.py"), recursive=True) + +setup( + cmdclass={"build_py": NoPyBuild}, + ext_modules=cythonize(py_files, compiler_directives={"language_level": "3"}, build_dir="build"), +) diff --git a/packages/connectors/stripe/pyproject.toml b/packages/connectors/stripe/pyproject.toml new file mode 100644 index 0000000..c6e0283 --- /dev/null +++ b/packages/connectors/stripe/pyproject.toml @@ -0,0 +1,22 @@ +[project] +name = "node-wire-stripe" +version = "0.1.0" +description = "Node Wire connector — Stripe payments" +requires-python = ">=3.11" +authors = [{ name = "AOT", email = "dev@aot.local" }] + +dependencies = [ + "node-wire-runtime>=0.1.0", + "stripe>=10.0.0", +] + +[project.entry-points."node_wire.connectors"] +stripe = "node_wire_stripe.logic" + +[build-system] +requires = ["setuptools>=69.0.0", "cython>=3.0", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +where = ["../../../src"] +include = ["node_wire_stripe*"] diff --git a/packages/connectors/stripe/setup.py b/packages/connectors/stripe/setup.py new file mode 100644 index 0000000..08d00de --- /dev/null +++ b/packages/connectors/stripe/setup.py @@ -0,0 +1,16 @@ +import glob, os +from Cython.Build import cythonize +from setuptools import setup +from setuptools.command.build_py import build_py as _BuildPy + +class NoPyBuild(_BuildPy): + def find_package_modules(self, package, package_dir): + return [] + +src_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../src/node_wire_stripe")) +py_files = glob.glob(os.path.join(src_root, "**", "*.py"), recursive=True) + +setup( + cmdclass={"build_py": NoPyBuild}, + ext_modules=cythonize(py_files, compiler_directives={"language_level": "3"}, build_dir="build"), +) diff --git a/packages/runtime/pyproject.toml b/packages/runtime/pyproject.toml new file mode 100644 index 0000000..d5e51c7 --- /dev/null +++ b/packages/runtime/pyproject.toml @@ -0,0 +1,38 @@ +[project] +name = "node-wire-runtime" +version = "0.1.0" +description = "Node Wire runtime — connector framework, resilience, observability, and pluggable secrets" +requires-python = ">=3.11" +authors = [{ name = "AOT", email = "dev@aot.local" }] +readme = "README.md" + +dependencies = [ + "pydantic>=2.6.0,<3.0.0", + "tenacity>=8.2.0", + "pybreaker>=1.0.0", + "opentelemetry-api>=1.24.0", + "opentelemetry-sdk>=1.24.0", + "opentelemetry-exporter-otlp>=1.24.0", + "traceloop-sdk>=0.53.0", + "pyyaml>=6.0.1", + "python-dotenv>=1.0.0", +] + +[project.optional-dependencies] +# Cloud secret backends — install only what your deployment needs +aws = ["boto3>=1.34.0"] +vault = ["hvac>=2.1.0"] +azure = ["azure-keyvault-secrets>=4.8.0", "azure-identity>=1.16.0"] +gcp = ["google-cloud-secret-manager>=2.20.0"] + +[build-system] +requires = ["setuptools>=69.0.0", "cython>=3.0", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +where = ["../../src"] +include = ["node_wire_runtime*"] + +# Ship connectors.yaml.sample as a non-Python package resource +[tool.setuptools.package-data] +node_wire_runtime = ["connectors.yaml.sample"] diff --git a/packages/runtime/setup.py b/packages/runtime/setup.py new file mode 100644 index 0000000..23edeed --- /dev/null +++ b/packages/runtime/setup.py @@ -0,0 +1,46 @@ +""" +Cython build for node-wire-runtime. + +Compiles all .py files to .so/.pyd extensions and overrides build_py +so that source .py files are NOT copied into the wheel — only the compiled +binary extensions are included. + +Build with: + python -m build --wheel --no-isolation + +Verify no .py files leaked: + unzip -l dist/node_wire_runtime-*.whl | grep '\.py$' +""" + +import glob +import os + +from Cython.Build import cythonize +from setuptools import setup +from setuptools.command.build_py import build_py as _BuildPy + + +class NoPyBuild(_BuildPy): + """Override that skips copying .py source files into the build tree. + + Setuptools would normally copy every .py file into the wheel alongside + the compiled extension. Returning [] here ensures the wheel contains + only .so/.pyd binaries. + """ + + def find_package_modules(self, package, package_dir): + return [] + + +src_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../src/node_wire_runtime")) +py_files = glob.glob(os.path.join(src_root, "**", "*.py"), recursive=True) + +setup( + cmdclass={"build_py": NoPyBuild}, + ext_modules=cythonize( + py_files, + compiler_directives={"language_level": "3"}, + build_dir="build", + annotate=False, + ), +) diff --git a/playground/README.md b/playground/README.md index 1ba1290..43b3490 100644 --- a/playground/README.md +++ b/playground/README.md @@ -9,7 +9,7 @@ The demo provides a modern, interactive web interface to trigger, monitor, and v ### Core Technologies - **Frontend**: Vanilla HTML5, CSS3 (Glassmorphism), and Javascript. - **Backend API**: FastAPI (Python) serving orchestration logic via `playground/scenarios.py`. -- **Connector Layer**: Integrated with `connectors` using the `fhir_epic`, `fhir_cerner`, and `http_generic` bindings. +- **Connector layer**: Uses Node Wire connectors (`fhir_epic`, `fhir_cerner`, `http_generic`, and others) via the platform REST API and `ConnectorFactory` (see [docs/connectors.md](docs/connectors.md)). --- @@ -97,9 +97,8 @@ The demo is pre-configured with mock/sandbox endpoints for immediate use. To tes To test the Google Drive integration manually, follow these specialized setup steps: 1. **Service Account**: Create a Service Account in the Google Cloud Console with the **Google Drive API** enabled. Download the JSON key. 2. **Secret Configuration**: - * Place the JSON key file in your project directory (e.g., `D:\connector-platform\service_account.json`). - * Update your `.env` file: `GOOGLE_DRIVE_SA_JSON=D:\connector-platform\service_account.json`. - * *Note: The platform now supports direct file paths for easier local configuration.* + * Place the JSON key file somewhere safe on your machine. + * Set `GOOGLE_DRIVE_SA_JSON` to the **full JSON content** (paste minified JSON in `.env`, or load the file into the variable in your shell — see [docs/google_drive_connector.md](../docs/google_drive_connector.md)). 3. **Permissions**: If using a specific **Vault Folder ID**, ensure that folder is shared with the Service Account's email address (found in the JSON) with "Editor" or "Manager" permissions. 4. **Workflow Verification**: * **Direct Upload**: Drag a PDF or Image into the "Upload File" zone. Verify the file appears in the drive with correct metadata. @@ -110,7 +109,7 @@ To test the Google Drive integration manually, follow these specialized setup st To enable the AI Agent chat, you need to configure an LLM provider: 1. **Select Provider**: Set `LLM_PROVIDER` to `groq` (default) or `openai` in your `.env`. 2. **Add API Key**: Provide the corresponding key, e.g., `GROQ_API_KEY=your_key_here`. -3. **SMTP Setup**: (Optional) Add SMTP credentials (`SMTP_HOST`, `SMTP_PORT`, `SMTP_USER`, `SMTP_PASS`) to enable the agent to send emails. +3. **SMTP Setup**: (Optional) Add SMTP credentials (`SMTP_HOST`, `SMTP_PORT`, `SMTP_USERNAME`, `SMTP_PASSWORD`) to enable the agent to send emails. 4. **MCP URL**: (Optional) If running the MCP server in a separate container, set `TOOLHIVE_MCP_URL` to point to the MCP proxy. --- @@ -119,8 +118,14 @@ To enable the AI Agent chat, you need to configure an LLM provider: 1. Navigate to the project root. 2. Start the FastAPI server: - ```bash - set MODE=API&& python -m bindings_entrypoint - ``` + +```bash +# Recommended +uv run node-wire + +# Equivalent (no uv) +MODE=API python -m bindings_entrypoint +``` + 3. Open your browser to `http://localhost:8000/playground/` (or the configured port). 4. Switch between **EHR**, **IT Ops**, **Cerner**, **Google Drive Vault**, and **AI Agent** tabs to explore the different workflows. diff --git a/playground/index.html b/playground/index.html index 46978e8..a18a660 100644 --- a/playground/index.html +++ b/playground/index.html @@ -4,7 +4,7 @@ - Node-wire Playground + node-wire Playground @@ -28,7 +28,7 @@
-

Node-Wire

+

node-wire

Autonomous Connector Orchestration Platform

@@ -93,7 +93,7 @@

Connectors

-

Node-Wire MCP via ToolHive

+

node-wire MCP via ToolHive

MCP Agent — Guardrailed