From a4ac9713dfd66fe9e540db1a089de08c37d160cd Mon Sep 17 00:00:00 2001 From: Jim Schaff Date: Sun, 12 Jul 2026 13:42:50 -0400 Subject: [PATCH 01/11] feat(k8s): add LiteLLM proxy + Ollama for the PR #72 architecture PR #72 routes chat through a LiteLLM proxy with per-user virtual keys/budgets. Extend the kustomize deployment to match: - base/litellm.yaml: LiteLLM Deployment+Service (:4000), config mounted from a litellm-config-file ConfigMap (base/litellm_config.yaml), secrets from litellm-secrets - base/ollama.yaml: Ollama StatefulSet+Service (:11434, 10Gi PVC) serving the local-model fallback; pulls phi4-mini on start (CPU-friendly) - backend gets LITELLM_URL (config) + LITELLM_MASTER_KEY (backend-secrets) to provision per-user virtual keys; still uses Azure for KB embeddings - config//litellm.env: LOCAL_LLM_MODEL=ollama/phi4-mini, LOCAL_LLM_API_BASE, LANGFUSE_HOST - new sealed secret litellm-secrets (master key, OpenAI key, Langfuse, Supabase DATABASE_URL) via scripts/sealed_secret_litellm.sh; secrets.sh + templates updated across overlays - bump overlay image tags to 0.1.7 (must be tagged/built before deploy) - fix .gitignore: anchor the docker-compose `/litellm.env` so it no longer ignores the committed kustomize/config/*/litellm.env Validated: kubectl kustomize renders all three overlays (20 objects each); scripts pass bash -n. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01XpNzobVi83p3YKL9sqtGwZ --- .gitignore | 5 +- kustomize/README.md | 32 +++++-- kustomize/base/backend.yaml | 6 ++ kustomize/base/kustomization.yaml | 9 ++ kustomize/base/litellm.yaml | 93 +++++++++++++++++++ kustomize/base/litellm_config.yaml | 19 ++++ kustomize/base/ollama.yaml | 80 ++++++++++++++++ kustomize/config/vcell-ai-local/backend.env | 3 + .../config/vcell-ai-local/kustomization.yaml | 3 + kustomize/config/vcell-ai-local/litellm.env | 10 ++ kustomize/config/vcell-ai-rke-dev/backend.env | 3 + .../vcell-ai-rke-dev/kustomization.yaml | 3 + kustomize/config/vcell-ai-rke-dev/litellm.env | 10 ++ kustomize/config/vcell-ai-rke/backend.env | 3 + .../config/vcell-ai-rke/kustomization.yaml | 3 + kustomize/config/vcell-ai-rke/litellm.env | 10 ++ .../vcell-ai-local/kustomization.yaml | 5 +- .../vcell-ai-local/secrets.dat.template | 7 ++ kustomize/overlays/vcell-ai-local/secrets.sh | 11 ++- .../vcell-ai-rke-dev/kustomization.yaml | 5 +- .../vcell-ai-rke-dev/secrets.dat.template | 7 ++ .../overlays/vcell-ai-rke-dev/secrets.sh | 11 ++- .../overlays/vcell-ai-rke/kustomization.yaml | 5 +- .../vcell-ai-rke/secrets.dat.template | 7 ++ kustomize/overlays/vcell-ai-rke/secrets.sh | 11 ++- kustomize/scripts/sealed_secret_backend.sh | 8 +- kustomize/scripts/sealed_secret_litellm.sh | 66 +++++++++++++ 27 files changed, 415 insertions(+), 20 deletions(-) create mode 100644 kustomize/base/litellm.yaml create mode 100644 kustomize/base/litellm_config.yaml create mode 100644 kustomize/base/ollama.yaml create mode 100644 kustomize/config/vcell-ai-local/litellm.env create mode 100644 kustomize/config/vcell-ai-rke-dev/litellm.env create mode 100644 kustomize/config/vcell-ai-rke/litellm.env create mode 100755 kustomize/scripts/sealed_secret_litellm.sh diff --git a/.gitignore b/.gitignore index dbd4c2c..ea30efc 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,9 @@ kustomize/overlays/*/secrets.dat kustomize/overlays/*/secret-backend.yaml kustomize/overlays/*/secret-frontend.yaml kustomize/overlays/*/secret-ghcr.yaml +kustomize/overlays/*/secret-litellm.yaml qdrant_storage -litellm.env \ No newline at end of file +# repo-root docker-compose secrets file — anchored with a leading slash so it does +# NOT also ignore the non-secret kustomize/config/*/litellm.env config files. +/litellm.env \ No newline at end of file diff --git a/kustomize/README.md b/kustomize/README.md index 5226c1e..5bad7f8 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -1,41 +1,53 @@ # VCell-AI — Kubernetes deployment (kustomize + sealed secrets) -Deploys the three VCell-AI services from `docker-compose.yml` to Kubernetes: +Deploys the VCell-AI services from `docker-compose.yml` to Kubernetes: | Component | Image | Port | Notes | |-----------|-------|------|-------| | `qdrant` | `qdrant/qdrant` | 6333 / 6334 | StatefulSet + 10Gi PVC | | `backend` | `ghcr.io/virtualcell/vcell-ai-backend` | 8000 | FastAPI | | `frontend` | `ghcr.io/virtualcell/vcell-ai-frontend` | 3000 | Next.js | +| `litellm` | `ghcr.io/berriai/litellm:main-stable` | 4000 | LLM gateway; chat routes through it with per-user virtual keys/budgets | +| `ollama` | `ollama/ollama` | 11434 | StatefulSet + 20Gi PVC; serves the `local-model` fallback (CPU; pulls `llama3.1:8b` on start) | -Structured after `../sms-api/kustomize`. +The backend calls `litellm` for chat (`LITELLM_URL`, per-user virtual keys via +`LITELLM_MASTER_KEY`) but still uses Azure directly for KB **embeddings**. LiteLLM +routes `openai-model` (hosted OpenAI gpt-4o) with a fallback to `local-model` +(in-cluster Ollama), and persists virtual keys/budgets to a Postgres +(`DATABASE_URL`, e.g. Supabase). Structured after `../sms-api/kustomize`. ## Layout ``` kustomize/ -├── base/ # Deployments/StatefulSet + Services (env-agnostic) +├── base/ # Deployments/StatefulSets + Services (env-agnostic) │ ├── qdrant.yaml │ ├── backend.yaml │ ├── frontend.yaml +│ ├── litellm.yaml +│ ├── ollama.yaml +│ ├── litellm_config.yaml # -> litellm-config-file ConfigMap (mounted at /app/config.yaml) │ └── kustomization.yaml -├── config// # non-secret config -> ConfigMaps (backend/frontend .env) +├── config// # non-secret config -> ConfigMaps │ ├── backend.env │ ├── frontend.env +│ ├── litellm.env │ └── kustomization.yaml ├── overlays// # per-environment: namespace, images, ingress, secrets │ ├── kustomization.yaml │ ├── ingress.yaml -│ ├── secrets.sh # master script -> generates the 3 sealed secrets +│ ├── secrets.sh # master script -> generates the 4 sealed secrets │ ├── secrets.dat.template # copy to secrets.dat (gitignored) and fill in │ ├── secrets.dat # (gitignored) plaintext inputs │ ├── secret-backend.yaml # (gitignored) generated SealedSecret │ ├── secret-frontend.yaml # (gitignored) generated SealedSecret -│ └── secret-ghcr.yaml # (gitignored) generated SealedSecret +│ ├── secret-ghcr.yaml # (gitignored) generated SealedSecret +│ └── secret-litellm.yaml # (gitignored) generated SealedSecret └── scripts/ # sealing + build helpers ├── sealed_secret_backend.sh ├── sealed_secret_frontend.sh ├── sealed_secret_ghcr.sh + ├── sealed_secret_litellm.sh └── build_and_push.sh ``` @@ -47,10 +59,16 @@ Environments (``): `vcell-ai-rke` (prod, on-prem UCHC RKE), `vcell-ai-rke-d Non-secret configuration lives in `config//*.env` (committed). Only truly sensitive values are sealed: -- **backend-secrets** → `AZURE_API_KEY`, `LANGFUSE_SECRET_KEY`, `LANGFUSE_PUBLIC_KEY`, `SUPABASE_SERVICE_ROLE_KEY` +- **backend-secrets** → `AZURE_API_KEY`, `LANGFUSE_SECRET_KEY`, `LANGFUSE_PUBLIC_KEY`, `SUPABASE_SERVICE_ROLE_KEY`, `LITELLM_MASTER_KEY` - **frontend-secrets** → `AUTH0_SECRET`, `AUTH0_CLIENT_SECRET` +- **litellm-secrets** → `LITELLM_MASTER_KEY`, `OPENAI_API_KEY`, `LANGFUSE_SECRET_KEY`, `LANGFUSE_PUBLIC_KEY`, `DATABASE_URL` - **ghcr-secret** → ghcr.io image-pull credentials (`.dockerconfigjson`) +(`LITELLM_MASTER_KEY` and the Langfuse keys are intentionally in more than one +sealed secret — the backend needs the master key to provision virtual keys, and +LiteLLM needs it plus Langfuse for tracing. Provide each value once in +`secrets.dat`; `secrets.sh` seals it into the right secrets.) + `secrets.dat` (plaintext) and the generated `secret-*.yaml` are **gitignored**. Run `secrets.sh` to (re)generate the sealed manifests before applying an overlay. diff --git a/kustomize/base/backend.yaml b/kustomize/base/backend.yaml index 5a1b31c..73951bd 100644 --- a/kustomize/base/backend.yaml +++ b/kustomize/base/backend.yaml @@ -49,6 +49,12 @@ spec: secretKeyRef: name: backend-secrets key: supabase-service-role-key + # Backend provisions per-user LiteLLM virtual keys using the master key. + - name: LITELLM_MASTER_KEY + valueFrom: + secretKeyRef: + name: backend-secrets + key: litellm-master-key resources: requests: memory: "512Mi" diff --git a/kustomize/base/kustomization.yaml b/kustomize/base/kustomization.yaml index 3db2db9..8d4ee1c 100644 --- a/kustomize/base/kustomization.yaml +++ b/kustomize/base/kustomization.yaml @@ -5,3 +5,12 @@ resources: - qdrant.yaml - backend.yaml - frontend.yaml + - litellm.yaml + - ollama.yaml + +# litellm_config.yaml is env-agnostic, so it is generated here (mounted read-only +# into the litellm pod at /app/config.yaml). +configMapGenerator: + - name: litellm-config-file + files: + - config.yaml=litellm_config.yaml diff --git a/kustomize/base/litellm.yaml b/kustomize/base/litellm.yaml new file mode 100644 index 0000000..fcf3f81 --- /dev/null +++ b/kustomize/base/litellm.yaml @@ -0,0 +1,93 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: litellm + labels: + app: litellm +spec: + replicas: 1 + selector: + matchLabels: + app: litellm + template: + metadata: + labels: + app: litellm + spec: + containers: + - name: litellm + image: ghcr.io/berriai/litellm:main-stable + imagePullPolicy: IfNotPresent + args: ["--config", "/app/config.yaml"] + ports: + - containerPort: 4000 + # Non-secret config (local-model alias + api_base, Langfuse host) comes + # from the litellm-config ConfigMap generated per-environment. + envFrom: + - configMapRef: + name: litellm-config + # Secrets come from the sealed "litellm-secrets" secret. + env: + - name: LITELLM_MASTER_KEY + valueFrom: + secretKeyRef: + name: litellm-secrets + key: master-key + - name: OPENAI_API_KEY + valueFrom: + secretKeyRef: + name: litellm-secrets + key: openai-api-key + - name: LANGFUSE_SECRET_KEY + valueFrom: + secretKeyRef: + name: litellm-secrets + key: langfuse-secret-key + - name: LANGFUSE_PUBLIC_KEY + valueFrom: + secretKeyRef: + name: litellm-secrets + key: langfuse-public-key + - name: DATABASE_URL + valueFrom: + secretKeyRef: + name: litellm-secrets + key: database-url + volumeMounts: + - name: litellm-config-file + mountPath: /app/config.yaml + subPath: config.yaml + resources: + requests: + memory: "256Mi" + cpu: "100m" + limits: + memory: "1Gi" + cpu: "500m" + readinessProbe: + httpGet: + path: /health/liveliness + port: 4000 + initialDelaySeconds: 15 + periodSeconds: 15 + volumes: + - name: litellm-config-file + configMap: + name: litellm-config-file + imagePullSecrets: + - name: ghcr-secret + +--- +apiVersion: v1 +kind: Service +metadata: + name: litellm + labels: + app: litellm +spec: + ports: + - name: "4000" + port: 4000 + targetPort: 4000 + selector: + app: litellm diff --git a/kustomize/base/litellm_config.yaml b/kustomize/base/litellm_config.yaml new file mode 100644 index 0000000..c4d0581 --- /dev/null +++ b/kustomize/base/litellm_config.yaml @@ -0,0 +1,19 @@ +model_list: + - model_name: openai-model + litellm_params: + model: openai/gpt-4o + api_key: os.environ/OPENAI_API_KEY + + - model_name: local-model + litellm_params: + model: os.environ/LOCAL_LLM_MODEL + api_base: os.environ/LOCAL_LLM_API_BASE + +litellm_settings: + success_callback: ["langfuse"] + fallbacks: + - openai-model: ["local-model"] + +general_settings: + master_key: os.environ/LITELLM_MASTER_KEY + database_url: os.environ/DATABASE_URL diff --git a/kustomize/base/ollama.yaml b/kustomize/base/ollama.yaml new file mode 100644 index 0000000..1cd4e91 --- /dev/null +++ b/kustomize/base/ollama.yaml @@ -0,0 +1,80 @@ +# In-cluster Ollama, serving the "local-model" fallback that LiteLLM routes to +# when the primary (openai-model) fails. CPU inference — no GPU is assumed on the +# node; phi4-mini (~3.8B) is small enough to respond acceptably on CPU. The model +# is pulled on startup into the PVC so it survives restarts. +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: ollama + labels: + app: ollama +spec: + serviceName: ollama + replicas: 1 + selector: + matchLabels: + app: ollama + template: + metadata: + labels: + app: ollama + spec: + containers: + - name: ollama + image: ollama/ollama:latest + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 11434 + # Start the server, then pull the model in the background so it lands + # in the mounted PVC (idempotent — a no-op once cached). + command: ["/bin/sh", "-c"] + args: + - | + ollama serve & + pid=$! + sleep 5 + ollama pull "${OLLAMA_PULL_MODEL}" || echo "model pull failed; will retry on demand" + wait $pid + env: + - name: OLLAMA_PULL_MODEL + value: "phi4-mini" + volumeMounts: + - name: ollama-models + mountPath: /root/.ollama + resources: + requests: + memory: "3Gi" + cpu: "500m" + limits: + memory: "6Gi" + cpu: "2" + readinessProbe: + httpGet: + path: / + port: 11434 + initialDelaySeconds: 10 + periodSeconds: 15 + volumeClaimTemplates: + - metadata: + name: ollama-models + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 10Gi + +--- +apiVersion: v1 +kind: Service +metadata: + name: ollama + labels: + app: ollama +spec: + ports: + - name: http + port: 11434 + targetPort: 11434 + selector: + app: ollama diff --git a/kustomize/config/vcell-ai-local/backend.env b/kustomize/config/vcell-ai-local/backend.env index fa78aa8..369c444 100644 --- a/kustomize/config/vcell-ai-local/backend.env +++ b/kustomize/config/vcell-ai-local/backend.env @@ -23,5 +23,8 @@ AUTH0_AUDIENCE=https://vcell-ai-api # Supabase (URL is public; service-role key is secret) SUPABASE_URL=https://your-project.supabase.co +# LiteLLM proxy (in-cluster service; master key is secret) +LITELLM_URL=http://litellm:4000 + # CORS / frontend origin (add "127.0.0.1 vcell-ai.local" to /etc/hosts for minikube) FRONTEND_URL=http://vcell-ai.local diff --git a/kustomize/config/vcell-ai-local/kustomization.yaml b/kustomize/config/vcell-ai-local/kustomization.yaml index 3b0b7b5..f3a99f9 100644 --- a/kustomize/config/vcell-ai-local/kustomization.yaml +++ b/kustomize/config/vcell-ai-local/kustomization.yaml @@ -8,3 +8,6 @@ configMapGenerator: - name: frontend-config envs: - frontend.env + - name: litellm-config + envs: + - litellm.env diff --git a/kustomize/config/vcell-ai-local/litellm.env b/kustomize/config/vcell-ai-local/litellm.env new file mode 100644 index 0000000..9d136b1 --- /dev/null +++ b/kustomize/config/vcell-ai-local/litellm.env @@ -0,0 +1,10 @@ +# Non-secret LiteLLM proxy config for vcell-ai-local (minikube). +# Secrets (master key, OpenAI key, Langfuse keys, DATABASE_URL) come from the +# sealed litellm-secrets, not this file. + +# local-model alias -> in-cluster Ollama (fallback when openai-model fails) +LOCAL_LLM_MODEL=ollama/phi4-mini +LOCAL_LLM_API_BASE=http://ollama:11434 + +# Langfuse tracing callback (host only; keys are secret) +LANGFUSE_HOST=https://cloud.langfuse.com diff --git a/kustomize/config/vcell-ai-rke-dev/backend.env b/kustomize/config/vcell-ai-rke-dev/backend.env index 6ef6e09..8d5acec 100644 --- a/kustomize/config/vcell-ai-rke-dev/backend.env +++ b/kustomize/config/vcell-ai-rke-dev/backend.env @@ -23,5 +23,8 @@ AUTH0_AUDIENCE=https://vcell-ai-api # Supabase (URL is public; service-role key is secret) SUPABASE_URL=https://your-project.supabase.co +# LiteLLM proxy (in-cluster service; master key is secret) +LITELLM_URL=http://litellm:4000 + # CORS / frontend origin FRONTEND_URL=https://vcell-ai-dev.cam.uchc.edu diff --git a/kustomize/config/vcell-ai-rke-dev/kustomization.yaml b/kustomize/config/vcell-ai-rke-dev/kustomization.yaml index 3b0b7b5..f3a99f9 100644 --- a/kustomize/config/vcell-ai-rke-dev/kustomization.yaml +++ b/kustomize/config/vcell-ai-rke-dev/kustomization.yaml @@ -8,3 +8,6 @@ configMapGenerator: - name: frontend-config envs: - frontend.env + - name: litellm-config + envs: + - litellm.env diff --git a/kustomize/config/vcell-ai-rke-dev/litellm.env b/kustomize/config/vcell-ai-rke-dev/litellm.env new file mode 100644 index 0000000..b658215 --- /dev/null +++ b/kustomize/config/vcell-ai-rke-dev/litellm.env @@ -0,0 +1,10 @@ +# Non-secret LiteLLM proxy config for vcell-ai-rke-dev (dev). +# Secrets (master key, OpenAI key, Langfuse keys, DATABASE_URL) come from the +# sealed litellm-secrets, not this file. + +# local-model alias -> in-cluster Ollama (fallback when openai-model fails) +LOCAL_LLM_MODEL=ollama/phi4-mini +LOCAL_LLM_API_BASE=http://ollama:11434 + +# Langfuse tracing callback (host only; keys are secret) +LANGFUSE_HOST=https://cloud.langfuse.com diff --git a/kustomize/config/vcell-ai-rke/backend.env b/kustomize/config/vcell-ai-rke/backend.env index d7d5488..653622b 100644 --- a/kustomize/config/vcell-ai-rke/backend.env +++ b/kustomize/config/vcell-ai-rke/backend.env @@ -23,5 +23,8 @@ AUTH0_AUDIENCE=https://vcell-ai-api # Supabase (URL is public; service-role key is secret) SUPABASE_URL=https://your-project.supabase.co +# LiteLLM proxy (in-cluster service; master key is secret) +LITELLM_URL=http://litellm:4000 + # CORS / frontend origin FRONTEND_URL=https://vcell-ai.cam.uchc.edu diff --git a/kustomize/config/vcell-ai-rke/kustomization.yaml b/kustomize/config/vcell-ai-rke/kustomization.yaml index 3b0b7b5..f3a99f9 100644 --- a/kustomize/config/vcell-ai-rke/kustomization.yaml +++ b/kustomize/config/vcell-ai-rke/kustomization.yaml @@ -8,3 +8,6 @@ configMapGenerator: - name: frontend-config envs: - frontend.env + - name: litellm-config + envs: + - litellm.env diff --git a/kustomize/config/vcell-ai-rke/litellm.env b/kustomize/config/vcell-ai-rke/litellm.env new file mode 100644 index 0000000..eb62cc0 --- /dev/null +++ b/kustomize/config/vcell-ai-rke/litellm.env @@ -0,0 +1,10 @@ +# Non-secret LiteLLM proxy config for vcell-ai-rke (prod). +# Secrets (master key, OpenAI key, Langfuse keys, DATABASE_URL) come from the +# sealed litellm-secrets, not this file. + +# local-model alias -> in-cluster Ollama (fallback when openai-model fails) +LOCAL_LLM_MODEL=ollama/phi4-mini +LOCAL_LLM_API_BASE=http://ollama:11434 + +# Langfuse tracing callback (host only; keys are secret) +LANGFUSE_HOST=https://cloud.langfuse.com diff --git a/kustomize/overlays/vcell-ai-local/kustomization.yaml b/kustomize/overlays/vcell-ai-local/kustomization.yaml index cd7c820..a5e34c6 100644 --- a/kustomize/overlays/vcell-ai-local/kustomization.yaml +++ b/kustomize/overlays/vcell-ai-local/kustomization.yaml @@ -5,9 +5,9 @@ namespace: vcell-ai-local images: - name: ghcr.io/virtualcell/vcell-ai-backend - newTag: 0.1.6.2 + newTag: 0.1.7 - name: ghcr.io/virtualcell/vcell-ai-frontend - newTag: 0.1.6.2 + newTag: 0.1.7 replicas: - count: 1 @@ -22,5 +22,6 @@ resources: - secret-backend.yaml - secret-frontend.yaml - secret-ghcr.yaml + - secret-litellm.yaml - ../../config/vcell-ai-local - ../../base diff --git a/kustomize/overlays/vcell-ai-local/secrets.dat.template b/kustomize/overlays/vcell-ai-local/secrets.dat.template index 19c272b..4bccced 100644 --- a/kustomize/overlays/vcell-ai-local/secrets.dat.template +++ b/kustomize/overlays/vcell-ai-local/secrets.dat.template @@ -9,6 +9,13 @@ LANGFUSE_SECRET_KEY=sk-lf-your-langfuse-secret-key LANGFUSE_PUBLIC_KEY=pk-lf-your-langfuse-public-key SUPABASE_SERVICE_ROLE_KEY=your_supabase_service_role_key +# --- LiteLLM (litellm-secrets; master key also injected into the backend) --- +# LITELLM_MASTER_KEY: strong random value, e.g. `openssl rand -hex 32` (sk- prefix) +LITELLM_MASTER_KEY=sk-litellm-change-me +OPENAI_API_KEY=sk-your-openai-api-key +# Postgres for persistent virtual keys/budgets (Supabase direct/session pooler, SSL) +DATABASE_URL=postgresql://postgres:your-db-password@db.your-project-ref.supabase.co:5432/postgres?sslmode=require + # --- Frontend (frontend-secrets) --- # AUTH0_SECRET: 32-byte random string, e.g. `openssl rand -hex 32` AUTH0_SECRET=your_auth0_session_secret diff --git a/kustomize/overlays/vcell-ai-local/secrets.sh b/kustomize/overlays/vcell-ai-local/secrets.sh index 2882b83..998662c 100755 --- a/kustomize/overlays/vcell-ai-local/secrets.sh +++ b/kustomize/overlays/vcell-ai-local/secrets.sh @@ -43,7 +43,7 @@ echo "Generating backend secrets..." ${SCRIPTS_DIR}/sealed_secret_backend.sh \ --controller-name ${SEALED_SECRETS_CONTROLLER_NAME} \ --controller-namespace ${SEALED_SECRETS_CONTROLLER_NAMESPACE} \ - ${NAMESPACE} "${AZURE_API_KEY}" "${LANGFUSE_SECRET_KEY}" "${LANGFUSE_PUBLIC_KEY}" "${SUPABASE_SERVICE_ROLE_KEY}" \ + ${NAMESPACE} "${AZURE_API_KEY}" "${LANGFUSE_SECRET_KEY}" "${LANGFUSE_PUBLIC_KEY}" "${SUPABASE_SERVICE_ROLE_KEY}" "${LITELLM_MASTER_KEY}" \ > ${SECRETS_DIR}/secret-backend.yaml echo "✓ secret-backend.yaml generated" @@ -65,5 +65,14 @@ ${SCRIPTS_DIR}/sealed_secret_ghcr.sh \ > ${SECRETS_DIR}/secret-ghcr.yaml echo "✓ secret-ghcr.yaml generated" +# litellm-secrets: +echo "Generating LiteLLM secrets..." +${SCRIPTS_DIR}/sealed_secret_litellm.sh \ + --controller-name ${SEALED_SECRETS_CONTROLLER_NAME} \ + --controller-namespace ${SEALED_SECRETS_CONTROLLER_NAMESPACE} \ + ${NAMESPACE} "${LITELLM_MASTER_KEY}" "${OPENAI_API_KEY}" "${LANGFUSE_SECRET_KEY}" "${LANGFUSE_PUBLIC_KEY}" "${DATABASE_URL}" \ + > ${SECRETS_DIR}/secret-litellm.yaml +echo "✓ secret-litellm.yaml generated" + echo "" echo "=== All sealed secrets generated successfully! ===" diff --git a/kustomize/overlays/vcell-ai-rke-dev/kustomization.yaml b/kustomize/overlays/vcell-ai-rke-dev/kustomization.yaml index 2948ab7..b8caef7 100644 --- a/kustomize/overlays/vcell-ai-rke-dev/kustomization.yaml +++ b/kustomize/overlays/vcell-ai-rke-dev/kustomization.yaml @@ -5,9 +5,9 @@ namespace: vcell-ai-rke-dev images: - name: ghcr.io/virtualcell/vcell-ai-backend - newTag: 0.1.6.2 + newTag: 0.1.7 - name: ghcr.io/virtualcell/vcell-ai-frontend - newTag: 0.1.6.2 + newTag: 0.1.7 replicas: - count: 1 @@ -22,5 +22,6 @@ resources: - secret-backend.yaml - secret-frontend.yaml - secret-ghcr.yaml + - secret-litellm.yaml - ../../config/vcell-ai-rke-dev - ../../base diff --git a/kustomize/overlays/vcell-ai-rke-dev/secrets.dat.template b/kustomize/overlays/vcell-ai-rke-dev/secrets.dat.template index 4f51806..2e62bed 100644 --- a/kustomize/overlays/vcell-ai-rke-dev/secrets.dat.template +++ b/kustomize/overlays/vcell-ai-rke-dev/secrets.dat.template @@ -9,6 +9,13 @@ LANGFUSE_SECRET_KEY=sk-lf-your-langfuse-secret-key LANGFUSE_PUBLIC_KEY=pk-lf-your-langfuse-public-key SUPABASE_SERVICE_ROLE_KEY=your_supabase_service_role_key +# --- LiteLLM (litellm-secrets; master key also injected into the backend) --- +# LITELLM_MASTER_KEY: strong random value, e.g. `openssl rand -hex 32` (sk- prefix) +LITELLM_MASTER_KEY=sk-litellm-change-me +OPENAI_API_KEY=sk-your-openai-api-key +# Postgres for persistent virtual keys/budgets (Supabase direct/session pooler, SSL) +DATABASE_URL=postgresql://postgres:your-db-password@db.your-project-ref.supabase.co:5432/postgres?sslmode=require + # --- Frontend (frontend-secrets) --- # AUTH0_SECRET: 32-byte random string, e.g. `openssl rand -hex 32` AUTH0_SECRET=your_auth0_session_secret diff --git a/kustomize/overlays/vcell-ai-rke-dev/secrets.sh b/kustomize/overlays/vcell-ai-rke-dev/secrets.sh index 31f8138..6e8dd8c 100755 --- a/kustomize/overlays/vcell-ai-rke-dev/secrets.sh +++ b/kustomize/overlays/vcell-ai-rke-dev/secrets.sh @@ -42,7 +42,7 @@ echo "Generating backend secrets..." ${SCRIPTS_DIR}/sealed_secret_backend.sh \ --controller-name ${SEALED_SECRETS_CONTROLLER_NAME} \ --controller-namespace ${SEALED_SECRETS_CONTROLLER_NAMESPACE} \ - ${NAMESPACE} "${AZURE_API_KEY}" "${LANGFUSE_SECRET_KEY}" "${LANGFUSE_PUBLIC_KEY}" "${SUPABASE_SERVICE_ROLE_KEY}" \ + ${NAMESPACE} "${AZURE_API_KEY}" "${LANGFUSE_SECRET_KEY}" "${LANGFUSE_PUBLIC_KEY}" "${SUPABASE_SERVICE_ROLE_KEY}" "${LITELLM_MASTER_KEY}" \ > ${SECRETS_DIR}/secret-backend.yaml echo "✓ secret-backend.yaml generated" @@ -64,5 +64,14 @@ ${SCRIPTS_DIR}/sealed_secret_ghcr.sh \ > ${SECRETS_DIR}/secret-ghcr.yaml echo "✓ secret-ghcr.yaml generated" +# litellm-secrets: +echo "Generating LiteLLM secrets..." +${SCRIPTS_DIR}/sealed_secret_litellm.sh \ + --controller-name ${SEALED_SECRETS_CONTROLLER_NAME} \ + --controller-namespace ${SEALED_SECRETS_CONTROLLER_NAMESPACE} \ + ${NAMESPACE} "${LITELLM_MASTER_KEY}" "${OPENAI_API_KEY}" "${LANGFUSE_SECRET_KEY}" "${LANGFUSE_PUBLIC_KEY}" "${DATABASE_URL}" \ + > ${SECRETS_DIR}/secret-litellm.yaml +echo "✓ secret-litellm.yaml generated" + echo "" echo "=== All sealed secrets generated successfully! ===" diff --git a/kustomize/overlays/vcell-ai-rke/kustomization.yaml b/kustomize/overlays/vcell-ai-rke/kustomization.yaml index 02747eb..46fe6a4 100644 --- a/kustomize/overlays/vcell-ai-rke/kustomization.yaml +++ b/kustomize/overlays/vcell-ai-rke/kustomization.yaml @@ -5,9 +5,9 @@ namespace: vcell-ai-rke images: - name: ghcr.io/virtualcell/vcell-ai-backend - newTag: 0.1.6.2 + newTag: 0.1.7 - name: ghcr.io/virtualcell/vcell-ai-frontend - newTag: 0.1.6.2 + newTag: 0.1.7 replicas: - count: 1 @@ -22,5 +22,6 @@ resources: - secret-backend.yaml - secret-frontend.yaml - secret-ghcr.yaml + - secret-litellm.yaml - ../../config/vcell-ai-rke - ../../base diff --git a/kustomize/overlays/vcell-ai-rke/secrets.dat.template b/kustomize/overlays/vcell-ai-rke/secrets.dat.template index f0feb01..94ff305 100644 --- a/kustomize/overlays/vcell-ai-rke/secrets.dat.template +++ b/kustomize/overlays/vcell-ai-rke/secrets.dat.template @@ -9,6 +9,13 @@ LANGFUSE_SECRET_KEY=sk-lf-your-langfuse-secret-key LANGFUSE_PUBLIC_KEY=pk-lf-your-langfuse-public-key SUPABASE_SERVICE_ROLE_KEY=your_supabase_service_role_key +# --- LiteLLM (litellm-secrets; master key also injected into the backend) --- +# LITELLM_MASTER_KEY: strong random value, e.g. `openssl rand -hex 32` (sk- prefix) +LITELLM_MASTER_KEY=sk-litellm-change-me +OPENAI_API_KEY=sk-your-openai-api-key +# Postgres for persistent virtual keys/budgets (Supabase direct/session pooler, SSL) +DATABASE_URL=postgresql://postgres:your-db-password@db.your-project-ref.supabase.co:5432/postgres?sslmode=require + # --- Frontend (frontend-secrets) --- # AUTH0_SECRET: 32-byte random string, e.g. `openssl rand -hex 32` AUTH0_SECRET=your_auth0_session_secret diff --git a/kustomize/overlays/vcell-ai-rke/secrets.sh b/kustomize/overlays/vcell-ai-rke/secrets.sh index f93d6ad..08dfa36 100755 --- a/kustomize/overlays/vcell-ai-rke/secrets.sh +++ b/kustomize/overlays/vcell-ai-rke/secrets.sh @@ -42,7 +42,7 @@ echo "Generating backend secrets..." ${SCRIPTS_DIR}/sealed_secret_backend.sh \ --controller-name ${SEALED_SECRETS_CONTROLLER_NAME} \ --controller-namespace ${SEALED_SECRETS_CONTROLLER_NAMESPACE} \ - ${NAMESPACE} "${AZURE_API_KEY}" "${LANGFUSE_SECRET_KEY}" "${LANGFUSE_PUBLIC_KEY}" "${SUPABASE_SERVICE_ROLE_KEY}" \ + ${NAMESPACE} "${AZURE_API_KEY}" "${LANGFUSE_SECRET_KEY}" "${LANGFUSE_PUBLIC_KEY}" "${SUPABASE_SERVICE_ROLE_KEY}" "${LITELLM_MASTER_KEY}" \ > ${SECRETS_DIR}/secret-backend.yaml echo "✓ secret-backend.yaml generated" @@ -64,5 +64,14 @@ ${SCRIPTS_DIR}/sealed_secret_ghcr.sh \ > ${SECRETS_DIR}/secret-ghcr.yaml echo "✓ secret-ghcr.yaml generated" +# litellm-secrets: +echo "Generating LiteLLM secrets..." +${SCRIPTS_DIR}/sealed_secret_litellm.sh \ + --controller-name ${SEALED_SECRETS_CONTROLLER_NAME} \ + --controller-namespace ${SEALED_SECRETS_CONTROLLER_NAMESPACE} \ + ${NAMESPACE} "${LITELLM_MASTER_KEY}" "${OPENAI_API_KEY}" "${LANGFUSE_SECRET_KEY}" "${LANGFUSE_PUBLIC_KEY}" "${DATABASE_URL}" \ + > ${SECRETS_DIR}/secret-litellm.yaml +echo "✓ secret-litellm.yaml generated" + echo "" echo "=== All sealed secrets generated successfully! ===" diff --git a/kustomize/scripts/sealed_secret_backend.sh b/kustomize/scripts/sealed_secret_backend.sh index 937feec..fd5dabb 100755 --- a/kustomize/scripts/sealed_secret_backend.sh +++ b/kustomize/scripts/sealed_secret_backend.sh @@ -7,7 +7,7 @@ set -u # Usage: # ./sealed_secret_backend.sh [--cert ] \ # [--controller-name ] [--controller-namespace ] \ -# \ +# \ # > secret-backend.yaml # # For GKE / AWS GovCloud the controller cert is needed and can be fetched with: @@ -43,9 +43,9 @@ while [[ "$1" == --* ]]; do done # Validate the number of positional arguments -if [ "$#" -ne 5 ]; then +if [ "$#" -ne 6 ]; then echo "Illegal number of parameters" - echo "Usage: ./sealed_secret_backend.sh [--cert ] " + echo "Usage: ./sealed_secret_backend.sh [--cert ] " exit 1 fi @@ -55,6 +55,7 @@ AZURE_API_KEY=$2 LANGFUSE_SECRET_KEY=$3 LANGFUSE_PUBLIC_KEY=$4 SUPABASE_SERVICE_ROLE_KEY=$5 +LITELLM_MASTER_KEY=$6 # Create the generic secret and seal it kubectl create secret generic ${SECRET_NAME} --dry-run=client \ @@ -62,4 +63,5 @@ kubectl create secret generic ${SECRET_NAME} --dry-run=client \ --from-literal=langfuse-secret-key="${LANGFUSE_SECRET_KEY}" \ --from-literal=langfuse-public-key="${LANGFUSE_PUBLIC_KEY}" \ --from-literal=supabase-service-role-key="${SUPABASE_SERVICE_ROLE_KEY}" \ + --from-literal=litellm-master-key="${LITELLM_MASTER_KEY}" \ --namespace="${NAMESPACE}" -o yaml | kubeseal --controller-name=${CONTROLLER_NAME} --controller-namespace=${CONTROLLER_NAMESPACE} --format yaml ${CERT_ARG:+--cert=$CERT_ARG} diff --git a/kustomize/scripts/sealed_secret_litellm.sh b/kustomize/scripts/sealed_secret_litellm.sh new file mode 100755 index 0000000..6ce4de1 --- /dev/null +++ b/kustomize/scripts/sealed_secret_litellm.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash + +set -u + +# Create a SealedSecret for the LiteLLM proxy's sensitive environment values: +# master key, the hosted OpenAI provider key, Langfuse tracing keys, and the +# Postgres DATABASE_URL used to persist virtual keys / budgets. +# +# Usage: +# ./sealed_secret_litellm.sh [--cert ] \ +# [--controller-name ] [--controller-namespace ] \ +# \ +# > secret-litellm.yaml +# +# For GKE / AWS GovCloud the controller cert is needed and can be fetched with: +# kubeseal --fetch-cert > filename.pem + +CERT_ARG="" +CONTROLLER_NAME="sealed-secrets-controller" +CONTROLLER_NAMESPACE="sealed-secrets" + +# Parse optional arguments +while [[ "$1" == --* ]]; do + case "$1" in + --cert) + CERT_ARG="$2" + shift 2 + ;; + --controller-name) + CONTROLLER_NAME="$2" + shift 2 + ;; + --controller-namespace) + CONTROLLER_NAMESPACE="$2" + shift 2 + ;; + *) + echo "Unknown option: $1" + exit 1 + ;; + esac +done + +# Validate the number of positional arguments +if [ "$#" -ne 6 ]; then + echo "Illegal number of parameters" + echo "Usage: ./sealed_secret_litellm.sh [--cert ] " + exit 1 +fi + +SECRET_NAME="litellm-secrets" +NAMESPACE=$1 +MASTER_KEY=$2 +OPENAI_API_KEY=$3 +LANGFUSE_SECRET_KEY=$4 +LANGFUSE_PUBLIC_KEY=$5 +DATABASE_URL=$6 + +# Create the generic secret and seal it +kubectl create secret generic ${SECRET_NAME} --dry-run=client \ + --from-literal=master-key="${MASTER_KEY}" \ + --from-literal=openai-api-key="${OPENAI_API_KEY}" \ + --from-literal=langfuse-secret-key="${LANGFUSE_SECRET_KEY}" \ + --from-literal=langfuse-public-key="${LANGFUSE_PUBLIC_KEY}" \ + --from-literal=database-url="${DATABASE_URL}" \ + --namespace="${NAMESPACE}" -o yaml | kubeseal --controller-name=${CONTROLLER_NAME} --controller-namespace=${CONTROLLER_NAMESPACE} --format yaml ${CERT_ARG:+--cert=$CERT_ARG} From 1cc0823cfd8ee64e6cfee860780302bec989afbc Mon Sep 17 00:00:00 2001 From: Jim Schaff Date: Sun, 12 Jul 2026 15:27:53 -0400 Subject: [PATCH 02/11] feat(k8s): back LiteLLM "openai-model" with Azure instead of hosted OpenAI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The project uses Azure OpenAI (already used for embeddings) and has no separate hosted-OpenAI key, so point LiteLLM's openai-model alias at Azure. The alias name stays "openai-model" (hardcoded in the backend LLMModel Literal + frontend). - litellm_config.yaml (kustomize base + repo root, kept in sync): openai-model litellm_params -> model=azure/ via os.environ/AZURE_MODEL, plus api_base/api_version/api_key from env - config//litellm.env: add AZURE_MODEL, AZURE_API_BASE, AZURE_API_VERSION - litellm-secrets: openai-api-key -> azure-api-key; base/litellm.yaml env and sealed_secret_litellm.sh updated; secrets.sh passes the existing AZURE_API_KEY - secrets.dat.template: drop OPENAI_API_KEY (Azure key reused); fix DATABASE_URL example to the Supabase SESSION POOLER (IPv4, 5432) — the direct host is IPv6-only - litellm.env.example: OpenAI -> Azure vars Validated: kubectl kustomize renders all three overlays (20 objects); the litellm ConfigMap carries the AZURE_* vars and the pod reads azure-api-key. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01XpNzobVi83p3YKL9sqtGwZ --- kustomize/README.md | 11 ++++++----- kustomize/base/litellm.yaml | 4 ++-- kustomize/base/litellm_config.yaml | 8 ++++++-- kustomize/config/vcell-ai-local/litellm.env | 5 +++++ kustomize/config/vcell-ai-rke-dev/litellm.env | 5 +++++ kustomize/config/vcell-ai-rke/litellm.env | 5 +++++ .../overlays/vcell-ai-local/secrets.dat.template | 7 ++++--- kustomize/overlays/vcell-ai-local/secrets.sh | 4 ++-- .../overlays/vcell-ai-rke-dev/secrets.dat.template | 7 ++++--- kustomize/overlays/vcell-ai-rke-dev/secrets.sh | 4 ++-- .../overlays/vcell-ai-rke/secrets.dat.template | 7 ++++--- kustomize/overlays/vcell-ai-rke/secrets.sh | 4 ++-- kustomize/scripts/sealed_secret_litellm.sh | 13 +++++++------ litellm.env.example | 8 ++++++-- litellm_config.yaml | 8 ++++++-- 15 files changed, 66 insertions(+), 34 deletions(-) diff --git a/kustomize/README.md b/kustomize/README.md index 5bad7f8..367b277 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -11,10 +11,11 @@ Deploys the VCell-AI services from `docker-compose.yml` to Kubernetes: | `ollama` | `ollama/ollama` | 11434 | StatefulSet + 20Gi PVC; serves the `local-model` fallback (CPU; pulls `llama3.1:8b` on start) | The backend calls `litellm` for chat (`LITELLM_URL`, per-user virtual keys via -`LITELLM_MASTER_KEY`) but still uses Azure directly for KB **embeddings**. LiteLLM -routes `openai-model` (hosted OpenAI gpt-4o) with a fallback to `local-model` -(in-cluster Ollama), and persists virtual keys/budgets to a Postgres -(`DATABASE_URL`, e.g. Supabase). Structured after `../sms-api/kustomize`. +`LITELLM_MASTER_KEY`) and also uses Azure directly for KB **embeddings**. LiteLLM +routes `openai-model` (backed by **Azure OpenAI** — the same account, so no separate +hosted-OpenAI key) with a fallback to `local-model` (in-cluster Ollama), and +persists virtual keys/budgets to a Postgres (`DATABASE_URL`, e.g. Supabase's +session pooler). Structured after `../sms-api/kustomize`. ## Layout @@ -61,7 +62,7 @@ sensitive values are sealed: - **backend-secrets** → `AZURE_API_KEY`, `LANGFUSE_SECRET_KEY`, `LANGFUSE_PUBLIC_KEY`, `SUPABASE_SERVICE_ROLE_KEY`, `LITELLM_MASTER_KEY` - **frontend-secrets** → `AUTH0_SECRET`, `AUTH0_CLIENT_SECRET` -- **litellm-secrets** → `LITELLM_MASTER_KEY`, `OPENAI_API_KEY`, `LANGFUSE_SECRET_KEY`, `LANGFUSE_PUBLIC_KEY`, `DATABASE_URL` +- **litellm-secrets** → `LITELLM_MASTER_KEY`, `AZURE_API_KEY`, `LANGFUSE_SECRET_KEY`, `LANGFUSE_PUBLIC_KEY`, `DATABASE_URL` - **ghcr-secret** → ghcr.io image-pull credentials (`.dockerconfigjson`) (`LITELLM_MASTER_KEY` and the Langfuse keys are intentionally in more than one diff --git a/kustomize/base/litellm.yaml b/kustomize/base/litellm.yaml index fcf3f81..4cbcdc9 100644 --- a/kustomize/base/litellm.yaml +++ b/kustomize/base/litellm.yaml @@ -33,11 +33,11 @@ spec: secretKeyRef: name: litellm-secrets key: master-key - - name: OPENAI_API_KEY + - name: AZURE_API_KEY valueFrom: secretKeyRef: name: litellm-secrets - key: openai-api-key + key: azure-api-key - name: LANGFUSE_SECRET_KEY valueFrom: secretKeyRef: diff --git a/kustomize/base/litellm_config.yaml b/kustomize/base/litellm_config.yaml index c4d0581..80641a0 100644 --- a/kustomize/base/litellm_config.yaml +++ b/kustomize/base/litellm_config.yaml @@ -1,8 +1,12 @@ model_list: + # "openai-model" is the alias the app requests (backend LLMModel Literal + + # frontend). It is backed by Azure OpenAI here — no hosted-OpenAI key needed. - model_name: openai-model litellm_params: - model: openai/gpt-4o - api_key: os.environ/OPENAI_API_KEY + model: os.environ/AZURE_MODEL # e.g. azure/ + api_base: os.environ/AZURE_API_BASE + api_key: os.environ/AZURE_API_KEY + api_version: os.environ/AZURE_API_VERSION - model_name: local-model litellm_params: diff --git a/kustomize/config/vcell-ai-local/litellm.env b/kustomize/config/vcell-ai-local/litellm.env index 9d136b1..d83695b 100644 --- a/kustomize/config/vcell-ai-local/litellm.env +++ b/kustomize/config/vcell-ai-local/litellm.env @@ -2,6 +2,11 @@ # Secrets (master key, OpenAI key, Langfuse keys, DATABASE_URL) come from the # sealed litellm-secrets, not this file. +# openai-model alias -> Azure OpenAI (api key is secret, in litellm-secrets) +AZURE_MODEL=azure/your-chat-deployment-name +AZURE_API_BASE=https://your-resource.openai.azure.com +AZURE_API_VERSION=2024-02-01 + # local-model alias -> in-cluster Ollama (fallback when openai-model fails) LOCAL_LLM_MODEL=ollama/phi4-mini LOCAL_LLM_API_BASE=http://ollama:11434 diff --git a/kustomize/config/vcell-ai-rke-dev/litellm.env b/kustomize/config/vcell-ai-rke-dev/litellm.env index b658215..da7a653 100644 --- a/kustomize/config/vcell-ai-rke-dev/litellm.env +++ b/kustomize/config/vcell-ai-rke-dev/litellm.env @@ -2,6 +2,11 @@ # Secrets (master key, OpenAI key, Langfuse keys, DATABASE_URL) come from the # sealed litellm-secrets, not this file. +# openai-model alias -> Azure OpenAI (api key is secret, in litellm-secrets) +AZURE_MODEL=azure/your-chat-deployment-name +AZURE_API_BASE=https://your-resource.openai.azure.com +AZURE_API_VERSION=2024-02-01 + # local-model alias -> in-cluster Ollama (fallback when openai-model fails) LOCAL_LLM_MODEL=ollama/phi4-mini LOCAL_LLM_API_BASE=http://ollama:11434 diff --git a/kustomize/config/vcell-ai-rke/litellm.env b/kustomize/config/vcell-ai-rke/litellm.env index eb62cc0..88ab749 100644 --- a/kustomize/config/vcell-ai-rke/litellm.env +++ b/kustomize/config/vcell-ai-rke/litellm.env @@ -2,6 +2,11 @@ # Secrets (master key, OpenAI key, Langfuse keys, DATABASE_URL) come from the # sealed litellm-secrets, not this file. +# openai-model alias -> Azure OpenAI (api key is secret, in litellm-secrets) +AZURE_MODEL=azure/your-chat-deployment-name +AZURE_API_BASE=https://your-resource.openai.azure.com +AZURE_API_VERSION=2024-02-01 + # local-model alias -> in-cluster Ollama (fallback when openai-model fails) LOCAL_LLM_MODEL=ollama/phi4-mini LOCAL_LLM_API_BASE=http://ollama:11434 diff --git a/kustomize/overlays/vcell-ai-local/secrets.dat.template b/kustomize/overlays/vcell-ai-local/secrets.dat.template index 4bccced..c953e52 100644 --- a/kustomize/overlays/vcell-ai-local/secrets.dat.template +++ b/kustomize/overlays/vcell-ai-local/secrets.dat.template @@ -12,9 +12,10 @@ SUPABASE_SERVICE_ROLE_KEY=your_supabase_service_role_key # --- LiteLLM (litellm-secrets; master key also injected into the backend) --- # LITELLM_MASTER_KEY: strong random value, e.g. `openssl rand -hex 32` (sk- prefix) LITELLM_MASTER_KEY=sk-litellm-change-me -OPENAI_API_KEY=sk-your-openai-api-key -# Postgres for persistent virtual keys/budgets (Supabase direct/session pooler, SSL) -DATABASE_URL=postgresql://postgres:your-db-password@db.your-project-ref.supabase.co:5432/postgres?sslmode=require +# The "openai-model" alias is backed by Azure — LiteLLM reuses AZURE_API_KEY above. +# Postgres for persistent virtual keys/budgets. Use the Supabase SESSION POOLER +# (IPv4, port 5432) — the direct db..supabase.co host is IPv6-only. +DATABASE_URL=postgresql://postgres.your-project-ref:your-db-password@aws-0-your-region.pooler.supabase.com:5432/postgres?sslmode=require # --- Frontend (frontend-secrets) --- # AUTH0_SECRET: 32-byte random string, e.g. `openssl rand -hex 32` diff --git a/kustomize/overlays/vcell-ai-local/secrets.sh b/kustomize/overlays/vcell-ai-local/secrets.sh index 998662c..b0aaf02 100755 --- a/kustomize/overlays/vcell-ai-local/secrets.sh +++ b/kustomize/overlays/vcell-ai-local/secrets.sh @@ -65,12 +65,12 @@ ${SCRIPTS_DIR}/sealed_secret_ghcr.sh \ > ${SECRETS_DIR}/secret-ghcr.yaml echo "✓ secret-ghcr.yaml generated" -# litellm-secrets: +# litellm-secrets: echo "Generating LiteLLM secrets..." ${SCRIPTS_DIR}/sealed_secret_litellm.sh \ --controller-name ${SEALED_SECRETS_CONTROLLER_NAME} \ --controller-namespace ${SEALED_SECRETS_CONTROLLER_NAMESPACE} \ - ${NAMESPACE} "${LITELLM_MASTER_KEY}" "${OPENAI_API_KEY}" "${LANGFUSE_SECRET_KEY}" "${LANGFUSE_PUBLIC_KEY}" "${DATABASE_URL}" \ + ${NAMESPACE} "${LITELLM_MASTER_KEY}" "${AZURE_API_KEY}" "${LANGFUSE_SECRET_KEY}" "${LANGFUSE_PUBLIC_KEY}" "${DATABASE_URL}" \ > ${SECRETS_DIR}/secret-litellm.yaml echo "✓ secret-litellm.yaml generated" diff --git a/kustomize/overlays/vcell-ai-rke-dev/secrets.dat.template b/kustomize/overlays/vcell-ai-rke-dev/secrets.dat.template index 2e62bed..88059a1 100644 --- a/kustomize/overlays/vcell-ai-rke-dev/secrets.dat.template +++ b/kustomize/overlays/vcell-ai-rke-dev/secrets.dat.template @@ -12,9 +12,10 @@ SUPABASE_SERVICE_ROLE_KEY=your_supabase_service_role_key # --- LiteLLM (litellm-secrets; master key also injected into the backend) --- # LITELLM_MASTER_KEY: strong random value, e.g. `openssl rand -hex 32` (sk- prefix) LITELLM_MASTER_KEY=sk-litellm-change-me -OPENAI_API_KEY=sk-your-openai-api-key -# Postgres for persistent virtual keys/budgets (Supabase direct/session pooler, SSL) -DATABASE_URL=postgresql://postgres:your-db-password@db.your-project-ref.supabase.co:5432/postgres?sslmode=require +# The "openai-model" alias is backed by Azure — LiteLLM reuses AZURE_API_KEY above. +# Postgres for persistent virtual keys/budgets. Use the Supabase SESSION POOLER +# (IPv4, port 5432) — the direct db..supabase.co host is IPv6-only. +DATABASE_URL=postgresql://postgres.your-project-ref:your-db-password@aws-0-your-region.pooler.supabase.com:5432/postgres?sslmode=require # --- Frontend (frontend-secrets) --- # AUTH0_SECRET: 32-byte random string, e.g. `openssl rand -hex 32` diff --git a/kustomize/overlays/vcell-ai-rke-dev/secrets.sh b/kustomize/overlays/vcell-ai-rke-dev/secrets.sh index 6e8dd8c..c8b741a 100755 --- a/kustomize/overlays/vcell-ai-rke-dev/secrets.sh +++ b/kustomize/overlays/vcell-ai-rke-dev/secrets.sh @@ -64,12 +64,12 @@ ${SCRIPTS_DIR}/sealed_secret_ghcr.sh \ > ${SECRETS_DIR}/secret-ghcr.yaml echo "✓ secret-ghcr.yaml generated" -# litellm-secrets: +# litellm-secrets: echo "Generating LiteLLM secrets..." ${SCRIPTS_DIR}/sealed_secret_litellm.sh \ --controller-name ${SEALED_SECRETS_CONTROLLER_NAME} \ --controller-namespace ${SEALED_SECRETS_CONTROLLER_NAMESPACE} \ - ${NAMESPACE} "${LITELLM_MASTER_KEY}" "${OPENAI_API_KEY}" "${LANGFUSE_SECRET_KEY}" "${LANGFUSE_PUBLIC_KEY}" "${DATABASE_URL}" \ + ${NAMESPACE} "${LITELLM_MASTER_KEY}" "${AZURE_API_KEY}" "${LANGFUSE_SECRET_KEY}" "${LANGFUSE_PUBLIC_KEY}" "${DATABASE_URL}" \ > ${SECRETS_DIR}/secret-litellm.yaml echo "✓ secret-litellm.yaml generated" diff --git a/kustomize/overlays/vcell-ai-rke/secrets.dat.template b/kustomize/overlays/vcell-ai-rke/secrets.dat.template index 94ff305..2d8501f 100644 --- a/kustomize/overlays/vcell-ai-rke/secrets.dat.template +++ b/kustomize/overlays/vcell-ai-rke/secrets.dat.template @@ -12,9 +12,10 @@ SUPABASE_SERVICE_ROLE_KEY=your_supabase_service_role_key # --- LiteLLM (litellm-secrets; master key also injected into the backend) --- # LITELLM_MASTER_KEY: strong random value, e.g. `openssl rand -hex 32` (sk- prefix) LITELLM_MASTER_KEY=sk-litellm-change-me -OPENAI_API_KEY=sk-your-openai-api-key -# Postgres for persistent virtual keys/budgets (Supabase direct/session pooler, SSL) -DATABASE_URL=postgresql://postgres:your-db-password@db.your-project-ref.supabase.co:5432/postgres?sslmode=require +# The "openai-model" alias is backed by Azure — LiteLLM reuses AZURE_API_KEY above. +# Postgres for persistent virtual keys/budgets. Use the Supabase SESSION POOLER +# (IPv4, port 5432) — the direct db..supabase.co host is IPv6-only. +DATABASE_URL=postgresql://postgres.your-project-ref:your-db-password@aws-0-your-region.pooler.supabase.com:5432/postgres?sslmode=require # --- Frontend (frontend-secrets) --- # AUTH0_SECRET: 32-byte random string, e.g. `openssl rand -hex 32` diff --git a/kustomize/overlays/vcell-ai-rke/secrets.sh b/kustomize/overlays/vcell-ai-rke/secrets.sh index 08dfa36..d2589fa 100755 --- a/kustomize/overlays/vcell-ai-rke/secrets.sh +++ b/kustomize/overlays/vcell-ai-rke/secrets.sh @@ -64,12 +64,12 @@ ${SCRIPTS_DIR}/sealed_secret_ghcr.sh \ > ${SECRETS_DIR}/secret-ghcr.yaml echo "✓ secret-ghcr.yaml generated" -# litellm-secrets: +# litellm-secrets: echo "Generating LiteLLM secrets..." ${SCRIPTS_DIR}/sealed_secret_litellm.sh \ --controller-name ${SEALED_SECRETS_CONTROLLER_NAME} \ --controller-namespace ${SEALED_SECRETS_CONTROLLER_NAMESPACE} \ - ${NAMESPACE} "${LITELLM_MASTER_KEY}" "${OPENAI_API_KEY}" "${LANGFUSE_SECRET_KEY}" "${LANGFUSE_PUBLIC_KEY}" "${DATABASE_URL}" \ + ${NAMESPACE} "${LITELLM_MASTER_KEY}" "${AZURE_API_KEY}" "${LANGFUSE_SECRET_KEY}" "${LANGFUSE_PUBLIC_KEY}" "${DATABASE_URL}" \ > ${SECRETS_DIR}/secret-litellm.yaml echo "✓ secret-litellm.yaml generated" diff --git a/kustomize/scripts/sealed_secret_litellm.sh b/kustomize/scripts/sealed_secret_litellm.sh index 6ce4de1..72a3878 100755 --- a/kustomize/scripts/sealed_secret_litellm.sh +++ b/kustomize/scripts/sealed_secret_litellm.sh @@ -3,13 +3,14 @@ set -u # Create a SealedSecret for the LiteLLM proxy's sensitive environment values: -# master key, the hosted OpenAI provider key, Langfuse tracing keys, and the -# Postgres DATABASE_URL used to persist virtual keys / budgets. +# master key, the Azure OpenAI key (openai-model alias is backed by Azure), +# Langfuse tracing keys, and the Postgres DATABASE_URL used to persist virtual +# keys / budgets. # # Usage: # ./sealed_secret_litellm.sh [--cert ] \ # [--controller-name ] [--controller-namespace ] \ -# \ +# \ # > secret-litellm.yaml # # For GKE / AWS GovCloud the controller cert is needed and can be fetched with: @@ -44,14 +45,14 @@ done # Validate the number of positional arguments if [ "$#" -ne 6 ]; then echo "Illegal number of parameters" - echo "Usage: ./sealed_secret_litellm.sh [--cert ] " + echo "Usage: ./sealed_secret_litellm.sh [--cert ] " exit 1 fi SECRET_NAME="litellm-secrets" NAMESPACE=$1 MASTER_KEY=$2 -OPENAI_API_KEY=$3 +AZURE_API_KEY=$3 LANGFUSE_SECRET_KEY=$4 LANGFUSE_PUBLIC_KEY=$5 DATABASE_URL=$6 @@ -59,7 +60,7 @@ DATABASE_URL=$6 # Create the generic secret and seal it kubectl create secret generic ${SECRET_NAME} --dry-run=client \ --from-literal=master-key="${MASTER_KEY}" \ - --from-literal=openai-api-key="${OPENAI_API_KEY}" \ + --from-literal=azure-api-key="${AZURE_API_KEY}" \ --from-literal=langfuse-secret-key="${LANGFUSE_SECRET_KEY}" \ --from-literal=langfuse-public-key="${LANGFUSE_PUBLIC_KEY}" \ --from-literal=database-url="${DATABASE_URL}" \ diff --git a/litellm.env.example b/litellm.env.example index 5e3d230..307bbfc 100644 --- a/litellm.env.example +++ b/litellm.env.example @@ -3,8 +3,12 @@ # openssl rand -hex 32 LITELLM_MASTER_KEY=sk-litellm-change-me -# Hosted OpenAI provider for the "openai-model" LiteLLM alias. -OPENAI_API_KEY=sk-your-openai-api-key +# Azure OpenAI provider for the "openai-model" LiteLLM alias (reuses the same +# Azure account the backend uses for embeddings — no separate OpenAI key needed). +AZURE_MODEL=azure/your-chat-deployment-name +AZURE_API_BASE=https://your-resource.openai.azure.com +AZURE_API_KEY=your_azure_openai_api_key +AZURE_API_VERSION=2024-02-01 # Local provider for the "local-model" LiteLLM alias. # For Ollama, keep the model prefixed with "ollama/". diff --git a/litellm_config.yaml b/litellm_config.yaml index c4d0581..80641a0 100644 --- a/litellm_config.yaml +++ b/litellm_config.yaml @@ -1,8 +1,12 @@ model_list: + # "openai-model" is the alias the app requests (backend LLMModel Literal + + # frontend). It is backed by Azure OpenAI here — no hosted-OpenAI key needed. - model_name: openai-model litellm_params: - model: openai/gpt-4o - api_key: os.environ/OPENAI_API_KEY + model: os.environ/AZURE_MODEL # e.g. azure/ + api_base: os.environ/AZURE_API_BASE + api_key: os.environ/AZURE_API_KEY + api_version: os.environ/AZURE_API_VERSION - model_name: local-model litellm_params: From f513cf831e52ef80da3f7e513ea940969bafc87f Mon Sep 17 00:00:00 2001 From: Jim Schaff Date: Sun, 12 Jul 2026 19:07:39 -0400 Subject: [PATCH 03/11] feat(k8s): LiteLLM openai-model accepts either Azure or OpenAI key Make the "openai-model" alias a group of two interchangeable deployments (Azure OpenAI + hosted OpenAI). Provide a key for either (or both) in secrets.dat; LiteLLM routes to whichever authenticates. Gives deploy-time flexibility without committing to a provider now. - litellm_config.yaml (base + root): second openai-model deployment (openai/gpt-4o) - litellm-secrets regains openai-api-key alongside azure-api-key; base/litellm.yaml, sealed_secret_litellm.sh (7 args), secrets.sh and secrets.dat.template updated - unify the endpoint var: LiteLLM now reads AZURE_ENDPOINT (the value the backend already uses for embeddings) instead of a separate AZURE_API_BASE, and the backend/litellm placeholders are aligned so the shared value can't drift - note in configs/README that KB embeddings still require the Azure key specifically, and root litellm.env.example local model -> phi4-mini Validated: kubectl kustomize renders all overlays (20 objects); the mounted config has both openai-model deployments and the pod reads both api keys. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01XpNzobVi83p3YKL9sqtGwZ --- kustomize/README.md | 8 +++---- kustomize/base/litellm.yaml | 6 ++++++ kustomize/base/litellm_config.yaml | 14 +++++++++++-- kustomize/config/vcell-ai-local/litellm.env | 4 ++-- kustomize/config/vcell-ai-rke-dev/litellm.env | 4 ++-- kustomize/config/vcell-ai-rke/litellm.env | 4 ++-- .../vcell-ai-local/secrets.dat.template | 5 ++++- kustomize/overlays/vcell-ai-local/secrets.sh | 4 ++-- .../vcell-ai-rke-dev/secrets.dat.template | 5 ++++- .../overlays/vcell-ai-rke-dev/secrets.sh | 4 ++-- .../vcell-ai-rke/secrets.dat.template | 5 ++++- kustomize/overlays/vcell-ai-rke/secrets.sh | 4 ++-- kustomize/scripts/sealed_secret_litellm.sh | 21 +++++++++++-------- litellm.env.example | 11 ++++++---- litellm_config.yaml | 14 +++++++++++-- 15 files changed, 77 insertions(+), 36 deletions(-) diff --git a/kustomize/README.md b/kustomize/README.md index 367b277..95f0450 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -12,9 +12,9 @@ Deploys the VCell-AI services from `docker-compose.yml` to Kubernetes: The backend calls `litellm` for chat (`LITELLM_URL`, per-user virtual keys via `LITELLM_MASTER_KEY`) and also uses Azure directly for KB **embeddings**. LiteLLM -routes `openai-model` (backed by **Azure OpenAI** — the same account, so no separate -hosted-OpenAI key) with a fallback to `local-model` (in-cluster Ollama), and -persists virtual keys/budgets to a Postgres (`DATABASE_URL`, e.g. Supabase's +routes `openai-model` (a group backed by **Azure OpenAI and/or hosted OpenAI** — +provide a key for either) with a fallback to `local-model` (in-cluster Ollama), +and persists virtual keys/budgets to a Postgres (`DATABASE_URL`, e.g. Supabase's session pooler). Structured after `../sms-api/kustomize`. ## Layout @@ -62,7 +62,7 @@ sensitive values are sealed: - **backend-secrets** → `AZURE_API_KEY`, `LANGFUSE_SECRET_KEY`, `LANGFUSE_PUBLIC_KEY`, `SUPABASE_SERVICE_ROLE_KEY`, `LITELLM_MASTER_KEY` - **frontend-secrets** → `AUTH0_SECRET`, `AUTH0_CLIENT_SECRET` -- **litellm-secrets** → `LITELLM_MASTER_KEY`, `AZURE_API_KEY`, `LANGFUSE_SECRET_KEY`, `LANGFUSE_PUBLIC_KEY`, `DATABASE_URL` +- **litellm-secrets** → `LITELLM_MASTER_KEY`, `AZURE_API_KEY`, `OPENAI_API_KEY`, `LANGFUSE_SECRET_KEY`, `LANGFUSE_PUBLIC_KEY`, `DATABASE_URL` (the `openai-model` alias accepts **either** an Azure key or a hosted-OpenAI key — provide whichever you have; embeddings still need the Azure one) - **ghcr-secret** → ghcr.io image-pull credentials (`.dockerconfigjson`) (`LITELLM_MASTER_KEY` and the Langfuse keys are intentionally in more than one diff --git a/kustomize/base/litellm.yaml b/kustomize/base/litellm.yaml index 4cbcdc9..a342587 100644 --- a/kustomize/base/litellm.yaml +++ b/kustomize/base/litellm.yaml @@ -33,11 +33,17 @@ spec: secretKeyRef: name: litellm-secrets key: master-key + # Provide a key for EITHER provider (or both); see litellm_config.yaml. - name: AZURE_API_KEY valueFrom: secretKeyRef: name: litellm-secrets key: azure-api-key + - name: OPENAI_API_KEY + valueFrom: + secretKeyRef: + name: litellm-secrets + key: openai-api-key - name: LANGFUSE_SECRET_KEY valueFrom: secretKeyRef: diff --git a/kustomize/base/litellm_config.yaml b/kustomize/base/litellm_config.yaml index 80641a0..193732f 100644 --- a/kustomize/base/litellm_config.yaml +++ b/kustomize/base/litellm_config.yaml @@ -1,12 +1,22 @@ model_list: # "openai-model" is the alias the app requests (backend LLMModel Literal + - # frontend). It is backed by Azure OpenAI here — no hosted-OpenAI key needed. + # frontend). It is a GROUP of two interchangeable chat backends — Azure OpenAI + # and hosted OpenAI. Provide a key for EITHER (or both); LiteLLM routes to + # whichever authenticates and cools down one that fails. This gives deploy-time + # flexibility without committing to a provider up front. + # + # NOTE: knowledge-base *embeddings* still use Azure directly (backend), so the + # RAG features specifically require an Azure key regardless of the above. - model_name: openai-model litellm_params: model: os.environ/AZURE_MODEL # e.g. azure/ - api_base: os.environ/AZURE_API_BASE + api_base: os.environ/AZURE_ENDPOINT # same value the backend uses for embeddings api_key: os.environ/AZURE_API_KEY api_version: os.environ/AZURE_API_VERSION + - model_name: openai-model + litellm_params: + model: openai/gpt-4o + api_key: os.environ/OPENAI_API_KEY - model_name: local-model litellm_params: diff --git a/kustomize/config/vcell-ai-local/litellm.env b/kustomize/config/vcell-ai-local/litellm.env index d83695b..82fb7c1 100644 --- a/kustomize/config/vcell-ai-local/litellm.env +++ b/kustomize/config/vcell-ai-local/litellm.env @@ -2,9 +2,9 @@ # Secrets (master key, OpenAI key, Langfuse keys, DATABASE_URL) come from the # sealed litellm-secrets, not this file. -# openai-model alias -> Azure OpenAI (api key is secret, in litellm-secrets) +# openai-model alias -> Azure and/or hosted OpenAI (api keys are secret, in litellm-secrets) AZURE_MODEL=azure/your-chat-deployment-name -AZURE_API_BASE=https://your-resource.openai.azure.com +AZURE_ENDPOINT=https://your-azure-openai.openai.azure.com AZURE_API_VERSION=2024-02-01 # local-model alias -> in-cluster Ollama (fallback when openai-model fails) diff --git a/kustomize/config/vcell-ai-rke-dev/litellm.env b/kustomize/config/vcell-ai-rke-dev/litellm.env index da7a653..b49e6d2 100644 --- a/kustomize/config/vcell-ai-rke-dev/litellm.env +++ b/kustomize/config/vcell-ai-rke-dev/litellm.env @@ -2,9 +2,9 @@ # Secrets (master key, OpenAI key, Langfuse keys, DATABASE_URL) come from the # sealed litellm-secrets, not this file. -# openai-model alias -> Azure OpenAI (api key is secret, in litellm-secrets) +# openai-model alias -> Azure and/or hosted OpenAI (api keys are secret, in litellm-secrets) AZURE_MODEL=azure/your-chat-deployment-name -AZURE_API_BASE=https://your-resource.openai.azure.com +AZURE_ENDPOINT=https://your-azure-openai.openai.azure.com AZURE_API_VERSION=2024-02-01 # local-model alias -> in-cluster Ollama (fallback when openai-model fails) diff --git a/kustomize/config/vcell-ai-rke/litellm.env b/kustomize/config/vcell-ai-rke/litellm.env index 88ab749..0b3ea18 100644 --- a/kustomize/config/vcell-ai-rke/litellm.env +++ b/kustomize/config/vcell-ai-rke/litellm.env @@ -2,9 +2,9 @@ # Secrets (master key, OpenAI key, Langfuse keys, DATABASE_URL) come from the # sealed litellm-secrets, not this file. -# openai-model alias -> Azure OpenAI (api key is secret, in litellm-secrets) +# openai-model alias -> Azure and/or hosted OpenAI (api keys are secret, in litellm-secrets) AZURE_MODEL=azure/your-chat-deployment-name -AZURE_API_BASE=https://your-resource.openai.azure.com +AZURE_ENDPOINT=https://your-azure-openai.openai.azure.com AZURE_API_VERSION=2024-02-01 # local-model alias -> in-cluster Ollama (fallback when openai-model fails) diff --git a/kustomize/overlays/vcell-ai-local/secrets.dat.template b/kustomize/overlays/vcell-ai-local/secrets.dat.template index c953e52..b6e9651 100644 --- a/kustomize/overlays/vcell-ai-local/secrets.dat.template +++ b/kustomize/overlays/vcell-ai-local/secrets.dat.template @@ -12,7 +12,10 @@ SUPABASE_SERVICE_ROLE_KEY=your_supabase_service_role_key # --- LiteLLM (litellm-secrets; master key also injected into the backend) --- # LITELLM_MASTER_KEY: strong random value, e.g. `openssl rand -hex 32` (sk- prefix) LITELLM_MASTER_KEY=sk-litellm-change-me -# The "openai-model" alias is backed by Azure — LiteLLM reuses AZURE_API_KEY above. +# The "openai-model" alias accepts EITHER provider: it reuses AZURE_API_KEY above +# and/or OPENAI_API_KEY here. Fill in whichever you have (at least one for chat; +# KB embeddings still require the Azure key). Leave the unused one as-is. +OPENAI_API_KEY=sk-your-openai-api-key # Postgres for persistent virtual keys/budgets. Use the Supabase SESSION POOLER # (IPv4, port 5432) — the direct db..supabase.co host is IPv6-only. DATABASE_URL=postgresql://postgres.your-project-ref:your-db-password@aws-0-your-region.pooler.supabase.com:5432/postgres?sslmode=require diff --git a/kustomize/overlays/vcell-ai-local/secrets.sh b/kustomize/overlays/vcell-ai-local/secrets.sh index b0aaf02..c9ad9ae 100755 --- a/kustomize/overlays/vcell-ai-local/secrets.sh +++ b/kustomize/overlays/vcell-ai-local/secrets.sh @@ -65,12 +65,12 @@ ${SCRIPTS_DIR}/sealed_secret_ghcr.sh \ > ${SECRETS_DIR}/secret-ghcr.yaml echo "✓ secret-ghcr.yaml generated" -# litellm-secrets: +# litellm-secrets: echo "Generating LiteLLM secrets..." ${SCRIPTS_DIR}/sealed_secret_litellm.sh \ --controller-name ${SEALED_SECRETS_CONTROLLER_NAME} \ --controller-namespace ${SEALED_SECRETS_CONTROLLER_NAMESPACE} \ - ${NAMESPACE} "${LITELLM_MASTER_KEY}" "${AZURE_API_KEY}" "${LANGFUSE_SECRET_KEY}" "${LANGFUSE_PUBLIC_KEY}" "${DATABASE_URL}" \ + ${NAMESPACE} "${LITELLM_MASTER_KEY}" "${AZURE_API_KEY}" "${OPENAI_API_KEY}" "${LANGFUSE_SECRET_KEY}" "${LANGFUSE_PUBLIC_KEY}" "${DATABASE_URL}" \ > ${SECRETS_DIR}/secret-litellm.yaml echo "✓ secret-litellm.yaml generated" diff --git a/kustomize/overlays/vcell-ai-rke-dev/secrets.dat.template b/kustomize/overlays/vcell-ai-rke-dev/secrets.dat.template index 88059a1..fc21d5a 100644 --- a/kustomize/overlays/vcell-ai-rke-dev/secrets.dat.template +++ b/kustomize/overlays/vcell-ai-rke-dev/secrets.dat.template @@ -12,7 +12,10 @@ SUPABASE_SERVICE_ROLE_KEY=your_supabase_service_role_key # --- LiteLLM (litellm-secrets; master key also injected into the backend) --- # LITELLM_MASTER_KEY: strong random value, e.g. `openssl rand -hex 32` (sk- prefix) LITELLM_MASTER_KEY=sk-litellm-change-me -# The "openai-model" alias is backed by Azure — LiteLLM reuses AZURE_API_KEY above. +# The "openai-model" alias accepts EITHER provider: it reuses AZURE_API_KEY above +# and/or OPENAI_API_KEY here. Fill in whichever you have (at least one for chat; +# KB embeddings still require the Azure key). Leave the unused one as-is. +OPENAI_API_KEY=sk-your-openai-api-key # Postgres for persistent virtual keys/budgets. Use the Supabase SESSION POOLER # (IPv4, port 5432) — the direct db..supabase.co host is IPv6-only. DATABASE_URL=postgresql://postgres.your-project-ref:your-db-password@aws-0-your-region.pooler.supabase.com:5432/postgres?sslmode=require diff --git a/kustomize/overlays/vcell-ai-rke-dev/secrets.sh b/kustomize/overlays/vcell-ai-rke-dev/secrets.sh index c8b741a..e46896e 100755 --- a/kustomize/overlays/vcell-ai-rke-dev/secrets.sh +++ b/kustomize/overlays/vcell-ai-rke-dev/secrets.sh @@ -64,12 +64,12 @@ ${SCRIPTS_DIR}/sealed_secret_ghcr.sh \ > ${SECRETS_DIR}/secret-ghcr.yaml echo "✓ secret-ghcr.yaml generated" -# litellm-secrets: +# litellm-secrets: echo "Generating LiteLLM secrets..." ${SCRIPTS_DIR}/sealed_secret_litellm.sh \ --controller-name ${SEALED_SECRETS_CONTROLLER_NAME} \ --controller-namespace ${SEALED_SECRETS_CONTROLLER_NAMESPACE} \ - ${NAMESPACE} "${LITELLM_MASTER_KEY}" "${AZURE_API_KEY}" "${LANGFUSE_SECRET_KEY}" "${LANGFUSE_PUBLIC_KEY}" "${DATABASE_URL}" \ + ${NAMESPACE} "${LITELLM_MASTER_KEY}" "${AZURE_API_KEY}" "${OPENAI_API_KEY}" "${LANGFUSE_SECRET_KEY}" "${LANGFUSE_PUBLIC_KEY}" "${DATABASE_URL}" \ > ${SECRETS_DIR}/secret-litellm.yaml echo "✓ secret-litellm.yaml generated" diff --git a/kustomize/overlays/vcell-ai-rke/secrets.dat.template b/kustomize/overlays/vcell-ai-rke/secrets.dat.template index 2d8501f..173cbc7 100644 --- a/kustomize/overlays/vcell-ai-rke/secrets.dat.template +++ b/kustomize/overlays/vcell-ai-rke/secrets.dat.template @@ -12,7 +12,10 @@ SUPABASE_SERVICE_ROLE_KEY=your_supabase_service_role_key # --- LiteLLM (litellm-secrets; master key also injected into the backend) --- # LITELLM_MASTER_KEY: strong random value, e.g. `openssl rand -hex 32` (sk- prefix) LITELLM_MASTER_KEY=sk-litellm-change-me -# The "openai-model" alias is backed by Azure — LiteLLM reuses AZURE_API_KEY above. +# The "openai-model" alias accepts EITHER provider: it reuses AZURE_API_KEY above +# and/or OPENAI_API_KEY here. Fill in whichever you have (at least one for chat; +# KB embeddings still require the Azure key). Leave the unused one as-is. +OPENAI_API_KEY=sk-your-openai-api-key # Postgres for persistent virtual keys/budgets. Use the Supabase SESSION POOLER # (IPv4, port 5432) — the direct db..supabase.co host is IPv6-only. DATABASE_URL=postgresql://postgres.your-project-ref:your-db-password@aws-0-your-region.pooler.supabase.com:5432/postgres?sslmode=require diff --git a/kustomize/overlays/vcell-ai-rke/secrets.sh b/kustomize/overlays/vcell-ai-rke/secrets.sh index d2589fa..3ddb969 100755 --- a/kustomize/overlays/vcell-ai-rke/secrets.sh +++ b/kustomize/overlays/vcell-ai-rke/secrets.sh @@ -64,12 +64,12 @@ ${SCRIPTS_DIR}/sealed_secret_ghcr.sh \ > ${SECRETS_DIR}/secret-ghcr.yaml echo "✓ secret-ghcr.yaml generated" -# litellm-secrets: +# litellm-secrets: echo "Generating LiteLLM secrets..." ${SCRIPTS_DIR}/sealed_secret_litellm.sh \ --controller-name ${SEALED_SECRETS_CONTROLLER_NAME} \ --controller-namespace ${SEALED_SECRETS_CONTROLLER_NAMESPACE} \ - ${NAMESPACE} "${LITELLM_MASTER_KEY}" "${AZURE_API_KEY}" "${LANGFUSE_SECRET_KEY}" "${LANGFUSE_PUBLIC_KEY}" "${DATABASE_URL}" \ + ${NAMESPACE} "${LITELLM_MASTER_KEY}" "${AZURE_API_KEY}" "${OPENAI_API_KEY}" "${LANGFUSE_SECRET_KEY}" "${LANGFUSE_PUBLIC_KEY}" "${DATABASE_URL}" \ > ${SECRETS_DIR}/secret-litellm.yaml echo "✓ secret-litellm.yaml generated" diff --git a/kustomize/scripts/sealed_secret_litellm.sh b/kustomize/scripts/sealed_secret_litellm.sh index 72a3878..106f8bc 100755 --- a/kustomize/scripts/sealed_secret_litellm.sh +++ b/kustomize/scripts/sealed_secret_litellm.sh @@ -3,14 +3,15 @@ set -u # Create a SealedSecret for the LiteLLM proxy's sensitive environment values: -# master key, the Azure OpenAI key (openai-model alias is backed by Azure), -# Langfuse tracing keys, and the Postgres DATABASE_URL used to persist virtual -# keys / budgets. +# master key, the Azure OpenAI key and/or hosted-OpenAI key (the openai-model +# alias accepts either — pass real value(s) for whichever you have, leave the +# other as a placeholder), Langfuse tracing keys, and the Postgres DATABASE_URL +# used to persist virtual keys / budgets. # # Usage: # ./sealed_secret_litellm.sh [--cert ] \ # [--controller-name ] [--controller-namespace ] \ -# \ +# \ # > secret-litellm.yaml # # For GKE / AWS GovCloud the controller cert is needed and can be fetched with: @@ -43,9 +44,9 @@ while [[ "$1" == --* ]]; do done # Validate the number of positional arguments -if [ "$#" -ne 6 ]; then +if [ "$#" -ne 7 ]; then echo "Illegal number of parameters" - echo "Usage: ./sealed_secret_litellm.sh [--cert ] " + echo "Usage: ./sealed_secret_litellm.sh [--cert ] " exit 1 fi @@ -53,14 +54,16 @@ SECRET_NAME="litellm-secrets" NAMESPACE=$1 MASTER_KEY=$2 AZURE_API_KEY=$3 -LANGFUSE_SECRET_KEY=$4 -LANGFUSE_PUBLIC_KEY=$5 -DATABASE_URL=$6 +OPENAI_API_KEY=$4 +LANGFUSE_SECRET_KEY=$5 +LANGFUSE_PUBLIC_KEY=$6 +DATABASE_URL=$7 # Create the generic secret and seal it kubectl create secret generic ${SECRET_NAME} --dry-run=client \ --from-literal=master-key="${MASTER_KEY}" \ --from-literal=azure-api-key="${AZURE_API_KEY}" \ + --from-literal=openai-api-key="${OPENAI_API_KEY}" \ --from-literal=langfuse-secret-key="${LANGFUSE_SECRET_KEY}" \ --from-literal=langfuse-public-key="${LANGFUSE_PUBLIC_KEY}" \ --from-literal=database-url="${DATABASE_URL}" \ diff --git a/litellm.env.example b/litellm.env.example index 307bbfc..5d5461c 100644 --- a/litellm.env.example +++ b/litellm.env.example @@ -3,16 +3,19 @@ # openssl rand -hex 32 LITELLM_MASTER_KEY=sk-litellm-change-me -# Azure OpenAI provider for the "openai-model" LiteLLM alias (reuses the same -# Azure account the backend uses for embeddings — no separate OpenAI key needed). +# The "openai-model" alias accepts EITHER provider (Azure and/or hosted OpenAI) — +# fill in whichever you have. Azure reuses the same account the backend uses for +# embeddings. NOTE: KB embeddings specifically require the Azure key. AZURE_MODEL=azure/your-chat-deployment-name -AZURE_API_BASE=https://your-resource.openai.azure.com +AZURE_ENDPOINT=https://your-azure-openai.openai.azure.com AZURE_API_KEY=your_azure_openai_api_key AZURE_API_VERSION=2024-02-01 +OPENAI_API_KEY=sk-your-openai-api-key + # Local provider for the "local-model" LiteLLM alias. # For Ollama, keep the model prefixed with "ollama/". -LOCAL_LLM_MODEL=ollama/llama3.1:8b +LOCAL_LLM_MODEL=ollama/phi4-mini LOCAL_LLM_API_BASE=http://host.docker.internal:11434 # Langfuse tracing callback (litellm_settings.success_callback in litellm_config.yaml). diff --git a/litellm_config.yaml b/litellm_config.yaml index 80641a0..193732f 100644 --- a/litellm_config.yaml +++ b/litellm_config.yaml @@ -1,12 +1,22 @@ model_list: # "openai-model" is the alias the app requests (backend LLMModel Literal + - # frontend). It is backed by Azure OpenAI here — no hosted-OpenAI key needed. + # frontend). It is a GROUP of two interchangeable chat backends — Azure OpenAI + # and hosted OpenAI. Provide a key for EITHER (or both); LiteLLM routes to + # whichever authenticates and cools down one that fails. This gives deploy-time + # flexibility without committing to a provider up front. + # + # NOTE: knowledge-base *embeddings* still use Azure directly (backend), so the + # RAG features specifically require an Azure key regardless of the above. - model_name: openai-model litellm_params: model: os.environ/AZURE_MODEL # e.g. azure/ - api_base: os.environ/AZURE_API_BASE + api_base: os.environ/AZURE_ENDPOINT # same value the backend uses for embeddings api_key: os.environ/AZURE_API_KEY api_version: os.environ/AZURE_API_VERSION + - model_name: openai-model + litellm_params: + model: openai/gpt-4o + api_key: os.environ/OPENAI_API_KEY - model_name: local-model litellm_params: From 63e3e73398cb74e451961d6a2dfce08dd52171f7 Mon Sep 17 00:00:00 2001 From: Jim Schaff Date: Mon, 13 Jul 2026 14:34:34 -0400 Subject: [PATCH 04/11] feat(k8s): make LiteLLM openai-model configurable; point dev at hosted OpenAI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - litellm_config.yaml (base + root): openai-model deployment model is now os.environ/OPENAI_MODEL (default openai/gpt-4o-mini) instead of hardcoded gpt-4o - config//litellm.env + litellm.env.example: add OPENAI_MODEL=openai/gpt-4o-mini - config/vcell-ai-rke-dev/backend.env: switch dev to hosted OpenAI — PROVIDER=openai (singleton uses a plain OpenAI client with base_url=AZURE_ENDPOINT on the non-azure path), AZURE_ENDPOINT=https://api.openai.com/v1, gpt-4o-mini / text-embedding-3-small Dev now runs chat + embeddings on hosted OpenAI (verified the key against /v1/models: valid, both models accessible), so it no longer blocks on the Azure endpoint. The Azure deployment in the openai-model group stays dormant in dev (placeholder endpoint) and LiteLLM falls back to the OpenAI deployment. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01XpNzobVi83p3YKL9sqtGwZ --- kustomize/base/litellm_config.yaml | 2 +- kustomize/config/vcell-ai-local/litellm.env | 1 + kustomize/config/vcell-ai-rke-dev/backend.env | 15 +++++++++------ kustomize/config/vcell-ai-rke-dev/litellm.env | 1 + kustomize/config/vcell-ai-rke/litellm.env | 1 + litellm.env.example | 1 + litellm_config.yaml | 2 +- 7 files changed, 15 insertions(+), 8 deletions(-) diff --git a/kustomize/base/litellm_config.yaml b/kustomize/base/litellm_config.yaml index 193732f..e26c814 100644 --- a/kustomize/base/litellm_config.yaml +++ b/kustomize/base/litellm_config.yaml @@ -15,7 +15,7 @@ model_list: api_version: os.environ/AZURE_API_VERSION - model_name: openai-model litellm_params: - model: openai/gpt-4o + model: os.environ/OPENAI_MODEL api_key: os.environ/OPENAI_API_KEY - model_name: local-model diff --git a/kustomize/config/vcell-ai-local/litellm.env b/kustomize/config/vcell-ai-local/litellm.env index 82fb7c1..98ccfc1 100644 --- a/kustomize/config/vcell-ai-local/litellm.env +++ b/kustomize/config/vcell-ai-local/litellm.env @@ -6,6 +6,7 @@ AZURE_MODEL=azure/your-chat-deployment-name AZURE_ENDPOINT=https://your-azure-openai.openai.azure.com AZURE_API_VERSION=2024-02-01 +OPENAI_MODEL=openai/gpt-4o-mini # local-model alias -> in-cluster Ollama (fallback when openai-model fails) LOCAL_LLM_MODEL=ollama/phi4-mini diff --git a/kustomize/config/vcell-ai-rke-dev/backend.env b/kustomize/config/vcell-ai-rke-dev/backend.env index 8d5acec..5de4499 100644 --- a/kustomize/config/vcell-ai-rke-dev/backend.env +++ b/kustomize/config/vcell-ai-rke-dev/backend.env @@ -2,12 +2,15 @@ # Secret values (AZURE_API_KEY, LANGFUSE_*_KEY, SUPABASE_SERVICE_ROLE_KEY) are # injected from the sealed "backend-secrets" secret, not from this file. -# LLM provider -PROVIDER=azure -AZURE_API_VERSION=2024-02-01 -AZURE_ENDPOINT=https://your-azure-openai.openai.azure.com -AZURE_DEPLOYMENT_NAME=gpt-4o -AZURE_EMBEDDING_DEPLOYMENT_NAME=text-embedding-3-large +# LLM provider — hosted OpenAI. With PROVIDER != "azure" the backend builds a +# plain OpenAI client with base_url=AZURE_ENDPOINT (the AZURE_* names are reused +# for the OpenAI-compatible path; api_version is ignored here). AZURE_API_KEY +# (backend-secrets) must be an OpenAI sk-... key. +PROVIDER=openai +AZURE_API_VERSION=2020-10-01 +AZURE_ENDPOINT=https://api.openai.com/v1 +AZURE_DEPLOYMENT_NAME=gpt-4o-mini +AZURE_EMBEDDING_DEPLOYMENT_NAME=text-embedding-3-small # Qdrant (in-cluster StatefulSet service) QDRANT_URL=http://qdrant:6333 diff --git a/kustomize/config/vcell-ai-rke-dev/litellm.env b/kustomize/config/vcell-ai-rke-dev/litellm.env index b49e6d2..7353177 100644 --- a/kustomize/config/vcell-ai-rke-dev/litellm.env +++ b/kustomize/config/vcell-ai-rke-dev/litellm.env @@ -6,6 +6,7 @@ AZURE_MODEL=azure/your-chat-deployment-name AZURE_ENDPOINT=https://your-azure-openai.openai.azure.com AZURE_API_VERSION=2024-02-01 +OPENAI_MODEL=openai/gpt-4o-mini # local-model alias -> in-cluster Ollama (fallback when openai-model fails) LOCAL_LLM_MODEL=ollama/phi4-mini diff --git a/kustomize/config/vcell-ai-rke/litellm.env b/kustomize/config/vcell-ai-rke/litellm.env index 0b3ea18..62eff79 100644 --- a/kustomize/config/vcell-ai-rke/litellm.env +++ b/kustomize/config/vcell-ai-rke/litellm.env @@ -6,6 +6,7 @@ AZURE_MODEL=azure/your-chat-deployment-name AZURE_ENDPOINT=https://your-azure-openai.openai.azure.com AZURE_API_VERSION=2024-02-01 +OPENAI_MODEL=openai/gpt-4o-mini # local-model alias -> in-cluster Ollama (fallback when openai-model fails) LOCAL_LLM_MODEL=ollama/phi4-mini diff --git a/litellm.env.example b/litellm.env.example index 5d5461c..db0c9c2 100644 --- a/litellm.env.example +++ b/litellm.env.example @@ -12,6 +12,7 @@ AZURE_API_KEY=your_azure_openai_api_key AZURE_API_VERSION=2024-02-01 OPENAI_API_KEY=sk-your-openai-api-key +OPENAI_MODEL=openai/gpt-4o-mini # Local provider for the "local-model" LiteLLM alias. # For Ollama, keep the model prefixed with "ollama/". diff --git a/litellm_config.yaml b/litellm_config.yaml index 193732f..e26c814 100644 --- a/litellm_config.yaml +++ b/litellm_config.yaml @@ -15,7 +15,7 @@ model_list: api_version: os.environ/AZURE_API_VERSION - model_name: openai-model litellm_params: - model: openai/gpt-4o + model: os.environ/OPENAI_MODEL api_key: os.environ/OPENAI_API_KEY - model_name: local-model From f6961cd5696df38bc9c60a0abc3ef8c6fcb7b551 Mon Sep 17 00:00:00 2001 From: Jim Schaff Date: Mon, 13 Jul 2026 15:48:56 -0400 Subject: [PATCH 05/11] fix(k8s): raise LiteLLM memory limit to 2Gi (was OOMKilled at 1Gi) The litellm pod was CrashLoopBackOff / OOMKilled (exit 137) on the vxrails cluster: Prisma migrate + proxy startup spikes past 1Gi. Bump requests to 512Mi/250m and limits to 2Gi/1cpu. Verified: litellm reaches 1/1 Ready and completes its DB migrations. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01XpNzobVi83p3YKL9sqtGwZ --- kustomize/base/litellm.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/kustomize/base/litellm.yaml b/kustomize/base/litellm.yaml index a342587..1996e29 100644 --- a/kustomize/base/litellm.yaml +++ b/kustomize/base/litellm.yaml @@ -65,11 +65,13 @@ spec: subPath: config.yaml resources: requests: - memory: "256Mi" - cpu: "100m" + memory: "512Mi" + cpu: "250m" limits: - memory: "1Gi" - cpu: "500m" + # LiteLLM runs Prisma migrate + the proxy at startup, which spikes + # well past 1Gi (was getting OOMKilled/137). 2Gi gives headroom. + memory: "2Gi" + cpu: "1" readinessProbe: httpGet: path: /health/liveliness From 96b960463917118b9829bf921fa9c68258d4eeee Mon Sep 17 00:00:00 2001 From: Jim Schaff Date: Mon, 13 Jul 2026 15:56:54 -0400 Subject: [PATCH 06/11] chore(k8s): switch dev ingress cert to letsencrypt-prod DNS for vcell-ai-dev.cam.uchc.edu now resolves to the ingress, so switch the dev overlay from letsencrypt-staging (browser-untrusted) to letsencrypt-prod for a trusted cert. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01XpNzobVi83p3YKL9sqtGwZ --- kustomize/overlays/vcell-ai-rke-dev/ingress.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kustomize/overlays/vcell-ai-rke-dev/ingress.yaml b/kustomize/overlays/vcell-ai-rke-dev/ingress.yaml index aaf886e..32e1c70 100644 --- a/kustomize/overlays/vcell-ai-rke-dev/ingress.yaml +++ b/kustomize/overlays/vcell-ai-rke-dev/ingress.yaml @@ -21,7 +21,7 @@ spec: tls: - hosts: - vcell-ai-dev.cam.uchc.edu - secretName: letsencrypt-staging-vcell-ai-dev-tls + secretName: letsencrypt-prod-vcell-ai-dev-tls rules: - host: vcell-ai-dev.cam.uchc.edu http: @@ -41,13 +41,13 @@ metadata: labels: app: frontend-ingress annotations: - cert-manager.io/cluster-issuer: "letsencrypt-staging" + cert-manager.io/cluster-issuer: "letsencrypt-prod" spec: ingressClassName: nginx tls: - hosts: - vcell-ai-dev.cam.uchc.edu - secretName: letsencrypt-staging-vcell-ai-dev-tls + secretName: letsencrypt-prod-vcell-ai-dev-tls rules: - host: vcell-ai-dev.cam.uchc.edu http: From b591e7f5e4d8aef002c8c3c6ca5c990e896a4982 Mon Sep 17 00:00:00 2001 From: Jim Schaff Date: Mon, 13 Jul 2026 17:14:37 -0400 Subject: [PATCH 07/11] chore(k8s): fill dev Auth0 + Supabase config (non-secret) Real dev values for vcell-ai-rke-dev: AUTH0_DOMAIN/CLIENT_ID/AUDIENCE and SUPABASE_URL (derived from the DB project ref). These are public OIDC/Supabase identifiers, not secrets. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01XpNzobVi83p3YKL9sqtGwZ --- kustomize/config/vcell-ai-rke-dev/backend.env | 6 +++--- kustomize/config/vcell-ai-rke-dev/frontend.env | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/kustomize/config/vcell-ai-rke-dev/backend.env b/kustomize/config/vcell-ai-rke-dev/backend.env index 5de4499..4c88767 100644 --- a/kustomize/config/vcell-ai-rke-dev/backend.env +++ b/kustomize/config/vcell-ai-rke-dev/backend.env @@ -20,11 +20,11 @@ QDRANT_COLLECTION_NAME=vcell_kb_dev LANGFUSE_HOST=https://cloud.langfuse.com # Auth0 (domain/audience are public; client/session secrets are on the frontend) -AUTH0_DOMAIN=your-tenant.us.auth0.com -AUTH0_AUDIENCE=https://vcell-ai-api +AUTH0_DOMAIN=dev-dzhx7i2db3x3kkvq.us.auth0.com +AUTH0_AUDIENCE=https://vcell.cam.uchc.edu # Supabase (URL is public; service-role key is secret) -SUPABASE_URL=https://your-project.supabase.co +SUPABASE_URL=https://jgihatfppinstknxzdkk.supabase.co # LiteLLM proxy (in-cluster service; master key is secret) LITELLM_URL=http://litellm:4000 diff --git a/kustomize/config/vcell-ai-rke-dev/frontend.env b/kustomize/config/vcell-ai-rke-dev/frontend.env index d8afc3c..e8c4801 100644 --- a/kustomize/config/vcell-ai-rke-dev/frontend.env +++ b/kustomize/config/vcell-ai-rke-dev/frontend.env @@ -8,6 +8,6 @@ NEXT_PUBLIC_API_URL=https://vcell-ai-dev.cam.uchc.edu/api APP_BASE_URL=https://vcell-ai-dev.cam.uchc.edu -AUTH0_DOMAIN=your-tenant.us.auth0.com -AUTH0_CLIENT_ID=your_auth0_client_id -AUTH0_AUDIENCE=https://vcell-ai-api +AUTH0_DOMAIN=dev-dzhx7i2db3x3kkvq.us.auth0.com +AUTH0_CLIENT_ID=3IrBy6zwTALi6o4vMclPO8OxVVUTR5eA +AUTH0_AUDIENCE=https://vcell.cam.uchc.edu From d2897c8098a480a095ba6abbe3ae2906de190310 Mon Sep 17 00:00:00 2001 From: Jim Schaff Date: Mon, 13 Jul 2026 17:18:56 -0400 Subject: [PATCH 08/11] fix(frontend): bake relative NEXT_PUBLIC_API_URL=/api (fixes #70) The frontend image hardcoded NEXT_PUBLIC_API_URL=http://k8s-wn-01...:30001 at build time, so the browser's data/chat calls never hit the environment's own ingress. Since the ingress serves the frontend at / and the backend at /api on the same host, bake a RELATIVE "/api" instead: the browser calls the same origin it's served from, the ingress routes /api -> backend, and one image works in every environment. - frontend/Dockerfile: ARG/ENV NEXT_PUBLIC_API_URL defaulting to /api; drop the COPY .env step - build_containers.yml: pass --build-arg NEXT_PUBLIC_API_URL=/api instead of writing a hardcoded NodePort .env Fixes #70. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01XpNzobVi83p3YKL9sqtGwZ --- .github/workflows/build_containers.yml | 2 +- frontend/Dockerfile | 13 +++++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build_containers.yml b/.github/workflows/build_containers.yml index 905d723..c21f4c0 100644 --- a/.github/workflows/build_containers.yml +++ b/.github/workflows/build_containers.yml @@ -49,9 +49,9 @@ jobs: set -ux echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin pushd frontend - echo "NEXT_PUBLIC_API_URL=http://k8s-wn-01.cam.uchc.edu:30001" > .env docker build \ --no-cache \ + --build-arg NEXT_PUBLIC_API_URL=/api \ --file Dockerfile \ --tag ghcr.io/virtualcell/vcell-ai-frontend:${TAG_NAME} \ --label org.opencontainers.image.created=${CREATED} \ diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 47dcb95..cf71e02 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -8,12 +8,17 @@ WORKDIR /app COPY package.json package-lock.json* pnpm-lock.yaml* ./ RUN npm install -# 4. Copy .env file -COPY .env ./ - -# 5. Copy the rest of the app +# 4. Copy the rest of the app COPY . . +# 5. NEXT_PUBLIC_* vars are inlined into the browser bundle at build time. Default +# to a RELATIVE "/api" so the browser calls the same origin it's served from — the +# ingress routes /api -> backend — which makes ONE image work in every environment +# (dev/prod/local). Override with --build-arg NEXT_PUBLIC_API_URL= +# only if the frontend is served on a different origin than the API. +ARG NEXT_PUBLIC_API_URL=/api +ENV NEXT_PUBLIC_API_URL=${NEXT_PUBLIC_API_URL} + # 6. Expose the dev server port EXPOSE 3000 From 57d94185acba7e04dcdcd1bdca144526ac985e26 Mon Sep 17 00:00:00 2001 From: Jim Schaff Date: Wed, 15 Jul 2026 11:09:28 -0400 Subject: [PATCH 09/11] chore(k8s): use 0.1.8 images; NEXT_PUBLIC_API_URL=/api Point all overlays at 0.1.8 (frontend built with the relative /api base from #74) and set NEXT_PUBLIC_API_URL=/api in the frontend config so browser data calls hit the environment's own ingress instead of a baked NodePort. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01XpNzobVi83p3YKL9sqtGwZ --- kustomize/config/vcell-ai-local/frontend.env | 2 +- kustomize/config/vcell-ai-rke-dev/frontend.env | 2 +- kustomize/config/vcell-ai-rke/frontend.env | 2 +- kustomize/overlays/vcell-ai-local/kustomization.yaml | 4 ++-- kustomize/overlays/vcell-ai-rke-dev/kustomization.yaml | 4 ++-- kustomize/overlays/vcell-ai-rke/kustomization.yaml | 4 ++-- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/kustomize/config/vcell-ai-local/frontend.env b/kustomize/config/vcell-ai-local/frontend.env index 6616210..f83c100 100644 --- a/kustomize/config/vcell-ai-local/frontend.env +++ b/kustomize/config/vcell-ai-local/frontend.env @@ -7,7 +7,7 @@ # the value here is used for SSR and as documentation of the expected origin. # Add "127.0.0.1 vcell-ai.local" to /etc/hosts and run `minikube tunnel`. -NEXT_PUBLIC_API_URL=http://vcell-ai.local/api +NEXT_PUBLIC_API_URL=/api APP_BASE_URL=http://vcell-ai.local AUTH0_DOMAIN=your-tenant.us.auth0.com AUTH0_CLIENT_ID=your_auth0_client_id diff --git a/kustomize/config/vcell-ai-rke-dev/frontend.env b/kustomize/config/vcell-ai-rke-dev/frontend.env index e8c4801..0de16fb 100644 --- a/kustomize/config/vcell-ai-rke-dev/frontend.env +++ b/kustomize/config/vcell-ai-rke-dev/frontend.env @@ -6,7 +6,7 @@ # effect they must be set when the frontend image is built (see build_and_push.sh); # the value here is used for SSR and as documentation of the expected origin. -NEXT_PUBLIC_API_URL=https://vcell-ai-dev.cam.uchc.edu/api +NEXT_PUBLIC_API_URL=/api APP_BASE_URL=https://vcell-ai-dev.cam.uchc.edu AUTH0_DOMAIN=dev-dzhx7i2db3x3kkvq.us.auth0.com AUTH0_CLIENT_ID=3IrBy6zwTALi6o4vMclPO8OxVVUTR5eA diff --git a/kustomize/config/vcell-ai-rke/frontend.env b/kustomize/config/vcell-ai-rke/frontend.env index 6a5ac29..5290dc5 100644 --- a/kustomize/config/vcell-ai-rke/frontend.env +++ b/kustomize/config/vcell-ai-rke/frontend.env @@ -6,7 +6,7 @@ # effect they must be set when the frontend image is built (see build_and_push.sh); # the value here is used for SSR and as documentation of the expected origin. -NEXT_PUBLIC_API_URL=https://vcell-ai.cam.uchc.edu/api +NEXT_PUBLIC_API_URL=/api APP_BASE_URL=https://vcell-ai.cam.uchc.edu AUTH0_DOMAIN=your-tenant.us.auth0.com AUTH0_CLIENT_ID=your_auth0_client_id diff --git a/kustomize/overlays/vcell-ai-local/kustomization.yaml b/kustomize/overlays/vcell-ai-local/kustomization.yaml index a5e34c6..a518d34 100644 --- a/kustomize/overlays/vcell-ai-local/kustomization.yaml +++ b/kustomize/overlays/vcell-ai-local/kustomization.yaml @@ -5,9 +5,9 @@ namespace: vcell-ai-local images: - name: ghcr.io/virtualcell/vcell-ai-backend - newTag: 0.1.7 + newTag: 0.1.8 - name: ghcr.io/virtualcell/vcell-ai-frontend - newTag: 0.1.7 + newTag: 0.1.8 replicas: - count: 1 diff --git a/kustomize/overlays/vcell-ai-rke-dev/kustomization.yaml b/kustomize/overlays/vcell-ai-rke-dev/kustomization.yaml index b8caef7..8af28fc 100644 --- a/kustomize/overlays/vcell-ai-rke-dev/kustomization.yaml +++ b/kustomize/overlays/vcell-ai-rke-dev/kustomization.yaml @@ -5,9 +5,9 @@ namespace: vcell-ai-rke-dev images: - name: ghcr.io/virtualcell/vcell-ai-backend - newTag: 0.1.7 + newTag: 0.1.8 - name: ghcr.io/virtualcell/vcell-ai-frontend - newTag: 0.1.7 + newTag: 0.1.8 replicas: - count: 1 diff --git a/kustomize/overlays/vcell-ai-rke/kustomization.yaml b/kustomize/overlays/vcell-ai-rke/kustomization.yaml index 46fe6a4..d794015 100644 --- a/kustomize/overlays/vcell-ai-rke/kustomization.yaml +++ b/kustomize/overlays/vcell-ai-rke/kustomization.yaml @@ -5,9 +5,9 @@ namespace: vcell-ai-rke images: - name: ghcr.io/virtualcell/vcell-ai-backend - newTag: 0.1.7 + newTag: 0.1.8 - name: ghcr.io/virtualcell/vcell-ai-frontend - newTag: 0.1.7 + newTag: 0.1.8 replicas: - count: 1 From e1d540f52830bcf5ec081612d7620b2ca9c15061 Mon Sep 17 00:00:00 2001 From: Jim Schaff Date: Wed, 15 Jul 2026 11:39:04 -0400 Subject: [PATCH 10/11] fix(k8s): bump nginx proxy-buffer-size for the Auth0 callback (502) Login returned 502 Bad Gateway at /auth/callback: on a successful callback the @auth0/nextjs-auth0 SDK sets a large encrypted session cookie, and nginx's default 4k proxy buffer overflows on that Set-Cookie response header ("upstream sent too big header"). Add nginx.ingress.kubernetes.io/proxy-buffer-size: 16k to the frontend ingress in all overlays. Applied live to dev. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01XpNzobVi83p3YKL9sqtGwZ --- kustomize/overlays/vcell-ai-local/ingress.yaml | 4 ++++ kustomize/overlays/vcell-ai-rke-dev/ingress.yaml | 4 ++++ kustomize/overlays/vcell-ai-rke/ingress.yaml | 3 +++ 3 files changed, 11 insertions(+) diff --git a/kustomize/overlays/vcell-ai-local/ingress.yaml b/kustomize/overlays/vcell-ai-local/ingress.yaml index fd88991..d551eee 100644 --- a/kustomize/overlays/vcell-ai-local/ingress.yaml +++ b/kustomize/overlays/vcell-ai-local/ingress.yaml @@ -31,6 +31,10 @@ metadata: name: frontend-ingress labels: app: frontend-ingress + annotations: + # Auth0 sets a large encrypted session cookie on /auth/callback; bump the + # nginx proxy buffer so it doesn't 502 ("upstream sent too big header"). + nginx.ingress.kubernetes.io/proxy-buffer-size: "16k" spec: ingressClassName: nginx rules: diff --git a/kustomize/overlays/vcell-ai-rke-dev/ingress.yaml b/kustomize/overlays/vcell-ai-rke-dev/ingress.yaml index 32e1c70..934e4d1 100644 --- a/kustomize/overlays/vcell-ai-rke-dev/ingress.yaml +++ b/kustomize/overlays/vcell-ai-rke-dev/ingress.yaml @@ -42,6 +42,10 @@ metadata: app: frontend-ingress annotations: cert-manager.io/cluster-issuer: "letsencrypt-prod" + # Auth0 (@auth0/nextjs-auth0) sets a large encrypted session cookie on + # /auth/callback; the default 4k nginx proxy buffer overflows on that + # Set-Cookie response header -> 502 "upstream sent too big header". Bump it. + nginx.ingress.kubernetes.io/proxy-buffer-size: "16k" spec: ingressClassName: nginx tls: diff --git a/kustomize/overlays/vcell-ai-rke/ingress.yaml b/kustomize/overlays/vcell-ai-rke/ingress.yaml index 17af68a..44fde29 100644 --- a/kustomize/overlays/vcell-ai-rke/ingress.yaml +++ b/kustomize/overlays/vcell-ai-rke/ingress.yaml @@ -42,6 +42,9 @@ metadata: app: frontend-ingress annotations: cert-manager.io/cluster-issuer: "letsencrypt-prod" + # Auth0 sets a large encrypted session cookie on /auth/callback; bump the + # nginx proxy buffer so it doesn't 502 ("upstream sent too big header"). + nginx.ingress.kubernetes.io/proxy-buffer-size: "16k" spec: ingressClassName: nginx tls: From ef77b07fca7430b4580f2b0c5273eb02909bb310 Mon Sep 17 00:00:00 2001 From: Jim Schaff Date: Wed, 15 Jul 2026 11:55:02 -0400 Subject: [PATCH 11/11] chore(k8s): bump overlays to 0.1.9 (fresh images from up-to-date main) Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01XpNzobVi83p3YKL9sqtGwZ --- kustomize/overlays/vcell-ai-local/kustomization.yaml | 4 ++-- kustomize/overlays/vcell-ai-rke-dev/kustomization.yaml | 4 ++-- kustomize/overlays/vcell-ai-rke/kustomization.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/kustomize/overlays/vcell-ai-local/kustomization.yaml b/kustomize/overlays/vcell-ai-local/kustomization.yaml index a518d34..4c03f41 100644 --- a/kustomize/overlays/vcell-ai-local/kustomization.yaml +++ b/kustomize/overlays/vcell-ai-local/kustomization.yaml @@ -5,9 +5,9 @@ namespace: vcell-ai-local images: - name: ghcr.io/virtualcell/vcell-ai-backend - newTag: 0.1.8 + newTag: 0.1.9 - name: ghcr.io/virtualcell/vcell-ai-frontend - newTag: 0.1.8 + newTag: 0.1.9 replicas: - count: 1 diff --git a/kustomize/overlays/vcell-ai-rke-dev/kustomization.yaml b/kustomize/overlays/vcell-ai-rke-dev/kustomization.yaml index 8af28fc..520f643 100644 --- a/kustomize/overlays/vcell-ai-rke-dev/kustomization.yaml +++ b/kustomize/overlays/vcell-ai-rke-dev/kustomization.yaml @@ -5,9 +5,9 @@ namespace: vcell-ai-rke-dev images: - name: ghcr.io/virtualcell/vcell-ai-backend - newTag: 0.1.8 + newTag: 0.1.9 - name: ghcr.io/virtualcell/vcell-ai-frontend - newTag: 0.1.8 + newTag: 0.1.9 replicas: - count: 1 diff --git a/kustomize/overlays/vcell-ai-rke/kustomization.yaml b/kustomize/overlays/vcell-ai-rke/kustomization.yaml index d794015..b2ea165 100644 --- a/kustomize/overlays/vcell-ai-rke/kustomization.yaml +++ b/kustomize/overlays/vcell-ai-rke/kustomization.yaml @@ -5,9 +5,9 @@ namespace: vcell-ai-rke images: - name: ghcr.io/virtualcell/vcell-ai-backend - newTag: 0.1.8 + newTag: 0.1.9 - name: ghcr.io/virtualcell/vcell-ai-frontend - newTag: 0.1.8 + newTag: 0.1.9 replicas: - count: 1