feat(k8s): LiteLLM + Ollama deployment, validated on dev (incl. #70 /api fix)

.github/workflows/build_containers.yml

@@ -49,9 +49,9 @@ jobs:
           set -ux
           echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
           pushd frontend
-          echo "NEXT_PUBLIC_API_URL=http://k8s-wn-01.cam.uchc.edu:30001" > .env
           docker build \
           --no-cache \
+          --build-arg NEXT_PUBLIC_API_URL=/api \
           --file Dockerfile \
           --tag ghcr.io/virtualcell/vcell-ai-frontend:${TAG_NAME} \
           --label org.opencontainers.image.created=${CREATED} \

.gitignore

@@ -20,6 +20,9 @@ kustomize/overlays/*/secrets.dat
 kustomize/overlays/*/secret-backend.yaml
 kustomize/overlays/*/secret-frontend.yaml
 kustomize/overlays/*/secret-ghcr.yaml
+kustomize/overlays/*/secret-litellm.yaml
 
 qdrant_storage
-litellm.env
+# repo-root docker-compose secrets file — anchored with a leading slash so it does
+# NOT also ignore the non-secret kustomize/config/*/litellm.env config files.
+/litellm.env

frontend/Dockerfile

@@ -8,12 +8,17 @@ WORKDIR /app
 COPY package.json package-lock.json* pnpm-lock.yaml* ./
 RUN npm install
 
-# 4. Copy .env file
-COPY .env ./
-
-# 5. Copy the rest of the app
+# 4. Copy the rest of the app
 COPY . .
 
+# 5. NEXT_PUBLIC_* vars are inlined into the browser bundle at build time. Default
+# to a RELATIVE "/api" so the browser calls the same origin it's served from — the
+# ingress routes /api -> backend — which makes ONE image work in every environment
+# (dev/prod/local). Override with --build-arg NEXT_PUBLIC_API_URL=<absolute-url>
+# only if the frontend is served on a different origin than the API.
+ARG NEXT_PUBLIC_API_URL=/api
+ENV NEXT_PUBLIC_API_URL=${NEXT_PUBLIC_API_URL}
+
 # 6. Expose the dev server port
 EXPOSE 3000
 

kustomize/README.md

@@ -1,41 +1,54 @@
 # VCell-AI — Kubernetes deployment (kustomize + sealed secrets)
 
-Deploys the three VCell-AI services from `docker-compose.yml` to Kubernetes:
+Deploys the VCell-AI services from `docker-compose.yml` to Kubernetes:
 
 | Component | Image | Port | Notes |
 |-----------|-------|------|-------|
 | `qdrant`   | `qdrant/qdrant`                          | 6333 / 6334 | StatefulSet + 10Gi PVC |
 | `backend`  | `ghcr.io/virtualcell/vcell-ai-backend`   | 8000 | FastAPI |
 | `frontend` | `ghcr.io/virtualcell/vcell-ai-frontend`  | 3000 | Next.js |
+| `litellm`  | `ghcr.io/berriai/litellm:main-stable`    | 4000 | LLM gateway; chat routes through it with per-user virtual keys/budgets |
+| `ollama`   | `ollama/ollama`                          | 11434 | StatefulSet + 20Gi PVC; serves the `local-model` fallback (CPU; pulls `llama3.1:8b` on start) |
 
-Structured after `../sms-api/kustomize`.
+The backend calls `litellm` for chat (`LITELLM_URL`, per-user virtual keys via
+`LITELLM_MASTER_KEY`) and also uses Azure directly for KB **embeddings**. LiteLLM
+routes `openai-model` (a group backed by **Azure OpenAI and/or hosted OpenAI** —
+provide a key for either) with a fallback to `local-model` (in-cluster Ollama),
+and persists virtual keys/budgets to a Postgres (`DATABASE_URL`, e.g. Supabase's
+session pooler). Structured after `../sms-api/kustomize`.
 
 ## Layout
 
 ```
 kustomize/
-├── base/                     # Deployments/StatefulSet + Services (env-agnostic)
+├── base/                     # Deployments/StatefulSets + Services (env-agnostic)
 │   ├── qdrant.yaml
 │   ├── backend.yaml
 │   ├── frontend.yaml
+│   ├── litellm.yaml
+│   ├── ollama.yaml
+│   ├── litellm_config.yaml   # -> litellm-config-file ConfigMap (mounted at /app/config.yaml)
 │   └── kustomization.yaml
-├── config/<env>/             # non-secret config -> ConfigMaps (backend/frontend .env)
+├── config/<env>/             # non-secret config -> ConfigMaps
 │   ├── backend.env
 │   ├── frontend.env
+│   ├── litellm.env
 │   └── kustomization.yaml
 ├── overlays/<env>/           # per-environment: namespace, images, ingress, secrets
 │   ├── kustomization.yaml
 │   ├── ingress.yaml
-│   ├── secrets.sh            # master script -> generates the 3 sealed secrets
+│   ├── secrets.sh            # master script -> generates the 4 sealed secrets
 │   ├── secrets.dat.template  # copy to secrets.dat (gitignored) and fill in
 │   ├── secrets.dat           # (gitignored) plaintext inputs
 │   ├── secret-backend.yaml   # (gitignored) generated SealedSecret
 │   ├── secret-frontend.yaml  # (gitignored) generated SealedSecret
-│   └── secret-ghcr.yaml      # (gitignored) generated SealedSecret
+│   ├── secret-ghcr.yaml      # (gitignored) generated SealedSecret
+│   └── secret-litellm.yaml   # (gitignored) generated SealedSecret
 └── scripts/                  # sealing + build helpers
     ├── sealed_secret_backend.sh
     ├── sealed_secret_frontend.sh
     ├── sealed_secret_ghcr.sh
+    ├── sealed_secret_litellm.sh
     └── build_and_push.sh
 ```
 
@@ -47,10 +60,16 @@ Environments (`<env>`): `vcell-ai-rke` (prod, on-prem UCHC RKE), `vcell-ai-rke-d
 Non-secret configuration lives in `config/<env>/*.env` (committed). Only truly
 sensitive values are sealed:
 
-- **backend-secrets** → `AZURE_API_KEY`, `LANGFUSE_SECRET_KEY`, `LANGFUSE_PUBLIC_KEY`, `SUPABASE_SERVICE_ROLE_KEY`
+- **backend-secrets** → `AZURE_API_KEY`, `LANGFUSE_SECRET_KEY`, `LANGFUSE_PUBLIC_KEY`, `SUPABASE_SERVICE_ROLE_KEY`, `LITELLM_MASTER_KEY`
 - **frontend-secrets** → `AUTH0_SECRET`, `AUTH0_CLIENT_SECRET`
+- **litellm-secrets** → `LITELLM_MASTER_KEY`, `AZURE_API_KEY`, `OPENAI_API_KEY`, `LANGFUSE_SECRET_KEY`, `LANGFUSE_PUBLIC_KEY`, `DATABASE_URL` (the `openai-model` alias accepts **either** an Azure key or a hosted-OpenAI key — provide whichever you have; embeddings still need the Azure one)
 - **ghcr-secret** → ghcr.io image-pull credentials (`.dockerconfigjson`)
 
+(`LITELLM_MASTER_KEY` and the Langfuse keys are intentionally in more than one
+sealed secret — the backend needs the master key to provision virtual keys, and
+LiteLLM needs it plus Langfuse for tracing. Provide each value once in
+`secrets.dat`; `secrets.sh` seals it into the right secrets.)
+
 `secrets.dat` (plaintext) and the generated `secret-*.yaml` are **gitignored**.
 Run `secrets.sh` to (re)generate the sealed manifests before applying an overlay.
 

kustomize/base/backend.yaml

@@ -49,6 +49,12 @@ spec:
                 secretKeyRef:
                   name: backend-secrets
                   key: supabase-service-role-key
+            # Backend provisions per-user LiteLLM virtual keys using the master key.
+            - name: LITELLM_MASTER_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: backend-secrets
+                  key: litellm-master-key
           resources:
             requests:
               memory: "512Mi"

kustomize/base/kustomization.yaml

@@ -5,3 +5,12 @@ resources:
   - qdrant.yaml
   - backend.yaml
   - frontend.yaml
+  - litellm.yaml
+  - ollama.yaml
+
+# litellm_config.yaml is env-agnostic, so it is generated here (mounted read-only
+# into the litellm pod at /app/config.yaml).
+configMapGenerator:
+  - name: litellm-config-file
+    files:
+      - config.yaml=litellm_config.yaml

kustomize/base/litellm.yaml

@@ -0,0 +1,101 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: litellm
+  labels:
+    app: litellm
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: litellm
+  template:
+    metadata:
+      labels:
+        app: litellm
+    spec:
+      containers:
+        - name: litellm
+          image: ghcr.io/berriai/litellm:main-stable
+          imagePullPolicy: IfNotPresent
+          args: ["--config", "/app/config.yaml"]
+          ports:
+            - containerPort: 4000
+          # Non-secret config (local-model alias + api_base, Langfuse host) comes
+          # from the litellm-config ConfigMap generated per-environment.
+          envFrom:
+            - configMapRef:
+                name: litellm-config
+          # Secrets come from the sealed "litellm-secrets" secret.
+          env:
+            - name: LITELLM_MASTER_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: litellm-secrets
+                  key: master-key
+            # Provide a key for EITHER provider (or both); see litellm_config.yaml.
+            - name: AZURE_API_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: litellm-secrets
+                  key: azure-api-key
+            - name: OPENAI_API_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: litellm-secrets
+                  key: openai-api-key
+            - name: LANGFUSE_SECRET_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: litellm-secrets
+                  key: langfuse-secret-key
+            - name: LANGFUSE_PUBLIC_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: litellm-secrets
+                  key: langfuse-public-key
+            - name: DATABASE_URL
+              valueFrom:
+                secretKeyRef:
+                  name: litellm-secrets
+                  key: database-url
+          volumeMounts:
+            - name: litellm-config-file
+              mountPath: /app/config.yaml
+              subPath: config.yaml
+          resources:
+            requests:
+              memory: "512Mi"
+              cpu: "250m"
+            limits:
+              # LiteLLM runs Prisma migrate + the proxy at startup, which spikes
+              # well past 1Gi (was getting OOMKilled/137). 2Gi gives headroom.
+              memory: "2Gi"
+              cpu: "1"
+          readinessProbe:
+            httpGet:
+              path: /health/liveliness
+              port: 4000
+            initialDelaySeconds: 15
+            periodSeconds: 15
+      volumes:
+        - name: litellm-config-file
+          configMap:
+            name: litellm-config-file
+      imagePullSecrets:
+        - name: ghcr-secret
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: litellm
+  labels:
+    app: litellm
+spec:
+  ports:
+    - name: "4000"
+      port: 4000
+      targetPort: 4000
+  selector:
+    app: litellm

kustomize/base/litellm_config.yaml

@@ -0,0 +1,33 @@
+model_list:
+  # "openai-model" is the alias the app requests (backend LLMModel Literal +
+  # frontend). It is a GROUP of two interchangeable chat backends — Azure OpenAI
+  # and hosted OpenAI. Provide a key for EITHER (or both); LiteLLM routes to
+  # whichever authenticates and cools down one that fails. This gives deploy-time
+  # flexibility without committing to a provider up front.
+  #
+  # NOTE: knowledge-base *embeddings* still use Azure directly (backend), so the
+  # RAG features specifically require an Azure key regardless of the above.
+  - model_name: openai-model
+    litellm_params:
+      model: os.environ/AZURE_MODEL          # e.g. azure/<chat-deployment-name>
+      api_base: os.environ/AZURE_ENDPOINT    # same value the backend uses for embeddings
+      api_key: os.environ/AZURE_API_KEY
+      api_version: os.environ/AZURE_API_VERSION
+  - model_name: openai-model
+    litellm_params:
+      model: os.environ/OPENAI_MODEL
+      api_key: os.environ/OPENAI_API_KEY
+
+  - model_name: local-model
+    litellm_params:
+      model: os.environ/LOCAL_LLM_MODEL
+      api_base: os.environ/LOCAL_LLM_API_BASE
+
+litellm_settings:
+  success_callback: ["langfuse"]
+  fallbacks:
+    - openai-model: ["local-model"]
+
+general_settings:
+  master_key: os.environ/LITELLM_MASTER_KEY
+  database_url: os.environ/DATABASE_URL

kustomize/base/ollama.yaml

@@ -0,0 +1,80 @@
+# In-cluster Ollama, serving the "local-model" fallback that LiteLLM routes to
+# when the primary (openai-model) fails. CPU inference — no GPU is assumed on the
+# node; phi4-mini (~3.8B) is small enough to respond acceptably on CPU. The model
+# is pulled on startup into the PVC so it survives restarts.
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: ollama
+  labels:
+    app: ollama
+spec:
+  serviceName: ollama
+  replicas: 1
+  selector:
+    matchLabels:
+      app: ollama
+  template:
+    metadata:
+      labels:
+        app: ollama
+    spec:
+      containers:
+        - name: ollama
+          image: ollama/ollama:latest
+          imagePullPolicy: IfNotPresent
+          ports:
+            - name: http
+              containerPort: 11434
+          # Start the server, then pull the model in the background so it lands
+          # in the mounted PVC (idempotent — a no-op once cached).
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              ollama serve &
+              pid=$!
+              sleep 5
+              ollama pull "${OLLAMA_PULL_MODEL}" || echo "model pull failed; will retry on demand"
+              wait $pid
+          env:
+            - name: OLLAMA_PULL_MODEL
+              value: "phi4-mini"
+          volumeMounts:
+            - name: ollama-models
+              mountPath: /root/.ollama
+          resources:
+            requests:
+              memory: "3Gi"
+              cpu: "500m"
+            limits:
+              memory: "6Gi"
+              cpu: "2"
+          readinessProbe:
+            httpGet:
+              path: /
+              port: 11434
+            initialDelaySeconds: 10
+            periodSeconds: 15
+  volumeClaimTemplates:
+    - metadata:
+        name: ollama-models
+      spec:
+        accessModes: ["ReadWriteOnce"]
+        resources:
+          requests:
+            storage: 10Gi
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: ollama
+  labels:
+    app: ollama
+spec:
+  ports:
+    - name: http
+      port: 11434
+      targetPort: 11434
+  selector:
+    app: ollama

kustomize/config/vcell-ai-local/backend.env

@@ -23,5 +23,8 @@ AUTH0_AUDIENCE=https://vcell-ai-api
 # Supabase (URL is public; service-role key is secret)
 SUPABASE_URL=https://your-project.supabase.co
 
+# LiteLLM proxy (in-cluster service; master key is secret)
+LITELLM_URL=http://litellm:4000
+
 # CORS / frontend origin (add "127.0.0.1 vcell-ai.local" to /etc/hosts for minikube)
 FRONTEND_URL=http://vcell-ai.local

kustomize/config/vcell-ai-local/frontend.env

@@ -7,7 +7,7 @@
 # the value here is used for SSR and as documentation of the expected origin.
 # Add "127.0.0.1 vcell-ai.local" to /etc/hosts and run `minikube tunnel`.
 
-NEXT_PUBLIC_API_URL=http://vcell-ai.local/api
+NEXT_PUBLIC_API_URL=/api
 APP_BASE_URL=http://vcell-ai.local
 AUTH0_DOMAIN=your-tenant.us.auth0.com
 AUTH0_CLIENT_ID=your_auth0_client_id

kustomize/config/vcell-ai-local/kustomization.yaml

@@ -8,3 +8,6 @@ configMapGenerator:
   - name: frontend-config
     envs:
       - frontend.env
+  - name: litellm-config
+    envs:
+      - litellm.env