feat: optional AWS Lambda MicroVM stateful sandbox backend

.github/workflows/ci.yml

@@ -33,15 +33,15 @@ jobs:
 
       - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
         with:
-          bun-version: 1.3.13
+          bun-version: 1.3.14
 
       - name: Cache Bun packages
         uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
         with:
           path: ~/.bun/install/cache
-          key: ${{ runner.os }}-codeapi-api-bun-1.3.13-${{ hashFiles('api/bun.lock') }}
+          key: ${{ runner.os }}-codeapi-api-bun-1.3.14-${{ hashFiles('api/bun.lock') }}
           restore-keys: |
-            ${{ runner.os }}-codeapi-api-bun-1.3.13-
+            ${{ runner.os }}-codeapi-api-bun-1.3.14-
 
       - name: Install dependencies
         run: bun ci
@@ -82,18 +82,108 @@ jobs:
 
       - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
         with:
-          bun-version: 1.3.13
+          bun-version: 1.3.14
 
       - name: Cache Bun packages
         uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
         with:
           path: ~/.bun/install/cache
-          key: ${{ runner.os }}-codeapi-service-bun-1.3.13-${{ hashFiles('service/bun.lock') }}
+          key: ${{ runner.os }}-codeapi-service-bun-1.3.14-${{ hashFiles('service/bun.lock') }}
           restore-keys: |
-            ${{ runner.os }}-codeapi-service-bun-1.3.13-
+            ${{ runner.os }}-codeapi-service-bun-1.3.14-
 
       - name: Install dependencies
         run: bun ci
 
+      - name: Build service
+        run: bun run build
+
       - name: Bun tests
         run: bun run test
+
+  lambda-microvm-provisioning:
+    name: Lambda MicroVM Provisioning
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
+
+      - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
+        with:
+          bun-version: 1.3.14
+
+      - name: Install service dependencies
+        working-directory: service
+        run: bun ci
+
+      - name: Type-check MicroVM image helper
+        working-directory: service
+        run: >-
+          bunx tsc --noEmit
+          --moduleResolution node16
+          --module node16
+          --target es2022
+          --strict
+          --esModuleInterop
+          --skipLibCheck
+          --types node,bun-types
+          scripts/create-microvm-image.ts
+
+      - name: Test MicroVM image helper state machine
+        working-directory: service
+        run: bun test scripts/create-microvm-image.test.ts
+
+      - name: Validate artifact build script
+        run: |
+          bash -n scripts/build-lambda-microvm-artifact.sh
+          shellcheck scripts/build-lambda-microvm-artifact.sh
+
+      - name: Validate runner Dockerfile
+        run: >-
+          docker buildx build --check
+          --platform linux/arm64
+          --target lambda-microvm-runner
+          -f api/Dockerfile
+          .
+
+      - uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1
+        with:
+          terraform_version: 1.15.2
+          terraform_wrapper: false
+
+      - name: Validate Terraform
+        working-directory: docs/lambda-microvm/terraform
+        run: |
+          terraform fmt -check -recursive
+          terraform init -backend=false -input=false -lockfile=readonly
+          terraform validate
+
+  lambda-microvm-runner-build:
+    name: Lambda MicroVM Runner Image (arm64)
+    needs: lambda-microvm-provisioning
+    runs-on: ubuntu-24.04-arm
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
+
+      - name: Create native BuildKit builder
+        run: |
+          test "$(uname -m)" = "aarch64"
+          docker buildx create \
+            --name lambda-microvm-ci \
+            --driver docker-container \
+            --use
+          docker buildx inspect --bootstrap
+
+      - name: Build Lambda MicroVM runner image
+        # The target pins Bun 1.3.14 and installs api/bun.lock frozen. Building
+        # natively catches architecture-specific package and Docker layer
+        # failures without retaining a second multi-gigabyte image copy.
+        run: |
+          docker buildx build \
+            --builder lambda-microvm-ci \
+            --platform linux/arm64 \
+            --target lambda-microvm-runner \
+            --output type=cacheonly \
+            --progress plain \
+            -f api/Dockerfile \
+            .

.gitignore

@@ -7,6 +7,7 @@ node_modules
 /file-server-logs/**/*
 /service-logs/**/*
 /tool-call-server-logs/**/*
+/.build-lambda-microvm/
 
 # Helm artifacts
 helm/*/charts/*.tgz

api/Dockerfile

@@ -121,8 +121,8 @@ COPY --from=node:22-slim /usr/local/bin/node /usr/local/bin/node
 
 WORKDIR /sandbox_api
 
-COPY api/package.json ./
-RUN bun install
+COPY api/package.json api/bun.lock ./
+RUN bun install --frozen-lockfile
 
 COPY api/tsconfig.json ./
 COPY api/src ./src
@@ -148,6 +148,27 @@ RUN rm -f /usr/bin/nsenter /usr/bin/unshare /usr/bin/chroot /usr/sbin/chroot \
 COPY api/src/entrypoint.sh ./entrypoint.sh
 RUN chmod +x ./entrypoint.sh
 
+# ============================================================================
+# Stage 2b: AWS Lambda MicroVM container base image
+#
+# Lambda MicroVM IS the VM boundary: no libkrun launcher, no guest kernel.
+# This target is pushed to a same-account ECR repo and referenced by the
+# FROM line of the code-artifact Dockerfile that Lambda builds on top of
+# the AL2023 MicroVM base image (see scripts/build-lambda-microvm-artifact.sh).
+# Inbound traffic defaults to port 8080; lifecycle/build hooks are served
+# by the same app at /aws/lambda-microvms/runtime/v1/*.
+# ============================================================================
+FROM sandbox-build AS lambda-microvm-runner
+
+COPY --from=package-builder /pkgs /pkgs
+
+ENV PORT=8080 \
+    SANDBOX_PACKAGES_DIRECTORY=/pkgs \
+    SANDBOX_SESSION_WORKSPACE_ENABLED=true
+
+EXPOSE 8080/tcp
+ENTRYPOINT ["/sandbox_api/entrypoint.sh"]
+
 # ============================================================================
 # Stage 3: Build the Rust launcher binary (Fedora for libkrun ABI)
 # ============================================================================

api/README.md

@@ -40,7 +40,7 @@ All prefixed with `SANDBOX_` unless noted:
 | `SANDBOX_PACKAGES_DIRECTORY` | `/pkgs` | Directory containing language packages |
 | `SANDBOX_DISABLE_NETWORKING` | `true` | Isolate sandbox from the network |
 | `SANDBOX_ALLOWED_LOCAL_NETWORK_PORT` | `0` | Allow sandbox to reach this host port (for tool calling) |
-| `SANDBOX_OUTPUT_MAX_SIZE` | `1024` | Max stdout/stderr bytes before truncation |
+| `SANDBOX_OUTPUT_MAX_SIZE` | `1024` | Per-stream output cap. stderr truncates at this size (the job keeps running); stdout overflow kills the job (`status: OL`), since stdout is the result. Every shipped compose/helm config sets `65536` — the `1024` fallback only applies when running the runner bare. |
 | `SANDBOX_MAX_PROCESS_COUNT` | `64` | Max PIDs inside the sandbox |
 | `SANDBOX_MAX_OPEN_FILES` | `2048` | rlimit nofile |
 | `SANDBOX_MAX_FILE_SIZE` | `10000000` | rlimit fsize (bytes) |

api/src/api/lifecycle.test.ts

@@ -0,0 +1,59 @@
+import { afterEach, describe, expect, test } from 'bun:test';
+import {
+  LIFECYCLE_HOOK_BASE_PATH,
+  applyRunHook,
+  getMicrovmRunContext,
+  resetMicrovmRunContextForTests,
+} from './lifecycle';
+
+afterEach(resetMicrovmRunContextForTests);
+
+describe('MicroVM /run hook context', () => {
+  test('captures microvmId and runHookPayload from the platform body', () => {
+    const context = applyRunHook({
+      microvmId: 'mvm-0123',
+      runHookPayload: '{"runtime_session_id":"rt_x"}',
+    });
+    expect(context.microvmId).toBe('mvm-0123');
+    expect(context.runHookPayload).toBe('{"runtime_session_id":"rt_x"}');
+    expect(getMicrovmRunContext()).toBe(context);
+  });
+
+  test('first run wins: a different microvmId does not overwrite the context', () => {
+    applyRunHook({ microvmId: 'mvm-first', runHookPayload: 'a' });
+    applyRunHook({ microvmId: 'mvm-second', runHookPayload: 'b' });
+    expect(getMicrovmRunContext()?.microvmId).toBe('mvm-first');
+    expect(getMicrovmRunContext()?.runHookPayload).toBe('a');
+  });
+
+  test('retries with the same microvmId are idempotent', () => {
+    const first = applyRunHook({ microvmId: 'mvm-0123' });
+    const second = applyRunHook({ microvmId: 'mvm-0123' });
+    expect(second).toBe(first);
+  });
+
+  test('tolerates malformed and empty bodies', () => {
+    expect(applyRunHook(undefined).microvmId).toBeUndefined();
+    resetMicrovmRunContextForTests();
+    expect(applyRunHook('not-an-object').microvmId).toBeUndefined();
+    resetMicrovmRunContextForTests();
+    expect(applyRunHook({ microvmId: 42, runHookPayload: {} }).microvmId).toBeUndefined();
+  });
+
+  test('a malformed first delivery does not block a valid retry', () => {
+    const malformed = applyRunHook({ runHookPayload: '{"runtime_session_id":"rt_bad"}' });
+    expect(malformed.microvmId).toBeUndefined();
+    expect(getMicrovmRunContext()).toBeUndefined();
+
+    const retried = applyRunHook({
+      microvmId: 'mvm-retried',
+      runHookPayload: '{"runtime_session_id":"rt_good"}',
+    });
+    expect(retried.microvmId).toBe('mvm-retried');
+    expect(getMicrovmRunContext()).toBe(retried);
+  });
+
+  test('hook base path matches the AWS well-known prefix', () => {
+    expect(LIFECYCLE_HOOK_BASE_PATH).toBe('/aws/lambda-microvms/runtime/v1');
+  });
+});

api/src/api/lifecycle.ts

@@ -0,0 +1,104 @@
+import express, { Router, type Request, type Response } from 'express';
+import { logger } from '../logger';
+import { bindSessionWorkspace, parseSessionBinding, unbindSessionWorkspace } from '../session-workspace';
+
+/**
+ * AWS Lambda MicroVM hook endpoints. The platform POSTs to
+ * `/aws/lambda-microvms/runtime/v1/<hook>` on the configured hook port:
+ *
+ *   /ready, /validate   image build hooks (200 = proceed, 503 = retry)
+ *   /run                after start; body {microvmId, runHookPayload};
+ *                       external traffic only flows after this returns 200
+ *   /resume, /suspend, /terminate   lifecycle transitions
+ *
+ * Phase 1-2: structured no-ops that log and 200. /suspend and /terminate
+ * are the Phase 3 checkpoint-flush attachment points; /run captures the
+ * per-VM payload for later runtime-session wiring.
+ */
+
+export const LIFECYCLE_HOOK_BASE_PATH = '/aws/lambda-microvms/runtime/v1';
+
+export interface MicrovmRunContext {
+  microvmId?: string;
+  runHookPayload?: string;
+  receivedAt: number;
+}
+
+let runContext: MicrovmRunContext | undefined;
+
+export function getMicrovmRunContext(): MicrovmRunContext | undefined {
+  return runContext;
+}
+
+export function resetMicrovmRunContextForTests(): void {
+  runContext = undefined;
+}
+
+/** First /run wins; retries with the same microvmId are idempotent, a
+ *  different id is logged and ignored. Always 200 — a non-200 would fail
+ *  the platform's lifecycle operation. */
+export function applyRunHook(body: unknown): MicrovmRunContext {
+  const parsed = (typeof body === 'object' && body !== null ? body : {}) as {
+    microvmId?: unknown;
+    runHookPayload?: unknown;
+  };
+  const microvmId = typeof parsed.microvmId === 'string' ? parsed.microvmId : undefined;
+  const runHookPayload = typeof parsed.runHookPayload === 'string' ? parsed.runHookPayload : undefined;
+
+  /* A malformed first delivery must not permanently win the idempotency slot.
+   * Lambda can retry lifecycle hooks, so only a payload carrying the platform
+   * identity becomes the immutable run context. Keep returning an ephemeral
+   * context for malformed calls because the hook must still acknowledge them
+   * with 200 rather than failing the lifecycle operation. */
+  if (microvmId == null) {
+    logger.warn('Ignoring /run hook without a valid microvmId');
+    return runContext ?? { microvmId, runHookPayload, receivedAt: Date.now() };
+  }
+
+  if (runContext == null) {
+    runContext = { microvmId, runHookPayload, receivedAt: Date.now() };
+    const binding = parseSessionBinding(runHookPayload);
+    if (binding) {
+      bindSessionWorkspace(binding);
+      logger.info({ runtimeSessionId: binding.runtimeSessionId }, 'Bound persistent session workspace');
+    }
+  } else if (runContext.microvmId != null && microvmId != null && runContext.microvmId !== microvmId) {
+    logger.warn(
+      { existing: runContext.microvmId, incoming: microvmId },
+      'Ignoring /run hook for a different microvmId',
+    );
+  }
+  return runContext;
+}
+
+const lifecycleRouter = Router();
+
+function ackHook(hook: string) {
+  return (_req: Request, res: Response): Response => {
+    logger.info({ hook }, 'MicroVM lifecycle hook invoked');
+    return res.status(200).json({ hook, status: 'ok' });
+  };
+}
+
+lifecycleRouter.post('/ready', ackHook('ready'));
+lifecycleRouter.post('/validate', ackHook('validate'));
+
+lifecycleRouter.post('/run', express.json({ limit: '32kb' }), (req: Request, res: Response) => {
+  const context = applyRunHook(req.body);
+  logger.info(
+    { hook: 'run', microvmId: context.microvmId, hasPayload: context.runHookPayload != null },
+    'MicroVM lifecycle hook invoked',
+  );
+  return res.status(200).json({ hook: 'run', status: 'ok' });
+});
+
+lifecycleRouter.post('/resume', ackHook('resume'));
+lifecycleRouter.post('/suspend', ackHook('suspend'));
+
+lifecycleRouter.post('/terminate', (_req: Request, res: Response) => {
+  logger.info({ hook: 'terminate' }, 'MicroVM lifecycle hook invoked');
+  void unbindSessionWorkspace().catch((err) => logger.error({ err }, 'Failed to unbind session workspace on terminate'));
+  return res.status(200).json({ hook: 'terminate', status: 'ok' });
+});
+
+export default lifecycleRouter;