feat(cdk): integ-tests Phase 1 — core lifecycle E2E (#317)

.dockerignore

@@ -9,6 +9,14 @@ cdk/cdk.out/
 cdk/lib/
 cdk/node_modules/
 
+# integ-runner output dirs. The agent artifact's build context is the repo
+# root, and integ-runner writes its synth/snapshot output UNDER that root
+# (cdk/test/integ/cdk-integ.out.<test>.ts[.snapshot]/). Without these excludes,
+# staging the root copies its own output dir into itself recursively until the
+# path overflows (ENAMETOOLONG). Mirrors .gitignore lines 70-71.
+cdk/test/integ/cdk-integ.out.*/
+cdk/test/integ/*.snapshot/
+
 # CLI and docs build artifacts
 cli/lib/
 cli/node_modules/

.github/workflows/integ-sweeper.yml

@@ -0,0 +1,204 @@
+name: integ-sweeper
+# Reclaims stranded ephemeral integ stacks (issue #317 / PR #348 follow-up).
+#
+# The Phase-1 lifecycle integ test (integ.yml + cdk/test/integ/integ.task-lifecycle.ts)
+# deploys a per-run `int-<commit-sha>` stack running the AgentCore Runtime in VPC
+# mode. That runtime injects AWS-service-managed `agentic_ai` ENIs into the private
+# subnets, which AWS releases only ASYNCHRONOUSLY (observed: 1+ hours after the
+# runtime is deleted). So the in-run `cdk destroy` reliably fails the subnet/SG/VPC
+# deletes (DependencyViolation) and the integ run tolerates that failure
+# (destroy.expectError) rather than blocking on a wait it can't win. The per-run
+# UNIQUE stack name means a stranded stack never blocks a later run — but nothing
+# in the run reclaims it either.
+#
+# THIS workflow is that reclaimer: on a schedule (after the ENIs have had time to
+# detach), it deletes every `int-*` stack, and FAILS LOUDLY + opens a tracking
+# issue for any `int-*` stack older than the alarm threshold that still won't
+# delete — so a genuine leak (cost in the shared account) surfaces instead of
+# accumulating silently.
+on:
+  workflow_dispatch: {}
+  schedule:
+    # Every 2 hours. Frequent enough that a normal stranded stack (ENIs release in
+    # ~1-2h) is reclaimed within a cycle or two, well before the 6h alarm age.
+    - cron: "0 */2 * * *"
+
+concurrency:
+  group: integ-sweeper
+  cancel-in-progress: false
+
+permissions:
+  contents: none
+
+jobs:
+  sweep:
+    name: Reclaim stranded int-* stacks
+    runs-on: ubuntu-latest
+    # The integ deploy role (secrets.AWS_ROLE_TO_ASSUME) is scoped to the `integ`
+    # environment — same as integ.yml. The environment's protection rules must
+    # permit this scheduled run to assume the role (no manual approval is possible
+    # on a cron trigger).
+    environment: integ
+    timeout-minutes: 30
+    permissions:
+      id-token: write   # OIDC role assumption
+      contents: read
+      issues: write     # open a tracking issue on a genuine leak
+    env:
+      # Stacks older than this (hours) that STILL fail to delete are treated as a
+      # genuine leak → fail the job + file an issue. Comfortably past the observed
+      # ENI-release window so normal teardown lag never false-alarms.
+      ALARM_AGE_HOURS: "6"
+      AWS_REGION: ${{ vars.AWS_REGION || 'us-east-1' }}
+      AWS_DEFAULT_REGION: ${{ vars.AWS_REGION || 'us-east-1' }}
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0
+        with:
+          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
+          aws-region: ${{ vars.AWS_REGION || 'us-east-1' }}
+
+      - name: Sweep int-* stacks
+        id: sweep
+        run: |
+          set -uo pipefail
+
+          # Only the integ test's own per-run stacks are eligible. The test names
+          # them `int-<commit-hash>` where the hash is the 8-char short SHA
+          # (integ.task-lifecycle.ts: COMMIT_HASH.slice(0,8)). We therefore sweep
+          # ONLY names matching `int-<8 lowercase hex>` — NOT a bare `int-*` glob.
+          # `int-` is a short prefix; an unguarded glob in a shared account could
+          # delete an unrelated stack that merely starts with those 4 chars. The
+          # `int-local` fallback name (local dev runs) is intentionally NOT swept:
+          # CI never produces it, so a match would be someone's local stack.
+          STACK_RE='^int-[0-9a-f]{8}$'
+
+          # All non-deleted int-* stacks (active, DELETE_FAILED, or rollback states);
+          # the JMESPath prefilter narrows the API page, the regex below is the
+          # authoritative guard.
+          mapfile -t candidates < <(
+            aws cloudformation list-stacks \
+              --stack-status-filter CREATE_COMPLETE CREATE_FAILED ROLLBACK_COMPLETE ROLLBACK_FAILED \
+                UPDATE_COMPLETE UPDATE_ROLLBACK_COMPLETE UPDATE_ROLLBACK_FAILED DELETE_FAILED \
+              --query 'StackSummaries[?starts_with(StackName, `int-`)].StackName' \
+              --output text 2>/dev/null | tr '\t' '\n' | sort -u
+          )
+
+          stacks=()
+          for c in "${candidates[@]}"; do
+            [ -n "$c" ] || continue
+            if [[ "$c" =~ $STACK_RE ]]; then
+              stacks+=("$c")
+            else
+              echo "Skipping '$c' — does not match ${STACK_RE} (not a sweepable integ stack)."
+            fi
+          done
+
+          if [ "${#stacks[@]}" -eq 0 ]; then
+            echo "No int-* stacks present. Nothing to sweep."
+            exit 0
+          fi
+
+          echo "Found ${#stacks[@]} int-* stack(s): ${stacks[*]}"
+          now_epoch="$(date -u +%s)"
+          alarm_secs=$(( ALARM_AGE_HOURS * 3600 ))
+          leaked=""
+
+          for stack in "${stacks[@]}"; do
+            [ -n "$stack" ] || continue
+            echo "::group::$stack"
+
+            # Best-effort delete (idempotent; no-op if already deleting/gone).
+            aws cloudformation delete-stack --stack-name "$stack" || true
+            # Give CloudFormation a moment, then read the resulting status.
+            sleep 15
+            status="$(aws cloudformation describe-stacks --stack-name "$stack" \
+              --query 'Stacks[0].StackStatus' --output text 2>&1 || true)"
+
+            if echo "$status" | grep -qiE 'does not exist|ValidationError'; then
+              echo "✅ $stack deleted (or gone)."
+              echo "::endgroup::"
+              continue
+            fi
+
+            # Still present — how old is it? Alarm only if past the threshold.
+            created="$(aws cloudformation describe-stacks --stack-name "$stack" \
+              --query 'Stacks[0].CreationTime' --output text 2>/dev/null || true)"
+            created_epoch="$(date -u -d "$created" +%s 2>/dev/null || echo 0)"
+            age_secs=$(( now_epoch - created_epoch ))
+            age_hours=$(( age_secs / 3600 ))
+
+            if [ "$created_epoch" -gt 0 ] && [ "$age_secs" -ge "$alarm_secs" ]; then
+              echo "❌ $stack still present (status: $status), age ${age_hours}h ≥ ${ALARM_AGE_HOURS}h — LEAK."
+              leaked="${leaked}\n- \`${stack}\` — status \`${status}\`, age ~${age_hours}h"
+            else
+              echo "⏳ $stack still present (status: $status), age ~${age_hours}h — within ${ALARM_AGE_HOURS}h window; ENIs likely not yet released. Will retry next cycle."
+            fi
+            echo "::endgroup::"
+          done
+
+          if [ -n "$leaked" ]; then
+            {
+              echo "leaked<<EOF"
+              echo -e "$leaked"
+              echo "EOF"
+            } >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Open issue on genuine leak
+        if: steps.sweep.outputs.leaked != ''
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          # Pass via env (not inline ${{ }} interpolation) so the value never
+          # expands into the shell script body — avoids template injection
+          # (zizmor template-injection). Stack names are AWS-controlled, but env
+          # is the correct, lint-clean pattern regardless.
+          LEAKED: ${{ steps.sweep.outputs.leaked }}
+          # Stable label used both to tag the tracking issue and to find an
+          # existing open one — this is the dedup key, so it must not change.
+          LEAK_LABEL: integ-leak
+        run: |
+          set -euo pipefail
+          body_file="$(mktemp)"
+          {
+            echo "The integ-sweeper found stranded \`int-*\` CloudFormation stacks older than ${ALARM_AGE_HOURS}h that still fail to delete — likely a real leak in the shared integ account (each carries a VPC + NAT gateway + interface endpoints + the AgentCore runtime, billing hourly)."
+            echo ""
+            echo "These are normally reclaimed automatically once the AgentCore \`agentic_ai\` ENIs detach (~1-2h). Past ${ALARM_AGE_HOURS}h, investigate: the ENIs may be genuinely stuck (needs manual ENI/VPC cleanup) or the deploy role lacks teardown permissions."
+            echo ""
+            echo "### Stranded stacks (as of this run)"
+            echo -e "${LEAKED}"
+            echo ""
+            echo "| Field | Value |"
+            echo "| --- | --- |"
+            echo "| Workflow run | [integ-sweeper #${GITHUB_RUN_NUMBER}](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}) |"
+            echo "| Region | \`${AWS_REGION}\` |"
+            echo ""
+            echo "Close this issue once the stacks are deleted and the sweeper run is green."
+          } > "${body_file}"
+
+          # Dedup: a stuck stack re-alarms every 2h cycle. Without this guard each
+          # cycle files a fresh duplicate. Find an existing OPEN issue carrying the
+          # stable leak label and comment on it instead of opening another; only
+          # open a new issue when none exists. `--search` scopes to open issues with
+          # the label; `--json number --jq '.[0].number'` yields the first match (or
+          # empty). Ensure the label exists first (idempotent; ignore "already exists").
+          gh label create "${LEAK_LABEL}" \
+            --description "Stranded integ stacks flagged by integ-sweeper" \
+            --color B60205 2>/dev/null || true
+
+          existing="$(gh issue list --state open --label "${LEAK_LABEL}" \
+            --json number --jq '.[0].number // empty' 2>/dev/null || true)"
+
+          if [ -n "${existing}" ]; then
+            echo "Existing open leak issue #${existing} — commenting instead of opening a duplicate."
+            gh issue comment "${existing}" --body-file "${body_file}"
+          else
+            gh issue create \
+              --title "Stranded integ stacks not reclaimed (>${ALARM_AGE_HOURS}h)" \
+              --label "${LEAK_LABEL}" \
+              --body-file "${body_file}"
+          fi
+
+      - name: Fail job on genuine leak
+        if: steps.sweep.outputs.leaked != ''
+        run: exit 1

.github/workflows/integ.yml

@@ -13,6 +13,11 @@ name: integ
 # `integ-smoke` is posted back to the PR head so it shows up as a (required)
 # check that blocks merge.
 #
+# The scheduled integ-sweeper (.github/workflows/integ-sweeper.yml) runs under its
+# own `environment: integ`; a cron trigger cannot satisfy a manual approval, so
+# keeping the sweeper on a separate environment from this workflow's `deploy` gate
+# is deliberate.
+#
 # Local dev path is unchanged: run `mise //cdk:integ` with your own AWS creds.
 #
 # Nightly schedule was intentionally dropped (previously 07:00 UTC) — the per-PR
@@ -161,7 +166,7 @@ jobs:
             exit 1
           fi
           if echo "$CHANGED" | grep -Eq '^(cdk|agent)/'; then
-            post_status pending "awaiting admin approval / running"
+            post_status pending "awaiting integ run"
             echo "applicable=true" >> "$GITHUB_OUTPUT"
             echo "PR #$PR_NUMBER touches cdk/** or agent/** — integ applies."
           else
@@ -180,7 +185,12 @@ jobs:
     name: CDK integ smoke (Task API)
     runs-on: ubuntu-latest
     environment: deploy
-    timeout-minutes: 45
+    # The lifecycle test (integ.task-lifecycle.ts) deploys the full AgentStack
+    # (orchestrator + AgentCore runtime + Docker image build) and drives real
+    # agent runs through their terminal states before destroying — far heavier
+    # than the Phase-0 trimmed smoke test. 90 min covers deploy + cold Docker
+    # build + agent runs + teardown with margin.
+    timeout-minutes: 90
     permissions:
       id-token: write
       contents: read
@@ -219,36 +229,71 @@ jobs:
       - name: Install dependencies
         run: yarn install --immutable
 
+      - name: Run integ tests (deploy → assert → destroy)
+        # Two integ tests deploy in this one integ-runner pass and MUST get
+        # distinct CloudFormation names, so each reads its own env var:
+        #   • Phase-0 smoke (integ.task-api-smoke.ts) → INTEG_STACK_NAME
+        #     (`int-<sha>-<run>`; the run number distinguishes same-commit
+        #     re-runs). Fixed default `backgroundagent-integ-local` for local runs.
+        #   • Phase-1 lifecycle (integ.task-lifecycle.ts) → COMMIT_HASH
+        #     (`int-<hash>`, see cdk/mise.toml). Read via process.env because
+        #     integ-runner synths in a subprocess that inherits the env but not
+        #     our shell's CDK context.
+        # Per-run-unique names mean a stranded stack from a cancelled/crashed run
+        # never collides with or head-of-line blocks a later run.
+        #
+        # INTEG_SANDBOX_REPO / INTEG_PAT_SECRET_ID bind the gate scenarios (3 & 4)
+        # to the account's provisioned sandbox repo + PAT secret instead of a
+        # hardcoded contributor repo. Same vars the sandbox-cleanup step reads.
+        # When unset, the test falls back to its literals and the gates degrade to
+        # clone-failures (still synthesizes).
+        env:
+          COMMIT_HASH: ${{ needs.resolve.outputs.head_sha }}
+          INTEG_STACK_NAME: int-${{ needs.resolve.outputs.head_sha }}-${{ github.run_number }}
+          INTEG_SANDBOX_REPO: ${{ vars.INTEG_SANDBOX_REPO }}
+          INTEG_PAT_SECRET_ID: ${{ vars.INTEG_PAT_SECRET_ID }}
+        run: mise //cdk:integ
+
       # Teardown: integ-runner forces `cdk destroy` on success and failure (via
       # the CDK bootstrap roles, which hold the CloudFormation permissions). No
-      # raw-CLI safety net here: the GitHub OIDC role has no
-      # cloudformation:DeleteStack/DescribeStacks, so such a step can only fail
-      # (#566). If a cancelled/crashed run strands the stack, reclaim is the
-      # stranded-stack cleanup owned by #400/#72 — and the run-unique name below
-      # means a stranded stack never blocks a later run.
-      #
-      # INTEG_STACK_NAME maps the stack to the commit under test plus the run
-      # number (`int-<sha>-<run>`), replacing the single fixed
-      # `backgroundagent-integ` name. The run number makes same-commit re-runs
-      # distinct; the `int-` prefix marks it ephemeral for #72's sweep. HEAD_SHA
-      # is the resolved PR head (or dispatched ref) and RUN_NUMBER is a positive
-      # integer, so both are already lowercase-alphanumeric — no sanitizing
-      # needed to stay CloudFormation-valid.
-      - name: Run integ tests (deploy → assert → destroy)
+      # raw-CLI CloudFormation safety net here — the GitHub OIDC role has no
+      # cloudformation:DeleteStack/DescribeStacks, so such a step can only
+      # AccessDenied and fail an otherwise-passing run (#566/#567). If a
+      # cancelled/crashed run strands a stack, the per-run-unique `int-*` name
+      # keeps it from blocking later runs; reclaim is owned by the scheduled
+      # sweep (integ-sweeper.yml / #72) and non-blocking cleanup (#400).
+
+      # Sandbox cleanup for the gate scenarios (3 & 4): coding/new-task-v1 pushes
+      # a `bgagent/<task_id>/<slug>` branch and (on approve) opens a PR on the
+      # sandbox repo. The agent never closes these, so each run would accumulate
+      # stale branches/PRs. Reconstructing the exact branch name in the test is
+      # fragile (it depends on the agent-side slug), so we sweep by prefix here:
+      # delete every `bgagent/*` branch on the sandbox, which also closes the
+      # associated PRs. Reads the same PAT the agent used, from the pre-seeded
+      # secret. Gated on the repo vars being set so this is a no-op until the
+      # sandbox + secret are provisioned. Never fails the job — best-effort.
+      - name: Clean up sandbox PRs/branches
+        if: always() && vars.INTEG_SANDBOX_REPO != '' && vars.INTEG_PAT_SECRET_ID != ''
         env:
-          HEAD_SHA: ${{ needs.resolve.outputs.head_sha }}
-          RUN_NUMBER: ${{ github.run_number }}
+          AWS_REGION: ${{ vars.AWS_REGION || 'us-east-1' }}
+          AWS_DEFAULT_REGION: ${{ vars.AWS_REGION || 'us-east-1' }}
+          SANDBOX_REPO: ${{ vars.INTEG_SANDBOX_REPO }}
+          PAT_SECRET_ID: ${{ vars.INTEG_PAT_SECRET_ID }}
         run: |
           set -euo pipefail
-          export INTEG_STACK_NAME="int-${HEAD_SHA:0:12}-${RUN_NUMBER}"
-          echo "Integ stack name: $INTEG_STACK_NAME"
-          mise //cdk:integ -- --all --require-approval never \
-            -c stackName="integ-${{ github.run_id }}-${{ github.run_attempt }}" \
-            -c github:sha="$(git rev-parse HEAD)" \
-            -c github:ref="$(git branch --show-current || echo "detached")" \
-            -c github:actor="${{ github.triggering_actor }}" \
-            -c github:repository="$(gh repo view --json nameWithOwner --jq .nameWithOwner)" \
-            -c github:clean="$([ -z "$(git status --porcelain)" ] && echo true || echo false)"
+          GH_TOKEN="$(aws secretsmanager get-secret-value \
+            --secret-id "$PAT_SECRET_ID" \
+            --query SecretString --output text)"
+          export GH_TOKEN
+          # List bgagent/* branch refs; delete each (deleting the branch closes
+          # any open PR from it). Best-effort: never fail the job on cleanup.
+          gh api "repos/${SANDBOX_REPO}/git/matching-refs/heads/bgagent/" \
+            --jq '.[].ref | sub("^refs/heads/"; "")' 2>/dev/null \
+          | while read -r branch; do
+              [ -n "$branch" ] || continue
+              echo "Deleting sandbox branch: $branch"
+              gh api -X DELETE "repos/${SANDBOX_REPO}/git/refs/heads/${branch}" || true
+            done || true
 
   # Post the final integ-smoke status back to the PR head so the check flips from
   # pending to success/failure. Skipped for workflow_dispatch (no PR to gate).

cdk/mise.toml

@@ -63,10 +63,26 @@ description = "CDK deploy-then-verify integration tests (integ-runner). Needs AW
 depends = [":compile"]
 run = [
   "mkdir -p $TMPDIR",
+  # Per-run unique stack naming: the lifecycle test names its stack `int-<hash>`
+  # from the COMMIT_HASH env var (read directly via process.env in the test —
+  # integ-runner synths in a subprocess that inherits the env but not our shell's
+  # CDK context). A stranded stack (the AgentCore ENI teardown race) then never
+  # blocks the next run. Source: COMMIT_HASH (set by CI from the resolved head
+  # SHA), falling back to the local git SHA, then "local" outside a checkout.
+  #
   # No --update-on-failed: .snapshot/ is gitignored, so there is no committed
   # snapshot to diff against or update. --force re-runs the deploy-then-verify
   # unconditionally, which is what we want in CI.
-  "npx integ-runner --language typescript --directory test/integ --force",
+  #
+  # --verbose: integ-runner otherwise prints only a one-line pass/fail per test,
+  # which hides WHICH assertion failed and its actual-vs-expected payload. The
+  # lifecycle test polls DynamoDB for terminal task status; without --verbose a
+  # failure (e.g. task stuck at SUBMITTED instead of COMPLETED) is undiagnosable
+  # from the log alone. Verbose surfaces the assertion diffs we need.
+  '''
+  export COMMIT_HASH="${COMMIT_HASH:-$(git rev-parse HEAD 2>/dev/null || echo local)}"
+  npx integ-runner --language typescript --directory test/integ --force --verbose
+  ''',
 ]
 
 [tasks.bundle]