From e1bdbef21b1d040a554f770647645fcb651e0e82 Mon Sep 17 00:00:00 2001
From: Matt Rossman <22670878+mattrossman@users.noreply.github.com>
Date: Thu, 16 Jul 2026 11:54:41 -0400
Subject: [PATCH 01/14] feat: add approved-profiles RLS regression eval

---
 .../EVAL.ts                                   | 119 ++++++++++++++++++
 .../PROMPT.md                                 |  14 +++
 .../remote/project.sql                        |  13 ++
 3 files changed, 146 insertions(+)
 create mode 100644 evals/build-rls-004-approved-profiles-visibility/EVAL.ts
 create mode 100644 evals/build-rls-004-approved-profiles-visibility/PROMPT.md
 create mode 100644 evals/build-rls-004-approved-profiles-visibility/remote/project.sql

diff --git a/evals/build-rls-004-approved-profiles-visibility/EVAL.ts b/evals/build-rls-004-approved-profiles-visibility/EVAL.ts
new file mode 100644
index 00000000..0ba15796
--- /dev/null
+++ b/evals/build-rls-004-approved-profiles-visibility/EVAL.ts
@@ -0,0 +1,119 @@
+import type { CheckResult, SupabaseClient, ToolScorer } from "@supabase-evals/core";
+import { stripIndent } from "common-tags";
+
+const PASSWORD = "secret123";
+
+const scorer: ToolScorer = async (ctx) => {
+  try {
+    const userA = await signUpUser(ctx.client, "profiles-a@example.com");
+    if ("failure" in userA) return { passed: false, checks: [userA.failure] };
+    const userB = await signUpUser(ctx.getClient(), "profiles-b@example.com");
+    if ("failure" in userB) return { passed: false, checks: [userB.failure] };
+    const userC = await signUpUser(ctx.getClient(), "profiles-c@example.com");
+    if ("failure" in userC) return { passed: false, checks: [userC.failure] };
+
+    const clientA = userA.client;
+    const clientB = userB.client;
+    const clientC = userC.client;
+
+    await ctx.query(stripIndent`
+      INSERT INTO profiles (user_id, display_name, is_approved) VALUES
+        ('${userA.id}', 'Alice', true),
+        ('${userB.id}', 'Bob', true),
+        ('${userC.id}', 'Cara', false);
+    `);
+
+    const { rows: rls } = await ctx.query(
+      `SELECT relrowsecurity FROM pg_class WHERE relname = 'profiles';`,
+    );
+
+    const aOwnSelect = await clientA.from("profiles").select("id").eq("user_id", userA.id);
+    const aOwnUpdate = await clientA
+      .from("profiles")
+      .update({ bio: "hello" })
+      .eq("user_id", userA.id)
+      .select("bio");
+    const cOwnSelect = await clientC.from("profiles").select("id").eq("user_id", userC.id);
+    const cOwnUpdate = await clientC
+      .from("profiles")
+      .update({ bio: "not approved yet" })
+      .eq("user_id", userC.id)
+      .select("bio");
+    const bReadsA = await clientB.from("profiles").select("id").eq("user_id", userA.id);
+    const aReadsB = await clientA.from("profiles").select("id").eq("user_id", userB.id);
+    const bUpdatesA = await clientB
+      .from("profiles")
+      .update({ display_name: "hijacked" })
+      .eq("user_id", userA.id)
+      .select("id");
+    const bDeletesA = await clientB.from("profiles").delete().eq("user_id", userA.id).select("id");
+
+    const checks: CheckResult[] = [
+      { name: "RLS enabled on profiles", passed: rls[0]?.relrowsecurity === true },
+      {
+        name: "user A can select own profile",
+        passed: !aOwnSelect.error && aOwnSelect.data?.length === 1,
+        notes: aOwnSelect.error?.message,
+      },
+      {
+        name: "user A can update own profile",
+        passed: !aOwnUpdate.error && aOwnUpdate.data?.[0]?.bio === "hello",
+        notes: aOwnUpdate.error?.message,
+      },
+      {
+        name: "unapproved user C can still select own profile",
+        passed: !cOwnSelect.error && cOwnSelect.data?.length === 1,
+        notes: cOwnSelect.error?.message,
+      },
+      {
+        name: "unapproved user C can still update own profile",
+        passed: !cOwnUpdate.error && cOwnUpdate.data?.[0]?.bio === "not approved yet",
+        notes: cOwnUpdate.error?.message,
+      },
+      {
+        name: "unrelated user B cannot select user A's approved profile",
+        passed: Boolean(bReadsA.error) || bReadsA.data?.length === 0,
+      },
+      {
+        name: "unrelated user A cannot select user B's approved profile",
+        passed: Boolean(aReadsB.error) || aReadsB.data?.length === 0,
+      },
+      {
+        name: "user B cannot update user A's profile",
+        passed: Boolean(bUpdatesA.error) || !bUpdatesA.data || bUpdatesA.data.length === 0,
+      },
+      {
+        name: "user B cannot delete user A's profile",
+        passed: Boolean(bDeletesA.error) || !bDeletesA.data || bDeletesA.data.length === 0,
+      },
+    ];
+
+    return { passed: checks.every((check) => check.passed), checks };
+  } catch (error) {
+    const msg = error instanceof Error ? error.message : String(error);
+    return {
+      passed: false,
+      checks: [{ name: "scorer evaluated profiles RLS behavior", passed: false, notes: msg }],
+    };
+  }
+};
+
+export default scorer;
+
+/** Signs up a fresh auth user on the given client so tests can exercise per-user RLS. */
+async function signUpUser(
+  client: SupabaseClient,
+  email: string,
+): Promise<{ client: SupabaseClient; id: string } | { failure: CheckResult }> {
+  const { data, error } = await client.auth.signUp({ email, password: PASSWORD });
+  if (error || !data.user?.id || !data.session) {
+    return {
+      failure: {
+        name: `created auth session for ${email}`,
+        passed: false,
+        notes: error?.message ?? "missing session",
+      },
+    };
+  }
+  return { client, id: data.user.id };
+}
diff --git a/evals/build-rls-004-approved-profiles-visibility/PROMPT.md b/evals/build-rls-004-approved-profiles-visibility/PROMPT.md
new file mode 100644
index 00000000..6bc73bd1
--- /dev/null
+++ b/evals/build-rls-004-approved-profiles-visibility/PROMPT.md
@@ -0,0 +1,14 @@
+---
+stage: build
+suite: regression
+interface: mcp
+product:
+  - database
+  - auth
+topic:
+  - rls
+  - security
+motivation: AI-676
+---
+
+Add RLS policies to our profiles table. Users should be able to see approved profiles and manage their own.
diff --git a/evals/build-rls-004-approved-profiles-visibility/remote/project.sql b/evals/build-rls-004-approved-profiles-visibility/remote/project.sql
new file mode 100644
index 00000000..4a910c0e
--- /dev/null
+++ b/evals/build-rls-004-approved-profiles-visibility/remote/project.sql
@@ -0,0 +1,13 @@
+CREATE TABLE profiles (
+  id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+  user_id uuid NOT NULL,
+  display_name text,
+  bio text,
+  is_approved boolean NOT NULL DEFAULT false
+);
+
+ALTER TABLE profiles ENABLE ROW LEVEL SECURITY;
+
+-- Baseline privileges. The eval measures RLS policy correctness, not whether
+-- the agent remembered to GRANT standard table access.
+GRANT SELECT, INSERT, UPDATE, DELETE ON profiles TO authenticated;

From 1557ec006af2436f770a2b88098d317ba17769fd Mon Sep 17 00:00:00 2001
From: Matt Rossman <22670878+mattrossman@users.noreply.github.com>
Date: Thu, 16 Jul 2026 12:04:36 -0400
Subject: [PATCH 02/14] chore: drop superfluous comment in profiles seed SQL

---
 .../remote/project.sql                                          | 2 --
 1 file changed, 2 deletions(-)

diff --git a/evals/build-rls-004-approved-profiles-visibility/remote/project.sql b/evals/build-rls-004-approved-profiles-visibility/remote/project.sql
index 4a910c0e..6da8cf57 100644
--- a/evals/build-rls-004-approved-profiles-visibility/remote/project.sql
+++ b/evals/build-rls-004-approved-profiles-visibility/remote/project.sql
@@ -8,6 +8,4 @@ CREATE TABLE profiles (
 
 ALTER TABLE profiles ENABLE ROW LEVEL SECURITY;
 
--- Baseline privileges. The eval measures RLS policy correctness, not whether
--- the agent remembered to GRANT standard table access.
 GRANT SELECT, INSERT, UPDATE, DELETE ON profiles TO authenticated;

From 451eb6595b6f5f69d2b35b62a6299a95f68d4a47 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Thu, 16 Jul 2026 16:09:14 +0000
Subject: [PATCH 03/14] chore: refresh eval results

---
 .../web/src/data/regression-eval-results.json | 143 ++++++++++++++++++
 1 file changed, 143 insertions(+)

diff --git a/apps/web/src/data/regression-eval-results.json b/apps/web/src/data/regression-eval-results.json
index b6322b41..0383fe6f 100644
--- a/apps/web/src/data/regression-eval-results.json
+++ b/apps/web/src/data/regression-eval-results.json
@@ -131,6 +131,80 @@
     "attempts": 1,
     "sourcePath": "claude-code-sonnet-5/build-realtime-001-live-chat-updates.json"
   },
+  {
+    "experiment": "claude-code-sonnet-5",
+    "experimentSuite": "regression",
+    "experimentDisplay": {
+      "agent": "claude-code",
+      "modelProvider": "anthropic",
+      "modelId": "claude-sonnet-5",
+      "reasoningEffort": "high"
+    },
+    "eval": "build-rls-004-approved-profiles-visibility",
+    "stage": "build",
+    "product": [
+      "database",
+      "auth"
+    ],
+    "topic": [
+      "rls",
+      "security"
+    ],
+    "suite": "regression",
+    "interface": "mcp",
+    "passed": false,
+    "checks": [
+      {
+        "name": "RLS enabled on profiles",
+        "passed": true
+      },
+      {
+        "name": "user A can select own profile",
+        "passed": true
+      },
+      {
+        "name": "user A can update own profile",
+        "passed": true
+      },
+      {
+        "name": "unapproved user C can still select own profile",
+        "passed": true
+      },
+      {
+        "name": "unapproved user C can still update own profile",
+        "passed": true
+      },
+      {
+        "name": "unrelated user B cannot select user A's approved profile",
+        "passed": false
+      },
+      {
+        "name": "unrelated user A cannot select user B's approved profile",
+        "passed": false
+      },
+      {
+        "name": "user B cannot update user A's profile",
+        "passed": true
+      },
+      {
+        "name": "user B cannot delete user A's profile",
+        "passed": true
+      }
+    ],
+    "skills": {
+      "available": [
+        "supabase",
+        "supabase-postgres-best-practices"
+      ],
+      "loaded": [
+        "supabase"
+      ]
+    },
+    "prompt": "Add RLS policies to our profiles table. Users should be able to see approved profiles and manage their own.",
+    "promptSourcePath": "evals/build-rls-004-approved-profiles-visibility/PROMPT.md",
+    "attempts": 2,
+    "sourcePath": "claude-code-sonnet-5/build-rls-004-approved-profiles-visibility.json"
+  },
   {
     "experiment": "claude-code-sonnet-5",
     "experimentSuite": "regression",
@@ -467,6 +541,75 @@
     "attempts": 1,
     "sourcePath": "claude-code-sonnet-5-no-skills/build-realtime-001-live-chat-updates.json"
   },
+  {
+    "experiment": "claude-code-sonnet-5-no-skills",
+    "experimentSuite": "regression",
+    "experimentDisplay": {
+      "agent": "claude-code",
+      "modelProvider": "anthropic",
+      "modelId": "claude-sonnet-5",
+      "reasoningEffort": "high"
+    },
+    "eval": "build-rls-004-approved-profiles-visibility",
+    "stage": "build",
+    "product": [
+      "database",
+      "auth"
+    ],
+    "topic": [
+      "rls",
+      "security"
+    ],
+    "suite": "regression",
+    "interface": "mcp",
+    "passed": false,
+    "checks": [
+      {
+        "name": "RLS enabled on profiles",
+        "passed": true
+      },
+      {
+        "name": "user A can select own profile",
+        "passed": true
+      },
+      {
+        "name": "user A can update own profile",
+        "passed": true
+      },
+      {
+        "name": "unapproved user C can still select own profile",
+        "passed": true
+      },
+      {
+        "name": "unapproved user C can still update own profile",
+        "passed": true
+      },
+      {
+        "name": "unrelated user B cannot select user A's approved profile",
+        "passed": false
+      },
+      {
+        "name": "unrelated user A cannot select user B's approved profile",
+        "passed": false
+      },
+      {
+        "name": "user B cannot update user A's profile",
+        "passed": true
+      },
+      {
+        "name": "user B cannot delete user A's profile",
+        "passed": true
+      }
+    ],
+    "skills": {
+      "available": [],
+      "loaded": []
+    },
+    "prompt": "Add RLS policies to our profiles table. Users should be able to see approved profiles and manage their own.",
+    "promptSourcePath": "evals/build-rls-004-approved-profiles-visibility/PROMPT.md",
+    "attempts": 2,
+    "sourcePath": "claude-code-sonnet-5-no-skills/build-rls-004-approved-profiles-visibility.json"
+  },
   {
     "experiment": "claude-code-sonnet-5-no-skills",
     "experimentSuite": "regression",

From d3078b5f7522d39945415f1ff5e329fabd70ca8b Mon Sep 17 00:00:00 2001
From: Matt Rossman <22670878+mattrossman@users.noreply.github.com>
Date: Thu, 16 Jul 2026 12:38:50 -0400
Subject: [PATCH 04/14] Revert "chore: refresh eval results"

This reverts commit 451eb6595b6f5f69d2b35b62a6299a95f68d4a47.
---
 .../web/src/data/regression-eval-results.json | 143 ------------------
 1 file changed, 143 deletions(-)

diff --git a/apps/web/src/data/regression-eval-results.json b/apps/web/src/data/regression-eval-results.json
index 0383fe6f..b6322b41 100644
--- a/apps/web/src/data/regression-eval-results.json
+++ b/apps/web/src/data/regression-eval-results.json
@@ -131,80 +131,6 @@
     "attempts": 1,
     "sourcePath": "claude-code-sonnet-5/build-realtime-001-live-chat-updates.json"
   },
-  {
-    "experiment": "claude-code-sonnet-5",
-    "experimentSuite": "regression",
-    "experimentDisplay": {
-      "agent": "claude-code",
-      "modelProvider": "anthropic",
-      "modelId": "claude-sonnet-5",
-      "reasoningEffort": "high"
-    },
-    "eval": "build-rls-004-approved-profiles-visibility",
-    "stage": "build",
-    "product": [
-      "database",
-      "auth"
-    ],
-    "topic": [
-      "rls",
-      "security"
-    ],
-    "suite": "regression",
-    "interface": "mcp",
-    "passed": false,
-    "checks": [
-      {
-        "name": "RLS enabled on profiles",
-        "passed": true
-      },
-      {
-        "name": "user A can select own profile",
-        "passed": true
-      },
-      {
-        "name": "user A can update own profile",
-        "passed": true
-      },
-      {
-        "name": "unapproved user C can still select own profile",
-        "passed": true
-      },
-      {
-        "name": "unapproved user C can still update own profile",
-        "passed": true
-      },
-      {
-        "name": "unrelated user B cannot select user A's approved profile",
-        "passed": false
-      },
-      {
-        "name": "unrelated user A cannot select user B's approved profile",
-        "passed": false
-      },
-      {
-        "name": "user B cannot update user A's profile",
-        "passed": true
-      },
-      {
-        "name": "user B cannot delete user A's profile",
-        "passed": true
-      }
-    ],
-    "skills": {
-      "available": [
-        "supabase",
-        "supabase-postgres-best-practices"
-      ],
-      "loaded": [
-        "supabase"
-      ]
-    },
-    "prompt": "Add RLS policies to our profiles table. Users should be able to see approved profiles and manage their own.",
-    "promptSourcePath": "evals/build-rls-004-approved-profiles-visibility/PROMPT.md",
-    "attempts": 2,
-    "sourcePath": "claude-code-sonnet-5/build-rls-004-approved-profiles-visibility.json"
-  },
   {
     "experiment": "claude-code-sonnet-5",
     "experimentSuite": "regression",
@@ -541,75 +467,6 @@
     "attempts": 1,
     "sourcePath": "claude-code-sonnet-5-no-skills/build-realtime-001-live-chat-updates.json"
   },
-  {
-    "experiment": "claude-code-sonnet-5-no-skills",
-    "experimentSuite": "regression",
-    "experimentDisplay": {
-      "agent": "claude-code",
-      "modelProvider": "anthropic",
-      "modelId": "claude-sonnet-5",
-      "reasoningEffort": "high"
-    },
-    "eval": "build-rls-004-approved-profiles-visibility",
-    "stage": "build",
-    "product": [
-      "database",
-      "auth"
-    ],
-    "topic": [
-      "rls",
-      "security"
-    ],
-    "suite": "regression",
-    "interface": "mcp",
-    "passed": false,
-    "checks": [
-      {
-        "name": "RLS enabled on profiles",
-        "passed": true
-      },
-      {
-        "name": "user A can select own profile",
-        "passed": true
-      },
-      {
-        "name": "user A can update own profile",
-        "passed": true
-      },
-      {
-        "name": "unapproved user C can still select own profile",
-        "passed": true
-      },
-      {
-        "name": "unapproved user C can still update own profile",
-        "passed": true
-      },
-      {
-        "name": "unrelated user B cannot select user A's approved profile",
-        "passed": false
-      },
-      {
-        "name": "unrelated user A cannot select user B's approved profile",
-        "passed": false
-      },
-      {
-        "name": "user B cannot update user A's profile",
-        "passed": true
-      },
-      {
-        "name": "user B cannot delete user A's profile",
-        "passed": true
-      }
-    ],
-    "skills": {
-      "available": [],
-      "loaded": []
-    },
-    "prompt": "Add RLS policies to our profiles table. Users should be able to see approved profiles and manage their own.",
-    "promptSourcePath": "evals/build-rls-004-approved-profiles-visibility/PROMPT.md",
-    "attempts": 2,
-    "sourcePath": "claude-code-sonnet-5-no-skills/build-rls-004-approved-profiles-visibility.json"
-  },
   {
     "experiment": "claude-code-sonnet-5-no-skills",
     "experimentSuite": "regression",

From 8cdaf19192b2c0bb3fb1d7ebc14f6eb698a84b64 Mon Sep 17 00:00:00 2001
From: Matt Rossman <22670878+mattrossman@users.noreply.github.com>
Date: Thu, 16 Jul 2026 12:39:12 -0400
Subject: [PATCH 05/14] feat: promote approved-profiles RLS eval to benchmark
 suite

---
 evals/build-rls-004-approved-profiles-visibility/PROMPT.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/evals/build-rls-004-approved-profiles-visibility/PROMPT.md b/evals/build-rls-004-approved-profiles-visibility/PROMPT.md
index 6bc73bd1..813d314d 100644
--- a/evals/build-rls-004-approved-profiles-visibility/PROMPT.md
+++ b/evals/build-rls-004-approved-profiles-visibility/PROMPT.md
@@ -1,6 +1,6 @@
 ---
 stage: build
-suite: regression
+suite: benchmark
 interface: mcp
 product:
   - database

From c36b1fa344edd30e47ba320f40fa990117b272fc Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Thu, 16 Jul 2026 16:45:18 +0000
Subject: [PATCH 06/14] chore: refresh eval results

---
 apps/web/src/data/eval-results.json | 572 ++++++++++++++++++++++++++++
 1 file changed, 572 insertions(+)

diff --git a/apps/web/src/data/eval-results.json b/apps/web/src/data/eval-results.json
index cd6b8942..ce2ed1f5 100644
--- a/apps/web/src/data/eval-results.json
+++ b/apps/web/src/data/eval-results.json
@@ -383,6 +383,80 @@
     "attempts": 1,
     "sourcePath": "claude-code-opus-4.8/build-functions-005-dual-auth-user-secret.json"
   },
+  {
+    "experiment": "claude-code-opus-4.8",
+    "experimentSuite": "benchmark",
+    "experimentDisplay": {
+      "agent": "claude-code",
+      "modelProvider": "anthropic",
+      "modelId": "claude-opus-4-8",
+      "reasoningEffort": "high"
+    },
+    "eval": "build-rls-004-approved-profiles-visibility",
+    "stage": "build",
+    "product": [
+      "database",
+      "auth"
+    ],
+    "topic": [
+      "rls",
+      "security"
+    ],
+    "suite": "benchmark",
+    "interface": "mcp",
+    "passed": false,
+    "checks": [
+      {
+        "name": "RLS enabled on profiles",
+        "passed": true
+      },
+      {
+        "name": "user A can select own profile",
+        "passed": true
+      },
+      {
+        "name": "user A can update own profile",
+        "passed": true
+      },
+      {
+        "name": "unapproved user C can still select own profile",
+        "passed": true
+      },
+      {
+        "name": "unapproved user C can still update own profile",
+        "passed": true
+      },
+      {
+        "name": "unrelated user B cannot select user A's approved profile",
+        "passed": false
+      },
+      {
+        "name": "unrelated user A cannot select user B's approved profile",
+        "passed": false
+      },
+      {
+        "name": "user B cannot update user A's profile",
+        "passed": true
+      },
+      {
+        "name": "user B cannot delete user A's profile",
+        "passed": true
+      }
+    ],
+    "skills": {
+      "available": [
+        "supabase",
+        "supabase-postgres-best-practices"
+      ],
+      "loaded": [
+        "supabase"
+      ]
+    },
+    "prompt": "Add RLS policies to our profiles table. Users should be able to see approved profiles and manage their own.",
+    "promptSourcePath": "evals/build-rls-004-approved-profiles-visibility/PROMPT.md",
+    "attempts": 2,
+    "sourcePath": "claude-code-opus-4.8/build-rls-004-approved-profiles-visibility.json"
+  },
   {
     "experiment": "claude-code-opus-4.8",
     "experimentSuite": "benchmark",
@@ -1533,6 +1607,75 @@
     "attempts": 2,
     "sourcePath": "claude-code-opus-4.8-no-skills/build-functions-005-dual-auth-user-secret.json"
   },
+  {
+    "experiment": "claude-code-opus-4.8-no-skills",
+    "experimentSuite": "no-skills",
+    "experimentDisplay": {
+      "agent": "claude-code",
+      "modelProvider": "anthropic",
+      "modelId": "claude-opus-4-8",
+      "reasoningEffort": "high"
+    },
+    "eval": "build-rls-004-approved-profiles-visibility",
+    "stage": "build",
+    "product": [
+      "database",
+      "auth"
+    ],
+    "topic": [
+      "rls",
+      "security"
+    ],
+    "suite": "benchmark",
+    "interface": "mcp",
+    "passed": false,
+    "checks": [
+      {
+        "name": "RLS enabled on profiles",
+        "passed": true
+      },
+      {
+        "name": "user A can select own profile",
+        "passed": true
+      },
+      {
+        "name": "user A can update own profile",
+        "passed": true
+      },
+      {
+        "name": "unapproved user C can still select own profile",
+        "passed": true
+      },
+      {
+        "name": "unapproved user C can still update own profile",
+        "passed": true
+      },
+      {
+        "name": "unrelated user B cannot select user A's approved profile",
+        "passed": false
+      },
+      {
+        "name": "unrelated user A cannot select user B's approved profile",
+        "passed": false
+      },
+      {
+        "name": "user B cannot update user A's profile",
+        "passed": true
+      },
+      {
+        "name": "user B cannot delete user A's profile",
+        "passed": true
+      }
+    ],
+    "skills": {
+      "available": [],
+      "loaded": []
+    },
+    "prompt": "Add RLS policies to our profiles table. Users should be able to see approved profiles and manage their own.",
+    "promptSourcePath": "evals/build-rls-004-approved-profiles-visibility/PROMPT.md",
+    "attempts": 2,
+    "sourcePath": "claude-code-opus-4.8-no-skills/build-rls-004-approved-profiles-visibility.json"
+  },
   {
     "experiment": "claude-code-opus-4.8-no-skills",
     "experimentSuite": "no-skills",
@@ -2648,6 +2791,80 @@
     "attempts": 1,
     "sourcePath": "claude-code-sonnet-5/build-functions-005-dual-auth-user-secret.json"
   },
+  {
+    "experiment": "claude-code-sonnet-5",
+    "experimentSuite": "benchmark",
+    "experimentDisplay": {
+      "agent": "claude-code",
+      "modelProvider": "anthropic",
+      "modelId": "claude-sonnet-5",
+      "reasoningEffort": "high"
+    },
+    "eval": "build-rls-004-approved-profiles-visibility",
+    "stage": "build",
+    "product": [
+      "database",
+      "auth"
+    ],
+    "topic": [
+      "rls",
+      "security"
+    ],
+    "suite": "benchmark",
+    "interface": "mcp",
+    "passed": false,
+    "checks": [
+      {
+        "name": "RLS enabled on profiles",
+        "passed": true
+      },
+      {
+        "name": "user A can select own profile",
+        "passed": true
+      },
+      {
+        "name": "user A can update own profile",
+        "passed": true
+      },
+      {
+        "name": "unapproved user C can still select own profile",
+        "passed": true
+      },
+      {
+        "name": "unapproved user C can still update own profile",
+        "passed": true
+      },
+      {
+        "name": "unrelated user B cannot select user A's approved profile",
+        "passed": false
+      },
+      {
+        "name": "unrelated user A cannot select user B's approved profile",
+        "passed": false
+      },
+      {
+        "name": "user B cannot update user A's profile",
+        "passed": true
+      },
+      {
+        "name": "user B cannot delete user A's profile",
+        "passed": true
+      }
+    ],
+    "skills": {
+      "available": [
+        "supabase",
+        "supabase-postgres-best-practices"
+      ],
+      "loaded": [
+        "supabase"
+      ]
+    },
+    "prompt": "Add RLS policies to our profiles table. Users should be able to see approved profiles and manage their own.",
+    "promptSourcePath": "evals/build-rls-004-approved-profiles-visibility/PROMPT.md",
+    "attempts": 2,
+    "sourcePath": "claude-code-sonnet-5/build-rls-004-approved-profiles-visibility.json"
+  },
   {
     "experiment": "claude-code-sonnet-5",
     "experimentSuite": "benchmark",
@@ -3798,6 +4015,75 @@
     "attempts": 2,
     "sourcePath": "claude-code-sonnet-5-no-skills/build-functions-005-dual-auth-user-secret.json"
   },
+  {
+    "experiment": "claude-code-sonnet-5-no-skills",
+    "experimentSuite": "no-skills",
+    "experimentDisplay": {
+      "agent": "claude-code",
+      "modelProvider": "anthropic",
+      "modelId": "claude-sonnet-5",
+      "reasoningEffort": "high"
+    },
+    "eval": "build-rls-004-approved-profiles-visibility",
+    "stage": "build",
+    "product": [
+      "database",
+      "auth"
+    ],
+    "topic": [
+      "rls",
+      "security"
+    ],
+    "suite": "benchmark",
+    "interface": "mcp",
+    "passed": false,
+    "checks": [
+      {
+        "name": "RLS enabled on profiles",
+        "passed": true
+      },
+      {
+        "name": "user A can select own profile",
+        "passed": true
+      },
+      {
+        "name": "user A can update own profile",
+        "passed": true
+      },
+      {
+        "name": "unapproved user C can still select own profile",
+        "passed": true
+      },
+      {
+        "name": "unapproved user C can still update own profile",
+        "passed": true
+      },
+      {
+        "name": "unrelated user B cannot select user A's approved profile",
+        "passed": false
+      },
+      {
+        "name": "unrelated user A cannot select user B's approved profile",
+        "passed": false
+      },
+      {
+        "name": "user B cannot update user A's profile",
+        "passed": true
+      },
+      {
+        "name": "user B cannot delete user A's profile",
+        "passed": true
+      }
+    ],
+    "skills": {
+      "available": [],
+      "loaded": []
+    },
+    "prompt": "Add RLS policies to our profiles table. Users should be able to see approved profiles and manage their own.",
+    "promptSourcePath": "evals/build-rls-004-approved-profiles-visibility/PROMPT.md",
+    "attempts": 2,
+    "sourcePath": "claude-code-sonnet-5-no-skills/build-rls-004-approved-profiles-visibility.json"
+  },
   {
     "experiment": "claude-code-sonnet-5-no-skills",
     "experimentSuite": "no-skills",
@@ -4914,6 +5200,80 @@
     "attempts": 2,
     "sourcePath": "codex-gpt-5.4-mini/build-functions-005-dual-auth-user-secret.json"
   },
+  {
+    "experiment": "codex-gpt-5.4-mini",
+    "experimentSuite": "benchmark",
+    "experimentDisplay": {
+      "agent": "codex",
+      "modelProvider": "openai",
+      "modelId": "gpt-5.4-mini",
+      "reasoningEffort": "medium"
+    },
+    "eval": "build-rls-004-approved-profiles-visibility",
+    "stage": "build",
+    "product": [
+      "database",
+      "auth"
+    ],
+    "topic": [
+      "rls",
+      "security"
+    ],
+    "suite": "benchmark",
+    "interface": "mcp",
+    "passed": false,
+    "checks": [
+      {
+        "name": "RLS enabled on profiles",
+        "passed": true
+      },
+      {
+        "name": "user A can select own profile",
+        "passed": true
+      },
+      {
+        "name": "user A can update own profile",
+        "passed": true
+      },
+      {
+        "name": "unapproved user C can still select own profile",
+        "passed": true
+      },
+      {
+        "name": "unapproved user C can still update own profile",
+        "passed": true
+      },
+      {
+        "name": "unrelated user B cannot select user A's approved profile",
+        "passed": false
+      },
+      {
+        "name": "unrelated user A cannot select user B's approved profile",
+        "passed": false
+      },
+      {
+        "name": "user B cannot update user A's profile",
+        "passed": true
+      },
+      {
+        "name": "user B cannot delete user A's profile",
+        "passed": true
+      }
+    ],
+    "skills": {
+      "available": [
+        "supabase",
+        "supabase-postgres-best-practices"
+      ],
+      "loaded": [
+        "supabase"
+      ]
+    },
+    "prompt": "Add RLS policies to our profiles table. Users should be able to see approved profiles and manage their own.",
+    "promptSourcePath": "evals/build-rls-004-approved-profiles-visibility/PROMPT.md",
+    "attempts": 2,
+    "sourcePath": "codex-gpt-5.4-mini/build-rls-004-approved-profiles-visibility.json"
+  },
   {
     "experiment": "codex-gpt-5.4-mini",
     "experimentSuite": "benchmark",
@@ -6071,6 +6431,75 @@
     "attempts": 2,
     "sourcePath": "codex-gpt-5.4-mini-no-skills/build-functions-005-dual-auth-user-secret.json"
   },
+  {
+    "experiment": "codex-gpt-5.4-mini-no-skills",
+    "experimentSuite": "no-skills",
+    "experimentDisplay": {
+      "agent": "codex",
+      "modelProvider": "openai",
+      "modelId": "gpt-5.4-mini",
+      "reasoningEffort": "medium"
+    },
+    "eval": "build-rls-004-approved-profiles-visibility",
+    "stage": "build",
+    "product": [
+      "database",
+      "auth"
+    ],
+    "topic": [
+      "rls",
+      "security"
+    ],
+    "suite": "benchmark",
+    "interface": "mcp",
+    "passed": false,
+    "checks": [
+      {
+        "name": "RLS enabled on profiles",
+        "passed": true
+      },
+      {
+        "name": "user A can select own profile",
+        "passed": true
+      },
+      {
+        "name": "user A can update own profile",
+        "passed": true
+      },
+      {
+        "name": "unapproved user C can still select own profile",
+        "passed": true
+      },
+      {
+        "name": "unapproved user C can still update own profile",
+        "passed": true
+      },
+      {
+        "name": "unrelated user B cannot select user A's approved profile",
+        "passed": false
+      },
+      {
+        "name": "unrelated user A cannot select user B's approved profile",
+        "passed": false
+      },
+      {
+        "name": "user B cannot update user A's profile",
+        "passed": true
+      },
+      {
+        "name": "user B cannot delete user A's profile",
+        "passed": true
+      }
+    ],
+    "skills": {
+      "available": [],
+      "loaded": []
+    },
+    "prompt": "Add RLS policies to our profiles table. Users should be able to see approved profiles and manage their own.",
+    "promptSourcePath": "evals/build-rls-004-approved-profiles-visibility/PROMPT.md",
+    "attempts": 2,
+    "sourcePath": "codex-gpt-5.4-mini-no-skills/build-rls-004-approved-profiles-visibility.json"
+  },
   {
     "experiment": "codex-gpt-5.4-mini-no-skills",
     "experimentSuite": "no-skills",
@@ -7189,6 +7618,80 @@
     "attempts": 2,
     "sourcePath": "codex-gpt-5.5/build-functions-005-dual-auth-user-secret.json"
   },
+  {
+    "experiment": "codex-gpt-5.5",
+    "experimentSuite": "benchmark",
+    "experimentDisplay": {
+      "agent": "codex",
+      "modelProvider": "openai",
+      "modelId": "gpt-5.5",
+      "reasoningEffort": "medium"
+    },
+    "eval": "build-rls-004-approved-profiles-visibility",
+    "stage": "build",
+    "product": [
+      "database",
+      "auth"
+    ],
+    "topic": [
+      "rls",
+      "security"
+    ],
+    "suite": "benchmark",
+    "interface": "mcp",
+    "passed": false,
+    "checks": [
+      {
+        "name": "RLS enabled on profiles",
+        "passed": true
+      },
+      {
+        "name": "user A can select own profile",
+        "passed": true
+      },
+      {
+        "name": "user A can update own profile",
+        "passed": true
+      },
+      {
+        "name": "unapproved user C can still select own profile",
+        "passed": true
+      },
+      {
+        "name": "unapproved user C can still update own profile",
+        "passed": true
+      },
+      {
+        "name": "unrelated user B cannot select user A's approved profile",
+        "passed": false
+      },
+      {
+        "name": "unrelated user A cannot select user B's approved profile",
+        "passed": false
+      },
+      {
+        "name": "user B cannot update user A's profile",
+        "passed": true
+      },
+      {
+        "name": "user B cannot delete user A's profile",
+        "passed": true
+      }
+    ],
+    "skills": {
+      "available": [
+        "supabase",
+        "supabase-postgres-best-practices"
+      ],
+      "loaded": [
+        "supabase"
+      ]
+    },
+    "prompt": "Add RLS policies to our profiles table. Users should be able to see approved profiles and manage their own.",
+    "promptSourcePath": "evals/build-rls-004-approved-profiles-visibility/PROMPT.md",
+    "attempts": 2,
+    "sourcePath": "codex-gpt-5.5/build-rls-004-approved-profiles-visibility.json"
+  },
   {
     "experiment": "codex-gpt-5.5",
     "experimentSuite": "benchmark",
@@ -8343,6 +8846,75 @@
     "attempts": 2,
     "sourcePath": "codex-gpt-5.5-no-skills/build-functions-005-dual-auth-user-secret.json"
   },
+  {
+    "experiment": "codex-gpt-5.5-no-skills",
+    "experimentSuite": "no-skills",
+    "experimentDisplay": {
+      "agent": "codex",
+      "modelProvider": "openai",
+      "modelId": "gpt-5.5",
+      "reasoningEffort": "medium"
+    },
+    "eval": "build-rls-004-approved-profiles-visibility",
+    "stage": "build",
+    "product": [
+      "database",
+      "auth"
+    ],
+    "topic": [
+      "rls",
+      "security"
+    ],
+    "suite": "benchmark",
+    "interface": "mcp",
+    "passed": false,
+    "checks": [
+      {
+        "name": "RLS enabled on profiles",
+        "passed": true
+      },
+      {
+        "name": "user A can select own profile",
+        "passed": true
+      },
+      {
+        "name": "user A can update own profile",
+        "passed": true
+      },
+      {
+        "name": "unapproved user C can still select own profile",
+        "passed": true
+      },
+      {
+        "name": "unapproved user C can still update own profile",
+        "passed": true
+      },
+      {
+        "name": "unrelated user B cannot select user A's approved profile",
+        "passed": false
+      },
+      {
+        "name": "unrelated user A cannot select user B's approved profile",
+        "passed": false
+      },
+      {
+        "name": "user B cannot update user A's profile",
+        "passed": true
+      },
+      {
+        "name": "user B cannot delete user A's profile",
+        "passed": true
+      }
+    ],
+    "skills": {
+      "available": [],
+      "loaded": []
+    },
+    "prompt": "Add RLS policies to our profiles table. Users should be able to see approved profiles and manage their own.",
+    "promptSourcePath": "evals/build-rls-004-approved-profiles-visibility/PROMPT.md",
+    "attempts": 2,
+    "sourcePath": "codex-gpt-5.5-no-skills/build-rls-004-approved-profiles-visibility.json"
+  },
   {
     "experiment": "codex-gpt-5.5-no-skills",
     "experimentSuite": "no-skills",

From 8015ed1601cb3f77a5ae35f66c81ea75baad109c Mon Sep 17 00:00:00 2001
From: Matt Rossman <22670878+mattrossman@users.noreply.github.com>
Date: Thu, 16 Jul 2026 14:03:09 -0400
Subject: [PATCH 07/14] fix: scope approved-profiles visibility to company, not
 just approval

Seeds two named companies and a company_id FK so the multi-tenant
boundary is unambiguous in the data model rather than the prompt. A
same-company coworker must still see an approved profile (a positive
requirement the old ownership-only seed couldn't test), while a
different-company user must not, closing the "maybe this is a single
company directory" reading of the old 3-user, no-company seed.
---
 .../EVAL.ts                                   | 51 +++++++++++++------
 .../remote/project.sql                        | 11 ++++
 2 files changed, 46 insertions(+), 16 deletions(-)

diff --git a/evals/build-rls-004-approved-profiles-visibility/EVAL.ts b/evals/build-rls-004-approved-profiles-visibility/EVAL.ts
index 0ba15796..948dd0ba 100644
--- a/evals/build-rls-004-approved-profiles-visibility/EVAL.ts
+++ b/evals/build-rls-004-approved-profiles-visibility/EVAL.ts
@@ -2,6 +2,8 @@ import type { CheckResult, SupabaseClient, ToolScorer } from "@supabase-evals/co
 import { stripIndent } from "common-tags";
 
 const PASSWORD = "secret123";
+const ACME = "11111111-1111-1111-1111-111111111111";
+const GLOBEX = "22222222-2222-2222-2222-222222222222";
 
 const scorer: ToolScorer = async (ctx) => {
   try {
@@ -11,16 +13,21 @@ const scorer: ToolScorer = async (ctx) => {
     if ("failure" in userB) return { passed: false, checks: [userB.failure] };
     const userC = await signUpUser(ctx.getClient(), "profiles-c@example.com");
     if ("failure" in userC) return { passed: false, checks: [userC.failure] };
+    const userD = await signUpUser(ctx.getClient(), "profiles-d@example.com");
+    if ("failure" in userD) return { passed: false, checks: [userD.failure] };
 
     const clientA = userA.client;
     const clientB = userB.client;
     const clientC = userC.client;
+    const clientD = userD.client;
 
+    // A and B are coworkers at Acme, C is an unapproved Acme hire, D works at Globex.
     await ctx.query(stripIndent`
-      INSERT INTO profiles (user_id, display_name, is_approved) VALUES
-        ('${userA.id}', 'Alice', true),
-        ('${userB.id}', 'Bob', true),
-        ('${userC.id}', 'Cara', false);
+      INSERT INTO profiles (user_id, company_id, display_name, is_approved) VALUES
+        ('${userA.id}', '${ACME}', 'Alice', true),
+        ('${userB.id}', '${ACME}', 'Bob', true),
+        ('${userC.id}', '${ACME}', 'Cara', false),
+        ('${userD.id}', '${GLOBEX}', 'Dana', true);
     `);
 
     const { rows: rls } = await ctx.query(
@@ -39,14 +46,16 @@ const scorer: ToolScorer = async (ctx) => {
       .update({ bio: "not approved yet" })
       .eq("user_id", userC.id)
       .select("bio");
-    const bReadsA = await clientB.from("profiles").select("id").eq("user_id", userA.id);
-    const aReadsB = await clientA.from("profiles").select("id").eq("user_id", userB.id);
-    const bUpdatesA = await clientB
+    const bReadsCoworkerA = await clientB.from("profiles").select("id").eq("user_id", userA.id);
+    const aReadsCoworkerB = await clientA.from("profiles").select("id").eq("user_id", userB.id);
+    const dReadsOtherCompanyA = await clientD.from("profiles").select("id").eq("user_id", userA.id);
+    const aReadsOtherCompanyD = await clientA.from("profiles").select("id").eq("user_id", userD.id);
+    const dUpdatesA = await clientD
       .from("profiles")
       .update({ display_name: "hijacked" })
       .eq("user_id", userA.id)
       .select("id");
-    const bDeletesA = await clientB.from("profiles").delete().eq("user_id", userA.id).select("id");
+    const dDeletesA = await clientD.from("profiles").delete().eq("user_id", userA.id).select("id");
 
     const checks: CheckResult[] = [
       { name: "RLS enabled on profiles", passed: rls[0]?.relrowsecurity === true },
@@ -71,20 +80,30 @@ const scorer: ToolScorer = async (ctx) => {
         notes: cOwnUpdate.error?.message,
       },
       {
-        name: "unrelated user B cannot select user A's approved profile",
-        passed: Boolean(bReadsA.error) || bReadsA.data?.length === 0,
+        name: "same-company coworker B can select A's approved profile",
+        passed: !bReadsCoworkerA.error && bReadsCoworkerA.data?.length === 1,
+        notes: bReadsCoworkerA.error?.message,
       },
       {
-        name: "unrelated user A cannot select user B's approved profile",
-        passed: Boolean(aReadsB.error) || aReadsB.data?.length === 0,
+        name: "same-company coworker A can select B's approved profile",
+        passed: !aReadsCoworkerB.error && aReadsCoworkerB.data?.length === 1,
+        notes: aReadsCoworkerB.error?.message,
       },
       {
-        name: "user B cannot update user A's profile",
-        passed: Boolean(bUpdatesA.error) || !bUpdatesA.data || bUpdatesA.data.length === 0,
+        name: "different-company user D cannot select A's approved profile",
+        passed: Boolean(dReadsOtherCompanyA.error) || dReadsOtherCompanyA.data?.length === 0,
       },
       {
-        name: "user B cannot delete user A's profile",
-        passed: Boolean(bDeletesA.error) || !bDeletesA.data || bDeletesA.data.length === 0,
+        name: "different-company user A cannot select D's approved profile",
+        passed: Boolean(aReadsOtherCompanyD.error) || aReadsOtherCompanyD.data?.length === 0,
+      },
+      {
+        name: "different-company user D cannot update A's profile",
+        passed: Boolean(dUpdatesA.error) || !dUpdatesA.data || dUpdatesA.data.length === 0,
+      },
+      {
+        name: "different-company user D cannot delete A's profile",
+        passed: Boolean(dDeletesA.error) || !dDeletesA.data || dDeletesA.data.length === 0,
       },
     ];
 
diff --git a/evals/build-rls-004-approved-profiles-visibility/remote/project.sql b/evals/build-rls-004-approved-profiles-visibility/remote/project.sql
index 6da8cf57..193391a9 100644
--- a/evals/build-rls-004-approved-profiles-visibility/remote/project.sql
+++ b/evals/build-rls-004-approved-profiles-visibility/remote/project.sql
@@ -1,6 +1,16 @@
+CREATE TABLE companies (
+  id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+  name text NOT NULL
+);
+
+INSERT INTO companies (id, name) VALUES
+  ('11111111-1111-1111-1111-111111111111', 'Acme Corp'),
+  ('22222222-2222-2222-2222-222222222222', 'Globex Inc');
+
 CREATE TABLE profiles (
   id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
   user_id uuid NOT NULL,
+  company_id uuid NOT NULL REFERENCES companies(id),
   display_name text,
   bio text,
   is_approved boolean NOT NULL DEFAULT false
@@ -8,4 +18,5 @@ CREATE TABLE profiles (
 
 ALTER TABLE profiles ENABLE ROW LEVEL SECURITY;
 
+GRANT SELECT ON companies TO authenticated;
 GRANT SELECT, INSERT, UPDATE, DELETE ON profiles TO authenticated;

From 2f45953a0a277c24d30b3d5ca75b1c78205ed0bb Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Thu, 16 Jul 2026 18:12:07 +0000
Subject: [PATCH 08/14] chore: refresh eval results

---
 apps/web/src/data/eval-results.json | 128 +++++++++++++++++++++-------
 1 file changed, 96 insertions(+), 32 deletions(-)

diff --git a/apps/web/src/data/eval-results.json b/apps/web/src/data/eval-results.json
index ce2ed1f5..a1e9ae84 100644
--- a/apps/web/src/data/eval-results.json
+++ b/apps/web/src/data/eval-results.json
@@ -427,19 +427,27 @@
         "passed": true
       },
       {
-        "name": "unrelated user B cannot select user A's approved profile",
+        "name": "same-company coworker B can select A's approved profile",
+        "passed": true
+      },
+      {
+        "name": "same-company coworker A can select B's approved profile",
+        "passed": true
+      },
+      {
+        "name": "different-company user D cannot select A's approved profile",
         "passed": false
       },
       {
-        "name": "unrelated user A cannot select user B's approved profile",
+        "name": "different-company user A cannot select D's approved profile",
         "passed": false
       },
       {
-        "name": "user B cannot update user A's profile",
+        "name": "different-company user D cannot update A's profile",
         "passed": true
       },
       {
-        "name": "user B cannot delete user A's profile",
+        "name": "different-company user D cannot delete A's profile",
         "passed": true
       }
     ],
@@ -1651,19 +1659,27 @@
         "passed": true
       },
       {
-        "name": "unrelated user B cannot select user A's approved profile",
+        "name": "same-company coworker B can select A's approved profile",
+        "passed": true
+      },
+      {
+        "name": "same-company coworker A can select B's approved profile",
+        "passed": true
+      },
+      {
+        "name": "different-company user D cannot select A's approved profile",
         "passed": false
       },
       {
-        "name": "unrelated user A cannot select user B's approved profile",
+        "name": "different-company user A cannot select D's approved profile",
         "passed": false
       },
       {
-        "name": "user B cannot update user A's profile",
+        "name": "different-company user D cannot update A's profile",
         "passed": true
       },
       {
-        "name": "user B cannot delete user A's profile",
+        "name": "different-company user D cannot delete A's profile",
         "passed": true
       }
     ],
@@ -2835,19 +2851,27 @@
         "passed": true
       },
       {
-        "name": "unrelated user B cannot select user A's approved profile",
+        "name": "same-company coworker B can select A's approved profile",
+        "passed": true
+      },
+      {
+        "name": "same-company coworker A can select B's approved profile",
+        "passed": true
+      },
+      {
+        "name": "different-company user D cannot select A's approved profile",
         "passed": false
       },
       {
-        "name": "unrelated user A cannot select user B's approved profile",
+        "name": "different-company user A cannot select D's approved profile",
         "passed": false
       },
       {
-        "name": "user B cannot update user A's profile",
+        "name": "different-company user D cannot update A's profile",
         "passed": true
       },
       {
-        "name": "user B cannot delete user A's profile",
+        "name": "different-company user D cannot delete A's profile",
         "passed": true
       }
     ],
@@ -4059,19 +4083,27 @@
         "passed": true
       },
       {
-        "name": "unrelated user B cannot select user A's approved profile",
+        "name": "same-company coworker B can select A's approved profile",
+        "passed": true
+      },
+      {
+        "name": "same-company coworker A can select B's approved profile",
+        "passed": true
+      },
+      {
+        "name": "different-company user D cannot select A's approved profile",
         "passed": false
       },
       {
-        "name": "unrelated user A cannot select user B's approved profile",
+        "name": "different-company user A cannot select D's approved profile",
         "passed": false
       },
       {
-        "name": "user B cannot update user A's profile",
+        "name": "different-company user D cannot update A's profile",
         "passed": true
       },
       {
-        "name": "user B cannot delete user A's profile",
+        "name": "different-company user D cannot delete A's profile",
         "passed": true
       }
     ],
@@ -5244,19 +5276,27 @@
         "passed": true
       },
       {
-        "name": "unrelated user B cannot select user A's approved profile",
+        "name": "same-company coworker B can select A's approved profile",
+        "passed": true
+      },
+      {
+        "name": "same-company coworker A can select B's approved profile",
+        "passed": true
+      },
+      {
+        "name": "different-company user D cannot select A's approved profile",
         "passed": false
       },
       {
-        "name": "unrelated user A cannot select user B's approved profile",
+        "name": "different-company user A cannot select D's approved profile",
         "passed": false
       },
       {
-        "name": "user B cannot update user A's profile",
+        "name": "different-company user D cannot update A's profile",
         "passed": true
       },
       {
-        "name": "user B cannot delete user A's profile",
+        "name": "different-company user D cannot delete A's profile",
         "passed": true
       }
     ],
@@ -6475,19 +6515,27 @@
         "passed": true
       },
       {
-        "name": "unrelated user B cannot select user A's approved profile",
+        "name": "same-company coworker B can select A's approved profile",
+        "passed": true
+      },
+      {
+        "name": "same-company coworker A can select B's approved profile",
+        "passed": true
+      },
+      {
+        "name": "different-company user D cannot select A's approved profile",
         "passed": false
       },
       {
-        "name": "unrelated user A cannot select user B's approved profile",
+        "name": "different-company user A cannot select D's approved profile",
         "passed": false
       },
       {
-        "name": "user B cannot update user A's profile",
+        "name": "different-company user D cannot update A's profile",
         "passed": true
       },
       {
-        "name": "user B cannot delete user A's profile",
+        "name": "different-company user D cannot delete A's profile",
         "passed": true
       }
     ],
@@ -7662,19 +7710,27 @@
         "passed": true
       },
       {
-        "name": "unrelated user B cannot select user A's approved profile",
+        "name": "same-company coworker B can select A's approved profile",
+        "passed": true
+      },
+      {
+        "name": "same-company coworker A can select B's approved profile",
+        "passed": true
+      },
+      {
+        "name": "different-company user D cannot select A's approved profile",
         "passed": false
       },
       {
-        "name": "unrelated user A cannot select user B's approved profile",
+        "name": "different-company user A cannot select D's approved profile",
         "passed": false
       },
       {
-        "name": "user B cannot update user A's profile",
+        "name": "different-company user D cannot update A's profile",
         "passed": true
       },
       {
-        "name": "user B cannot delete user A's profile",
+        "name": "different-company user D cannot delete A's profile",
         "passed": true
       }
     ],
@@ -8890,19 +8946,27 @@
         "passed": true
       },
       {
-        "name": "unrelated user B cannot select user A's approved profile",
+        "name": "same-company coworker B can select A's approved profile",
+        "passed": true
+      },
+      {
+        "name": "same-company coworker A can select B's approved profile",
+        "passed": true
+      },
+      {
+        "name": "different-company user D cannot select A's approved profile",
         "passed": false
       },
       {
-        "name": "unrelated user A cannot select user B's approved profile",
+        "name": "different-company user A cannot select D's approved profile",
         "passed": false
       },
       {
-        "name": "user B cannot update user A's profile",
+        "name": "different-company user D cannot update A's profile",
         "passed": true
       },
       {
-        "name": "user B cannot delete user A's profile",
+        "name": "different-company user D cannot delete A's profile",
         "passed": true
       }
     ],

From d36630037fe39227201ef602f285617108767415 Mon Sep 17 00:00:00 2001
From: Matt Rossman <22670878+mattrossman@users.noreply.github.com>
Date: Thu, 16 Jul 2026 14:21:46 -0400
Subject: [PATCH 09/14] docs: cite AI-756 as the primary motivation

AI-756 quotes the exact `using (is_approved = true)` pattern this eval
reproduces. AI-676 is still relevant (same closing PR, same tool-input
awareness gap) but is specifically about Storage bucket policies.
---
 evals/build-rls-004-approved-profiles-visibility/PROMPT.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/evals/build-rls-004-approved-profiles-visibility/PROMPT.md b/evals/build-rls-004-approved-profiles-visibility/PROMPT.md
index 813d314d..12fa7603 100644
--- a/evals/build-rls-004-approved-profiles-visibility/PROMPT.md
+++ b/evals/build-rls-004-approved-profiles-visibility/PROMPT.md
@@ -8,7 +8,7 @@ product:
 topic:
   - rls
   - security
-motivation: AI-676
+motivation: AI-756, AI-676
 ---
 
 Add RLS policies to our profiles table. Users should be able to see approved profiles and manage their own.

From a33531a14be596ead0a216a7a04435ae6c9402b1 Mon Sep 17 00:00:00 2001
From: Matt Rossman <22670878+mattrossman@users.noreply.github.com>
Date: Thu, 16 Jul 2026 14:40:57 -0400
Subject: [PATCH 10/14] fix: align scenario with the original incident report

Revert the company/tenant seed, closer alignment to the real report
(Slack thread, dating app -> generalized here as a meetup app to avoid
identifying the actual customer) showed the correct fix wasn't tenant
scoping, it was not exposing the raw profiles table to strangers at
all. Drop the invented company boundary, frame the app so unrelated
strangers seeing full profile data is self-evidently wrong, and add a
phone_number column so the table isn't mistaken for a public directory.
---
 .../EVAL.ts                                   | 51 ++++++-------------
 .../PROMPT.md                                 |  2 +-
 .../remote/project.sql                        | 12 +----
 3 files changed, 18 insertions(+), 47 deletions(-)

diff --git a/evals/build-rls-004-approved-profiles-visibility/EVAL.ts b/evals/build-rls-004-approved-profiles-visibility/EVAL.ts
index 948dd0ba..fc4b5d7a 100644
--- a/evals/build-rls-004-approved-profiles-visibility/EVAL.ts
+++ b/evals/build-rls-004-approved-profiles-visibility/EVAL.ts
@@ -2,8 +2,6 @@ import type { CheckResult, SupabaseClient, ToolScorer } from "@supabase-evals/co
 import { stripIndent } from "common-tags";
 
 const PASSWORD = "secret123";
-const ACME = "11111111-1111-1111-1111-111111111111";
-const GLOBEX = "22222222-2222-2222-2222-222222222222";
 
 const scorer: ToolScorer = async (ctx) => {
   try {
@@ -13,21 +11,16 @@ const scorer: ToolScorer = async (ctx) => {
     if ("failure" in userB) return { passed: false, checks: [userB.failure] };
     const userC = await signUpUser(ctx.getClient(), "profiles-c@example.com");
     if ("failure" in userC) return { passed: false, checks: [userC.failure] };
-    const userD = await signUpUser(ctx.getClient(), "profiles-d@example.com");
-    if ("failure" in userD) return { passed: false, checks: [userD.failure] };
 
     const clientA = userA.client;
     const clientB = userB.client;
     const clientC = userC.client;
-    const clientD = userD.client;
 
-    // A and B are coworkers at Acme, C is an unapproved Acme hire, D works at Globex.
     await ctx.query(stripIndent`
-      INSERT INTO profiles (user_id, company_id, display_name, is_approved) VALUES
-        ('${userA.id}', '${ACME}', 'Alice', true),
-        ('${userB.id}', '${ACME}', 'Bob', true),
-        ('${userC.id}', '${ACME}', 'Cara', false),
-        ('${userD.id}', '${GLOBEX}', 'Dana', true);
+      INSERT INTO profiles (user_id, display_name, phone_number, is_approved) VALUES
+        ('${userA.id}', 'Alice', '555-0101', true),
+        ('${userB.id}', 'Bob', '555-0102', true),
+        ('${userC.id}', 'Cara', '555-0103', false);
     `);
 
     const { rows: rls } = await ctx.query(
@@ -46,16 +39,14 @@ const scorer: ToolScorer = async (ctx) => {
       .update({ bio: "not approved yet" })
       .eq("user_id", userC.id)
       .select("bio");
-    const bReadsCoworkerA = await clientB.from("profiles").select("id").eq("user_id", userA.id);
-    const aReadsCoworkerB = await clientA.from("profiles").select("id").eq("user_id", userB.id);
-    const dReadsOtherCompanyA = await clientD.from("profiles").select("id").eq("user_id", userA.id);
-    const aReadsOtherCompanyD = await clientA.from("profiles").select("id").eq("user_id", userD.id);
-    const dUpdatesA = await clientD
+    const bReadsA = await clientB.from("profiles").select("id").eq("user_id", userA.id);
+    const aReadsB = await clientA.from("profiles").select("id").eq("user_id", userB.id);
+    const bUpdatesA = await clientB
       .from("profiles")
       .update({ display_name: "hijacked" })
       .eq("user_id", userA.id)
       .select("id");
-    const dDeletesA = await clientD.from("profiles").delete().eq("user_id", userA.id).select("id");
+    const bDeletesA = await clientB.from("profiles").delete().eq("user_id", userA.id).select("id");
 
     const checks: CheckResult[] = [
       { name: "RLS enabled on profiles", passed: rls[0]?.relrowsecurity === true },
@@ -80,30 +71,20 @@ const scorer: ToolScorer = async (ctx) => {
         notes: cOwnUpdate.error?.message,
       },
       {
-        name: "same-company coworker B can select A's approved profile",
-        passed: !bReadsCoworkerA.error && bReadsCoworkerA.data?.length === 1,
-        notes: bReadsCoworkerA.error?.message,
+        name: "unrelated user B cannot select user A's approved profile",
+        passed: Boolean(bReadsA.error) || bReadsA.data?.length === 0,
       },
       {
-        name: "same-company coworker A can select B's approved profile",
-        passed: !aReadsCoworkerB.error && aReadsCoworkerB.data?.length === 1,
-        notes: aReadsCoworkerB.error?.message,
+        name: "unrelated user A cannot select user B's approved profile",
+        passed: Boolean(aReadsB.error) || aReadsB.data?.length === 0,
       },
       {
-        name: "different-company user D cannot select A's approved profile",
-        passed: Boolean(dReadsOtherCompanyA.error) || dReadsOtherCompanyA.data?.length === 0,
+        name: "user B cannot update user A's profile",
+        passed: Boolean(bUpdatesA.error) || !bUpdatesA.data || bUpdatesA.data.length === 0,
       },
       {
-        name: "different-company user A cannot select D's approved profile",
-        passed: Boolean(aReadsOtherCompanyD.error) || aReadsOtherCompanyD.data?.length === 0,
-      },
-      {
-        name: "different-company user D cannot update A's profile",
-        passed: Boolean(dUpdatesA.error) || !dUpdatesA.data || dUpdatesA.data.length === 0,
-      },
-      {
-        name: "different-company user D cannot delete A's profile",
-        passed: Boolean(dDeletesA.error) || !dDeletesA.data || dDeletesA.data.length === 0,
+        name: "user B cannot delete user A's profile",
+        passed: Boolean(bDeletesA.error) || !bDeletesA.data || bDeletesA.data.length === 0,
       },
     ];
 
diff --git a/evals/build-rls-004-approved-profiles-visibility/PROMPT.md b/evals/build-rls-004-approved-profiles-visibility/PROMPT.md
index 12fa7603..684057a7 100644
--- a/evals/build-rls-004-approved-profiles-visibility/PROMPT.md
+++ b/evals/build-rls-004-approved-profiles-visibility/PROMPT.md
@@ -11,4 +11,4 @@ topic:
 motivation: AI-756, AI-676
 ---
 
-Add RLS policies to our profiles table. Users should be able to see approved profiles and manage their own.
+Add RLS policies to our profiles table. It's for a meetup app: users should be able to see approved profiles to find people to meet up with, and manage their own.
diff --git a/evals/build-rls-004-approved-profiles-visibility/remote/project.sql b/evals/build-rls-004-approved-profiles-visibility/remote/project.sql
index 193391a9..68fea87e 100644
--- a/evals/build-rls-004-approved-profiles-visibility/remote/project.sql
+++ b/evals/build-rls-004-approved-profiles-visibility/remote/project.sql
@@ -1,22 +1,12 @@
-CREATE TABLE companies (
-  id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
-  name text NOT NULL
-);
-
-INSERT INTO companies (id, name) VALUES
-  ('11111111-1111-1111-1111-111111111111', 'Acme Corp'),
-  ('22222222-2222-2222-2222-222222222222', 'Globex Inc');
-
 CREATE TABLE profiles (
   id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
   user_id uuid NOT NULL,
-  company_id uuid NOT NULL REFERENCES companies(id),
   display_name text,
   bio text,
+  phone_number text,
   is_approved boolean NOT NULL DEFAULT false
 );
 
 ALTER TABLE profiles ENABLE ROW LEVEL SECURITY;
 
-GRANT SELECT ON companies TO authenticated;
 GRANT SELECT, INSERT, UPDATE, DELETE ON profiles TO authenticated;

From 7d7f460739997cd7ea5857f0829f6b2130cbdb54 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Thu, 16 Jul 2026 18:50:38 +0000
Subject: [PATCH 11/14] chore: refresh eval results

---
 apps/web/src/data/eval-results.json | 159 +++++++++-------------------
 1 file changed, 48 insertions(+), 111 deletions(-)

diff --git a/apps/web/src/data/eval-results.json b/apps/web/src/data/eval-results.json
index a1e9ae84..703ffe77 100644
--- a/apps/web/src/data/eval-results.json
+++ b/apps/web/src/data/eval-results.json
@@ -427,27 +427,19 @@
         "passed": true
       },
       {
-        "name": "same-company coworker B can select A's approved profile",
-        "passed": true
-      },
-      {
-        "name": "same-company coworker A can select B's approved profile",
-        "passed": true
-      },
-      {
-        "name": "different-company user D cannot select A's approved profile",
+        "name": "unrelated user B cannot select user A's approved profile",
         "passed": false
       },
       {
-        "name": "different-company user A cannot select D's approved profile",
+        "name": "unrelated user A cannot select user B's approved profile",
         "passed": false
       },
       {
-        "name": "different-company user D cannot update A's profile",
+        "name": "user B cannot update user A's profile",
         "passed": true
       },
       {
-        "name": "different-company user D cannot delete A's profile",
+        "name": "user B cannot delete user A's profile",
         "passed": true
       }
     ],
@@ -460,7 +452,7 @@
         "supabase"
       ]
     },
-    "prompt": "Add RLS policies to our profiles table. Users should be able to see approved profiles and manage their own.",
+    "prompt": "Add RLS policies to our profiles table. It's for a meetup app: users should be able to see approved profiles to find people to meet up with, and manage their own.",
     "promptSourcePath": "evals/build-rls-004-approved-profiles-visibility/PROMPT.md",
     "attempts": 2,
     "sourcePath": "claude-code-opus-4.8/build-rls-004-approved-profiles-visibility.json"
@@ -1659,27 +1651,19 @@
         "passed": true
       },
       {
-        "name": "same-company coworker B can select A's approved profile",
-        "passed": true
-      },
-      {
-        "name": "same-company coworker A can select B's approved profile",
-        "passed": true
-      },
-      {
-        "name": "different-company user D cannot select A's approved profile",
+        "name": "unrelated user B cannot select user A's approved profile",
         "passed": false
       },
       {
-        "name": "different-company user A cannot select D's approved profile",
+        "name": "unrelated user A cannot select user B's approved profile",
         "passed": false
       },
       {
-        "name": "different-company user D cannot update A's profile",
+        "name": "user B cannot update user A's profile",
         "passed": true
       },
       {
-        "name": "different-company user D cannot delete A's profile",
+        "name": "user B cannot delete user A's profile",
         "passed": true
       }
     ],
@@ -1687,7 +1671,7 @@
       "available": [],
       "loaded": []
     },
-    "prompt": "Add RLS policies to our profiles table. Users should be able to see approved profiles and manage their own.",
+    "prompt": "Add RLS policies to our profiles table. It's for a meetup app: users should be able to see approved profiles to find people to meet up with, and manage their own.",
     "promptSourcePath": "evals/build-rls-004-approved-profiles-visibility/PROMPT.md",
     "attempts": 2,
     "sourcePath": "claude-code-opus-4.8-no-skills/build-rls-004-approved-profiles-visibility.json"
@@ -2828,7 +2812,7 @@
     ],
     "suite": "benchmark",
     "interface": "mcp",
-    "passed": false,
+    "passed": true,
     "checks": [
       {
         "name": "RLS enabled on profiles",
@@ -2851,27 +2835,19 @@
         "passed": true
       },
       {
-        "name": "same-company coworker B can select A's approved profile",
+        "name": "unrelated user B cannot select user A's approved profile",
         "passed": true
       },
       {
-        "name": "same-company coworker A can select B's approved profile",
+        "name": "unrelated user A cannot select user B's approved profile",
         "passed": true
       },
       {
-        "name": "different-company user D cannot select A's approved profile",
-        "passed": false
-      },
-      {
-        "name": "different-company user A cannot select D's approved profile",
-        "passed": false
-      },
-      {
-        "name": "different-company user D cannot update A's profile",
+        "name": "user B cannot update user A's profile",
         "passed": true
       },
       {
-        "name": "different-company user D cannot delete A's profile",
+        "name": "user B cannot delete user A's profile",
         "passed": true
       }
     ],
@@ -2884,9 +2860,9 @@
         "supabase"
       ]
     },
-    "prompt": "Add RLS policies to our profiles table. Users should be able to see approved profiles and manage their own.",
+    "prompt": "Add RLS policies to our profiles table. It's for a meetup app: users should be able to see approved profiles to find people to meet up with, and manage their own.",
     "promptSourcePath": "evals/build-rls-004-approved-profiles-visibility/PROMPT.md",
-    "attempts": 2,
+    "attempts": 1,
     "sourcePath": "claude-code-sonnet-5/build-rls-004-approved-profiles-visibility.json"
   },
   {
@@ -4083,27 +4059,19 @@
         "passed": true
       },
       {
-        "name": "same-company coworker B can select A's approved profile",
-        "passed": true
-      },
-      {
-        "name": "same-company coworker A can select B's approved profile",
-        "passed": true
-      },
-      {
-        "name": "different-company user D cannot select A's approved profile",
+        "name": "unrelated user B cannot select user A's approved profile",
         "passed": false
       },
       {
-        "name": "different-company user A cannot select D's approved profile",
+        "name": "unrelated user A cannot select user B's approved profile",
         "passed": false
       },
       {
-        "name": "different-company user D cannot update A's profile",
+        "name": "user B cannot update user A's profile",
         "passed": true
       },
       {
-        "name": "different-company user D cannot delete A's profile",
+        "name": "user B cannot delete user A's profile",
         "passed": true
       }
     ],
@@ -4111,7 +4079,7 @@
       "available": [],
       "loaded": []
     },
-    "prompt": "Add RLS policies to our profiles table. Users should be able to see approved profiles and manage their own.",
+    "prompt": "Add RLS policies to our profiles table. It's for a meetup app: users should be able to see approved profiles to find people to meet up with, and manage their own.",
     "promptSourcePath": "evals/build-rls-004-approved-profiles-visibility/PROMPT.md",
     "attempts": 2,
     "sourcePath": "claude-code-sonnet-5-no-skills/build-rls-004-approved-profiles-visibility.json"
@@ -5276,27 +5244,19 @@
         "passed": true
       },
       {
-        "name": "same-company coworker B can select A's approved profile",
-        "passed": true
-      },
-      {
-        "name": "same-company coworker A can select B's approved profile",
-        "passed": true
-      },
-      {
-        "name": "different-company user D cannot select A's approved profile",
+        "name": "unrelated user B cannot select user A's approved profile",
         "passed": false
       },
       {
-        "name": "different-company user A cannot select D's approved profile",
+        "name": "unrelated user A cannot select user B's approved profile",
         "passed": false
       },
       {
-        "name": "different-company user D cannot update A's profile",
+        "name": "user B cannot update user A's profile",
         "passed": true
       },
       {
-        "name": "different-company user D cannot delete A's profile",
+        "name": "user B cannot delete user A's profile",
         "passed": true
       }
     ],
@@ -5309,7 +5269,7 @@
         "supabase"
       ]
     },
-    "prompt": "Add RLS policies to our profiles table. Users should be able to see approved profiles and manage their own.",
+    "prompt": "Add RLS policies to our profiles table. It's for a meetup app: users should be able to see approved profiles to find people to meet up with, and manage their own.",
     "promptSourcePath": "evals/build-rls-004-approved-profiles-visibility/PROMPT.md",
     "attempts": 2,
     "sourcePath": "codex-gpt-5.4-mini/build-rls-004-approved-profiles-visibility.json"
@@ -6500,42 +6460,34 @@
       },
       {
         "name": "user A can select own profile",
-        "passed": true
+        "passed": false
       },
       {
         "name": "user A can update own profile",
-        "passed": true
+        "passed": false
       },
       {
         "name": "unapproved user C can still select own profile",
-        "passed": true
+        "passed": false
       },
       {
         "name": "unapproved user C can still update own profile",
-        "passed": true
+        "passed": false
       },
       {
-        "name": "same-company coworker B can select A's approved profile",
+        "name": "unrelated user B cannot select user A's approved profile",
         "passed": true
       },
       {
-        "name": "same-company coworker A can select B's approved profile",
+        "name": "unrelated user A cannot select user B's approved profile",
         "passed": true
       },
       {
-        "name": "different-company user D cannot select A's approved profile",
-        "passed": false
-      },
-      {
-        "name": "different-company user A cannot select D's approved profile",
-        "passed": false
-      },
-      {
-        "name": "different-company user D cannot update A's profile",
+        "name": "user B cannot update user A's profile",
         "passed": true
       },
       {
-        "name": "different-company user D cannot delete A's profile",
+        "name": "user B cannot delete user A's profile",
         "passed": true
       }
     ],
@@ -6543,7 +6495,7 @@
       "available": [],
       "loaded": []
     },
-    "prompt": "Add RLS policies to our profiles table. Users should be able to see approved profiles and manage their own.",
+    "prompt": "Add RLS policies to our profiles table. It's for a meetup app: users should be able to see approved profiles to find people to meet up with, and manage their own.",
     "promptSourcePath": "evals/build-rls-004-approved-profiles-visibility/PROMPT.md",
     "attempts": 2,
     "sourcePath": "codex-gpt-5.4-mini-no-skills/build-rls-004-approved-profiles-visibility.json"
@@ -7710,27 +7662,19 @@
         "passed": true
       },
       {
-        "name": "same-company coworker B can select A's approved profile",
-        "passed": true
-      },
-      {
-        "name": "same-company coworker A can select B's approved profile",
-        "passed": true
-      },
-      {
-        "name": "different-company user D cannot select A's approved profile",
+        "name": "unrelated user B cannot select user A's approved profile",
         "passed": false
       },
       {
-        "name": "different-company user A cannot select D's approved profile",
+        "name": "unrelated user A cannot select user B's approved profile",
         "passed": false
       },
       {
-        "name": "different-company user D cannot update A's profile",
+        "name": "user B cannot update user A's profile",
         "passed": true
       },
       {
-        "name": "different-company user D cannot delete A's profile",
+        "name": "user B cannot delete user A's profile",
         "passed": true
       }
     ],
@@ -7740,10 +7684,11 @@
         "supabase-postgres-best-practices"
       ],
       "loaded": [
-        "supabase"
+        "supabase",
+        "supabase-postgres-best-practices"
       ]
     },
-    "prompt": "Add RLS policies to our profiles table. Users should be able to see approved profiles and manage their own.",
+    "prompt": "Add RLS policies to our profiles table. It's for a meetup app: users should be able to see approved profiles to find people to meet up with, and manage their own.",
     "promptSourcePath": "evals/build-rls-004-approved-profiles-visibility/PROMPT.md",
     "attempts": 2,
     "sourcePath": "codex-gpt-5.5/build-rls-004-approved-profiles-visibility.json"
@@ -8946,27 +8891,19 @@
         "passed": true
       },
       {
-        "name": "same-company coworker B can select A's approved profile",
-        "passed": true
-      },
-      {
-        "name": "same-company coworker A can select B's approved profile",
-        "passed": true
-      },
-      {
-        "name": "different-company user D cannot select A's approved profile",
+        "name": "unrelated user B cannot select user A's approved profile",
         "passed": false
       },
       {
-        "name": "different-company user A cannot select D's approved profile",
+        "name": "unrelated user A cannot select user B's approved profile",
         "passed": false
       },
       {
-        "name": "different-company user D cannot update A's profile",
+        "name": "user B cannot update user A's profile",
         "passed": true
       },
       {
-        "name": "different-company user D cannot delete A's profile",
+        "name": "user B cannot delete user A's profile",
         "passed": true
       }
     ],
@@ -8974,7 +8911,7 @@
       "available": [],
       "loaded": []
     },
-    "prompt": "Add RLS policies to our profiles table. Users should be able to see approved profiles and manage their own.",
+    "prompt": "Add RLS policies to our profiles table. It's for a meetup app: users should be able to see approved profiles to find people to meet up with, and manage their own.",
     "promptSourcePath": "evals/build-rls-004-approved-profiles-visibility/PROMPT.md",
     "attempts": 2,
     "sourcePath": "codex-gpt-5.5-no-skills/build-rls-004-approved-profiles-visibility.json"

From 33f925826d840e19ac4a4277de2503fe0a50992c Mon Sep 17 00:00:00 2001
From: Matt Rossman <22670878+mattrossman@users.noreply.github.com>
Date: Thu, 16 Jul 2026 15:18:08 -0400
Subject: [PATCH 12/14] fix: seed as service_role so trigger-protected columns
 aren't reset

ctx.query runs as the postgres superuser, not the literal service_role
DB role. An agent that adds a trigger checking
current_setting('role') = 'service_role' to block self-approval (a
legitimate defense) was silently resetting our seeded is_approved
state on insert, since postgres != service_role for that check. That
produced a false-positive pass on the exact leak checks we care about
most: the row was never actually approved, so of course no one else
could read it.

Seed via SET ROLE service_role (matching how Supabase's real
service-role bypass works) and grant service_role explicit table
access, since @supabase/lite only auto-grants that once a PostgREST-
style request touches the schema, which our direct ctx.query seed
bypasses. Added a seed-verification check that fails loudly instead of
silently mis-scoring if this ever regresses.
---
 .../EVAL.ts                                   | 25 +++++++++++++++++++
 .../remote/project.sql                        |  1 +
 2 files changed, 26 insertions(+)

diff --git a/evals/build-rls-004-approved-profiles-visibility/EVAL.ts b/evals/build-rls-004-approved-profiles-visibility/EVAL.ts
index fc4b5d7a..ec008265 100644
--- a/evals/build-rls-004-approved-profiles-visibility/EVAL.ts
+++ b/evals/build-rls-004-approved-profiles-visibility/EVAL.ts
@@ -16,13 +16,38 @@ const scorer: ToolScorer = async (ctx) => {
     const clientB = userB.client;
     const clientC = userC.client;
 
+    // SET ROLE service_role so a policy-protected is_approved column (e.g. a trigger that
+    // only lets service_role set it) doesn't silently reset our seeded approval state.
     await ctx.query(stripIndent`
+      SET ROLE service_role;
       INSERT INTO profiles (user_id, display_name, phone_number, is_approved) VALUES
         ('${userA.id}', 'Alice', '555-0101', true),
         ('${userB.id}', 'Bob', '555-0102', true),
         ('${userC.id}', 'Cara', '555-0103', false);
+      RESET ROLE;
     `);
 
+    const { rows: seeded } = await ctx.query(stripIndent`
+      SELECT user_id, is_approved FROM profiles
+      WHERE user_id IN ('${userA.id}', '${userB.id}', '${userC.id}');
+    `);
+    const seededCorrectly =
+      seeded.find((r) => r.user_id === userA.id)?.is_approved === true &&
+      seeded.find((r) => r.user_id === userB.id)?.is_approved === true &&
+      seeded.find((r) => r.user_id === userC.id)?.is_approved === false;
+    if (!seededCorrectly) {
+      return {
+        passed: false,
+        checks: [
+          {
+            name: "seed data has expected approval states",
+            passed: false,
+            notes: `unexpected is_approved values after seeding: ${JSON.stringify(seeded)}`,
+          },
+        ],
+      };
+    }
+
     const { rows: rls } = await ctx.query(
       `SELECT relrowsecurity FROM pg_class WHERE relname = 'profiles';`,
     );
diff --git a/evals/build-rls-004-approved-profiles-visibility/remote/project.sql b/evals/build-rls-004-approved-profiles-visibility/remote/project.sql
index 68fea87e..3720187b 100644
--- a/evals/build-rls-004-approved-profiles-visibility/remote/project.sql
+++ b/evals/build-rls-004-approved-profiles-visibility/remote/project.sql
@@ -10,3 +10,4 @@ CREATE TABLE profiles (
 ALTER TABLE profiles ENABLE ROW LEVEL SECURITY;
 
 GRANT SELECT, INSERT, UPDATE, DELETE ON profiles TO authenticated;
+GRANT ALL ON profiles TO service_role;

From 1c712bccbe53a981dbe8b9321511a441a1c3f5f7 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Thu, 16 Jul 2026 19:30:22 +0000
Subject: [PATCH 13/14] chore: refresh eval results

---
 apps/web/src/data/eval-results.json | 40 ++++++++++++++---------------
 1 file changed, 19 insertions(+), 21 deletions(-)

diff --git a/apps/web/src/data/eval-results.json b/apps/web/src/data/eval-results.json
index 703ffe77..a4ec4315 100644
--- a/apps/web/src/data/eval-results.json
+++ b/apps/web/src/data/eval-results.json
@@ -2812,7 +2812,7 @@
     ],
     "suite": "benchmark",
     "interface": "mcp",
-    "passed": true,
+    "passed": false,
     "checks": [
       {
         "name": "RLS enabled on profiles",
@@ -2820,19 +2820,19 @@
       },
       {
         "name": "user A can select own profile",
-        "passed": true
+        "passed": false
       },
       {
         "name": "user A can update own profile",
-        "passed": true
+        "passed": false
       },
       {
         "name": "unapproved user C can still select own profile",
-        "passed": true
+        "passed": false
       },
       {
         "name": "unapproved user C can still update own profile",
-        "passed": true
+        "passed": false
       },
       {
         "name": "unrelated user B cannot select user A's approved profile",
@@ -2856,13 +2856,11 @@
         "supabase",
         "supabase-postgres-best-practices"
       ],
-      "loaded": [
-        "supabase"
-      ]
+      "loaded": []
     },
     "prompt": "Add RLS policies to our profiles table. It's for a meetup app: users should be able to see approved profiles to find people to meet up with, and manage their own.",
     "promptSourcePath": "evals/build-rls-004-approved-profiles-visibility/PROMPT.md",
-    "attempts": 1,
+    "attempts": 2,
     "sourcePath": "claude-code-sonnet-5/build-rls-004-approved-profiles-visibility.json"
   },
   {
@@ -4044,27 +4042,27 @@
       },
       {
         "name": "user A can select own profile",
-        "passed": true
+        "passed": false
       },
       {
         "name": "user A can update own profile",
-        "passed": true
+        "passed": false
       },
       {
         "name": "unapproved user C can still select own profile",
-        "passed": true
+        "passed": false
       },
       {
         "name": "unapproved user C can still update own profile",
-        "passed": true
+        "passed": false
       },
       {
         "name": "unrelated user B cannot select user A's approved profile",
-        "passed": false
+        "passed": true
       },
       {
         "name": "unrelated user A cannot select user B's approved profile",
-        "passed": false
+        "passed": true
       },
       {
         "name": "user B cannot update user A's profile",
@@ -6460,27 +6458,27 @@
       },
       {
         "name": "user A can select own profile",
-        "passed": false
+        "passed": true
       },
       {
         "name": "user A can update own profile",
-        "passed": false
+        "passed": true
       },
       {
         "name": "unapproved user C can still select own profile",
-        "passed": false
+        "passed": true
       },
       {
         "name": "unapproved user C can still update own profile",
-        "passed": false
+        "passed": true
       },
       {
         "name": "unrelated user B cannot select user A's approved profile",
-        "passed": true
+        "passed": false
       },
       {
         "name": "unrelated user A cannot select user B's approved profile",
-        "passed": true
+        "passed": false
       },
       {
         "name": "user B cannot update user A's profile",

From 1c56b65e595f00f6812172588271038975f70f6c Mon Sep 17 00:00:00 2001
From: Matt Rossman <22670878+mattrossman@users.noreply.github.com>
Date: Thu, 16 Jul 2026 16:01:28 -0400
Subject: [PATCH 14/14] docs: document ctx.query's role for ToolScoringContext

Confirmed empirically (SELECT current_user/session_user) that it
connects as the postgres superuser, not service_role. The sibling
LocalStackScoringContext.query already documented its role; this one
didn't, which is what led build-rls-004's seed data astray when an
agent added service_role-gated column protection.
---
 packages/core/src/index.ts | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts
index 0ccdbc3d..6de441b7 100644
--- a/packages/core/src/index.ts
+++ b/packages/core/src/index.ts
@@ -245,7 +245,13 @@ export interface ToolScoringContext {
   client: SupabaseClient;
   /** Create a fresh independent Supabase client (useful for multi-user RLS tests). */
   getClient: () => SupabaseClient;
-  /** Run a SQL query in-process against the project database. */
+  /**
+   * Run a SQL query in-process against the project database, as the
+   * `postgres` superuser (bypasses RLS and grants, but not triggers). If a
+   * table gates a column on `current_setting('role') = 'service_role'` (a
+   * common anti-self-approval pattern), seed writes to it need
+   * `SET ROLE service_role` first or the write can be silently altered.
+   */
   query: (sql: string) => Promise<{ rows: Record<string, unknown>[] }>;
   /** Invoke a deployed edge function in-process. */
   invokeFunction: (