Fix indexed values going stale: module-edit fan-out misses and one-generation-old file content meta

.claude/settings.json

@@ -12,12 +12,23 @@
       "Bash(node --check *)",
       "Bash(npx vitest run *)",
       "Bash(pnpm exec prettier *)",
-      "mcp__linear__list_comments",
-      "mcp__linear__get_issue",
-      "mcp__linear__get_project",
-      "mcp__linear__get_user",
-      "mcp__linear__list_projects",
-      "mcp__linear__list_issue_statuses",
+      "mcp__Linear__get_issue",
+      "mcp__Linear__list_comments",
+      "mcp__Linear__get_project",
+      "mcp__Linear__get_user",
+      "mcp__Linear__list_projects",
+      "mcp__Linear__list_issue_statuses",
+      "mcp__Linear__list_issues",
+      "mcp__github__pull_request_read",
+      "mcp__github__list_pull_requests",
+      "mcp__github__get_file_contents",
+      "mcp__github__search_code",
+      "mcp__github__list_commits",
+      "mcp__github__get_commit",
+      "mcp__github__get_me",
+      "mcp__github__actions_list",
+      "mcp__github__actions_get",
+      "mcp__github__get_job_logs",
       "mcp__chrome-devtools__take_screenshot"
     ]
   },

packages/host/tests/unit/index-writer-test.ts

@@ -435,6 +435,66 @@ module('Unit | index-writer', function (hooks) {
     ]);
   });
 
+  test('invalidations reach dependents whose rows exist only in the production index', async function (assert) {
+    // A production row without a working counterpart is a reachable state:
+    // the working table is a staging area that only gains a row once a visit
+    // writes one in this deployment's lifetime — an index imported from a
+    // dump that carries only the production tables, or rows staged under a
+    // since-changed schema. The dependency fan-out must find such dependents
+    // or a module edit leaves their indexed computed values stale until each
+    // file is touched individually.
+    await setupIndex(
+      adapter,
+      [{ realm_url: testRealmURL, current_generation: 1 }],
+      {
+        working: [],
+        production: [
+          {
+            url: `${testRealmURL}1.json`,
+            file_alias: `${testRealmURL}1.json`,
+            type: 'instance',
+            generation: 1,
+            realm_url: testRealmURL,
+            deps: [`${testRealmURL}person`],
+          },
+          // Transitive: discoverable only through 1.json's production row.
+          {
+            url: `${testRealmURL}2.json`,
+            file_alias: `${testRealmURL}2.json`,
+            type: 'instance',
+            generation: 1,
+            realm_url: testRealmURL,
+            deps: [`${testRealmURL}1.json`],
+          },
+          // A promoted tombstone is a deleted file — it stays out of the
+          // fan-out.
+          {
+            url: `${testRealmURL}3.json`,
+            file_alias: `${testRealmURL}3.json`,
+            type: 'instance',
+            generation: 1,
+            realm_url: testRealmURL,
+            is_deleted: true,
+            deps: [`${testRealmURL}person`],
+          },
+        ],
+      },
+      { prerenderedHtml: false },
+    );
+
+    let batch = await indexWriter.createBatch(
+      new URL(testRealmURL),
+      virtualNetwork,
+    );
+    await batch.invalidate([new URL(`${testRealmURL}person.gts`)]);
+
+    assert.deepEqual(batch.invalidations.sort(), [
+      `${testRealmURL}1.json`,
+      `${testRealmURL}2.json`,
+      `${testRealmURL}person.gts`,
+    ]);
+  });
+
   test("invalidations don't cross realm boundaries", async function (assert) {
     await setupIndex(
       adapter,

packages/realm-server/tests/index.ts

@@ -111,6 +111,11 @@ if (testModules) {
   }
 }
 
+const qunitFilter = process.env.QUNIT_FILTER?.trim();
+if (qunitFilter) {
+  QUnit.config.filter = qunitFilter;
+}
+
 // Cleanup here ensures lingering servers/prerenderers/queues don't keep the
 // Node event loop alive after tests finish — and equivalently, don't leave
 // hardcoded test ports (4444-4471, etc.) bound after a test is aborted by

packages/realm-server/tests/indexing-test.ts

@@ -44,6 +44,7 @@ import {
 } from './helpers/indexing.ts';
 import stripScopedCSSAttributes from '@cardstack/runtime-common/helpers/strip-scoped-css-attributes';
 import { basename } from 'path';
+import { createHash } from 'crypto';
 import type { PgAdapter } from '@cardstack/postgres';
 
 function trimCardContainer(text: string) {
@@ -4489,6 +4490,37 @@ module(basename(import.meta.filename), function () {
         );
       });
 
+      test('file meta serves write-time contentHash/contentSize over indexed values', async function (assert) {
+        let content = 'bytes behind the file meta doc';
+        await realm.write('meta-note.txt', content);
+
+        // Emulate the state a mid-batch render observes: the production
+        // index row still carries the previous bytes' content values while
+        // `realm_file_meta` (written in the same critical section as the
+        // bytes) already has the current ones. The pristine resource is
+        // dropped so the served doc must choose between the indexed search
+        // doc and the persisted meta.
+        await testDbAdapter.execute(
+          `UPDATE boxel_index SET search_doc = COALESCE(search_doc, '{}'::jsonb) || '{"contentHash":"stale-hash","contentSize":1}'::jsonb, pristine_doc = NULL WHERE url = '${testRealm}meta-note.txt' AND type = 'file'`,
+        );
+
+        let response = await fetch(`${testRealm}meta-note.txt`, {
+          headers: { Accept: SupportedMimeType.FileMeta },
+        });
+        assert.strictEqual(response.status, 200, 'file meta response is ok');
+        let doc = (await response.json()) as LooseSingleCardDocument;
+        assert.strictEqual(
+          doc.data.attributes?.contentHash,
+          createHash('md5').update(content).digest('hex'),
+          'contentHash reflects the bytes on disk, not the indexed value',
+        );
+        assert.strictEqual(
+          doc.data.attributes?.contentSize,
+          content.length,
+          'contentSize reflects the bytes on disk, not the indexed value',
+        );
+      });
+
       test('can incrementally index deleted instance', async function (assert) {
         await realm.delete('mango.json');
 

packages/runtime-common/index-writer.ts

@@ -2420,7 +2420,7 @@ export class Batch {
       unresolvedPath !== resolvedPath &&
       this.isRegisteredPrefix(unresolvedPath);
     let scanTable = async (
-      tableName: 'boxel_index_working' | 'prerendered_html',
+      tableName: 'boxel_index_working' | 'prerendered_html' | 'boxel_index',
     ) => {
       let results: (Pick<BoxelIndexTable, 'url' | 'file_alias'> & {
         type: BoxelIndexTable['type'];
@@ -2476,6 +2476,19 @@ export class Batch {
             // probably need to reevaluate this condition when we get to cross
             // realm invalidation
             [`i.realm_url =`, param(this.realmURL.href)],
+            // A promoted tombstone is a deleted file — it has nothing to
+            // re-render, and pulling it into the fan-out would visit a path
+            // with no backing file. The working/prerendered scans carry
+            // tombstones from in-flight batches whose skip logic handles
+            // them, so only the production scan filters.
+            ...(tableName === 'boxel_index'
+              ? [
+                  any([
+                    ['i.is_deleted = false'],
+                    ['i.is_deleted IS NULL'],
+                  ]) as Expression,
+                ]
+              : []),
           ]),
           `LIMIT ${pageSize} OFFSET ${pageNumber * pageSize}`,
         ] as Expression)) as (Pick<BoxelIndexTable, 'url' | 'file_alias'> & {
@@ -2492,15 +2505,30 @@ export class Batch {
     // — a rendered non-searchable link, the scoped-CSS artifacts of linked
     // instances. Both edge sets must feed the fan-out or a change to a
     // render-only dependency would never re-render its consumers.
-    let [workingScan, prerenderedScan] = await Promise.all([
+    //
+    // Production `boxel_index` is scanned as well because the working table
+    // is a staging area whose coverage is not guaranteed: a row only exists
+    // there once a visit has written it in this deployment's lifetime, so
+    // promoted rows can lack a working counterpart (an index imported from a
+    // dump that carries only the production tables, or rows staged under a
+    // since-changed schema). Without the production scan, a module edit
+    // never reaches such dependents — their indexed computed values sit
+    // stale until each file is touched individually. Rows present in both
+    // tables collapse in the (url, type) dedup below.
+    let [workingScan, prerenderedScan, productionScan] = await Promise.all([
       scanTable('boxel_index_working'),
       scanTable('prerendered_html'),
+      scanTable('boxel_index'),
     ]);
     let seen = new Set<string>();
     let results: (Pick<BoxelIndexTable, 'url' | 'file_alias'> & {
       type: BoxelIndexTable['type'];
     })[] = [];
-    for (let row of [...workingScan.results, ...prerenderedScan.results]) {
+    for (let row of [
+      ...workingScan.results,
+      ...prerenderedScan.results,
+      ...productionScan.results,
+    ]) {
       let key = `${row.url}|${row.type}`;
       if (seen.has(key)) {
         continue;
@@ -2511,7 +2539,11 @@ export class Batch {
     this.#perfLog.debug(
       `${jobIdentity(this.jobInfo)} time to determine items that reference ${resolvedPath} ${
         Date.now() - start
-      } ms (page count=${workingScan.pageNumber + prerenderedScan.pageNumber})`,
+      } ms (page count=${
+        workingScan.pageNumber +
+        prerenderedScan.pageNumber +
+        productionScan.pageNumber
+      })`,
     );
     return results.map(({ url, file_alias, type }) => ({
       url,

packages/runtime-common/realm.ts

@@ -4479,13 +4479,18 @@ export class Realm {
       typeof searchDoc.contentSize === 'number'
         ? searchDoc.contentSize
         : undefined;
-    // Only hit the DB when the indexed searchDoc is missing a value.
-    let persistedMeta =
-      (searchHash === undefined || searchSize === undefined) && this.#dbAdapter
-        ? await getContentMeta(this.#dbAdapter, this.url, localPath)
-        : { contentHash: undefined, contentSize: undefined };
-    let contentHash = searchHash ?? persistedMeta.contentHash;
-    let contentSize = searchSize ?? persistedMeta.contentSize;
+    // `realm_file_meta` is written in the same critical section as the file's
+    // bytes, so it is authoritative for content-derived values. It is
+    // preferred over the indexed values, which lag one batch promotion behind:
+    // a render inside the batch that re-indexes this file reads the pre-swap
+    // production row, so a card linking the file sees the file's index row at
+    // its previous contentHash/contentSize. The indexed values remain as
+    // fallbacks for files whose bytes were never hashed at write time.
+    let persistedMeta = this.#dbAdapter
+      ? await getContentMeta(this.#dbAdapter, this.url, localPath)
+      : { contentHash: undefined, contentSize: undefined };
+    let contentHash = persistedMeta.contentHash ?? searchHash;
+    let contentSize = persistedMeta.contentSize ?? searchSize;
     let adoptsFrom =
       codeRefFromInternalKey(fileEntry.types?.[0]) ??
       (isCodeRef(fileEntry.resource?.meta?.adoptsFrom)
@@ -4501,8 +4506,11 @@ export class Realm {
         resourceAttributes.contentType ??
         searchDoc.contentType ??
         inferredContentType,
-      contentHash: resourceAttributes.contentHash ?? contentHash,
-      contentSize: resourceAttributes.contentSize ?? contentSize,
+      // The persisted write-time values win over the indexed resource's for
+      // the same staleness reason as above; the resource's extract-computed
+      // values cover files that predate write-time hashing.
+      contentHash: contentHash ?? resourceAttributes.contentHash,
+      contentSize: contentSize ?? resourceAttributes.contentSize,
       lastModified: fileEntry.lastModified ?? unixTime(Date.now()),
       createdAt: createdAt ?? unixTime(Date.now()),
     };