diff --git a/.claude/settings.json b/.claude/settings.json index bd09a945ce..45ecb0c64e 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -12,12 +12,23 @@ "Bash(node --check *)", "Bash(npx vitest run *)", "Bash(pnpm exec prettier *)", - "mcp__linear__list_comments", - "mcp__linear__get_issue", - "mcp__linear__get_project", - "mcp__linear__get_user", - "mcp__linear__list_projects", - "mcp__linear__list_issue_statuses", + "mcp__Linear__get_issue", + "mcp__Linear__list_comments", + "mcp__Linear__get_project", + "mcp__Linear__get_user", + "mcp__Linear__list_projects", + "mcp__Linear__list_issue_statuses", + "mcp__Linear__list_issues", + "mcp__github__pull_request_read", + "mcp__github__list_pull_requests", + "mcp__github__get_file_contents", + "mcp__github__search_code", + "mcp__github__list_commits", + "mcp__github__get_commit", + "mcp__github__get_me", + "mcp__github__actions_list", + "mcp__github__actions_get", + "mcp__github__get_job_logs", "mcp__chrome-devtools__take_screenshot" ] }, diff --git a/packages/host/tests/unit/index-writer-test.ts b/packages/host/tests/unit/index-writer-test.ts index 28392dbc39..9ce0299252 100644 --- a/packages/host/tests/unit/index-writer-test.ts +++ b/packages/host/tests/unit/index-writer-test.ts @@ -435,6 +435,66 @@ module('Unit | index-writer', function (hooks) { ]); }); + test('invalidations reach dependents whose rows exist only in the production index', async function (assert) { + // A production row without a working counterpart is a reachable state: + // the working table is a staging area that only gains a row once a visit + // writes one in this deployment's lifetime — an index imported from a + // dump that carries only the production tables, or rows staged under a + // since-changed schema. The dependency fan-out must find such dependents + // or a module edit leaves their indexed computed values stale until each + // file is touched individually. + await setupIndex( + adapter, + [{ realm_url: testRealmURL, current_generation: 1 }], + { + working: [], + production: [ + { + url: `${testRealmURL}1.json`, + file_alias: `${testRealmURL}1.json`, + type: 'instance', + generation: 1, + realm_url: testRealmURL, + deps: [`${testRealmURL}person`], + }, + // Transitive: discoverable only through 1.json's production row. + { + url: `${testRealmURL}2.json`, + file_alias: `${testRealmURL}2.json`, + type: 'instance', + generation: 1, + realm_url: testRealmURL, + deps: [`${testRealmURL}1.json`], + }, + // A promoted tombstone is a deleted file — it stays out of the + // fan-out. + { + url: `${testRealmURL}3.json`, + file_alias: `${testRealmURL}3.json`, + type: 'instance', + generation: 1, + realm_url: testRealmURL, + is_deleted: true, + deps: [`${testRealmURL}person`], + }, + ], + }, + { prerenderedHtml: false }, + ); + + let batch = await indexWriter.createBatch( + new URL(testRealmURL), + virtualNetwork, + ); + await batch.invalidate([new URL(`${testRealmURL}person.gts`)]); + + assert.deepEqual(batch.invalidations.sort(), [ + `${testRealmURL}1.json`, + `${testRealmURL}2.json`, + `${testRealmURL}person.gts`, + ]); + }); + test("invalidations don't cross realm boundaries", async function (assert) { await setupIndex( adapter, diff --git a/packages/realm-server/tests/index.ts b/packages/realm-server/tests/index.ts index c8ec404bb0..61b13d5d63 100644 --- a/packages/realm-server/tests/index.ts +++ b/packages/realm-server/tests/index.ts @@ -111,6 +111,11 @@ if (testModules) { } } +const qunitFilter = process.env.QUNIT_FILTER?.trim(); +if (qunitFilter) { + QUnit.config.filter = qunitFilter; +} + // Cleanup here ensures lingering servers/prerenderers/queues don't keep the // Node event loop alive after tests finish — and equivalently, don't leave // hardcoded test ports (4444-4471, etc.) bound after a test is aborted by diff --git a/packages/realm-server/tests/indexing-test.ts b/packages/realm-server/tests/indexing-test.ts index 18035ce9c6..567dc5efd3 100644 --- a/packages/realm-server/tests/indexing-test.ts +++ b/packages/realm-server/tests/indexing-test.ts @@ -44,6 +44,7 @@ import { } from './helpers/indexing.ts'; import stripScopedCSSAttributes from '@cardstack/runtime-common/helpers/strip-scoped-css-attributes'; import { basename } from 'path'; +import { createHash } from 'crypto'; import type { PgAdapter } from '@cardstack/postgres'; function trimCardContainer(text: string) { @@ -4489,6 +4490,37 @@ module(basename(import.meta.filename), function () { ); }); + test('file meta serves write-time contentHash/contentSize over indexed values', async function (assert) { + let content = 'bytes behind the file meta doc'; + await realm.write('meta-note.txt', content); + + // Emulate the state a mid-batch render observes: the production + // index row still carries the previous bytes' content values while + // `realm_file_meta` (written in the same critical section as the + // bytes) already has the current ones. The pristine resource is + // dropped so the served doc must choose between the indexed search + // doc and the persisted meta. + await testDbAdapter.execute( + `UPDATE boxel_index SET search_doc = COALESCE(search_doc, '{}'::jsonb) || '{"contentHash":"stale-hash","contentSize":1}'::jsonb, pristine_doc = NULL WHERE url = '${testRealm}meta-note.txt' AND type = 'file'`, + ); + + let response = await fetch(`${testRealm}meta-note.txt`, { + headers: { Accept: SupportedMimeType.FileMeta }, + }); + assert.strictEqual(response.status, 200, 'file meta response is ok'); + let doc = (await response.json()) as LooseSingleCardDocument; + assert.strictEqual( + doc.data.attributes?.contentHash, + createHash('md5').update(content).digest('hex'), + 'contentHash reflects the bytes on disk, not the indexed value', + ); + assert.strictEqual( + doc.data.attributes?.contentSize, + content.length, + 'contentSize reflects the bytes on disk, not the indexed value', + ); + }); + test('can incrementally index deleted instance', async function (assert) { await realm.delete('mango.json'); diff --git a/packages/runtime-common/index-writer.ts b/packages/runtime-common/index-writer.ts index 00ba3da02e..dae6c46b5b 100644 --- a/packages/runtime-common/index-writer.ts +++ b/packages/runtime-common/index-writer.ts @@ -2420,7 +2420,7 @@ export class Batch { unresolvedPath !== resolvedPath && this.isRegisteredPrefix(unresolvedPath); let scanTable = async ( - tableName: 'boxel_index_working' | 'prerendered_html', + tableName: 'boxel_index_working' | 'prerendered_html' | 'boxel_index', ) => { let results: (Pick & { type: BoxelIndexTable['type']; @@ -2476,6 +2476,19 @@ export class Batch { // probably need to reevaluate this condition when we get to cross // realm invalidation [`i.realm_url =`, param(this.realmURL.href)], + // A promoted tombstone is a deleted file — it has nothing to + // re-render, and pulling it into the fan-out would visit a path + // with no backing file. The working/prerendered scans carry + // tombstones from in-flight batches whose skip logic handles + // them, so only the production scan filters. + ...(tableName === 'boxel_index' + ? [ + any([ + ['i.is_deleted = false'], + ['i.is_deleted IS NULL'], + ]) as Expression, + ] + : []), ]), `LIMIT ${pageSize} OFFSET ${pageNumber * pageSize}`, ] as Expression)) as (Pick & { @@ -2492,15 +2505,30 @@ export class Batch { // — a rendered non-searchable link, the scoped-CSS artifacts of linked // instances. Both edge sets must feed the fan-out or a change to a // render-only dependency would never re-render its consumers. - let [workingScan, prerenderedScan] = await Promise.all([ + // + // Production `boxel_index` is scanned as well because the working table + // is a staging area whose coverage is not guaranteed: a row only exists + // there once a visit has written it in this deployment's lifetime, so + // promoted rows can lack a working counterpart (an index imported from a + // dump that carries only the production tables, or rows staged under a + // since-changed schema). Without the production scan, a module edit + // never reaches such dependents — their indexed computed values sit + // stale until each file is touched individually. Rows present in both + // tables collapse in the (url, type) dedup below. + let [workingScan, prerenderedScan, productionScan] = await Promise.all([ scanTable('boxel_index_working'), scanTable('prerendered_html'), + scanTable('boxel_index'), ]); let seen = new Set(); let results: (Pick & { type: BoxelIndexTable['type']; })[] = []; - for (let row of [...workingScan.results, ...prerenderedScan.results]) { + for (let row of [ + ...workingScan.results, + ...prerenderedScan.results, + ...productionScan.results, + ]) { let key = `${row.url}|${row.type}`; if (seen.has(key)) { continue; @@ -2511,7 +2539,11 @@ export class Batch { this.#perfLog.debug( `${jobIdentity(this.jobInfo)} time to determine items that reference ${resolvedPath} ${ Date.now() - start - } ms (page count=${workingScan.pageNumber + prerenderedScan.pageNumber})`, + } ms (page count=${ + workingScan.pageNumber + + prerenderedScan.pageNumber + + productionScan.pageNumber + })`, ); return results.map(({ url, file_alias, type }) => ({ url, diff --git a/packages/runtime-common/realm.ts b/packages/runtime-common/realm.ts index 913f6e4979..c7c1d444b7 100644 --- a/packages/runtime-common/realm.ts +++ b/packages/runtime-common/realm.ts @@ -4479,13 +4479,18 @@ export class Realm { typeof searchDoc.contentSize === 'number' ? searchDoc.contentSize : undefined; - // Only hit the DB when the indexed searchDoc is missing a value. - let persistedMeta = - (searchHash === undefined || searchSize === undefined) && this.#dbAdapter - ? await getContentMeta(this.#dbAdapter, this.url, localPath) - : { contentHash: undefined, contentSize: undefined }; - let contentHash = searchHash ?? persistedMeta.contentHash; - let contentSize = searchSize ?? persistedMeta.contentSize; + // `realm_file_meta` is written in the same critical section as the file's + // bytes, so it is authoritative for content-derived values. It is + // preferred over the indexed values, which lag one batch promotion behind: + // a render inside the batch that re-indexes this file reads the pre-swap + // production row, so a card linking the file sees the file's index row at + // its previous contentHash/contentSize. The indexed values remain as + // fallbacks for files whose bytes were never hashed at write time. + let persistedMeta = this.#dbAdapter + ? await getContentMeta(this.#dbAdapter, this.url, localPath) + : { contentHash: undefined, contentSize: undefined }; + let contentHash = persistedMeta.contentHash ?? searchHash; + let contentSize = persistedMeta.contentSize ?? searchSize; let adoptsFrom = codeRefFromInternalKey(fileEntry.types?.[0]) ?? (isCodeRef(fileEntry.resource?.meta?.adoptsFrom) @@ -4501,8 +4506,11 @@ export class Realm { resourceAttributes.contentType ?? searchDoc.contentType ?? inferredContentType, - contentHash: resourceAttributes.contentHash ?? contentHash, - contentSize: resourceAttributes.contentSize ?? contentSize, + // The persisted write-time values win over the indexed resource's for + // the same staleness reason as above; the resource's extract-computed + // values cover files that predate write-time hashing. + contentHash: contentHash ?? resourceAttributes.contentHash, + contentSize: contentSize ?? resourceAttributes.contentSize, lastModified: fileEntry.lastModified ?? unixTime(Date.now()), createdAt: createdAt ?? unixTime(Date.now()), };