diff --git a/.github/workflows/ci-host.yaml b/.github/workflows/ci-host.yaml index 8fa3dd48412..464d7573944 100644 --- a/.github/workflows/ci-host.yaml +++ b/.github/workflows/ci-host.yaml @@ -295,6 +295,9 @@ jobs: host-test: name: Host Tests runs-on: ubuntu-latest + # TEMP (CS-11450 diagnostic): cap the shard so a hang fails fast with + # retrievable logs instead of running to the 6h default. Revert before merge. + timeout-minutes: 35 needs: [test-web-assets, check-percy] strategy: fail-fast: false diff --git a/packages/base/card-api.gts b/packages/base/card-api.gts index 5ddca01807c..1faadb744f4 100644 --- a/packages/base/card-api.gts +++ b/packages/base/card-api.gts @@ -406,6 +406,9 @@ const stores = initSharedState( 'stores', () => new WeakMap(), ); +// TEMP (CS-11450 diagnostic): per-id deserialize-create counter to detect an +// identity-map miss storm. Revert. +const deserializeCreateCounts = new Map(); const subscribers = initSharedState( 'subscribers', () => new WeakMap>(), @@ -516,6 +519,14 @@ export interface CardStore { // undefined when the reference can't be resolved (no network available, or // an unresolvable reference) so callers can degrade to URL math. resolveURL(reference: string, base?: string): URL | undefined; + // Fold an id to its canonical RRI form: a mapped realm's URL collapses to + // its `@scope/name/...` prefix, an already-canonical RRI is returned + // unchanged, and anything with no registered mapping (an unmapped realm's + // URL, a local id) passes through as-is. The inverse of `resolveURL`'s + // boundary role — card code canonicalizes an incoming id to the opaque + // interior form without holding the network. Returns the input unchanged + // when no network is available. + canonicalizeId(id: string): string; getCard(url: string): CardDef | undefined; getFileMeta(url: string): FileDef | undefined; setCard(url: string, instance: CardDef): void; @@ -4135,7 +4146,14 @@ export async function updateFromSerialized( ): Promise> { stores.set(instance, store); if (!instance[relativeTo] && doc.data.id) { - instance[relativeTo] = rri(doc.data.id); + // Card ids fold to canonical RRI; FileDef ids stay URL (see + // `_createFromSerialized`). + let isFileLike = isFileDef( + Reflect.getPrototypeOf(instance)!.constructor as typeof BaseDef, + ); + instance[relativeTo] = rri( + isFileLike ? doc.data.id : store.canonicalizeId(doc.data.id), + ); } if (isCardInstance(instance)) { @@ -4182,23 +4200,45 @@ async function _createFromSerialized( if (!doc) { doc = { data: resource }; } + let isFileLike = isFileMetaResource(resource) || isFileDef(card); + // Fold the incoming id onto the canonical interior form (RRI for a mapped + // realm; unchanged for an unmapped realm or a local id) at this ingest + // boundary, so the in-memory `id` field, the identity-map key, and the + // relative-resolution base all share one opaque spelling. Scoped to card + // instances; FileDef ids stay in URL form (their identity is entangled with + // the file-extract invalidation contract). + let canonicalId = + resource.id != null && !isFileLike + ? (store.canonicalizeId(resource.id) as typeof resource.id) + : resource.id; let instance: BaseInstanceType | undefined; - if (resource.id != null || resource.lid != null) { - let resourceId = (resource.id ?? resource.lid)!; - let cachedInstance = - isFileMetaResource(resource) || isFileDef(card) - ? store.getFileMeta(resourceId) - : store.getCard(resourceId); + if (canonicalId != null || resource.lid != null) { + let resourceId = (canonicalId ?? resource.lid)!; + let cachedInstance = isFileLike + ? store.getFileMeta(resourceId) + : store.getCard(resourceId); if (cachedInstance && instanceOf(cachedInstance, card as any)) { instance = cachedInstance as BaseInstanceType; } } if (!instance) { instance = new card({ - id: resource.id, + id: canonicalId, [localId]: resource.lid, }) as BaseInstanceType; instance[relativeTo] = _relativeTo; + // TEMP (CS-11450 diagnostic): a re-creation storm (same id constructed + // many times) means the identity-map lookup above is missing — the + // signature of a store-key form divergence. Revert. + if (canonicalId != null) { + let n = (deserializeCreateCounts.get(canonicalId) ?? 0) + 1; + deserializeCreateCounts.set(canonicalId, n); + if (n === 25 || n === 100 || n % 500 === 0) { + console.log( + `[DESER-PROBE] created id=${JSON.stringify(canonicalId)} ${n} times (identity-map miss storm?)`, + ); + } + } } stores.set(instance, store); return await _updateFromSerialized({ @@ -4384,7 +4424,14 @@ async function _updateFromSerialized({ ...resource.attributes, ...nonNestedRelationships, ...linksToManyRelationships, - ...(resource.id !== undefined ? { id: resource.id } : {}), + ...(resource.id !== undefined + ? { + id: + isFileMetaResource(resource) || isFileDef(card) + ? resource.id + : store.canonicalizeId(resource.id), + } + : {}), }).map(async ([fieldName, value]) => { let field = getField(instance, fieldName); if (!field) { @@ -4923,6 +4970,16 @@ class FallbackCardStore implements CardStore { } } + canonicalizeId(id: string): string { + let vn: VirtualNetwork | undefined; + try { + vn = myLoader().getVirtualNetwork(); + } catch { + return id; + } + return vn ? vn.unresolveURL(id) : id; + } + getCard(id: string) { id = id.replace(/\.json$/, ''); return this.#instances.get(id); diff --git a/packages/host/app/lib/gc-card-store.ts b/packages/host/app/lib/gc-card-store.ts index 11224c31361..f19102676fc 100644 --- a/packages/host/app/lib/gc-card-store.ts +++ b/packages/host/app/lib/gc-card-store.ts @@ -324,6 +324,10 @@ export default class CardStoreWithGarbageCollection implements CardStore { } } + canonicalizeId(id: string): string { + return this.#virtualNetwork.unresolveURL(id); + } + getCard(id: string): CardDef | undefined { return this.getCardItem('instance', id) as CardDef | undefined; } @@ -698,6 +702,10 @@ export default class CardStoreWithGarbageCollection implements CardStore { this.deleteFileMeta(id); return; } + // Card buckets are keyed by canonical RRI (see getCardItem / setCardItem); + // fold the incoming id so a delete by the resolved URL (store.asURL) clears + // the RRI-keyed identity. No-op for a local id or an unmapped realm. + id = this.#virtualNetwork.unresolveURL(id); let localId = isLocalId(id) ? id : undefined; let remoteId = !isLocalId(id) ? id : undefined; @@ -894,9 +902,12 @@ export default class CardStoreWithGarbageCollection implements CardStore { } makeTracked(remoteId: string) { - // File-meta is keyed by the full URL; card buckets by the stripped id. + // File-meta is keyed by the full URL; card buckets by the stripped, + // canonical-RRI id (see getCardItem / setCardItem). let fileMetaId = remoteId; - remoteId = remoteId.replace(/\.json$/, ''); + remoteId = this.#virtualNetwork.unresolveURL( + remoteId.replace(/\.json$/, ''), + ); let instance = this.#nonTrackedCardInstances.get(remoteId); if (instance) { this.setCardItem(remoteId, instance); @@ -971,6 +982,11 @@ export default class CardStoreWithGarbageCollection implements CardStore { id: string, ): CardDef | CardErrorJSONAPI | undefined { id = id.replace(/\.json$/, ''); + // Key card instances by canonical RRI so a lookup lands regardless of the + // caller's spelling — a card's own `id` is canonical, while `store.asURL` + // hands us the resolved URL. `unresolveURL` is a no-op for a local id or an + // unmapped realm. File-meta buckets keep URL keys (handled separately). + id = this.#virtualNetwork.unresolveURL(id); let { item, localId } = this.tryFindingCardItem(type, id); if (!item && isLocalId(id)) { @@ -1068,6 +1084,8 @@ export default class CardStoreWithGarbageCollection implements CardStore { notTracked?: true, ) { id = id.replace(/\.json$/, ''); + // Match the canonical-RRI keying used by getCardItem so set/get agree. + id = this.#virtualNetwork.unresolveURL(id); let cardBucket = notTracked ? this.#nonTrackedCardInstances : this.#cardInstances; diff --git a/packages/host/app/services/render-service.ts b/packages/host/app/services/render-service.ts index b6b5975396c..71ba7c8a614 100644 --- a/packages/host/app/services/render-service.ts +++ b/packages/host/app/services/render-service.ts @@ -75,6 +75,10 @@ export class CardStoreWithErrors implements CardStore { } } + canonicalizeId(id: string): string { + return this.#virtualNetwork.unresolveURL(id); + } + getCard(id: string): CardDef | undefined { id = this.normalizeKey(id); return this.#cards.get(id); diff --git a/packages/host/app/services/store.ts b/packages/host/app/services/store.ts index 8eebb04edfe..938ba9208f7 100644 --- a/packages/host/app/services/store.ts +++ b/packages/host/app/services/store.ts @@ -1612,6 +1612,46 @@ export default class StoreService extends Service implements StoreInterface { return a === b || this.peek(a) === this.peek(b); } + // TEMPORARY hang tripwire (CS-11450 instance-id canonicalization): converts a + // silent store-key-divergence hang into a fast, self-describing failure so CI + // pinpoints the divergent key instead of timing out the shard after ~75min. + // Remove once the keying is confirmed consistent. + async #awaitWithTripwire( + promise: Promise, + label: string, + key: string, + ): Promise { + let settled = false; + promise.then( + () => (settled = true), + () => (settled = true), + ); + let timer: ReturnType | undefined; + let tripwire = new Promise((_resolve, reject) => { + timer = setTimeout(() => { + if (settled) { + return; + } + let diag = + `[HANG-TRIPWIRE] ${label} unsettled after 90s. ` + + `awaited key=${JSON.stringify(key)} ` + + `asURL(key)=${JSON.stringify(asURL(key, this.network.virtualNetwork))} ` + + `inflightGetCards=${JSON.stringify([...this.inflightGetCards.keys()])} ` + + `inflightCardLoads=${JSON.stringify([...this.inflightCardLoads.keys()])} ` + + `referenceCount=${JSON.stringify([...this.referenceCount.keys()])}`; + console.error(diag); + reject(new Error(diag)); + }, 90_000); + }); + try { + return await Promise.race([promise, tripwire]); + } finally { + if (timer) { + clearTimeout(timer); + } + } + } + async waitForCardLoad(cardId: string): Promise { let normalizedId = asURL(cardId, this.network.virtualNetwork); if (!normalizedId) { @@ -1619,7 +1659,11 @@ export default class StoreService extends Service implements StoreInterface { } let inflightLoad = this.inflightCardLoads.get(normalizedId); if (inflightLoad) { - await inflightLoad.promise; + await this.#awaitWithTripwire( + inflightLoad.promise, + 'waitForCardLoad', + normalizedId, + ); } } @@ -1688,9 +1732,13 @@ export default class StoreService extends Service implements StoreInterface { } let instanceOrError = this.peekError(url) ?? this.peek(url); if (!instanceOrError) { - instanceOrError = await this.getCardInstance({ - idOrDoc: url, - }); + instanceOrError = await this.#awaitWithTripwire( + this.getCardInstance({ + idOrDoc: url, + }), + 'wireUpNewReference:getCardInstance', + url, + ); this.setIdentityContext(instanceOrError); } await this.startAutoSaving(instanceOrError); diff --git a/packages/host/tests/integration/field-configuration-test.gts b/packages/host/tests/integration/field-configuration-test.gts index 858423deb10..d3a856de59f 100644 --- a/packages/host/tests/integration/field-configuration-test.gts +++ b/packages/host/tests/integration/field-configuration-test.gts @@ -52,6 +52,9 @@ class DeferredLinkStore implements CardStore { return undefined; } } + canonicalizeId(id: string): string { + return this.#virtualNetwork.unresolveURL(id); + } private cardInstances = new Map(); private fileMetaInstances = new Map(); private readyCardDocs = new Map(); diff --git a/packages/runtime-common/realm-index-query-engine.ts b/packages/runtime-common/realm-index-query-engine.ts index d530a36a401..bd5f833b1ac 100644 --- a/packages/runtime-common/realm-index-query-engine.ts +++ b/packages/runtime-common/realm-index-query-engine.ts @@ -1649,9 +1649,13 @@ export class RealmIndexQueryEngine { relationshipType === CardResourceType && !expectsFileMeta; let resolvedSelf: string; try { + // Resolve the base to a real URL first: `resource.id` may be a + // canonical RRI (mapped realm), which `resolveURL` only accepts as + // a base for a registered prefix — `toURL` yields the fetchable URL + // either form. Mirrors the `linkURL` base above. resolvedSelf = vn.resolveURL( relationship.links.self, - resource.id, + resource.id ? vn.toURL(resource.id) : realmURL, ).href; } catch { throw new Error( diff --git a/packages/runtime-common/virtual-network.ts b/packages/runtime-common/virtual-network.ts index ef4d8130f99..575b9ff67ad 100644 --- a/packages/runtime-common/virtual-network.ts +++ b/packages/runtime-common/virtual-network.ts @@ -34,6 +34,13 @@ export class VirtualNetwork { // a pure function of the realm mappings, so entries stay valid until a // mapping is added or removed (both clear the cache). private toURLHrefCache = new Map(); + // Memo for unresolveURL, the inverse of toURLHref. It runs on the store's + // hottest paths — every card deserialize and every gc-card-store keying / + // GC-sweep access folds its id through here — and each miss pays a native + // `new URL()` in the virtual→real mapping chase. Like toURLHrefCache, this is + // a pure function of the realm mappings, so entries stay valid until a + // mapping is added or removed (both clear it). + private unresolveURLCache = new Map(); // Notified whenever a realm-prefix mapping changes — added, removed, or // re-registered against a new target. Consumers that key caches by the RRI @@ -83,6 +90,10 @@ export class VirtualNetwork { addURLMapping(from: URL, to: URL) { this.urlMappings.push([from.href, to.href]); + // Both memos resolve through urlMappings (toURLHref via toURL, unresolveURL + // via its virtual→real chase), so a new URL mapping invalidates them. + this.toURLHrefCache.clear(); + this.unresolveURLCache.clear(); } mapURL( @@ -113,6 +124,7 @@ export class VirtualNetwork { let normalizedTarget = ensureTrailingSlash(targetURL); this.realmMappings.set(normalizedId, normalizedTarget); this.toURLHrefCache.clear(); + this.unresolveURLCache.clear(); this.addImportMap( normalizedId, (rest) => new URL(rest, normalizedTarget).href, @@ -131,6 +143,7 @@ export class VirtualNetwork { this.realmMappings.delete(normalizedId); this.importMap.delete(normalizedId); this.toURLHrefCache.clear(); + this.unresolveURLCache.clear(); this.notifyMappingChange(); } @@ -166,6 +179,22 @@ export class VirtualNetwork { * Inputs that match no prefix and no URL mapping are returned as-is. */ unresolveURL(url: string): RealmResourceIdentifier { + let cached = this.unresolveURLCache.get(url); + if (cached !== undefined) { + return cached; + } + let result = this.computeUnresolveURL(url); + this.unresolveURLCache.set(url, result); + // TEMP (CS-11450 diagnostic): surface monotonic cache growth. Revert. + if (this.unresolveURLCache.size % 2000 === 0) { + console.log( + `[CACHE-PROBE] unresolveURLCache.size=${this.unresolveURLCache.size} toURLHrefCache.size=${this.toURLHrefCache.size}`, + ); + } + return result; + } + + private computeUnresolveURL(url: string): RealmResourceIdentifier { for (let [prefix, target] of this.realmMappings) { if (url.startsWith(target)) { return (prefix + url.slice(target.length)) as RealmResourceIdentifier;