Add handling for RRIs in serialised forms

.github/workflows/ci-host.yaml

@@ -295,6 +295,9 @@ jobs:
   host-test:
     name: Host Tests
     runs-on: ubuntu-latest
+    # TEMP (CS-11450 diagnostic): cap the shard so a hang fails fast with
+    # retrievable logs instead of running to the 6h default. Revert before merge.
+    timeout-minutes: 35
     needs: [test-web-assets, check-percy]
     strategy:
       fail-fast: false

packages/base/card-api.gts

@@ -406,6 +406,9 @@ const stores = initSharedState(
   'stores',
   () => new WeakMap<BaseDef, CardStore>(),
 );
+// TEMP (CS-11450 diagnostic): per-id deserialize-create counter to detect an
+// identity-map miss storm. Revert.
+const deserializeCreateCounts = new Map<string, number>();
 const subscribers = initSharedState(
   'subscribers',
   () => new WeakMap<BaseDef, Set<CardChangeSubscriber>>(),
@@ -516,6 +519,14 @@ export interface CardStore {
   // undefined when the reference can't be resolved (no network available, or
   // an unresolvable reference) so callers can degrade to URL math.
   resolveURL(reference: string, base?: string): URL | undefined;
+  // Fold an id to its canonical RRI form: a mapped realm's URL collapses to
+  // its `@scope/name/...` prefix, an already-canonical RRI is returned
+  // unchanged, and anything with no registered mapping (an unmapped realm's
+  // URL, a local id) passes through as-is. The inverse of `resolveURL`'s
+  // boundary role — card code canonicalizes an incoming id to the opaque
+  // interior form without holding the network. Returns the input unchanged
+  // when no network is available.
+  canonicalizeId(id: string): string;
   getCard(url: string): CardDef | undefined;
   getFileMeta(url: string): FileDef | undefined;
   setCard(url: string, instance: CardDef): void;
@@ -4135,7 +4146,14 @@ export async function updateFromSerialized<T extends BaseDefConstructor>(
 ): Promise<BaseInstanceType<T>> {
   stores.set(instance, store);
   if (!instance[relativeTo] && doc.data.id) {
-    instance[relativeTo] = rri(doc.data.id);
+    // Card ids fold to canonical RRI; FileDef ids stay URL (see
+    // `_createFromSerialized`).
+    let isFileLike = isFileDef(
+      Reflect.getPrototypeOf(instance)!.constructor as typeof BaseDef,
+    );
+    instance[relativeTo] = rri(
+      isFileLike ? doc.data.id : store.canonicalizeId(doc.data.id),
+    );
   }
 
   if (isCardInstance(instance)) {
@@ -4182,23 +4200,45 @@ async function _createFromSerialized<T extends BaseDefConstructor>(
   if (!doc) {
     doc = { data: resource };
   }
+  let isFileLike = isFileMetaResource(resource) || isFileDef(card);
+  // Fold the incoming id onto the canonical interior form (RRI for a mapped
+  // realm; unchanged for an unmapped realm or a local id) at this ingest
+  // boundary, so the in-memory `id` field, the identity-map key, and the
+  // relative-resolution base all share one opaque spelling. Scoped to card
+  // instances; FileDef ids stay in URL form (their identity is entangled with
+  // the file-extract invalidation contract).
+  let canonicalId =
+    resource.id != null && !isFileLike
+      ? (store.canonicalizeId(resource.id) as typeof resource.id)
+      : resource.id;
   let instance: BaseInstanceType<T> | undefined;
-  if (resource.id != null || resource.lid != null) {
-    let resourceId = (resource.id ?? resource.lid)!;
-    let cachedInstance =
-      isFileMetaResource(resource) || isFileDef(card)
-        ? store.getFileMeta(resourceId)
-        : store.getCard(resourceId);
+  if (canonicalId != null || resource.lid != null) {
+    let resourceId = (canonicalId ?? resource.lid)!;
+    let cachedInstance = isFileLike
+      ? store.getFileMeta(resourceId)
+      : store.getCard(resourceId);
     if (cachedInstance && instanceOf(cachedInstance, card as any)) {
       instance = cachedInstance as BaseInstanceType<T>;
     }
   }
   if (!instance) {
     instance = new card({
-      id: resource.id,
+      id: canonicalId,
       [localId]: resource.lid,
     }) as BaseInstanceType<T>;
     instance[relativeTo] = _relativeTo;
+    // TEMP (CS-11450 diagnostic): a re-creation storm (same id constructed
+    // many times) means the identity-map lookup above is missing — the
+    // signature of a store-key form divergence. Revert.
+    if (canonicalId != null) {
+      let n = (deserializeCreateCounts.get(canonicalId) ?? 0) + 1;
+      deserializeCreateCounts.set(canonicalId, n);
+      if (n === 25 || n === 100 || n % 500 === 0) {
+        console.log(
+          `[DESER-PROBE] created id=${JSON.stringify(canonicalId)} ${n} times (identity-map miss storm?)`,
+        );
+      }
+    }
   }
   stores.set(instance, store);
   return await _updateFromSerialized({
@@ -4384,7 +4424,14 @@ async function _updateFromSerialized<T extends BaseDefConstructor>({
       ...resource.attributes,
       ...nonNestedRelationships,
       ...linksToManyRelationships,
-      ...(resource.id !== undefined ? { id: resource.id } : {}),
+      ...(resource.id !== undefined
+        ? {
+            id:
+              isFileMetaResource(resource) || isFileDef(card)
+                ? resource.id
+                : store.canonicalizeId(resource.id),
+          }
+        : {}),
     }).map(async ([fieldName, value]) => {
       let field = getField(instance, fieldName);
       if (!field) {
@@ -4923,6 +4970,16 @@ class FallbackCardStore implements CardStore {
     }
   }
 
+  canonicalizeId(id: string): string {
+    let vn: VirtualNetwork | undefined;
+    try {
+      vn = myLoader().getVirtualNetwork();
+    } catch {
+      return id;
+    }
+    return vn ? vn.unresolveURL(id) : id;
+  }
+
   getCard(id: string) {
     id = id.replace(/\.json$/, '');
     return this.#instances.get(id);

packages/host/app/lib/gc-card-store.ts

@@ -324,6 +324,10 @@ export default class CardStoreWithGarbageCollection implements CardStore {
     }
   }
 
+  canonicalizeId(id: string): string {
+    return this.#virtualNetwork.unresolveURL(id);
+  }
+
   getCard(id: string): CardDef | undefined {
     return this.getCardItem('instance', id) as CardDef | undefined;
   }
@@ -698,6 +702,10 @@ export default class CardStoreWithGarbageCollection implements CardStore {
       this.deleteFileMeta(id);
       return;
     }
+    // Card buckets are keyed by canonical RRI (see getCardItem / setCardItem);
+    // fold the incoming id so a delete by the resolved URL (store.asURL) clears
+    // the RRI-keyed identity. No-op for a local id or an unmapped realm.
+    id = this.#virtualNetwork.unresolveURL(id);
     let localId = isLocalId(id) ? id : undefined;
     let remoteId = !isLocalId(id) ? id : undefined;
 
@@ -894,9 +902,12 @@ export default class CardStoreWithGarbageCollection implements CardStore {
   }
 
   makeTracked(remoteId: string) {
-    // File-meta is keyed by the full URL; card buckets by the stripped id.
+    // File-meta is keyed by the full URL; card buckets by the stripped,
+    // canonical-RRI id (see getCardItem / setCardItem).
     let fileMetaId = remoteId;
-    remoteId = remoteId.replace(/\.json$/, '');
+    remoteId = this.#virtualNetwork.unresolveURL(
+      remoteId.replace(/\.json$/, ''),
+    );
     let instance = this.#nonTrackedCardInstances.get(remoteId);
     if (instance) {
       this.setCardItem(remoteId, instance);
@@ -971,6 +982,11 @@ export default class CardStoreWithGarbageCollection implements CardStore {
     id: string,
   ): CardDef | CardErrorJSONAPI | undefined {
     id = id.replace(/\.json$/, '');
+    // Key card instances by canonical RRI so a lookup lands regardless of the
+    // caller's spelling — a card's own `id` is canonical, while `store.asURL`
+    // hands us the resolved URL. `unresolveURL` is a no-op for a local id or an
+    // unmapped realm. File-meta buckets keep URL keys (handled separately).
+    id = this.#virtualNetwork.unresolveURL(id);
     let { item, localId } = this.tryFindingCardItem(type, id);
 
     if (!item && isLocalId(id)) {
@@ -1068,6 +1084,8 @@ export default class CardStoreWithGarbageCollection implements CardStore {
     notTracked?: true,
   ) {
     id = id.replace(/\.json$/, '');
+    // Match the canonical-RRI keying used by getCardItem so set/get agree.
+    id = this.#virtualNetwork.unresolveURL(id);
     let cardBucket = notTracked
       ? this.#nonTrackedCardInstances
       : this.#cardInstances;

packages/host/app/services/render-service.ts

@@ -75,6 +75,10 @@ export class CardStoreWithErrors implements CardStore {
     }
   }
 
+  canonicalizeId(id: string): string {
+    return this.#virtualNetwork.unresolveURL(id);
+  }
+
   getCard(id: string): CardDef | undefined {
     id = this.normalizeKey(id);
     return this.#cards.get(id);

packages/host/app/services/store.ts

@@ -1612,14 +1612,58 @@ export default class StoreService extends Service implements StoreInterface {
     return a === b || this.peek(a) === this.peek(b);
   }
 
+  // TEMPORARY hang tripwire (CS-11450 instance-id canonicalization): converts a
+  // silent store-key-divergence hang into a fast, self-describing failure so CI
+  // pinpoints the divergent key instead of timing out the shard after ~75min.
+  // Remove once the keying is confirmed consistent.
+  async #awaitWithTripwire<T>(
+    promise: Promise<T>,
+    label: string,
+    key: string,
+  ): Promise<T> {
+    let settled = false;
+    promise.then(
+      () => (settled = true),
+      () => (settled = true),
+    );
+    let timer: ReturnType<typeof setTimeout> | undefined;
+    let tripwire = new Promise<never>((_resolve, reject) => {
+      timer = setTimeout(() => {
+        if (settled) {
+          return;
+        }
+        let diag =
+          `[HANG-TRIPWIRE] ${label} unsettled after 90s. ` +
+          `awaited key=${JSON.stringify(key)} ` +
+          `asURL(key)=${JSON.stringify(asURL(key, this.network.virtualNetwork))} ` +
+          `inflightGetCards=${JSON.stringify([...this.inflightGetCards.keys()])} ` +
+          `inflightCardLoads=${JSON.stringify([...this.inflightCardLoads.keys()])} ` +
+          `referenceCount=${JSON.stringify([...this.referenceCount.keys()])}`;
+        console.error(diag);
+        reject(new Error(diag));
+      }, 90_000);
+    });
+    try {
+      return await Promise.race([promise, tripwire]);
+    } finally {
+      if (timer) {
+        clearTimeout(timer);
+      }
+    }
+  }
+
   async waitForCardLoad(cardId: string): Promise<void> {
     let normalizedId = asURL(cardId, this.network.virtualNetwork);
     if (!normalizedId) {
       return;
     }
     let inflightLoad = this.inflightCardLoads.get(normalizedId);
     if (inflightLoad) {
-      await inflightLoad.promise;
+      await this.#awaitWithTripwire(
+        inflightLoad.promise,
+        'waitForCardLoad',
+        normalizedId,
+      );
     }
   }
 
@@ -1688,9 +1732,13 @@ export default class StoreService extends Service implements StoreInterface {
         }
         let instanceOrError = this.peekError(url) ?? this.peek(url);
         if (!instanceOrError) {
-          instanceOrError = await this.getCardInstance({
-            idOrDoc: url,
-          });
+          instanceOrError = await this.#awaitWithTripwire(
+            this.getCardInstance({
+              idOrDoc: url,
+            }),
+            'wireUpNewReference:getCardInstance',
+            url,
+          );
           this.setIdentityContext(instanceOrError);
         }
         await this.startAutoSaving(instanceOrError);

packages/host/tests/integration/field-configuration-test.gts

@@ -52,6 +52,9 @@ class DeferredLinkStore implements CardStore {
       return undefined;
     }
   }
+  canonicalizeId(id: string): string {
+    return this.#virtualNetwork.unresolveURL(id);
+  }
   private cardInstances = new Map<string, CardDefType>();
   private fileMetaInstances = new Map<string, FileDef>();
   private readyCardDocs = new Map<string, SingleCardDocument>();

packages/runtime-common/realm-index-query-engine.ts

@@ -1649,9 +1649,13 @@ export class RealmIndexQueryEngine {
             relationshipType === CardResourceType && !expectsFileMeta;
           let resolvedSelf: string;
           try {
+            // Resolve the base to a real URL first: `resource.id` may be a
+            // canonical RRI (mapped realm), which `resolveURL` only accepts as
+            // a base for a registered prefix — `toURL` yields the fetchable URL
+            // either form. Mirrors the `linkURL` base above.
             resolvedSelf = vn.resolveURL(
               relationship.links.self,
-              resource.id,
+              resource.id ? vn.toURL(resource.id) : realmURL,
             ).href;
           } catch {
             throw new Error(

packages/runtime-common/virtual-network.ts

@@ -34,6 +34,13 @@ export class VirtualNetwork {
   // a pure function of the realm mappings, so entries stay valid until a
   // mapping is added or removed (both clear the cache).
   private toURLHrefCache = new Map<string, string>();
+  // Memo for unresolveURL, the inverse of toURLHref. It runs on the store's
+  // hottest paths — every card deserialize and every gc-card-store keying /
+  // GC-sweep access folds its id through here — and each miss pays a native
+  // `new URL()` in the virtual→real mapping chase. Like toURLHrefCache, this is
+  // a pure function of the realm mappings, so entries stay valid until a
+  // mapping is added or removed (both clear it).
+  private unresolveURLCache = new Map<string, RealmResourceIdentifier>();
 
   // Notified whenever a realm-prefix mapping changes — added, removed, or
   // re-registered against a new target. Consumers that key caches by the RRI
@@ -83,6 +90,10 @@ export class VirtualNetwork {
 
   addURLMapping(from: URL, to: URL) {
     this.urlMappings.push([from.href, to.href]);
+    // Both memos resolve through urlMappings (toURLHref via toURL, unresolveURL
+    // via its virtual→real chase), so a new URL mapping invalidates them.
+    this.toURLHrefCache.clear();
+    this.unresolveURLCache.clear();
   }
 
   mapURL(
@@ -113,6 +124,7 @@ export class VirtualNetwork {
     let normalizedTarget = ensureTrailingSlash(targetURL);
     this.realmMappings.set(normalizedId, normalizedTarget);
     this.toURLHrefCache.clear();
+    this.unresolveURLCache.clear();
     this.addImportMap(
       normalizedId,
       (rest) => new URL(rest, normalizedTarget).href,
@@ -131,6 +143,7 @@ export class VirtualNetwork {
     this.realmMappings.delete(normalizedId);
     this.importMap.delete(normalizedId);
     this.toURLHrefCache.clear();
+    this.unresolveURLCache.clear();
     this.notifyMappingChange();
   }
 
@@ -166,6 +179,22 @@ export class VirtualNetwork {
    * Inputs that match no prefix and no URL mapping are returned as-is.
    */
   unresolveURL(url: string): RealmResourceIdentifier {
+    let cached = this.unresolveURLCache.get(url);
+    if (cached !== undefined) {
+      return cached;
+    }
+    let result = this.computeUnresolveURL(url);
+    this.unresolveURLCache.set(url, result);
+    // TEMP (CS-11450 diagnostic): surface monotonic cache growth. Revert.
+    if (this.unresolveURLCache.size % 2000 === 0) {
+      console.log(
+        `[CACHE-PROBE] unresolveURLCache.size=${this.unresolveURLCache.size} toURLHrefCache.size=${this.toURLHrefCache.size}`,
+      );
+    }
+    return result;
+  }
+
+  private computeUnresolveURL(url: string): RealmResourceIdentifier {
     for (let [prefix, target] of this.realmMappings) {
       if (url.startsWith(target)) {
         return (prefix + url.slice(target.length)) as RealmResourceIdentifier;