From d54c8e10283cfc68150d1ea6b9f8c6326e06126f Mon Sep 17 00:00:00 2001 From: Yusuke Hirao Date: Tue, 14 Jul 2026 01:35:20 +0900 Subject: [PATCH] feat(page-cluster)!: add streaming API for corpora that overflow memory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BREAKING CHANGE: `resolvePageClusterKeys` is now async and takes a factory function `() => Iterable | AsyncIterable`. Callers with a materialized array can use the new `resolvePageClusterKeysFromArray` convenience wrapper; existing sync array-based use cases keep working with a one-line change. Corpora at or below `CORPUS_INLINE_THRESHOLD` (20,000 pages) still take the byte-identical in-memory path; larger corpora are processed via three passes: - Pass 0 (HTML-free) — read blocking signals only and compute block keys with the existing blocking / orphan-reassignment / first-party filter pipeline. - Pass 1 (per-block reservoir sampling) — draw up to `BLOCK_SAMPLE_SIZE` (100) pages per block, run Stage A on the sample, emit its CrossBlockUnits, and save learned artifacts (per-block `maxMainDepth`, local-chrome signatures, per-cluster member token sets) for the assignment step. - Pass 1b (streaming assignment) — Jaccard-assign every non-sample page to its block's nearest sample-derived cluster. Additional memory reductions: - `CrossBlockUnit.memberLandmarks: ExtractLandmarksResult[]` becomes `memberLandmarkInstances: PerPageLandmarkInstance[][]`. Landmarks are pre-tokenized at unit creation instead of being re-tokenized by shellQuorum each round. - `mergeCrossBlockClusters` accepts an opt-in `capMembers` option that down-samples merged groups after each merge. Streaming callers pass 100; the in-memory path omits it to keep validated corpora unchanged. Small-corpus regression preserved on the four historically validated corpora (302 / 8,936 / 1,416 / 89 pages → 9 / 21 / 63 / 3 clusters). New streaming path validated on a 176,046-page real crawl: completes in ~28 min at 1.4 GB peak RSS / 2.8 GB peak heap, well within an 8 GB budget. Co-Authored-By: Claude Opus 4.7 --- cspell.json | 2 + .../src/merge-cross-block-clusters.spec.ts | 60 +- .../src/merge-cross-block-clusters.ts | 65 +- .../page-cluster/src/pass0-blocking.spec.ts | 121 ++ .../page-cluster/src/pass0-blocking.ts | 125 ++ .../page-cluster/src/reservoir-sample.spec.ts | 66 ++ .../page-cluster/src/reservoir-sample.ts | 108 ++ ...esolve-page-cluster-keys-streaming.spec.ts | 125 ++ .../src/resolve-page-cluster-keys.spec.ts | 2 +- .../src/resolve-page-cluster-keys.ts | 1024 ++++++++++------- .../page-cluster/src/stage-a-per-block.ts | 283 +++++ 11 files changed, 1532 insertions(+), 449 deletions(-) create mode 100644 packages/@d-zero/page-cluster/src/pass0-blocking.spec.ts create mode 100644 packages/@d-zero/page-cluster/src/pass0-blocking.ts create mode 100644 packages/@d-zero/page-cluster/src/reservoir-sample.spec.ts create mode 100644 packages/@d-zero/page-cluster/src/reservoir-sample.ts create mode 100644 packages/@d-zero/page-cluster/src/resolve-page-cluster-keys-streaming.spec.ts create mode 100644 packages/@d-zero/page-cluster/src/stage-a-per-block.ts diff --git a/cspell.json b/cspell.json index ddad4398..6da4cc10 100644 --- a/cspell.json +++ b/cspell.json @@ -17,6 +17,8 @@ "hrefs", "Murtagh", "unioned", + "retags", + "Vitter", // "gaxios", diff --git a/packages/@d-zero/page-cluster/src/merge-cross-block-clusters.spec.ts b/packages/@d-zero/page-cluster/src/merge-cross-block-clusters.spec.ts index 482417d5..858a706a 100644 --- a/packages/@d-zero/page-cluster/src/merge-cross-block-clusters.spec.ts +++ b/packages/@d-zero/page-cluster/src/merge-cross-block-clusters.spec.ts @@ -4,6 +4,20 @@ import type { CrossBlockUnit } from './merge-cross-block-clusters.js'; import { describe, expect, test } from 'vitest'; import { mergeCrossBlockClusters } from './merge-cross-block-clusters.js'; +import { computePerPageLandmarkInstances } from './per-page-landmark-signatures.js'; + +/** + * Converts an old-style per-member `ExtractLandmarksResult[]` fixture into + * the pre-tokenized `PerPageLandmarkInstance[][]` shape that `CrossBlockUnit` + * now stores. Every test in this file historically constructed landmark + * fixtures as full `ExtractLandmarksResult` objects; this shim funnels them + * through the same tokenization Stage B used to do at read time, keeping + * every existing test's semantics intact under the new field name. + * @param landmarks + */ +function toInstances(landmarks: readonly ExtractLandmarksResult[]) { + return computePerPageLandmarkInstances(landmarks); +} const noLandmarks: ExtractLandmarksResult = { header: [], @@ -28,7 +42,7 @@ describe('mergeCrossBlockClusters', () => { const unit: CrossBlockUnit = { key: 'k1', memberTokenSets: [new Set(['body>main>.card'])], - memberLandmarks: [noLandmarks], + memberLandmarkInstances: toInstances([noLandmarks]), }; const result = mergeCrossBlockClusters([unit], {}); expect(result.get('k1')).toBe('k1'); @@ -39,12 +53,12 @@ describe('mergeCrossBlockClusters', () => { const unit1: CrossBlockUnit = { key: 'k1', memberTokenSets: [tokens], - memberLandmarks: [noLandmarks], + memberLandmarkInstances: toInstances([noLandmarks]), }; const unit2: CrossBlockUnit = { key: 'k2', memberTokenSets: [tokens], - memberLandmarks: [noLandmarks], + memberLandmarkInstances: toInstances([noLandmarks]), }; const result = mergeCrossBlockClusters([unit1, unit2], {}); expect(result.get('k1')).toBe(result.get('k2')); @@ -56,14 +70,14 @@ describe('mergeCrossBlockClusters', () => { memberTokenSets: [ new Set(['body>main>article', 'body>main>article>h1', 'body>main>article>p']), ], - memberLandmarks: [noLandmarks], + memberLandmarkInstances: toInstances([noLandmarks]), }; const unit2: CrossBlockUnit = { key: 'k2', memberTokenSets: [ new Set(['body>aside>section', 'body>aside>section>ul', 'body>footer>nav']), ], - memberLandmarks: [noLandmarks], + memberLandmarkInstances: toInstances([noLandmarks]), }; const result = mergeCrossBlockClusters([unit1, unit2], {}); expect(result.get('k1')).toBe('k1'); @@ -85,7 +99,7 @@ describe('mergeCrossBlockClusters', () => { 'body>main>section.c-reports>ul.c-reports__list', ]), ], - memberLandmarks: [noLandmarks, noLandmarks], + memberLandmarkInstances: toInstances([noLandmarks, noLandmarks]), }; const unit2: CrossBlockUnit = { key: 'projects', @@ -99,7 +113,7 @@ describe('mergeCrossBlockClusters', () => { 'body>main>section.c-projects>ul.c-projects__list', ]), ], - memberLandmarks: [noLandmarks, noLandmarks], + memberLandmarkInstances: toInstances([noLandmarks, noLandmarks]), }; const result = mergeCrossBlockClusters([unit1, unit2], {}); expect(result.get('reports')).toBe(result.get('projects')); @@ -113,14 +127,14 @@ describe('mergeCrossBlockClusters', () => { memberTokenSets: [ new Set(['body>main>section.type-a', 'body>main>section.type-a>p']), ], - memberLandmarks: [noLandmarks], + memberLandmarkInstances: toInstances([noLandmarks]), }; const unit2: CrossBlockUnit = { key: 'solo-b', memberTokenSets: [ new Set(['body>main>section.type-b', 'body>main>section.type-b>p']), ], - memberLandmarks: [noLandmarks], + memberLandmarkInstances: toInstances([noLandmarks]), }; const result = mergeCrossBlockClusters([unit1, unit2], {}); expect(result.get('solo-a')).toBe('solo-a'); @@ -131,17 +145,17 @@ describe('mergeCrossBlockClusters', () => { const unit1: CrossBlockUnit = { key: 'a', memberTokenSets: [new Set(['x'])], - memberLandmarks: [noLandmarks], + memberLandmarkInstances: toInstances([noLandmarks]), }; const unit2: CrossBlockUnit = { key: 'b', memberTokenSets: [new Set(['y'])], - memberLandmarks: [noLandmarks], + memberLandmarkInstances: toInstances([noLandmarks]), }; const unit3: CrossBlockUnit = { key: 'c', memberTokenSets: [new Set(['z'])], - memberLandmarks: [noLandmarks], + memberLandmarkInstances: toInstances([noLandmarks]), }; const result = mergeCrossBlockClusters([unit1, unit2, unit3], {}); expect(result.has('a')).toBe(true); @@ -154,12 +168,12 @@ describe('mergeCrossBlockClusters', () => { const unit1: CrossBlockUnit = { key: 'k1', memberTokenSets: [tokens], - memberLandmarks: [noLandmarks], + memberLandmarkInstances: toInstances([noLandmarks]), }; const unit2: CrossBlockUnit = { key: 'k2', memberTokenSets: [tokens], - memberLandmarks: [noLandmarks], + memberLandmarkInstances: toInstances([noLandmarks]), }; const r1 = mergeCrossBlockClusters([unit1, unit2], {}); const r2 = mergeCrossBlockClusters([unit1, unit2], {}); @@ -172,12 +186,12 @@ describe('mergeCrossBlockClusters', () => { const unit1: CrossBlockUnit = { key: 'block-a', memberTokenSets: [tokens], - memberLandmarks: [noLandmarks], + memberLandmarkInstances: toInstances([noLandmarks]), }; const unit2: CrossBlockUnit = { key: 'block-b', memberTokenSets: [tokens], - memberLandmarks: [noLandmarks], + memberLandmarkInstances: toInstances([noLandmarks]), }; const result = mergeCrossBlockClusters([unit1, unit2], {}); const rootKey = result.get('block-a')!; @@ -193,12 +207,12 @@ describe('mergeCrossBlockClusters', () => { const unit1: CrossBlockUnit = { key: 'l1', memberTokenSets: [tokens], - memberLandmarks: [landmarks], + memberLandmarkInstances: toInstances([landmarks]), }; const unit2: CrossBlockUnit = { key: 'l2', memberTokenSets: [tokens], - memberLandmarks: [landmarks], + memberLandmarkInstances: toInstances([landmarks]), }; const result = mergeCrossBlockClusters([unit1, unit2], {}); expect(result.get('l1')).toBe(result.get('l2')); @@ -216,7 +230,7 @@ describe('mergeCrossBlockClusters shellQuorum (via mergeCrossBlockClusters exerc const unit: CrossBlockUnit = { key: 'k', memberTokenSets: Array.from({ length: 5 }, () => tokens), - memberLandmarks: Array.from({ length: 5 }, () => landmarksAll), + memberLandmarkInstances: toInstances(Array.from({ length: 5 }, () => landmarksAll)), }; // The clustering itself just needs to run without crashing to prove // the histogram path works with a well-populated signature. @@ -244,12 +258,12 @@ describe('mergeCrossBlockClusters shellQuorum (via mergeCrossBlockClusters exerc const unitA: CrossBlockUnit = { key: 'A', memberTokenSets: Array.from({ length: 5 }, () => tokens), - memberLandmarks: landmarksList, + memberLandmarkInstances: toInstances(landmarksList), }; const unitB: CrossBlockUnit = { key: 'B', memberTokenSets: Array.from({ length: 5 }, () => tokens), - memberLandmarks: landmarksList, + memberLandmarkInstances: toInstances(landmarksList), }; const result = mergeCrossBlockClusters([unitA, unitB], {}); // Same tokens sets merge via fine stage (Jaccard = 1.0), independent @@ -275,12 +289,12 @@ describe('mergeCrossBlockClusters shellQuorum (via mergeCrossBlockClusters exerc const unitA: CrossBlockUnit = { key: 'A', memberTokenSets: Array.from({ length: 5 }, () => new Set(['body>main>.article'])), - memberLandmarks: landmarksList, + memberLandmarkInstances: toInstances(landmarksList), }; const unitB: CrossBlockUnit = { key: 'B', memberTokenSets: Array.from({ length: 5 }, () => new Set(['body>aside>.widget'])), - memberLandmarks: landmarksList, + memberLandmarkInstances: toInstances(landmarksList), }; const result = mergeCrossBlockClusters([unitA, unitB], {}); // Cores are disjoint and shells don't corroborate → A and B stay diff --git a/packages/@d-zero/page-cluster/src/merge-cross-block-clusters.ts b/packages/@d-zero/page-cluster/src/merge-cross-block-clusters.ts index 243e6f01..2a647a1a 100644 --- a/packages/@d-zero/page-cluster/src/merge-cross-block-clusters.ts +++ b/packages/@d-zero/page-cluster/src/merge-cross-block-clusters.ts @@ -1,4 +1,4 @@ -import type { ExtractLandmarksResult } from './extract-landmarks.js'; +import type { PerPageLandmarkInstance } from './per-page-landmark-signatures.js'; import { assignContainedClusters } from './assign-contained-clusters.js'; import { autoCutThreshold } from './auto-cut-threshold.js'; @@ -9,17 +9,28 @@ import { } from './complete-linkage-dendrogram.js'; import { computeDocumentFrequency } from './compute-document-frequency.js'; import { jaccardSimilarity } from './jaccard-similarity.js'; -import { computePerPageLandmarkInstances } from './per-page-landmark-signatures.js'; +import { reservoirSample } from './reservoir-sample.js'; import { shapeToken } from './shape-token.js'; import { splitTokensByFrequency } from './split-tokens-by-frequency.js'; /** * One cluster (post-Stage-A) entering cross-block comparison. + * + * ## Why `memberLandmarkInstances` rather than raw `ExtractLandmarksResult[]` + * + * Stage B never reads the full `ExtractLandmarksResult` — it only needs each + * member page's landmark instances (per-type, per-signature, per-token-set) + * for `shellQuorum`'s per-token page-frequency histogram. Callers pre-run + * {@link ./per-page-landmark-signatures.js | computePerPageLandmarkInstances} + * once at unit creation and hand Stage B the compact instance list instead + * of the ~10× larger raw landmark HTML strings. This is the single largest + * memory reduction the streaming path relies on: the difference between a + * 176k-page corpus fitting in an 8 GB heap and exhausting a 12 GB heap. */ export type CrossBlockUnit = { readonly key: string; readonly memberTokenSets: readonly ReadonlySet[]; - readonly memberLandmarks: readonly ExtractLandmarksResult[]; + readonly memberLandmarkInstances: readonly (readonly PerPageLandmarkInstance[])[]; }; /** @@ -247,15 +258,14 @@ function l2Contained(xSig: Map, ySig: Map): bool * shell-corroboration jaccard between two landmark-less pages is 0 rather * than 1 (which it would be if we handed back a ``-derived * `{body}` fallback set to both sides). - * @param memberLandmarks + * @param perPageInstances */ function shellQuorum( - memberLandmarks: readonly ExtractLandmarksResult[], + perPageInstances: readonly (readonly PerPageLandmarkInstance[])[], ): ReadonlySet { - const pageCount = memberLandmarks.length; + const pageCount = perPageInstances.length; if (pageCount === 0) return new Set(); - const perPageInstances = computePerPageLandmarkInstances(memberLandmarks); // Union all instance token sets per page (dedupe within page: a token // present on both header and footer of the same page still counts once // for that page's contribution). @@ -316,28 +326,42 @@ function shellQuorum( * @param units Post-Stage-A clusters. * @param options Forwarded `similarityThreshold` (defaults to 0.8). * @param options.similarityThreshold + * @param options.capMembers */ export function mergeCrossBlockClusters( units: readonly CrossBlockUnit[], - options?: { similarityThreshold?: number }, + options?: { + similarityThreshold?: number; + /** + * Opt-in post-merge cap on a group's retained member count. When + * set, each merged group is reservoir-sampled back down to this + * cap after every merge so a chain of merges cannot balloon past + * the per-unit cap Stage A applied at creation. Callers on the + * streaming path pass the same value as `capMembers` used in + * Stage A; the in-memory path omits it so validated corpora keep + * full-membership merge behavior unchanged. + */ + capMembers?: number; + }, ): Map { if (units.length <= 1) { return new Map(units.map((u) => [u.key, u.key])); } const threshold = options?.similarityThreshold ?? CROSS_BLOCK_THRESHOLD; + const capMembers = options?.capMembers; // Mutable group state: rootKey → combined member arrays type GroupMembers = { tokenSets: ReadonlySet[]; - landmarks: ExtractLandmarksResult[]; + landmarkInstances: (readonly PerPageLandmarkInstance[])[]; }; const groups = new Map(); for (const unit of units) { groups.set(unit.key, { tokenSets: [...unit.memberTokenSets], - landmarks: [...unit.memberLandmarks], + landmarkInstances: [...unit.memberLandmarkInstances], }); } @@ -355,8 +379,23 @@ export function mergeCrossBlockClusters( const rootG = groups.get(root); if (!absorbedG || !rootG) continue; - rootG.tokenSets = [...rootG.tokenSets, ...absorbedG.tokenSets]; - rootG.landmarks = [...rootG.landmarks, ...absorbedG.landmarks]; + const mergedTokenSets = [...rootG.tokenSets, ...absorbedG.tokenSets]; + const mergedLandmarkInstances = [ + ...rootG.landmarkInstances, + ...absorbedG.landmarkInstances, + ]; + // Only down-sample when the caller explicitly opts in (streaming + // path). Same-index sampling keeps memberTokenSets[i] and + // landmarkInstances[i] parallel. + if (capMembers !== undefined && mergedTokenSets.length > capMembers) { + const indices = mergedTokenSets.map((_, i) => i); + const kept = reservoirSample(indices, capMembers, root); + rootG.tokenSets = kept.map((i) => mergedTokenSets[i]!); + rootG.landmarkInstances = kept.map((i) => mergedLandmarkInstances[i]!); + } else { + rootG.tokenSets = mergedTokenSets; + rootG.landmarkInstances = mergedLandmarkInstances; + } groups.delete(absorbed); for (const [origKey, cur] of keyToRoot) { @@ -517,7 +556,7 @@ export function mergeCrossBlockClusters( const getShell = (key: string): ReadonlySet => { if (!shellCache.has(key)) { - shellCache.set(key, shellQuorum(groups.get(key)?.landmarks ?? [])); + shellCache.set(key, shellQuorum(groups.get(key)?.landmarkInstances ?? [])); } return shellCache.get(key) ?? new Set(); }; diff --git a/packages/@d-zero/page-cluster/src/pass0-blocking.spec.ts b/packages/@d-zero/page-cluster/src/pass0-blocking.spec.ts new file mode 100644 index 00000000..369ffc94 --- /dev/null +++ b/packages/@d-zero/page-cluster/src/pass0-blocking.spec.ts @@ -0,0 +1,121 @@ +import { describe, expect, test } from 'vitest'; + +import { groupIndicesByBlockKey, resolveBlockKeys } from './pass0-blocking.js'; + +describe('resolveBlockKeys', () => { + test('returns one block key per input page in order', () => { + // Third page with a *different* stylesheet keeps a.css from being + // 100%-common, so its frequency stays below the common-href cutoff + // and it survives as a distinctive css: signal for the first two. + const keys = resolveBlockKeys([ + { paths: ['news', '1'], stylesheetHrefs: ['https://example.com/a.css'] }, + { paths: ['news', '2'], stylesheetHrefs: ['https://example.com/a.css'] }, + { paths: ['about'], stylesheetHrefs: ['https://example.com/b.css'] }, + ]); + expect(keys).toHaveLength(3); + expect(keys[0]).toBe(keys[1]); + expect(keys[0]).toMatch(/^css:/); + // b.css appears on only one page — below minCssGroupSize=2, so it + // falls back to path. + expect(keys[2]).toBe('path:about'); + }); + + test('orphan (empty stylesheetHrefs) gets reassigned to same-section css block by default', () => { + const pages = [ + { + paths: ['news', '1'], + stylesheetHrefs: ['https://example.com/a.css', 'https://example.com/shared.css'], + }, + { + paths: ['news', '2'], + stylesheetHrefs: ['https://example.com/a.css', 'https://example.com/shared.css'], + }, + { + paths: ['news', '3'], + stylesheetHrefs: ['https://example.com/shared.css'], + }, + // Third-differently-styled page anchors the frequency filter so + // shared.css isn't 100% and a.css stays distinctive. + { + paths: ['about'], + stylesheetHrefs: ['https://example.com/b.css', 'https://example.com/shared.css'], + }, + // Orphan (no stylesheets) in the news section — should join the + // news css: block via orphan reassignment. + { paths: ['news', '4'], stylesheetHrefs: [] }, + ]; + const keys = resolveBlockKeys(pages); + // reassignOrphanBlockKeys retags every member of the reassigned + // css: block with the orphan-merge: prefix so the block pool is + // visibly renamed once an orphan joins — the whole news section + // shares one orphan-merge: key rather than a mix. + expect(keys[0]).toMatch(/^orphan-merge:/); + expect(keys[0]).toBe(keys[4]); + }); + + test('reassignOrphans: false leaves orphan on path key', () => { + const pages = [ + { + paths: ['news', '1'], + stylesheetHrefs: ['https://example.com/a.css', 'https://example.com/shared.css'], + }, + { + paths: ['news', '2'], + stylesheetHrefs: ['https://example.com/a.css', 'https://example.com/shared.css'], + }, + { + paths: ['about'], + stylesheetHrefs: ['https://example.com/b.css', 'https://example.com/shared.css'], + }, + { paths: ['news', '4'], stylesheetHrefs: [] }, + ]; + const keys = resolveBlockKeys(pages, { reassignOrphans: false }); + expect(keys[3]).toBe('path:news'); + }); + + test('per-page host lets third-party fonts get filtered out before blocking', () => { + // Both pages load their site's own first-party stylesheet plus a + // third-party font. Without filtering, the font shows up as a + // distinctive shared href and can affect css: key generation. + // With `host` provided, direct comparison drops fonts.googleapis.com. + const pages = [ + { + paths: ['a'], + host: 'example.com', + stylesheetHrefs: [ + 'https://example.com/a.css', + 'https://fonts.googleapis.com/css?family=x', + ], + }, + { + paths: ['a'], + host: 'example.com', + stylesheetHrefs: [ + 'https://example.com/a.css', + 'https://fonts.googleapis.com/css?family=x', + ], + }, + ]; + const keys = resolveBlockKeys(pages); + expect(keys[0]).toBe(keys[1]); + }); + + test('empty input returns empty array', () => { + expect(resolveBlockKeys([])).toEqual([]); + }); +}); + +describe('groupIndicesByBlockKey', () => { + test('groups indices by key, preserving first-seen order of keys and input order within each', () => { + const groups = groupIndicesByBlockKey(['a', 'b', 'a', 'c', 'a', 'b']); + expect([...groups.keys()]).toEqual(['a', 'b', 'c']); + expect(groups.get('a')).toEqual([0, 2, 4]); + expect(groups.get('b')).toEqual([1, 5]); + expect(groups.get('c')).toEqual([3]); + }); + + test('empty input returns empty map', () => { + const groups = groupIndicesByBlockKey([]); + expect(groups.size).toBe(0); + }); +}); diff --git a/packages/@d-zero/page-cluster/src/pass0-blocking.ts b/packages/@d-zero/page-cluster/src/pass0-blocking.ts new file mode 100644 index 00000000..d9a1c8c5 --- /dev/null +++ b/packages/@d-zero/page-cluster/src/pass0-blocking.ts @@ -0,0 +1,125 @@ +import type { ResolveBlockingGroupKeysOptions } from './resolve-blocking-group-keys.js'; + +import { filterFirstPartyStylesheetHrefs } from './filter-first-party-stylesheet-hrefs.js'; +import { reassignOrphanBlockKeys } from './reassign-orphan-block-keys.js'; +import { resolveBlockingGroupKeys } from './resolve-blocking-group-keys.js'; + +/** + * The corpus-wide, HTML-free inputs needed by + * {@link ./pass0-blocking.js | resolveBlockKeys}: the blocking signals plus + * the page's own URL host. The full `PageClusterSignals` type carries `html` + * on top of these, but Pass 0 deliberately does not read HTML — the whole + * point of extracting the blocking step is that it can be run before any + * per-page HTML is held in memory. + */ +export type Pass0PageSignals = { + readonly paths: readonly string[]; + readonly stylesheetHrefs: readonly string[]; + readonly host?: string; +}; + +/** + * @see resolveBlockKeys + */ +export type ResolveBlockKeysOptions = ResolveBlockingGroupKeysOptions & { + /** + * Apply {@link ./reassign-orphan-block-keys.js | reassignOrphanBlockKeys} + * to the blocking keys, so a page with no recorded stylesheets can rejoin + * a same-URL-section `css:` block instead of being stranded on its weaker + * `path:` fallback. Defaults to `true`. + */ + readonly reassignOrphans?: boolean; + /** + * Apply {@link ./filter-first-party-stylesheet-hrefs.js | + * filterFirstPartyStylesheetHrefs} to `pages` before computing blocking + * keys, so a page's incidental third-party embeds do not contaminate the + * blocking signal. Defaults to `true`. + */ + readonly restrictStylesheetsToFirstParty?: boolean; +}; + +/** + * Splits `resolvePageClusterKeys` into a size-flat first pass so the driver + * can decide per-block memory strategy before loading any page HTML. Runs the + * three corpus-wide, HTML-free stages of blocking in the same order the + * in-memory driver already uses — first-party stylesheet filtering, blocking- + * key derivation, orphan reassignment — and returns one final block key per + * input page in input order. + * + * ## Why extract this from resolvePageClusterKeys? + * + * The in-memory driver holds every page's `html`, `remainderHtml`, + * `landmarks[]`, and pre-Stage-A prepared HTML at once. At 176k pages × ~57 + * KB average, that alone breaks a 17 GB RAM machine well before Stage A + * starts (measured: OS SIGKILL at ~15,000 pages, before the resolve phase + * even began). All three blocking stages, in contrast, depend only on + * `paths` / `stylesheetHrefs` / `host` — a few hundred bytes per page. Running + * them first, HTML-free, lets the downstream per-block clustering hold HTML + * for only one block's pages at a time. + * + * ## Preserves in-memory driver semantics exactly + * + * The output of this function is byte-identical to the block-key portion of + * the current `resolvePageClusterKeys` for the same input, because it reuses + * the same three underlying functions in the same order with the same + * defaults. That guarantee is load-bearing: the size-threshold-gated Pass 1 + * that follows this function runs the current in-memory implementation + * unchanged for small blocks, and any drift in block-key computation between + * Pass 0 and Pass 1 would silently misroute pages between the two paths. + * @param pages + * @param options + * @example + * ```ts + * const blockKeys = resolveBlockKeys([ + * { paths: ['news', '1'], stylesheetHrefs: ['/a.css'], host: 'example.com' }, + * { paths: ['news', '2'], stylesheetHrefs: ['/a.css'], host: 'example.com' }, + * { paths: ['about'], stylesheetHrefs: [], host: 'example.com' }, + * ]); + * // ['css:', 'css:', 'path:about'] + * ``` + */ +export function resolveBlockKeys( + pages: readonly Pass0PageSignals[], + options?: ResolveBlockKeysOptions, +): string[] { + const restrictStylesheetsToFirstParty = + options?.restrictStylesheetsToFirstParty ?? true; + const blockingPages = restrictStylesheetsToFirstParty + ? filterFirstPartyStylesheetHrefs(pages) + : pages; + + const rawBlockKeys = resolveBlockingGroupKeys(blockingPages, options); + const reassignOrphans = options?.reassignOrphans ?? true; + return reassignOrphans + ? reassignOrphanBlockKeys(blockingPages, rawBlockKeys, options?.pathDepth) + : rawBlockKeys; +} + +/** + * Groups pages by their block key while preserving each block's members in + * input order. Returned as a `Map` so the caller can iterate blocks in + * insertion order (first-seen block first) — matching the order + * `resolvePageClusterKeys`'s own per-block loop already uses so cluster IDs + * assigned per block stay deterministic across in-memory and streaming + * paths. + * @param blockKeys + * @example + * ```ts + * const indices = groupIndicesByBlockKey(['a', 'b', 'a', 'c', 'a']); + * // Map { 'a' => [0, 2, 4], 'b' => [1], 'c' => [3] } + * ``` + */ +export function groupIndicesByBlockKey( + blockKeys: readonly string[], +): Map { + const groups = new Map(); + for (const [index, blockKey] of blockKeys.entries()) { + const list = groups.get(blockKey); + if (list) { + list.push(index); + } else { + groups.set(blockKey, [index]); + } + } + return groups; +} diff --git a/packages/@d-zero/page-cluster/src/reservoir-sample.spec.ts b/packages/@d-zero/page-cluster/src/reservoir-sample.spec.ts new file mode 100644 index 00000000..4ed45d3d --- /dev/null +++ b/packages/@d-zero/page-cluster/src/reservoir-sample.spec.ts @@ -0,0 +1,66 @@ +import { describe, expect, test } from 'vitest'; + +import { reservoirSample } from './reservoir-sample.js'; + +describe('reservoirSample', () => { + test('returns the requested number of items', () => { + const items = Array.from({ length: 100 }, (_, i) => i); + const result = reservoirSample(items, 10, 'seed'); + expect(result).toHaveLength(10); + }); + + test('is deterministic for the same seed', () => { + const items = Array.from({ length: 100 }, (_, i) => i); + const a = reservoirSample(items, 10, 'seed'); + const b = reservoirSample(items, 10, 'seed'); + expect(a).toEqual(b); + }); + + test('different seeds produce different samples on large inputs', () => { + const items = Array.from({ length: 100 }, (_, i) => i); + const a = reservoirSample(items, 10, 'block-a'); + const b = reservoirSample(items, 10, 'block-b'); + expect(a).not.toEqual(b); + }); + + test('preserves input order in the returned sample', () => { + const items = Array.from({ length: 100 }, (_, i) => i); + const result = reservoirSample(items, 10, 'seed'); + for (let i = 1; i < result.length; i++) { + expect(result[i]).toBeGreaterThan(result[i - 1] as number); + } + }); + + test('sampleSize >= items.length returns the full list in original order without invoking the PRNG', () => { + const items = [7, 2, 5, 1]; + const result = reservoirSample(items, 10, 'ignored-seed'); + expect(result).toEqual(items); + }); + + test('sampleSize === items.length returns full list in original order', () => { + const items = [7, 2, 5, 1]; + const result = reservoirSample(items, 4, 'ignored-seed'); + expect(result).toEqual(items); + }); + + test('sampleSize 0 returns empty', () => { + const items = [1, 2, 3]; + expect(reservoirSample(items, 0, 'x')).toEqual([]); + }); + + test('empty items returns empty', () => { + expect(reservoirSample([], 10, 'x')).toEqual([]); + }); + + test('numeric seed produces identical output to hashed string seed for the same underlying value (self-consistent)', () => { + const items = Array.from({ length: 100 }, (_, i) => i); + const withZero = reservoirSample(items, 5, 0); + const withZeroAgain = reservoirSample(items, 5, 0); + expect(withZero).toEqual(withZeroAgain); + }); + + test('rejects fractional or negative sampleSize', () => { + expect(() => reservoirSample([1, 2, 3], -1, 'x')).toThrow(RangeError); + expect(() => reservoirSample([1, 2, 3], 1.5, 'x')).toThrow(RangeError); + }); +}); diff --git a/packages/@d-zero/page-cluster/src/reservoir-sample.ts b/packages/@d-zero/page-cluster/src/reservoir-sample.ts new file mode 100644 index 00000000..5ab6a420 --- /dev/null +++ b/packages/@d-zero/page-cluster/src/reservoir-sample.ts @@ -0,0 +1,108 @@ +/** + * Mulberry32 — a tiny, well-known 32-bit PRNG. Chosen because it fits in a + * closure, seeds from a single 32-bit integer (so the caller can derive it + * deterministically from something stable like a block key), and produces a + * uniform enough sequence for reservoir sampling. Not cryptographic; that is + * not what this file needs. + * @param seed + */ +function mulberry32(seed: number): () => number { + let state = seed >>> 0; + return () => { + state = (state + 0x6d_2b_79_f5) >>> 0; + let t = state; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4_294_967_296; + }; +} + +/** + * FNV-1a 32-bit hash of a string, used as the default seed source for + * {@link ./reservoir-sample.js | reservoirSample} when the caller passes a + * `string` seed. Small, dependency-free, and deterministic across runs and + * machines — the whole point of using it as a seed is that a block key like + * `"orphan-merge:news"` always samples the same subset of members. + * @param input + */ +function fnv1a32(input: string): number { + let hash = 0x81_1c_9d_c5; + for (let i = 0; i < input.length; i++) { + hash ^= input.codePointAt(i) ?? 0; + hash = Math.imul(hash, 0x01_00_01_93); + } + return hash >>> 0; +} + +/** + * Runs Algorithm R (Vitter, 1985) — the standard one-pass reservoir sampling + * algorithm — to pick `sampleSize` items from `items`, deterministically for + * a given `seed`. Returns them in input order. + * + * ## Why deterministic + * + * `resolvePageClusterKeys`'s cluster keys must be reproducible: given the + * same input pages in the same order, subsequent runs must produce the same + * cluster keys, or downstream code that stores/compares them by value + * silently drifts across runs. `Math.random()` violates that outright. + * Reservoir sampling on top of a seedable PRNG (see {@link ./reservoir-sample.js | mulberry32}) + * preserves determinism while retaining reservoir sampling's O(1)-space, + * one-pass memory profile — the whole point of using it in the first place + * (a block too large for full in-memory processing must not require + * per-page auxiliary state for sampling either). + * + * ## Seed handling + * + * A `number` seed is used as-is; a `string` seed is hashed via + * {@link ./reservoir-sample.js | fnv1a32} first so the caller can pass a + * stable identifier (e.g. a block key like `"orphan-merge:news"`) without + * having to compute a numeric hash itself. Different blocks get different + * samples by passing each block's key as the seed. + * + * ## Edge cases + * + * - `sampleSize <= 0` — returns `[]`. + * - `sampleSize >= items.length` — returns all of `items` in input order, + * without invoking the PRNG. This matters for regression: a block small + * enough to fit its whole membership in the sample must not have items + * reordered by any sampling logic, so downstream cluster labels stay in + * the same first-seen order as the in-memory path. + * @param items + * @param sampleSize + * @param seed + * @example + * ```ts + * const sample = reservoirSample([0, 1, 2, 3, 4, 5, 6, 7, 8, 9], 3, 'block-a'); + * // deterministic 3-item subset, always the same for seed 'block-a' + * ``` + */ +export function reservoirSample( + items: readonly T[], + sampleSize: number, + seed: number | string = 0, +): T[] { + if (!(Number.isInteger(sampleSize) && sampleSize >= 0)) { + throw new RangeError( + `reservoirSample: sampleSize must be a non-negative integer, got ${sampleSize}`, + ); + } + if (sampleSize === 0 || items.length === 0) return []; + if (sampleSize >= items.length) return [...items]; + + const numericSeed = typeof seed === 'string' ? fnv1a32(seed) : seed >>> 0; + const rand = mulberry32(numericSeed); + + // Reservoir holds the picked positions (indices into `items`); we return + // items sorted by those positions to preserve input order in the result. + const reservoirIndices: number[] = Array.from({ length: sampleSize }, (_, i) => i); + + for (let i = sampleSize; i < items.length; i++) { + const j = Math.floor(rand() * (i + 1)); + if (j < sampleSize) { + reservoirIndices[j] = i; + } + } + + reservoirIndices.sort((a, b) => a - b); + return reservoirIndices.map((position) => items[position] as T); +} diff --git a/packages/@d-zero/page-cluster/src/resolve-page-cluster-keys-streaming.spec.ts b/packages/@d-zero/page-cluster/src/resolve-page-cluster-keys-streaming.spec.ts new file mode 100644 index 00000000..c5dc9b15 --- /dev/null +++ b/packages/@d-zero/page-cluster/src/resolve-page-cluster-keys-streaming.spec.ts @@ -0,0 +1,125 @@ +import type { PageClusterSignals } from './resolve-page-cluster-keys.js'; + +import { describe, expect, test } from 'vitest'; + +import { + CORPUS_INLINE_THRESHOLD, + resolvePageClusterKeys, + resolvePageClusterKeysFromArray, + resolvePageClusterKeysInMemory, +} from './resolve-page-cluster-keys.js'; + +/** + * Builds a small `PageClusterSignals[]` fixture with three visible template + * families and a section split so the returned cluster keys will actually + * vary. Small enough to stay under `CORPUS_INLINE_THRESHOLD`. + */ +function buildTinyCorpus(): PageClusterSignals[] { + const pages: PageClusterSignals[] = []; + // news template A + for (let i = 0; i < 3; i++) { + pages.push({ + paths: ['news', String(i)], + stylesheetHrefs: ['https://x.example.com/news.css'], + html: `

News ${i}

body

f
`, + }); + } + // about page + pages.push( + { + paths: ['about'], + stylesheetHrefs: ['https://x.example.com/about.css'], + html: `

About

text

f
`, + }, + { + paths: ['contact'], + stylesheetHrefs: ['https://x.example.com/contact.css'], + html: `
`, + }, + ); + return pages; +} + +describe('resolvePageClusterKeys (async factory)', () => { + test('empty input returns empty result', async () => { + const keys = await resolvePageClusterKeys(() => []); + expect(keys).toEqual([]); + }); + + test('small corpus matches resolvePageClusterKeysInMemory exactly', async () => { + const pages = buildTinyCorpus(); + const streamed = await resolvePageClusterKeys(() => pages); + const inMemory = resolvePageClusterKeysInMemory(pages); + expect(streamed).toEqual(inMemory); + }); + + test('async iterable factory works', async () => { + const pages = buildTinyCorpus(); + /** + * + */ + async function* asyncPages(): AsyncGenerator { + for (const page of pages) { + // An `await` on any expression suffices to make this a valid + // async generator per @typescript-eslint/require-await; passing + // a resolved Promise keeps behavior identical to a plain yield. + yield await Promise.resolve(page); + } + } + const streamed = await resolvePageClusterKeys(() => asyncPages()); + const inMemory = resolvePageClusterKeysInMemory(pages); + expect(streamed).toEqual(inMemory); + }); + + test('single-shot factory (small corpus stays under threshold, factory invoked twice)', async () => { + let calls = 0; + const pages = buildTinyCorpus(); + /** + * + */ + function factory() { + calls++; + return pages; + } + await resolvePageClusterKeys(factory); + // Small corpus takes the in-memory path, which invokes the factory + // exactly twice (once for blocking signals, once for full pages). + expect(calls).toBe(2); + }); + + test('mergeRareLandmarkClusters is rejected in streaming mode', async () => { + const pages: PageClusterSignals[] = Array.from( + { length: CORPUS_INLINE_THRESHOLD + 1 }, + (_, i) => ({ + paths: ['x', String(i)], + stylesheetHrefs: [], + html: `
${i}
`, + }), + ); + await expect( + resolvePageClusterKeys(() => pages, { mergeRareLandmarkClusters: true }), + ).rejects.toThrow(/mergeRareLandmarkClusters is not supported in streaming mode/); + }); + + test('mergeRareLandmarkClusters is allowed when the small-corpus path is taken', async () => { + const pages = buildTinyCorpus(); + // Should not throw — small corpus routes to resolvePageClusterKeysInMemory + // which accepts mergeRareLandmarkClusters. + await expect( + resolvePageClusterKeys(() => pages, { mergeRareLandmarkClusters: true }), + ).resolves.toBeInstanceOf(Array); + }); +}); + +describe('resolvePageClusterKeysFromArray', () => { + test('produces the same result as resolvePageClusterKeysInMemory on the same input', async () => { + const pages = buildTinyCorpus(); + const fromArray = await resolvePageClusterKeysFromArray(pages); + const inMemory = resolvePageClusterKeysInMemory(pages); + expect(fromArray).toEqual(inMemory); + }); + + test('empty input returns empty', async () => { + expect(await resolvePageClusterKeysFromArray([])).toEqual([]); + }); +}); diff --git a/packages/@d-zero/page-cluster/src/resolve-page-cluster-keys.spec.ts b/packages/@d-zero/page-cluster/src/resolve-page-cluster-keys.spec.ts index 40daaf56..716eac6b 100644 --- a/packages/@d-zero/page-cluster/src/resolve-page-cluster-keys.spec.ts +++ b/packages/@d-zero/page-cluster/src/resolve-page-cluster-keys.spec.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from 'vitest'; -import { resolvePageClusterKeys } from './resolve-page-cluster-keys.js'; +import { resolvePageClusterKeysInMemory as resolvePageClusterKeys } from './resolve-page-cluster-keys.js'; describe('resolvePageClusterKeys', () => { test('an empty page list returns an empty array', () => { diff --git a/packages/@d-zero/page-cluster/src/resolve-page-cluster-keys.ts b/packages/@d-zero/page-cluster/src/resolve-page-cluster-keys.ts index ed85bced..933b7e35 100644 --- a/packages/@d-zero/page-cluster/src/resolve-page-cluster-keys.ts +++ b/packages/@d-zero/page-cluster/src/resolve-page-cluster-keys.ts @@ -4,55 +4,131 @@ import type { ResolveBlockingGroupKeysOptions } from './resolve-blocking-group-k import type { ResolveStructuralClusterKeysOptions } from './resolve-structural-cluster-keys.js'; import type { TokenizeOptions } from './types.js'; -import { assignContainedClusters } from './assign-contained-clusters.js'; import { autoCutThreshold } from './auto-cut-threshold.js'; import { capContentDepth } from './cap-content-depth.js'; -import { collapseAnonymousDivs } from './collapse-anonymous-divs.js'; -import { - completeLinkageDendrogram, - labelsAtThreshold, -} from './complete-linkage-dendrogram.js'; -import { - deriveComparisonSets, - MIN_PAGE_COUNT_FOR_FREQUENCY_SPLIT, -} from './derive-comparison-sets.js'; import { detectContentDepthCap, validateDetectContentDepthCapOptions, } from './detect-content-depth-cap.js'; import { extractLandmarks } from './extract-landmarks.js'; import { filterFirstPartyStylesheetHrefs } from './filter-first-party-stylesheet-hrefs.js'; +import { jaccardSimilarity } from './jaccard-similarity.js'; import { mergeCrossBlockClusters } from './merge-cross-block-clusters.js'; import { mergeLandmarkAffinedClusters, validateMergeLandmarkAffinedClustersOptions, } from './merge-landmark-affined-clusters.js'; +import { groupIndicesByBlockKey, resolveBlockKeys } from './pass0-blocking.js'; import { computePerPageLandmarkInstances } from './per-page-landmark-signatures.js'; -import { reassignOrphanBlockKeys } from './reassign-orphan-block-keys.js'; import { removeContentBlocks } from './remove-content-blocks.js'; -import { resolveBlockingGroupKeys } from './resolve-blocking-group-keys.js'; +import { stageAPerBlock } from './stage-a-per-block.js'; import { tokenize } from './tokenize.js'; /** - * Reads `values[index]`, throwing instead of returning `undefined`. Every - * call site here indexes with a position this function generated itself - * (an entry from `Map#entries()`/`Array#entries()` over an array it just - * built), so the thrown branch is unreachable in practice; it exists to - * satisfy `noUncheckedIndexedAccess` without a non-null assertion. Not - * imported from `resolve-structural-cluster-keys.ts`'s own copy: that file's - * `export`s are its intended public API surface, and this ~7-line generic - * helper isn't worth carving an exception into that boundary for (same - * rationale as `readDpValue` in `array-edit-distance.ts` being its own - * independent copy rather than a shared import). - * @param values - * @param index + * FNV-1a 32-bit hash of a string, used to seed the per-block PRNG so + * reservoir sampling on the streaming path is deterministic for a given + * corpus (same input order → same sampled indices → same cluster keys). + * @param input + */ +function fnv1a32(input: string): number { + let hash = 0x81_1c_9d_c5; + for (let i = 0; i < input.length; i++) { + hash ^= input.codePointAt(i) ?? 0; + hash = Math.imul(hash, 0x01_00_01_93); + } + return hash >>> 0; +} + +/** + * Mulberry32 — small, well-known 32-bit PRNG. Kept independent per block + * (each block seeds from its own block key via {@link ./resolve-page-cluster-keys.js | fnv1a32}) + * so different blocks sample independently. + * @param seed + */ +function makeSeededPrng(seed: number | string): () => number { + let state = (typeof seed === 'string' ? fnv1a32(seed) : seed) >>> 0; + return () => { + state = (state + 0x6d_2b_79_f5) >>> 0; + let t = state; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4_294_967_296; + }; +} + +/** + * Tokenizes a non-sample page using the block's learned parameters + * (`maxMainDepth` and local-signature reinjection set) and returns the + * key of the sample-derived cluster whose members it most closely matches + * by Jaccard similarity. Ties break in first-seen order (JS `Map` + * iteration). Falls back to a block-scoped singleton key when the block + * has no clusters at all (edge case: an empty sample, which shouldn't + * happen for a non-empty block but is defended against here). + * @param html + * @param assignment + * @param assignment.maxMainDepth + * @param assignment.localSignatures + * @param assignment.clustersByUnitKey + * @param excludeLandmarks + * @param contentBlockAttribute + * @param tokenizeOptions + * @param blockKey */ -function requireIndex(values: ArrayLike, index: number): T { - const value = values[index]; - if (value === undefined) { - throw new Error('resolvePageClusterKeys: index out of bounds'); +function assignPageToNearestCluster( + html: string, + assignment: { + readonly maxMainDepth: number | undefined; + readonly localSignatures: ReadonlySet; + readonly clustersByUnitKey: ReadonlyMap[]>; + }, + excludeLandmarks: boolean, + contentBlockAttribute: string | undefined, + tokenizeOptions: TokenizeOptions | undefined, + blockKey: string, +): string { + const landmarkResult = extractLandmarks(html); + const landmarksExcised = excludeLandmarks ? landmarkResult.remainderHtml : html; + let prepared = + contentBlockAttribute === undefined + ? landmarksExcised + : removeContentBlocks(landmarksExcised, { blockAttribute: contentBlockAttribute }) + .remainderHtml; + if (assignment.maxMainDepth !== undefined) { + prepared = capContentDepth(prepared, { + landmark: 'main', + maxDepth: assignment.maxMainDepth, + }).remainderHtml; + } + + const pageTokens = new Set(tokenize(prepared, tokenizeOptions).tokens); + // Reinject tokens for landmark instances whose signature matches the + // block's learned local-signature set (same rule the sample-side Stage + // A applied via computeLocalChromeArtifacts). + if (assignment.localSignatures.size > 0) { + const instances = computePerPageLandmarkInstances( + [landmarkResult], + tokenizeOptions, + )[0]!; + for (const inst of instances) { + if (!assignment.localSignatures.has(inst.signature)) continue; + for (const t of inst.tokens) pageTokens.add(t); + } + } + + let bestKey: string | undefined; + let bestScore = -1; + for (const [unitKey, memberTokenSets] of assignment.clustersByUnitKey) { + let clusterBest = 0; + for (const memberTokens of memberTokenSets) { + const score = jaccardSimilarity(pageTokens, memberTokens); + if (score > clusterBest) clusterBest = score; + } + if (clusterBest > bestScore) { + bestScore = clusterBest; + bestKey = unitKey; + } } - return value; + return bestKey ?? JSON.stringify([blockKey, 'cluster:unassigned']); } /** @@ -104,12 +180,26 @@ function requireIndex(values: ArrayLike, index: number): T { * @param landmarks * @param tokenizeOptions */ -function computeLocalLandmarkTokens( +/** + * Companion to {@link ./resolve-page-cluster-keys.js | computeLocalLandmarkTokens} + * that also returns the local-signature *set* the streaming path needs to + * reuse when tokenizing non-sample pages during Pass 1b. The in-memory path + * only cares about the per-page token sets (which pages carry which + * chrome-below-the-cut tokens); the streaming path additionally needs to + * apply the *same* "which signatures are local" verdict to pages that were + * not part of the sample the verdict was learned from. + * @param landmarks + * @param tokenizeOptions + */ +export function computeLocalChromeArtifacts( landmarks: readonly ExtractLandmarksResult[], tokenizeOptions: TokenizeOptions | undefined, -): ReadonlySet[] { +): { + readonly localSignatures: ReadonlySet; + readonly localTokensByPage: readonly ReadonlySet[]; +} { const pageCount = landmarks.length; - if (pageCount === 0) return []; + if (pageCount === 0) return { localSignatures: new Set(), localTokensByPage: [] }; const perPageInstances = computePerPageLandmarkInstances(landmarks, tokenizeOptions); @@ -130,7 +220,12 @@ function computeLocalLandmarkTokens( } } } - if (corpusHistogram.size === 0) return landmarks.map(() => new Set()); + if (corpusHistogram.size === 0) { + return { + localSignatures: new Set(), + localTokensByPage: landmarks.map(() => new Set()), + }; + } const frequencies: number[] = []; for (const entry of corpusHistogram.values()) { @@ -146,10 +241,13 @@ function computeLocalLandmarkTokens( } } if (localSignatures.size === 0) { - return landmarks.map(() => new Set()); + return { + localSignatures: new Set(), + localTokensByPage: landmarks.map(() => new Set()), + }; } - return perPageInstances.map((instances) => { + const localTokensByPage = perPageInstances.map((instances) => { const out = new Set(); for (const inst of instances) { if (!localSignatures.has(inst.signature)) continue; @@ -157,16 +255,34 @@ function computeLocalLandmarkTokens( } return out; }); + + return { localSignatures, localTokensByPage }; +} + +/** + * Reinjects each page's *local* (non-corpus-wide) landmark-instance tokens + * into its block token set for Stage A clustering, restoring exactly the + * structural signal that landmark excision removed for those pages while + * keeping global chrome removed (the whole point of `excludeLandmarks`). + * + * See {@link ./resolve-page-cluster-keys.js | computeLocalChromeArtifacts} + * for the underlying algorithm — this function is a thin wrapper that + * discards the local-signature set, exposed for callers that only need the + * per-page tokens (the in-memory driver's use case). + * @param landmarks + * @param tokenizeOptions + */ +export function computeLocalLandmarkTokens( + landmarks: readonly ExtractLandmarksResult[], + tokenizeOptions: TokenizeOptions | undefined, +): ReadonlySet[] { + return [...computeLocalChromeArtifacts(landmarks, tokenizeOptions).localTokensByPage]; } /** * Per-page input to {@link ./resolve-page-cluster-keys.js | resolvePageClusterKeys}: * the blocking signals {@link ./resolve-blocking-group-keys.js | resolveBlockingGroupKeys} - * needs, plus the page's raw HTML. Raw HTML (rather than a pre-tokenized - * `Set`, as earlier versions of this type required) because - * `resolvePageClusterKeys` now needs to decide *how* to tokenize each page - * (see `excludeLandmarks` below) — a decision a caller handed a bare - * `Set` could no longer make correctly on its own. + * needs, plus the page's raw HTML. */ export type PageClusterSignals = { paths: readonly string[]; @@ -177,11 +293,7 @@ export type PageClusterSignals = { * `new URL(pageUrl).host`), forwarded to * {@link ./filter-first-party-stylesheet-hrefs.js | filterFirstPartyStylesheetHrefs} * so it can judge that page's `stylesheetHrefs` by direct comparison - * instead of inferring a batch-wide dominant host. Optional: omit if the - * caller doesn't have each page's URL on hand, at the cost of that - * function's dominant-host fallback and its known limitations (see its - * own JSDoc) — most crawlers already have this since they fetched the - * page from that URL, so providing it is the expected default. + * instead of inferring a batch-wide dominant host. */ host?: string; }; @@ -192,255 +304,99 @@ export type PageClusterSignals = { export type ResolvePageClusterKeysOptions = TokenizeOptions & ResolveBlockingGroupKeysOptions & ResolveStructuralClusterKeysOptions & { - /** - * Tokenize each page's `
`/`