From 542c53324c77d22e1ef90f799a12f1f03a6e2fc9 Mon Sep 17 00:00:00 2001 From: Yusuke Hirao Date: Wed, 8 Jul 2026 09:37:41 +0900 Subject: [PATCH 1/4] fix(page-cluster): resolve first-party stylesheet ties via page host MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit filterFirstPartyStylesheetHrefs picked a single "dominant host" by counting how many pages reference each stylesheet host, since it never had access to each page's own URL to compare against directly. When a third party (e.g. a sitewide Google Fonts request) is referenced by exactly as many pages as the real first-party host, the tie-break silently keeps whichever host happened to be counted first — confirmed on a real 302-page crawl where every single page loaded both, and the third party won. Adds an optional `host` field to PageClusterSignals: when a caller supplies it, each page's stylesheetHrefs are filtered by direct comparison against that host instead of the batch-wide vote, eliminating the tie case entirely. Omitting it keeps the existing dominant-host fallback unchanged. --- .gitignore | 3 + ...ilter-first-party-stylesheet-hrefs.spec.ts | 41 +++++++ .../filter-first-party-stylesheet-hrefs.ts | 108 ++++++++++++------ .../src/resolve-page-cluster-keys.spec.ts | 62 ++++++++++ .../src/resolve-page-cluster-keys.ts | 19 +++ 5 files changed, 198 insertions(+), 35 deletions(-) diff --git a/.gitignore b/.gitignore index 449a5d6b..a29bf50b 100644 --- a/.gitignore +++ b/.gitignore @@ -81,6 +81,9 @@ test/fixture/.__* # Print Result Directory .print +# page-cluster dogfooding scratch (results, throwaway scripts) +.page-cluster + # Secret Test Files ___* diff --git a/packages/@d-zero/page-cluster/src/filter-first-party-stylesheet-hrefs.spec.ts b/packages/@d-zero/page-cluster/src/filter-first-party-stylesheet-hrefs.spec.ts index 128f1e55..25c64d97 100644 --- a/packages/@d-zero/page-cluster/src/filter-first-party-stylesheet-hrefs.spec.ts +++ b/packages/@d-zero/page-cluster/src/filter-first-party-stylesheet-hrefs.spec.ts @@ -53,6 +53,47 @@ describe('filterFirstPartyStylesheetHrefs', () => { expect(result[1]?.stylesheetHrefs).toEqual([]); }); + test('a page-supplied host resolves a tie the dominant-host fallback would get wrong (regression test for the real crawl finding)', () => { + // Every page loads both its own first-party stylesheet and the same + // sitewide third-party webfont — tying "referenced by 100% of pages" + // between the real first party and the third party. Without `host`, + // the fallback's tie-break is arbitrary (see the test above); with + // `host` supplied, the third party is dropped correctly regardless of + // which host the batch-wide count would have favored. + const result = filterFirstPartyStylesheetHrefs([ + { + host: 'example.com', + stylesheetHrefs: [ + 'https://fonts.googleapis.com/css?family=x', + 'https://example.com/a.css', + ], + }, + { + host: 'example.com', + stylesheetHrefs: [ + 'https://fonts.googleapis.com/css?family=x', + 'https://example.com/b.css', + ], + }, + ]); + expect(result[0]?.stylesheetHrefs).toEqual(['https://example.com/a.css']); + expect(result[1]?.stylesheetHrefs).toEqual(['https://example.com/b.css']); + }); + + test('a page without a supplied host still falls back to dominant-host inference, unaffected by other pages that do supply one', () => { + const result = filterFirstPartyStylesheetHrefs([ + { host: 'example.com', stylesheetHrefs: ['https://example.com/a.css'] }, + { + stylesheetHrefs: [ + 'https://example.com/a.css', + 'https://fonts.googleapis.com/css', + ], + }, + ]); + expect(result[0]?.stylesheetHrefs).toEqual(['https://example.com/a.css']); + expect(result[1]?.stylesheetHrefs).toEqual(['https://example.com/a.css']); + }); + test('unparsable hrefs are dropped and never treated as an origin', () => { const result = filterFirstPartyStylesheetHrefs([ { stylesheetHrefs: ['https://example.com/a.css', 'not a url'] }, diff --git a/packages/@d-zero/page-cluster/src/filter-first-party-stylesheet-hrefs.ts b/packages/@d-zero/page-cluster/src/filter-first-party-stylesheet-hrefs.ts index fac6dc1f..4e34340b 100644 --- a/packages/@d-zero/page-cluster/src/filter-first-party-stylesheet-hrefs.ts +++ b/packages/@d-zero/page-cluster/src/filter-first-party-stylesheet-hrefs.ts @@ -16,8 +16,11 @@ function tryGetHost(href: string): string | undefined { /** * Narrows every page's `stylesheetHrefs` down to just the hrefs whose host - * matches the single most common host across the whole batch (the site's own - * first-party domain), dropping every other host. + * matches that page's own `host` field (direct comparison), when the caller + * provides it. Falls back to the single most common host across the whole + * batch (the site's own first-party domain, inferred rather than given) for + * any page that omits `host` — see this function's own JSDoc further down + * for that fallback's known limitations. * * Confirmed on real crawl data (302 pages): a handful of articles embedding * a YouTube video pulled in `youtube.com`'s own player stylesheet plus a @@ -36,20 +39,37 @@ function tryGetHost(href: string): string | undefined { * first-party hrefs before blocking removes that false signal at the * source, rather than trying to recognize its effects downstream. * - * Determining "first-party" from the batch's own href distribution (rather - * than, say, comparing each href's host against each page's own URL) means - * this needs no extra per-page input beyond what - * {@link ./resolve-blocking-group-keys.js | resolveBlockingGroupKeys} already - * takes — but it inherits that same function's "roughly homogeneous batch" - * precondition (see `computeDocumentFrequency`'s own JSDoc): a batch that - * mixes pages from more than one site in one call has no single genuine - * first-party host to find, and this function has no way to detect that - * it's been handed one — it will still confidently pick *a* dominant host - * (whichever site contributes more stylesheet-bearing pages) and silently - * strip every other site's real first-party hrefs. Splitting a - * multi-site/section batch into homogeneous groups before calling this is - * the caller's responsibility, same as it already is for - * `resolveBlockingGroupKeys`. + * `host`, when provided, is compared directly against each of that page's + * own stylesheet hrefs — no batch-wide inference involved, so this path + * cannot mistake a third party for the first party regardless of how many + * pages happen to also embed it. Added after a real crawl (302 pages) hit + * the dominant-host fallback's tie case: every single page loaded both its + * own first-party stylesheet *and* the same `fonts.googleapis.com` webfont + * request (a common sitewide pattern, not a rare misconfiguration), so both + * hosts tied at "referenced by 100% of pages" and the fallback's `>`-only + * tie-break (see below) picked whichever host happened to be counted first + * — silently the wrong one on that crawl. A caller that already knows each + * page's own host (e.g. it has the page's URL on hand, as most crawlers do) + * should always provide it; the inferred fallback exists only for callers + * that don't have that information available. + * + * The dominant-host fallback (used per-page whenever that page omits `host`) + * determines "first-party" from the batch's own href distribution instead + * — comparing each href's host against each page's own URL wasn't possible + * for callers that only ever had `stylesheetHrefs` on hand, not full page + * URLs (this is still true for anything built directly on + * {@link ./resolve-blocking-group-keys.js | resolveBlockingGroupKeys}'s + * `PageBlockingSignals`, which never carried a host field until this + * `host` option was added). It inherits `resolveBlockingGroupKeys`'s own + * "roughly homogeneous batch" precondition (see `computeDocumentFrequency`'s + * own JSDoc): a batch that mixes pages from more than one site in one call + * has no single genuine first-party host to find, and this function has no + * way to detect that it's been handed one — it will still confidently pick + * a* dominant host (whichever site contributes more stylesheet-bearing + * pages) and silently strip every other site's real first-party hrefs. + * Splitting a multi-site/section batch into homogeneous groups before + * calling this (or simply providing `host` per page) is the caller's + * responsibility, same as it already is for `resolveBlockingGroupKeys`. * * The dominant host is picked by how many *pages* reference it at least * once, not by how many stylesheet `` tags reference it — a page @@ -61,19 +81,23 @@ function tryGetHost(href: string): string | undefined { * unresolved protocol-relative URL) is still one site, not two competing * "hosts" splitting its own vote. * - * The trade-off: a site that legitimately serves its own stylesheets from - * more than one first-party host (e.g. a CDN subdomain alongside the main - * domain) will have its non-dominant host's hrefs dropped too, same as any - * genuinely-third-party host — not yet observed on real data, but a known - * limitation of picking a single dominant host rather than a set. + * The fallback's remaining trade-offs: a site that legitimately serves its + * own stylesheets from more than one first-party host (e.g. a CDN subdomain + * alongside the main domain) will have its non-dominant host's hrefs + * dropped too, same as any genuinely-third-party host; and a tie between + * two equally-common hosts silently keeps whichever was counted first + * (confirmed above to include real, common sitewide third parties, not + * just a theoretical edge case) — both are avoided entirely by providing + * `host`. * - * A batch where no page has any stylesheet href at all (or none of the - * hrefs are parseable absolute URLs) has no host to detect; every page's - * `stylesheetHrefs` is returned unchanged in that case, matching this + * A page with neither a provided `host` nor any batch-wide dominant host to + * fall back on (no page in the batch has any parseable stylesheet href at + * all) has its `stylesheetHrefs` returned unchanged, matching this * function's job of narrowing signal, not fabricating it. * @param pages * @example * ```ts + * // Without `host`: falls back to dominant-host inference (ties possible). * filterFirstPartyStylesheetHrefs([ * { stylesheetHrefs: ['https://example.com/a.css', 'https://example.com/b.css'] }, * { stylesheetHrefs: ['https://example.com/a.css', 'https://fonts.googleapis.com/css?family=x'] }, @@ -82,10 +106,19 @@ function tryGetHost(href: string): string | undefined { * // { stylesheetHrefs: ['https://example.com/a.css', 'https://example.com/b.css'] }, * // { stylesheetHrefs: ['https://example.com/a.css'] }, // fonts.googleapis.com dropped * // ] + * + * // With `host`: direct per-page comparison, immune to ties. + * filterFirstPartyStylesheetHrefs([ + * { + * host: 'example.com', + * stylesheetHrefs: ['https://example.com/a.css', 'https://fonts.googleapis.com/css?family=x'], + * }, + * ]); + * // [{ host: 'example.com', stylesheetHrefs: ['https://example.com/a.css'] }] * ``` */ export function filterFirstPartyStylesheetHrefs< - T extends { stylesheetHrefs: readonly string[] }, + T extends { stylesheetHrefs: readonly string[]; host?: string }, >(pages: readonly T[]): T[] { const pageHrefHosts = pages.map((page) => ({ page, @@ -113,14 +146,19 @@ export function filterFirstPartyStylesheetHrefs< } } - if (dominantHost === undefined) { - return [...pages]; - } - - return pageHrefHosts.map(({ page, hrefHosts }) => ({ - ...page, - stylesheetHrefs: hrefHosts - .filter(({ host }) => host === dominantHost) - .map(({ href }) => href), - })); + return pageHrefHosts.map(({ page, hrefHosts }) => { + // A page that supplies its own host is judged against that host + // alone, bypassing the batch-wide dominant-host inference (and its + // tie-breaking pitfall) entirely — see this function's own JSDoc. + const expectedHost = page.host ?? dominantHost; + if (expectedHost === undefined) { + return { ...page }; + } + return { + ...page, + stylesheetHrefs: hrefHosts + .filter(({ host }) => host === expectedHost) + .map(({ href }) => href), + }; + }); } diff --git a/packages/@d-zero/page-cluster/src/resolve-page-cluster-keys.spec.ts b/packages/@d-zero/page-cluster/src/resolve-page-cluster-keys.spec.ts index 56aab548..dea75334 100644 --- a/packages/@d-zero/page-cluster/src/resolve-page-cluster-keys.spec.ts +++ b/packages/@d-zero/page-cluster/src/resolve-page-cluster-keys.spec.ts @@ -366,6 +366,68 @@ describe('resolvePageClusterKeys', () => { expect(result[2]).not.toBe(result[0]); }); + test("providing each page's host avoids a dominant-host tie mistakenly keeping a sitewide third party over the real first party (regression test for a real crawl finding)", () => { + // 10 pages: every page loads *some* fonts.googleapis.com stylesheet + // plus its own first-party one, tying the third-party host and the + // first-party host at "referenced by 10/10 pages" — the exact shape + // confirmed on a real 302-page crawl. dept-a/dept-b share one + // identical googleapis query string (their only distinguishing + // signal if that third party wins the tie); the 8 filler pages each + // use a *different* googleapis query string (so it never looks like + // a shared template signal on its own) and their own unique + // first-party stylesheet. + // + // Without `host`, the tie-break keeps googleapis.com (inserted first + // below), so dept-a/dept-b's real, distinguishing first-party + // stylesheets (a.css vs b.css) are dropped and their identical + // surviving googleapis href wrongly becomes a shared css: block. + // With `host` supplied, each page's real first-party stylesheet + // survives instead, correctly keeping dept-a and dept-b apart. + const html = '
H
C
'; + const targetPages = [ + { + paths: ['dept-a', '1'], + host: 'example.com', + stylesheetHrefs: [ + 'https://fonts.googleapis.com/css?family=shared', + 'https://example.com/a.css', + ], + html, + }, + { + paths: ['dept-b', '1'], + host: 'example.com', + stylesheetHrefs: [ + 'https://fonts.googleapis.com/css?family=shared', + 'https://example.com/b.css', + ], + html, + }, + ]; + const fillerPages = Array.from({ length: 8 }, (_, i) => ({ + paths: [`filler-${i}`], + host: 'example.com', + stylesheetHrefs: [ + `https://fonts.googleapis.com/css?family=filler-${i}`, + `https://example.com/filler-${i}.css`, + ], + html, + })); + const pages = [...targetPages, ...fillerPages]; + + const withHost = resolvePageClusterKeys(pages); + expect(withHost[0]).not.toBe(withHost[1]); + + const withoutHost = resolvePageClusterKeys( + pages.map(({ paths, stylesheetHrefs, html: pageHtml }) => ({ + paths, + stylesheetHrefs, + html: pageHtml, + })), + ); + expect(withoutHost[0]).toBe(withoutHost[1]); + }); + test('autoCapMainDepth: true merges pages whose only difference is content nested deep inside
, requiring no site-specific configuration', () => { // Identical structure up to depth 3 inside
; a page-unique class // at depth 4 fragments every page apart unless that depth is capped. diff --git a/packages/@d-zero/page-cluster/src/resolve-page-cluster-keys.ts b/packages/@d-zero/page-cluster/src/resolve-page-cluster-keys.ts index 746af3c6..70f1484f 100644 --- a/packages/@d-zero/page-cluster/src/resolve-page-cluster-keys.ts +++ b/packages/@d-zero/page-cluster/src/resolve-page-cluster-keys.ts @@ -74,6 +74,18 @@ export type PageClusterSignals = { paths: readonly string[]; stylesheetHrefs: readonly string[]; html: string; + /** + * This page's own URL host (hostname, optionally `:port` — same shape as + * `new URL(pageUrl).host`), forwarded to + * {@link ./filter-first-party-stylesheet-hrefs.js | filterFirstPartyStylesheetHrefs} + * so it can judge that page's `stylesheetHrefs` by direct comparison + * instead of inferring a batch-wide dominant host. Optional: omit if the + * caller doesn't have each page's URL on hand, at the cost of that + * function's dominant-host fallback and its known limitations (see its + * own JSDoc) — most crawlers already have this since they fetched the + * page from that URL, so providing it is the expected default. + */ + host?: string; }; /** @@ -145,6 +157,13 @@ export type ResolvePageClusterKeysOptions = TokenizeOptions & * batch" precondition (see that function's own JSDoc): `pages` should * be one site or one section, the same expectation * `resolveBlockingGroupKeys` already places on its own input. + * + * Provide each page's `host` (see `PageClusterSignals`) so this filter + * can compare directly instead of inferring a batch-wide dominant + * host — the inferred fallback can pick the wrong host on a tie (e.g. + * a page loading its own first-party stylesheet plus the same + * sitewide webfont request as every other page, tying "referenced by + * 100% of pages" between the real first party and that third party). */ restrictStylesheetsToFirstParty?: boolean; /** From 4cb5e83cee0b9050d91c60e93368ea1b79768d86 Mon Sep 17 00:00:00 2001 From: Yusuke Hirao Date: Fri, 10 Jul 2026 16:58:14 +0900 Subject: [PATCH 2/4] feat(page-cluster): add self-tuning recursive clustering MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stage A: auto-cut threshold with clamping + containment assignment inside each block. Stage B: cross-block recursive merge using quorum-80% cores, shape-Jaccard ≥ 0.9, L2 multi-set signatures with shell corroboration, iterating until fixed point (MAX_ROUNDS=10). Also promotes autoCapMainDepth default to true. BREAKING CHANGE: cluster keys may differ from previous versions --- packages/@d-zero/page-cluster/README.md | 4 +- .../src/assign-contained-clusters.spec.ts | 116 ++++ .../src/assign-contained-clusters.ts | 183 ++++++ .../src/auto-cut-threshold.spec.ts | 38 ++ .../page-cluster/src/auto-cut-threshold.ts | 40 ++ .../src/collapse-anonymous-divs.spec.ts | 45 ++ .../src/collapse-anonymous-divs.ts | 46 ++ .../src/complete-linkage-dendrogram.spec.ts | 111 ++++ .../src/complete-linkage-dendrogram.ts | 180 ++++++ .../src/derive-comparison-sets.spec.ts | 30 + .../src/derive-comparison-sets.ts | 38 ++ .../src/merge-cross-block-clusters.spec.ts | 196 +++++++ .../src/merge-cross-block-clusters.ts | 532 ++++++++++++++++++ .../src/merge-landmark-affined-clusters.ts | 3 +- .../src/resolve-page-cluster-keys.spec.ts | 120 +++- .../src/resolve-page-cluster-keys.ts | 252 ++++++--- .../src/resolve-structural-cluster-keys.ts | 279 +-------- .../page-cluster/src/shape-token.spec.ts | 36 ++ .../@d-zero/page-cluster/src/shape-token.ts | 39 ++ 19 files changed, 1924 insertions(+), 364 deletions(-) create mode 100644 packages/@d-zero/page-cluster/src/assign-contained-clusters.spec.ts create mode 100644 packages/@d-zero/page-cluster/src/assign-contained-clusters.ts create mode 100644 packages/@d-zero/page-cluster/src/auto-cut-threshold.spec.ts create mode 100644 packages/@d-zero/page-cluster/src/auto-cut-threshold.ts create mode 100644 packages/@d-zero/page-cluster/src/collapse-anonymous-divs.spec.ts create mode 100644 packages/@d-zero/page-cluster/src/collapse-anonymous-divs.ts create mode 100644 packages/@d-zero/page-cluster/src/complete-linkage-dendrogram.spec.ts create mode 100644 packages/@d-zero/page-cluster/src/complete-linkage-dendrogram.ts create mode 100644 packages/@d-zero/page-cluster/src/derive-comparison-sets.spec.ts create mode 100644 packages/@d-zero/page-cluster/src/derive-comparison-sets.ts create mode 100644 packages/@d-zero/page-cluster/src/merge-cross-block-clusters.spec.ts create mode 100644 packages/@d-zero/page-cluster/src/merge-cross-block-clusters.ts create mode 100644 packages/@d-zero/page-cluster/src/shape-token.spec.ts create mode 100644 packages/@d-zero/page-cluster/src/shape-token.ts diff --git a/packages/@d-zero/page-cluster/README.md b/packages/@d-zero/page-cluster/README.md index cfcad883..9fa40511 100644 --- a/packages/@d-zero/page-cluster/README.md +++ b/packages/@d-zero/page-cluster/README.md @@ -35,7 +35,9 @@ tokenize(html, { 自由編集ブロックエディタ（CMSが各コンテンツブロックに固有のdata属性を付与するタイプ）を使うサイトでは、`contentBlockAttribute` オプションでその属性名を指定すると、ページごとに異なるブロック構成が構造比較のノイズになるのを防げる（既定は未指定＝無効、サイトごとの属性名を推測できないため）。詳細は `removeContentBlocks()` のJSDocを参照。 -CMSのブロック属性名が分からない・サイトごとに違う場合は `autoCapMainDepth: true` を使う。`
`/`role="main"`という標準タグを起点に、構造クラスタ数が急増する直前の深さをブロックごとに実データから自動検出して打ち切るため、サイト固有の設定が一切不要（既定はfalse。実データ検証では`contentBlockAttribute`より良い結果になる場合もあった一方、計算コストが実測で数倍〜1桁台後半になる。倍率はコーパスのブロック構成に依存する）。詳細は `detectContentDepthCap()` のJSDocを参照。 +CMSのブロック属性名が分からない・サイトごとに違う場合、`autoCapMainDepth` を使う。`
`/`role="main"`という標準タグを起点に、構造クラスタ数が急増する直前の深さをブロックごとに実データから自動検出して打ち切るため、サイト固有の設定が一切不要（既定はtrue。無効にするには `autoCapMainDepth: false` を渡す。実データ検証では`contentBlockAttribute`より良い結果になる場合もあった。計算コストは実測で数倍〜1桁台後半。倍率はコーパスのブロック構成に依存する）。詳細は `detectContentDepthCap()` のJSDocを参照。 + +ブロッキング後の各ブロックを個別に処理する Stage A に加え、`resolvePageClusterKeys()` はすべてのブロックを処理した後、ブロック境界を越えた Stage B（クロスブロック統合）を常時実行する。URLパス・スタイルシート集合が異なるだけで構造的に同一テンプレートと判定できるユニットを、クォーラムコア（メンバーページの80%以上が持つコーパス識別トークン）ベースの complete-linkage・包含・シェイプ類似度（クラス名剥きトークン）とL2シグネチャ比較（シェル裏付き）を組み合わせて不動点まで反復的に統合する（実コーパスで2〜5ラウンド収束）。詳細は `mergeCrossBlockClusters()` のJSDocを参照。 header/footer/nav/asideが一致するページ同士をさらに合流させたい場合は `mergeRareLandmarkClusters: true` を使う。ただし単純な一致判定は実データで過剰融合を招くことが分かっているため（header/footer/navは99%以上のページに存在し判別力を持たない）、コーパス全体で希少なランドマークバリアントが一致した場合に限り、より緩いコンテンツ類似度閾値（`landmarkGateSimilarityThreshold`）での合流を許可する（既定はfalse。実データでの検証は未実施で、合成フィクスチャでの単体・回帰テストのみ）。詳細・コスト特性は `mergeLandmarkAffinedClusters()` のJSDocを参照。 diff --git a/packages/@d-zero/page-cluster/src/assign-contained-clusters.spec.ts b/packages/@d-zero/page-cluster/src/assign-contained-clusters.spec.ts new file mode 100644 index 00000000..48fa08ba --- /dev/null +++ b/packages/@d-zero/page-cluster/src/assign-contained-clusters.spec.ts @@ -0,0 +1,116 @@ +import { describe, expect, test } from 'vitest'; + +import { assignContainedClusters } from './assign-contained-clusters.js'; + +describe('assignContainedClusters', () => { + test('single cluster maps to itself', () => { + const result = assignContainedClusters([ + { id: 0, tokens: new Set(['a', 'b', 'c']), pageCount: 3 }, + ]); + expect(result.get(0)).toBe(0); + }); + + test('empty cluster is never assigned (skipped in phase 1)', () => { + const result = assignContainedClusters([ + { id: 0, tokens: new Set(), pageCount: 0 }, + { id: 1, tokens: new Set(['a', 'b']), pageCount: 2 }, + ]); + expect(result.get(0)).toBe(0); + expect(result.get(1)).toBe(1); + }); + + test('X wholly contained by Y: X → Y, Y → itself', () => { + const result = assignContainedClusters([ + { id: 0, tokens: new Set(['a', 'b']), pageCount: 1 }, + { id: 1, tokens: new Set(['a', 'b', 'c', 'd']), pageCount: 5 }, + ]); + expect(result.get(0)).toBe(1); + expect(result.get(1)).toBe(1); + }); + + test('assignment is one-directional: Y does not absorb into X', () => { + // Containment(Y → X) = 2/4 = 0.5 < 0.9 → Y is NOT absorbed by X + const result = assignContainedClusters([ + { id: 0, tokens: new Set(['a', 'b']), pageCount: 1 }, + { id: 1, tokens: new Set(['a', 'b', 'c', 'd']), pageCount: 5 }, + ]); + expect(result.get(1)).toBe(1); // Y is NOT absorbed into X + }); + + test('containment below 0.9 is ignored', () => { + // 8 tokens shared out of 10 = 0.8 containment < 0.9 → no assignment + const xTokens = new Set(['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j']); + const yTokens = new Set(['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'x', 'y']); + const result = assignContainedClusters([ + { id: 0, tokens: xTokens, pageCount: 1 }, + { id: 1, tokens: yTokens, pageCount: 5 }, + ]); + expect(result.get(0)).toBe(0); + expect(result.get(1)).toBe(1); + }); + + test('hub pattern: X and Z both absorbed by hub, but not merged with each other', () => { + // Hub has all tokens; X ⊂ Hub, Z ⊂ Hub, but X and Z are disjoint + const result = assignContainedClusters([ + { id: 0, tokens: new Set(['a', 'b']), pageCount: 1 }, + { id: 1, tokens: new Set(['a', 'b', 'c', 'd']), pageCount: 5 }, + { id: 2, tokens: new Set(['c', 'd']), pageCount: 1 }, + ]); + expect(result.get(0)).toBe(1); // X → hub + expect(result.get(2)).toBe(1); // Z → hub + expect(result.get(1)).toBe(1); // hub → itself + // X and Z map to the same hub but are not connected to each other + }); + + test('chain A → B → C: all resolve to C', () => { + // A ⊂ B ⊂ C + const result = assignContainedClusters([ + { id: 0, tokens: new Set(['a']), pageCount: 1 }, + { id: 1, tokens: new Set(['a', 'b']), pageCount: 2 }, + { id: 2, tokens: new Set(['a', 'b', 'c', 'd']), pageCount: 3 }, + ]); + expect(result.get(0)).toBe(2); + expect(result.get(1)).toBe(2); + expect(result.get(2)).toBe(2); + }); + + test('cycle: mutual containment ≥ 0.9 → larger token set is root', () => { + // A containment in B = 10/10 = 1.0 → A → B + // B containment in A = 10/11 ≈ 0.909 → B → A + // Cycle detected; B has more tokens → B is root + const aTokens = new Set(['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j']); + const bTokens = new Set(['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k']); + const result = assignContainedClusters([ + { id: 0, tokens: aTokens, pageCount: 1 }, + { id: 1, tokens: bTokens, pageCount: 1 }, + ]); + expect(result.get(0)).toBe(1); + expect(result.get(1)).toBe(1); + }); + + test('cycle tie-break uses pageCount when token sizes are equal', () => { + // Both share 9/10 tokens each → containment 0.9 → mutual assignment → cycle + // A has 10 tokens, B has 10 tokens (equal); B has more pages → B wins + const sharedTokens = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i']; + const aTokens = new Set([...sharedTokens, 'unique-a']); + const bTokens = new Set([...sharedTokens, 'unique-b']); + const result = assignContainedClusters([ + { id: 0, tokens: aTokens, pageCount: 1 }, + { id: 1, tokens: bTokens, pageCount: 5 }, + ]); + // Both have 10 tokens; B has more pages → B is root + expect(result.get(0)).toBe(1); + expect(result.get(1)).toBe(1); + }); + + test('every id has an entry in the result map', () => { + const result = assignContainedClusters([ + { id: 0, tokens: new Set(['a']), pageCount: 1 }, + { id: 1, tokens: new Set(['b']), pageCount: 1 }, + { id: 2, tokens: new Set(['c']), pageCount: 1 }, + ]); + expect(result.has(0)).toBe(true); + expect(result.has(1)).toBe(true); + expect(result.has(2)).toBe(true); + }); +}); diff --git a/packages/@d-zero/page-cluster/src/assign-contained-clusters.ts b/packages/@d-zero/page-cluster/src/assign-contained-clusters.ts new file mode 100644 index 00000000..4ae8bf36 --- /dev/null +++ b/packages/@d-zero/page-cluster/src/assign-contained-clusters.ts @@ -0,0 +1,183 @@ +/** + * Minimum containment fraction for cluster X to be assigned into cluster Y. + * Containment is `|X ∩ Y| / |X|` — the fraction of X's tokens also present + * in Y's token set. At this threshold, X is "almost a subset" of Y. + * + * 0.9 was chosen because the observed containment scores for genuine + * template-subset relationships (e.g. a page missing one optional section) + * cluster tightly between 1.000 and 0.90–0.92 on real crawl data, while + * unrelated clusters score well below 0.8. A gap exists between ~0.92 and + * ~0.80 in practice, so the exact value within that gap is not sensitive. + */ +const CONTAINMENT_CUTOFF = 0.9; + +/** + * One cluster's entry for containment comparison. `tokens` is the union of + * comparison-set tokens for all pages in the cluster (Stage A uses + * frequency-narrowed comparison sets; Stage B uses quorum cores). + * `pageCount` is used only as a tiebreaker when containment and union size + * are tied between two candidate targets. + */ +export type ContainedClusterEntry = { + readonly id: number; + readonly tokens: ReadonlySet; + readonly pageCount: number; +}; + +/** + * Assigns each cluster to the "best" cluster that contains it (i.e., whose + * token set subsumes the cluster's token union at `>= CONTAINMENT_CUTOFF`). + * Returns a `Map` — clusters not assigned to anything map to + * themselves. + * + * This is a *directed* assignment (not union-find): X is absorbed by Y but Y + * is not absorbed by X, unless Y is independently assigned elsewhere too. + * This prevents hub-chaining: if /help/ is a structural superset of many + * clusters (because its HTML includes every nav variant), each of those + * clusters gets assigned to /help/, but they don't get merged with *each + * other* — only with /help/. Confirmed on real crawl data: a pure union-find + * approach produced a 9-cluster hub chain through one common-superset page. + * + * Principle: conditional rendering only *removes* elements from a template + * (an empty section, a missing paginator) — it never adds. So a + * conditionally-shorter page is always a structural subset of the + * full-featured template. Confirmed on real crawl data: a 43-page works + * cluster contained a 3-page outlier cluster at containment 1.000. + * + * Best target selection: highest containment → largest union size → most + * pages. Chain resolution and cycle breaking are applied after all raw + * assignments are computed (see the implementation). + * + * Cycles (mutual containment ≥ 0.9) mean the two clusters are practically + * identical token sets. The one with the larger token set (more pages as + * tiebreaker) becomes the root of the cycle. + * @param clusters + */ +export function assignContainedClusters( + clusters: readonly ContainedClusterEntry[], +): Map { + // Phase 1: find best raw assignment for each cluster + const raw = new Map(); + + for (const x of clusters) { + if (x.tokens.size === 0) continue; + + let bestTargetId = -1; + let bestContainment = CONTAINMENT_CUTOFF - 1e-9; + let bestUnionSize = 0; + let bestPageCount = 0; + + for (const y of clusters) { + if (y.id === x.id) continue; + + let intersection = 0; + for (const token of x.tokens) { + if (y.tokens.has(token)) intersection++; + } + const containment = intersection / x.tokens.size; + if (containment < CONTAINMENT_CUTOFF) continue; + + const unionSize = x.tokens.size + y.tokens.size - intersection; + if ( + containment > bestContainment || + (containment === bestContainment && unionSize > bestUnionSize) || + (containment === bestContainment && + unionSize === bestUnionSize && + y.pageCount > bestPageCount) + ) { + bestContainment = containment; + bestUnionSize = unionSize; + bestPageCount = y.pageCount; + bestTargetId = y.id; + } + } + + if (bestTargetId >= 0) { + raw.set(x.id, bestTargetId); + } + } + + // Phase 2: resolve chains and cycles + // Walk the raw assignment chain from each node; detect cycles by tracking + // the path walked so far. + const resolved = new Map(); + const idToEntry = new Map(clusters.map((cl) => [cl.id, cl])); + + for (const c of clusters) { + if (resolved.has(c.id)) continue; + + const path: number[] = []; + const pathSet = new Set(); + let current = c.id; + + // Walk until we reach a node with no further assignment or a cycle + while (!resolved.has(current)) { + const next = raw.get(current); + if (next === undefined) { + // No assignment → current is a root + resolved.set(current, current); + break; + } + if (pathSet.has(next)) { + // Cycle detected: find the cycle members and pick the root. + // Include `current` (the node that closed the back-edge) so it + // participates in root selection even if it has the largest token set. + const cycleStart = path.indexOf(next); + const cycleIds = [...path.slice(cycleStart), current]; + + // Root of the cycle: cluster with the largest token set + // (page count as tiebreaker) + let cycleRoot = cycleIds[0] ?? current; + for (const id of cycleIds) { + const bc = idToEntry.get(cycleRoot); + const cc = idToEntry.get(id); + const bcSize = bc?.tokens.size ?? 0; + const ccSize = cc?.tokens.size ?? 0; + if (ccSize > bcSize) { + cycleRoot = id; + } else if (ccSize === bcSize && (cc?.pageCount ?? 0) > (bc?.pageCount ?? 0)) { + cycleRoot = id; + } + } + + for (const id of cycleIds) { + resolved.set(id, cycleRoot); + } + // Everything before the cycle resolves to the cycle root too + for (const id of path.slice(0, cycleStart)) { + resolved.set(id, cycleRoot); + } + break; + } + if (resolved.has(next)) { + // Already resolved — propagate to everything in path + const root = resolved.get(next) ?? next; + resolved.set(current, root); + for (const id of path) { + resolved.set(id, root); + } + break; + } + path.push(current); + pathSet.add(current); + current = next; + } + + // If the path didn't resolve in the loop above, propagate what we know + if (!resolved.has(c.id) && resolved.has(current)) { + const root = resolved.get(current) ?? current; + for (const id of path) { + resolved.set(id, root); + } + } + } + + // Ensure every cluster id has an entry (fallback to self) + for (const c of clusters) { + if (!resolved.has(c.id)) { + resolved.set(c.id, c.id); + } + } + + return resolved; +} diff --git a/packages/@d-zero/page-cluster/src/auto-cut-threshold.spec.ts b/packages/@d-zero/page-cluster/src/auto-cut-threshold.spec.ts new file mode 100644 index 00000000..ff6b217c --- /dev/null +++ b/packages/@d-zero/page-cluster/src/auto-cut-threshold.spec.ts @@ -0,0 +1,38 @@ +import { describe, expect, test } from 'vitest'; + +import { autoCutThreshold } from './auto-cut-threshold.js'; + +describe('autoCutThreshold', () => { + test('empty array returns upperBound', () => { + expect(autoCutThreshold([], 0.8)).toBe(0.8); + }); + + test('single-element array returns upperBound', () => { + expect(autoCutThreshold([0.5], 0.8)).toBe(0.8); + }); + + test('all-equal heights return upperBound (no gap to measure)', () => { + expect(autoCutThreshold([0.5, 0.5, 0.5], 0.8)).toBe(0.8); + }); + + test('returns the midpoint of the largest gap', () => { + // sorted [0.9, 0.1]: gap = 0.8, midpoint = (0.9 + 0.1) / 2 = 0.5 + expect(autoCutThreshold([0.9, 0.1], 0.8)).toBe(0.5); + }); + + test('clamps midpoint to upperBound when midpoint exceeds it', () => { + // sorted [0.95, 0.85]: midpoint = 0.9, upperBound = 0.8 → clamped to 0.8 + expect(autoCutThreshold([0.95, 0.85], 0.8)).toBe(0.8); + }); + + test('returns midpoint unchanged when it is below upperBound', () => { + // sorted [0.6, 0.2]: 0.6 + 0.2 === 0.8 (exact in IEEE 754), midpoint = 0.4 + expect(autoCutThreshold([0.6, 0.2], 0.8)).toBe(0.4); + }); + + test('selects the largest gap, not the first', () => { + // sorted [0.9, 0.8, 0.2, 0.1]: gaps [0.1, 0.6, 0.1] — largest at index 1 + // midpoint = (0.8 + 0.2) / 2 = 0.5 + expect(autoCutThreshold([0.9, 0.8, 0.2, 0.1], 0.8)).toBe(0.5); + }); +}); diff --git a/packages/@d-zero/page-cluster/src/auto-cut-threshold.ts b/packages/@d-zero/page-cluster/src/auto-cut-threshold.ts new file mode 100644 index 00000000..a1fb2ff3 --- /dev/null +++ b/packages/@d-zero/page-cluster/src/auto-cut-threshold.ts @@ -0,0 +1,40 @@ +/** + * Finds the largest gap between adjacent merge heights and returns the + * midpoint of that gap as the cut threshold, clamped to `[0, upperBound]`. + * + * The clamp prevents the auto-cut from selecting a value *above* `upperBound` + * (the caller's intended default): this function only ever *loosens* the + * threshold relative to the default, never tightens it. Confirmed on real + * crawl data (8,936-page corpus): without the clamp, an 814-page block's + * auto-cut selected 0.952 — above the default 0.8 — and turned 46 clusters + * into 54; with the clamp it stays at 0.8 and the result is unchanged. + * + * Falls back to `upperBound` when the heights array has fewer than 2 entries + * (no gap to measure) or when all heights are equal (no gap exists). + * @param heights Merge heights from the dendrogram, in any order. + * @param upperBound Maximum allowed threshold (the caller's default). + */ +export function autoCutThreshold(heights: readonly number[], upperBound: number): number { + if (heights.length < 2) { + return upperBound; + } + + const sorted = [...heights].toSorted((a, b) => b - a); + + let maxGap = 0; + let gapIndex = 0; + for (let i = 0; i < sorted.length - 1; i++) { + const gap = (sorted[i] ?? 0) - (sorted[i + 1] ?? 0); + if (gap > maxGap) { + maxGap = gap; + gapIndex = i; + } + } + + if (maxGap === 0) { + return upperBound; + } + + const midpoint = ((sorted[gapIndex] ?? 0) + (sorted[gapIndex + 1] ?? 0)) / 2; + return Math.min(midpoint, upperBound); +} diff --git a/packages/@d-zero/page-cluster/src/collapse-anonymous-divs.spec.ts b/packages/@d-zero/page-cluster/src/collapse-anonymous-divs.spec.ts new file mode 100644 index 00000000..fc1622d2 --- /dev/null +++ b/packages/@d-zero/page-cluster/src/collapse-anonymous-divs.spec.ts @@ -0,0 +1,45 @@ +import { describe, expect, test } from 'vitest'; + +import { collapseAnonymousDivs } from './collapse-anonymous-divs.js'; + +describe('collapseAnonymousDivs', () => { + test('token with 2 segments is returned as-is', () => { + expect(collapseAnonymousDivs('body>div')).toBe('body>div'); + }); + + test('bare div in middle is removed', () => { + expect(collapseAnonymousDivs('body>div>section')).toBe('body>section'); + }); + + test('bare span in middle is removed', () => { + expect(collapseAnonymousDivs('body>span>section')).toBe('body>section'); + }); + + test('div with class in middle is preserved', () => { + // 'div.class' is not in FOLDABLE_TAGS as a whole string + expect(collapseAnonymousDivs('body>div.class>section')).toBe( + 'body>div.class>section', + ); + }); + + test('div with bracket in middle is preserved', () => { + // 'div[role=main]' is not in FOLDABLE_TAGS as a whole string + expect(collapseAnonymousDivs('body>div[role=main]>section')).toBe( + 'body>div[role=main]>section', + ); + }); + + test('bare div/span at first and last positions are always preserved', () => { + // only the middle segment 'main' is checked; first 'div' and last 'span' are untouched + expect(collapseAnonymousDivs('div>main>span')).toBe('div>main>span'); + }); + + test('non-foldable tag in middle is preserved', () => { + // 'main' and 'section' are not in FOLDABLE_TAGS + expect(collapseAnonymousDivs('body>main>section')).toBe('body>main>section'); + }); + + test('multiple bare divs in middle are all removed', () => { + expect(collapseAnonymousDivs('body>div>div>section')).toBe('body>section'); + }); +}); diff --git a/packages/@d-zero/page-cluster/src/collapse-anonymous-divs.ts b/packages/@d-zero/page-cluster/src/collapse-anonymous-divs.ts new file mode 100644 index 00000000..acf26490 --- /dev/null +++ b/packages/@d-zero/page-cluster/src/collapse-anonymous-divs.ts @@ -0,0 +1,46 @@ +import { FOLDABLE_TAGS } from './foldable-tags.js'; + +/** + * Only `FOLDABLE_TAGS` qualify — those are the tags `build-segment.ts` omits + * the name from when they have classes (producing `.c-x` instead of `div.c-x`), + * so a bare `div` or `span` with NO class is a structural-only wrapper. + * Non-foldable tags (`section`, `article`, …) always appear with their tag name + * and carry semantic meaning even when classless, so they are kept. + * @param segment + */ +function isAnonymousSegment(segment: string): boolean { + return FOLDABLE_TAGS.has(segment); +} + +/** + * Removes bare foldable-tag segments (e.g. plain `div`, `span`) from the + * interior of a path token, leaving the first and last segments untouched. + * Used before containment-assignment union construction so that two clusters + * whose paths differ only in intermediate anonymous wrapper `
`s are not + * spuriously separated. + * + * Intermediate bare `div`/`span` elements are anonymous wrappers with no + * class, role, or type — they carry no structural meaning beyond "something + * was nested here", and their presence varies freely across CMS themes and + * minor template revisions. Stripping them from the middle of a path keeps + * the union comparison focused on the meaningful skeleton (semantic tags, + * class-bearing wrappers, bracketed roles). + * + * First and last segments are always preserved: the first segment anchors the + * path in the document tree (`body`, `main`, …) and the last segment is the + * leaf element being tokenized — both carry meaning even when they are bare + * foldable tags. + * @param token A full path token, e.g. `body>main>div>section.c-x>div>.card`. + */ +export function collapseAnonymousDivs(token: string): string { + const segments = token.split('>'); + if (segments.length <= 2) { + return token; + } + + const first = segments[0] ?? ''; + const last = segments.at(-1) ?? ''; + const middle = segments.slice(1, -1).filter((seg) => !isAnonymousSegment(seg)); + + return [first, ...middle, last].join('>'); +} diff --git a/packages/@d-zero/page-cluster/src/complete-linkage-dendrogram.spec.ts b/packages/@d-zero/page-cluster/src/complete-linkage-dendrogram.spec.ts new file mode 100644 index 00000000..fb06f5e2 --- /dev/null +++ b/packages/@d-zero/page-cluster/src/complete-linkage-dendrogram.spec.ts @@ -0,0 +1,111 @@ +import { describe, expect, test } from 'vitest'; + +import { + completeLinkageDendrogram, + labelsAtThreshold, +} from './complete-linkage-dendrogram.js'; + +describe('completeLinkageDendrogram', () => { + test('empty input returns empty merge list', () => { + expect(completeLinkageDendrogram([])).toEqual([]); + }); + + test('single item returns empty merge list', () => { + expect(completeLinkageDendrogram([new Set(['a'])])).toEqual([]); + }); + + test('two identical items produce one merge at height 1', () => { + const merges = completeLinkageDendrogram([new Set(['a', 'b']), new Set(['a', 'b'])]); + expect(merges).toHaveLength(1); + expect(merges[0]?.height).toBe(1); + }); + + test('two disjoint items produce one merge at height 0', () => { + const merges = completeLinkageDendrogram([new Set(['a']), new Set(['b'])]); + expect(merges).toHaveLength(1); + expect(merges[0]?.height).toBe(0); + }); + + test('n items always produce exactly n-1 merges', () => { + const sets = Array.from({ length: 5 }, (_, i) => new Set([`unique-${i}`])); + expect(completeLinkageDendrogram(sets)).toHaveLength(4); + }); + + test('survivor index is always less than dead index', () => { + const merges = completeLinkageDendrogram([ + new Set(['a', 'b']), + new Set(['b', 'c']), + new Set(['c', 'd']), + ]); + for (const m of merges) { + expect(m.survivor).toBeLessThan(m.dead); + } + }); + + test('all merge heights are in [0, 1]', () => { + const merges = completeLinkageDendrogram([ + new Set(['a', 'b', 'c']), + new Set(['a', 'b']), + new Set(['x', 'y']), + ]); + for (const m of merges) { + expect(m.height).toBeGreaterThanOrEqual(0); + expect(m.height).toBeLessThanOrEqual(1); + } + }); + + test('high-similarity pair merges before low-similarity pair', () => { + // items 0 and 1 share 2/2 tokens (Jaccard=1); item 2 shares none + const merges = completeLinkageDendrogram([ + new Set(['a', 'b']), + new Set(['a', 'b']), + new Set(['x', 'y']), + ]); + // First merge should be 0-1 at height 1, then that cluster merges with 2 at height 0 + expect(merges[0]?.height).toBe(1); + expect(merges[1]?.height).toBe(0); + }); +}); + +describe('labelsAtThreshold', () => { + test('empty merge list returns each index as its own root', () => { + expect(labelsAtThreshold(3, [], 0.8)).toEqual([0, 1, 2]); + }); + + test('threshold 0: all items in one cluster', () => { + const merges = completeLinkageDendrogram([ + new Set(['a']), + new Set(['b']), + new Set(['c']), + ]); + const labels = labelsAtThreshold(3, merges, 0); + expect(new Set(labels).size).toBe(1); + }); + + test('threshold above all heights: every item is its own singleton', () => { + // Each item has unique tokens → all Jaccard = 0 → all heights = 0 < 1 + const merges = completeLinkageDendrogram([ + new Set(['a']), + new Set(['b']), + new Set(['c']), + ]); + const labels = labelsAtThreshold(3, merges, 1); + expect(new Set(labels).size).toBe(3); + }); + + test('identical items merge, disjoint stays separate at threshold 0.8', () => { + const merges = completeLinkageDendrogram([ + new Set(['a', 'b']), + new Set(['a', 'b']), + new Set(['c', 'd']), + ]); + const labels = labelsAtThreshold(3, merges, 0.8); + expect(labels[0]).toBe(labels[1]); // identical → merged at height 1 + expect(labels[0]).not.toBe(labels[2]); // disjoint → separate (height 0 < 0.8) + }); + + test('output length matches size parameter', () => { + const merges = completeLinkageDendrogram([new Set(['a']), new Set(['b'])]); + expect(labelsAtThreshold(2, merges, 0.5)).toHaveLength(2); + }); +}); diff --git a/packages/@d-zero/page-cluster/src/complete-linkage-dendrogram.ts b/packages/@d-zero/page-cluster/src/complete-linkage-dendrogram.ts new file mode 100644 index 00000000..315f34ac --- /dev/null +++ b/packages/@d-zero/page-cluster/src/complete-linkage-dendrogram.ts @@ -0,0 +1,180 @@ +import { jaccardSimilarity } from './jaccard-similarity.js'; + +/** + * One merge event in the complete-linkage dendrogram. `survivor` is the + * lower index (kept active), `dead` the higher (deactivated), `height` + * is the Jaccard similarity at which the merge occurred. All `n - 1` + * merges for `n` input sets are recorded, including those below any + * particular threshold — the caller decides which heights are meaningful + * via `labelsAtThreshold`. + */ +export type DendrogramMerge = { + readonly survivor: number; + readonly dead: number; + readonly height: number; +}; + +/** + * `jaccardSimilarity()` returns a floating-point division that can land a + * hair below the caller's threshold even when mathematically equal. Same + * epsilon as `resolve-structural-cluster-keys.ts`. + */ +const BOUNDARY_EPSILON = 1e-9; + +/** + * `noUncheckedIndexedAccess` makes indexed reads return `T | undefined`. + * Call sites here index positions this function itself generated (NN-chain + * internal arrays), so the throw branch is unreachable in practice — it exists + * solely to satisfy the compiler without a non-null assertion everywhere. + * @param values + * @param index + */ +function requireIndex(values: ArrayLike, index: number): T { + const value = values[index]; + if (value === undefined) { + throw new Error('completeLinkageDendrogram: index out of bounds'); + } + return value; +} + +/** + * Path-compressed union-find root lookup. `labelsAtThreshold` rebuilds the + * cluster label for each item by following its parent chain to the root; + * path compression keeps repeated lookups O(α) instead of O(n). + * @param parent + * @param index + */ +function find(parent: Int32Array, index: number): number { + let root = index; + while (requireIndex(parent, root) !== root) { + root = requireIndex(parent, root); + } + let current = index; + while (current !== root) { + const next = requireIndex(parent, current); + parent[current] = root; + current = next; + } + return root; +} + +/** + * Computes the complete-linkage dendrogram for `tokenSets` via the NN-chain + * algorithm (Murtagh, F., 1983) and returns all `n - 1` merge events in + * discovery order. This is the same O(n²) NN-chain implementation as + * `resolve-structural-cluster-keys.ts` — extracted here so that callers can + * record the full height sequence and apply an arbitrary threshold cut via + * `labelsAtThreshold`, rather than committing to a fixed threshold upfront. + * + * Recording all merges (not just threshold-clearing ones) is essential for + * `autoCutThreshold`: to find the largest gap in the height sequence, every + * height must be available regardless of where the eventual cut lands. + * @param tokenSets + */ +export function completeLinkageDendrogram( + tokenSets: readonly ReadonlySet[], +): DendrogramMerge[] { + const size = tokenSets.length; + if (size <= 1) return []; + + const similarity = new Float64Array(size * size); + for (let i = 0; i < size; i++) { + for (let j = i + 1; j < size; j++) { + const score = jaccardSimilarity( + requireIndex(tokenSets, i), + requireIndex(tokenSets, j), + ); + similarity[i * size + j] = score; + similarity[j * size + i] = score; + } + } + + const active = new Uint8Array(size).fill(1); + const chain: number[] = []; + const merges: DendrogramMerge[] = []; + + const findFreshStart = (): number => { + for (let index = 0; index < size; index++) { + if (requireIndex(active, index) === 1) return index; + } + throw new Error('completeLinkageDendrogram: no active cluster left to resume from'); + }; + + let activeCount = size; + while (activeCount > 1) { + if (chain.length === 0) chain.push(findFreshStart()); + + const top = requireIndex(chain, chain.length - 1); + let best = -1; + let bestScore = Number.NEGATIVE_INFINITY; + for (let candidate = 0; candidate < size; candidate++) { + if (candidate !== top && requireIndex(active, candidate) === 1) { + const score = requireIndex(similarity, top * size + candidate); + if (score > bestScore) { + bestScore = score; + best = candidate; + } + } + } + + const secondFromTop = chain.length >= 2 ? chain.at(-2) : undefined; + if (best === secondFromTop) { + chain.pop(); + chain.pop(); + + const survivor = Math.min(top, best); + const dead = Math.max(top, best); + for (let candidate = 0; candidate < size; candidate++) { + if ( + candidate !== top && + candidate !== best && + requireIndex(active, candidate) === 1 + ) { + const merged = Math.min( + requireIndex(similarity, top * size + candidate), + requireIndex(similarity, best * size + candidate), + ); + similarity[survivor * size + candidate] = merged; + similarity[candidate * size + survivor] = merged; + } + } + + active[dead] = 0; + merges.push({ survivor, dead, height: bestScore }); + activeCount--; + } else { + chain.push(best); + } + } + + return merges; +} + +/** + * Reconstructs cluster labels for all `size` original items at a given + * similarity threshold, using the merge list from `completeLinkageDendrogram`. + * + * Lance-Williams monotonicity guarantees that only merges at `>= threshold` + * need to be applied to reconstruct the correct threshold cut, regardless of + * the order in which NN-chain discovered them. This is the same invariant + * documented in `resolve-structural-cluster-keys.ts`'s JSDoc for + * `clusterByCompleteLinkage`. + * @param size Total number of items (= length of the original `tokenSets`). + * @param merges All merge events from `completeLinkageDendrogram`. + * @param threshold Similarity threshold; merges below this are ignored. + */ +export function labelsAtThreshold( + size: number, + merges: readonly DendrogramMerge[], + threshold: number, +): number[] { + const parent = Int32Array.from({ length: size }, (_, i) => i); + + for (const { survivor, dead, height } of merges) { + if (height >= threshold - BOUNDARY_EPSILON) { + parent[find(parent, dead)] = find(parent, survivor); + } + } + + return Array.from({ length: size }, (_, i) => find(parent, i)); +} diff --git a/packages/@d-zero/page-cluster/src/derive-comparison-sets.spec.ts b/packages/@d-zero/page-cluster/src/derive-comparison-sets.spec.ts new file mode 100644 index 00000000..b57aa966 --- /dev/null +++ b/packages/@d-zero/page-cluster/src/derive-comparison-sets.spec.ts @@ -0,0 +1,30 @@ +import { describe, expect, test } from 'vitest'; + +import { deriveComparisonSets } from './derive-comparison-sets.js'; + +describe('deriveComparisonSets', () => { + test('fewer than 10 pages returns the exact same array reference (no filtering)', () => { + const tokenSets = Array.from({ length: 9 }, (_, i) => new Set([`token-${i}`])); + expect(deriveComparisonSets(tokenSets)).toBe(tokenSets); + }); + + test('at 10 pages, tokens present in all pages are removed as chrome', () => { + // 'common' appears in all 10 pages → frequency 10/10 → chrome → stripped + // each 'page-N' appears only once → content → kept + const tokenSets = Array.from( + { length: 10 }, + (_, i) => new Set(['common', `page-${i}`]), + ); + const result = deriveComparisonSets(tokenSets); + expect(result[0]).toEqual(new Set(['page-0'])); + expect(result[9]).toEqual(new Set(['page-9'])); + }); + + test('falls back to original token set when all tokens resolve to chrome', () => { + // every page has only 'shared-only' → frequency 10/10 → chrome + // contentTokens = {} (size 0), tokens.size > 0 → fallback to original set + const tokenSets = Array.from({ length: 10 }, () => new Set(['shared-only'])); + const result = deriveComparisonSets(tokenSets); + expect(result[0]).toEqual(new Set(['shared-only'])); + }); +}); diff --git a/packages/@d-zero/page-cluster/src/derive-comparison-sets.ts b/packages/@d-zero/page-cluster/src/derive-comparison-sets.ts new file mode 100644 index 00000000..d4283649 --- /dev/null +++ b/packages/@d-zero/page-cluster/src/derive-comparison-sets.ts @@ -0,0 +1,38 @@ +import { computeDocumentFrequency } from './compute-document-frequency.js'; +import { splitTokensByFrequency } from './split-tokens-by-frequency.js'; + +/** + * Below this many pages, `computeDocumentFrequency`/`splitTokensByFrequency` + * (the default 90% cutoff) degenerate rather than usefully separate chrome + * from content — see `deriveComparisonSets` for the failure mode. Derived + * from `splitTokensByFrequency`'s own cutoff: a token missing from exactly + * one page out of `n` still counts as chrome only if + * `(n - 1) / n >= 0.9`, i.e. `n >= 10`. Below that, the caller should fall + * back to comparing token sets directly, unfiltered. + */ +export const MIN_PAGE_COUNT_FOR_FREQUENCY_SPLIT = 10; + +/** + * Narrows each page's token set to its page-specific content before + * clustering, so pages that only share site-wide chrome (header/nav/footer) + * don't read as more similar than they structurally are. Skipped below + * `MIN_PAGE_COUNT_FOR_FREQUENCY_SPLIT` pages — see that constant's JSDoc. + * + * A page whose entire token set narrows away falls back to its raw tokens + * rather than the empty result, for the same reason documented in + * `resolve-structural-cluster-keys.ts`. + * @param tokenSets + */ +export function deriveComparisonSets( + tokenSets: readonly ReadonlySet[], +): readonly ReadonlySet[] { + if (tokenSets.length < MIN_PAGE_COUNT_FOR_FREQUENCY_SPLIT) { + return tokenSets; + } + + const corpusFrequency = computeDocumentFrequency(tokenSets); + return tokenSets.map((tokens) => { + const { contentTokens } = splitTokensByFrequency(tokens, corpusFrequency); + return contentTokens.size === 0 && tokens.size > 0 ? tokens : contentTokens; + }); +} diff --git a/packages/@d-zero/page-cluster/src/merge-cross-block-clusters.spec.ts b/packages/@d-zero/page-cluster/src/merge-cross-block-clusters.spec.ts new file mode 100644 index 00000000..20cfd739 --- /dev/null +++ b/packages/@d-zero/page-cluster/src/merge-cross-block-clusters.spec.ts @@ -0,0 +1,196 @@ +import type { CrossBlockUnit } from './merge-cross-block-clusters.js'; + +import { describe, expect, test } from 'vitest'; + +import { mergeCrossBlockClusters } from './merge-cross-block-clusters.js'; + +const noLandmarks = { header: null, nav: null, footer: null, remainderHtml: '' }; + +describe('mergeCrossBlockClusters', () => { + test('empty input returns empty map', () => { + expect(mergeCrossBlockClusters([], {})).toEqual(new Map()); + }); + + test('single unit maps to itself', () => { + const unit: CrossBlockUnit = { + key: 'k1', + memberTokenSets: [new Set(['body>main>.card'])], + memberLandmarks: [noLandmarks], + }; + const result = mergeCrossBlockClusters([unit], {}); + expect(result.get('k1')).toBe('k1'); + }); + + test('two units with identical token sets merge into one', () => { + const tokens = new Set(['body>main>.card', 'body>main>.title', 'body>main>.body']); + const unit1: CrossBlockUnit = { + key: 'k1', + memberTokenSets: [tokens], + memberLandmarks: [noLandmarks], + }; + const unit2: CrossBlockUnit = { + key: 'k2', + memberTokenSets: [tokens], + memberLandmarks: [noLandmarks], + }; + const result = mergeCrossBlockClusters([unit1, unit2], {}); + expect(result.get('k1')).toBe(result.get('k2')); + }); + + test('two units with disjoint token sets stay separate', () => { + const unit1: CrossBlockUnit = { + key: 'k1', + memberTokenSets: [ + new Set(['body>main>article', 'body>main>article>h1', 'body>main>article>p']), + ], + memberLandmarks: [noLandmarks], + }; + const unit2: CrossBlockUnit = { + key: 'k2', + memberTokenSets: [ + new Set(['body>aside>section', 'body>aside>section>ul', 'body>footer>nav']), + ], + memberLandmarks: [noLandmarks], + }; + const result = mergeCrossBlockClusters([unit1, unit2], {}); + expect(result.get('k1')).toBe('k1'); + expect(result.get('k2')).toBe('k2'); + }); + + test('multi-page units with same shape but different class names merge via shape-Jaccard', () => { + // Two 2-page units: structurally identical skeleton, different BEM class names. + // Class-name Jaccard = 0; shape Jaccard = 1.0 → should merge. + const unit1: CrossBlockUnit = { + key: 'reports', + memberTokenSets: [ + new Set([ + 'body>main>section.c-reports', + 'body>main>section.c-reports>ul.c-reports__list', + ]), + new Set([ + 'body>main>section.c-reports', + 'body>main>section.c-reports>ul.c-reports__list', + ]), + ], + memberLandmarks: [noLandmarks, noLandmarks], + }; + const unit2: CrossBlockUnit = { + key: 'projects', + memberTokenSets: [ + new Set([ + 'body>main>section.c-projects', + 'body>main>section.c-projects>ul.c-projects__list', + ]), + new Set([ + 'body>main>section.c-projects', + 'body>main>section.c-projects>ul.c-projects__list', + ]), + ], + memberLandmarks: [noLandmarks, noLandmarks], + }; + const result = mergeCrossBlockClusters([unit1, unit2], {}); + expect(result.get('reports')).toBe(result.get('projects')); + }); + + test('single-page units are excluded from shape-Jaccard comparison', () => { + // Two 1-page units with same skeleton but different classes stay separate — + // SHAPE_MIN_PAGES = 2, so 1-page units never participate in shape merge. + const unit1: CrossBlockUnit = { + key: 'solo-a', + memberTokenSets: [ + new Set(['body>main>section.type-a', 'body>main>section.type-a>p']), + ], + memberLandmarks: [noLandmarks], + }; + const unit2: CrossBlockUnit = { + key: 'solo-b', + memberTokenSets: [ + new Set(['body>main>section.type-b', 'body>main>section.type-b>p']), + ], + memberLandmarks: [noLandmarks], + }; + const result = mergeCrossBlockClusters([unit1, unit2], {}); + expect(result.get('solo-a')).toBe('solo-a'); + expect(result.get('solo-b')).toBe('solo-b'); + }); + + test('result map has an entry for every input unit key', () => { + const unit1: CrossBlockUnit = { + key: 'a', + memberTokenSets: [new Set(['x'])], + memberLandmarks: [noLandmarks], + }; + const unit2: CrossBlockUnit = { + key: 'b', + memberTokenSets: [new Set(['y'])], + memberLandmarks: [noLandmarks], + }; + const unit3: CrossBlockUnit = { + key: 'c', + memberTokenSets: [new Set(['z'])], + memberLandmarks: [noLandmarks], + }; + const result = mergeCrossBlockClusters([unit1, unit2, unit3], {}); + expect(result.has('a')).toBe(true); + expect(result.has('b')).toBe(true); + expect(result.has('c')).toBe(true); + }); + + test('result is deterministic: same input twice produces identical output', () => { + const tokens = new Set(['body>main>.card', 'body>main>.title']); + const unit1: CrossBlockUnit = { + key: 'k1', + memberTokenSets: [tokens], + memberLandmarks: [noLandmarks], + }; + const unit2: CrossBlockUnit = { + key: 'k2', + memberTokenSets: [tokens], + memberLandmarks: [noLandmarks], + }; + const r1 = mergeCrossBlockClusters([unit1, unit2], {}); + const r2 = mergeCrossBlockClusters([unit1, unit2], {}); + expect(r1.get('k1')).toBe(r2.get('k1')); + expect(r1.get('k2')).toBe(r2.get('k2')); + }); + + test('merged result preserves an existing unit key (no freshly invented key)', () => { + const tokens = new Set(['body>main>.card', 'body>main>.title', 'body>main>.meta']); + const unit1: CrossBlockUnit = { + key: 'block-a', + memberTokenSets: [tokens], + memberLandmarks: [noLandmarks], + }; + const unit2: CrossBlockUnit = { + key: 'block-b', + memberTokenSets: [tokens], + memberLandmarks: [noLandmarks], + }; + const result = mergeCrossBlockClusters([unit1, unit2], {}); + const rootKey = result.get('block-a')!; + expect(rootKey === 'block-a' || rootKey === 'block-b').toBe(true); + }); + + test('units with landmarks merge when shells also match', () => { + // Both units have the same landmark header → shell corroboration passes + const landmarks = { + header: '
', + nav: null, + footer: null, + remainderHtml: '', + }; + const tokens = new Set(['body>main>.content']); + const unit1: CrossBlockUnit = { + key: 'l1', + memberTokenSets: [tokens], + memberLandmarks: [landmarks], + }; + const unit2: CrossBlockUnit = { + key: 'l2', + memberTokenSets: [tokens], + memberLandmarks: [landmarks], + }; + const result = mergeCrossBlockClusters([unit1, unit2], {}); + expect(result.get('l1')).toBe(result.get('l2')); + }); +}); diff --git a/packages/@d-zero/page-cluster/src/merge-cross-block-clusters.ts b/packages/@d-zero/page-cluster/src/merge-cross-block-clusters.ts new file mode 100644 index 00000000..7fa66478 --- /dev/null +++ b/packages/@d-zero/page-cluster/src/merge-cross-block-clusters.ts @@ -0,0 +1,532 @@ +import type { ExtractLandmarksResult } from './extract-landmarks.js'; + +import { assignContainedClusters } from './assign-contained-clusters.js'; +import { collapseAnonymousDivs } from './collapse-anonymous-divs.js'; +import { + completeLinkageDendrogram, + labelsAtThreshold, +} from './complete-linkage-dendrogram.js'; +import { computeDocumentFrequency } from './compute-document-frequency.js'; +import { jaccardSimilarity } from './jaccard-similarity.js'; +import { shapeToken } from './shape-token.js'; +import { splitTokensByFrequency } from './split-tokens-by-frequency.js'; +import { tokenize } from './tokenize.js'; + +/** + * One cluster (post-Stage-A) entering cross-block comparison. + */ +export type CrossBlockUnit = { + readonly key: string; + readonly memberTokenSets: readonly ReadonlySet[]; + readonly memberLandmarks: readonly ExtractLandmarksResult[]; +}; + +/** + * Fixed complete-linkage threshold for the cross-block fine stage. + * + * Not auto-cut: cross-block units are few (typically 10–100 for a whole + * site), so the merge-height distribution is too sparse for max-gap detection + * to produce a reliable cut. Confirmed on real crawl data: without this fixed + * floor, auto-cut selected 0.045 on an 18-unit corpus, causing spurious + * micro-merges. + */ +const CROSS_BLOCK_THRESHOLD = 0.8; + +/** + * Quorum fraction: a token must be present in at least this fraction of a + * unit's member pages to enter the unit's core. + * + * Strict intersection degenerates: a unit of 89 articles sharing only one + * common distinctive token produces jaccard 1.0 with everything — confirmed + * on real crawl data. Full union is shell-dominated: 298 pages collapsed into + * 4 clusters — also confirmed. 80% quorum avoids both failure modes. + */ +const QUORUM_FRACTION = 0.8; + +/** + * Shape-based Jaccard threshold for "same skeleton, different class names". + * Class-name Jaccard for reports/projects/news list pages: 0.000; shape + * Jaccard: 1.000 on real crawl data. + */ +const SHAPE_JACCARD_THRESHOLD = 0.9; + +/** + * Minimum member-page count for a unit to participate in shape-Jaccard + * comparison. Single-page units are excluded because their quorum core + * equals their raw token set with no frequency filtering — any two 1-page + * units with the same tag skeleton but completely different content will + * shape-merge spuriously. Multi-page units produce quorum cores that + * reflect a shared template rather than individual page noise, so shape + * comparison there is meaningful. + */ +const SHAPE_MIN_PAGES = 2; + +/** + * L2-stage shell corroboration threshold. Prevents cross-microsite false + * merges: a microsite with a different shell (header/nav/footer) from the + * main site would otherwise merge via L2 alone. Confirmed on real crawl + * data: two false merges blocked, correct merges (same shell) unaffected. + */ +const SHELL_CORROBORATION_THRESHOLD = 0.8; + +/** + * Maximum cross-block merge rounds. Real crawl data converged in ≤ 7 rounds + * on the two validation corpora (302 pages / 8,936 pages). + */ +const MAX_ROUNDS = 10; + +/** + * Segments carrying no structural information at L2 resolution. + * Tokens whose only non-`main` content segments are all generic are excluded + * from L2 signatures as uninformative. + */ +const GENERIC_SEGMENTS = new Set([ + 'div', + 'span', + '*', + 'script', + 'noscript', + 'style', + 'iframe', + 'a', + 'br', + 'img', + 'picture', + 'source', +]); + +// --------------------------------------------------------------------------- +// Internal helpers +// --------------------------------------------------------------------------- + +/** + * + * @param memberDistinctiveTokens + */ +function quorumCore( + memberDistinctiveTokens: readonly ReadonlySet[], +): ReadonlySet { + const n = memberDistinctiveTokens.length; + if (n === 0) return new Set(); + + const minCount = Math.ceil(QUORUM_FRACTION * n); + const tokenCount = new Map(); + for (const tokens of memberDistinctiveTokens) { + for (const token of tokens) { + tokenCount.set(token, (tokenCount.get(token) ?? 0) + 1); + } + } + + const core = new Set(); + for (const [token, count] of tokenCount) { + if (count >= minCount) core.add(token); + } + + if (core.size > 0) return core; + + // Fallback: union of all distinctive tokens (happens for very small units) + const union = new Set(); + for (const tokens of memberDistinctiveTokens) { + for (const t of tokens) union.add(t); + } + return union; +} + +/** + * + * @param core + */ +function shapedCoreSet(core: ReadonlySet): Set { + const shaped = new Set(); + for (const token of core) shaped.add(shapeToken(token)); + return shaped; +} + +/** + * + * @param core + */ +function l2Signature(core: ReadonlySet): Map | null { + const counts = new Map(); + for (const token of core) { + const shaped = shapeToken(token); + const segments = shaped.split('>'); + const mainIdx = segments.findIndex( + (s) => s === 'main' || s.startsWith('main[') || s.startsWith('main.'), + ); + if (mainIdx === -1) continue; + + // Take main + up to 2 levels after it + const truncated = segments.slice(mainIdx, mainIdx + 3); + const contentSegments = truncated.slice(1); + + // Skip if all content segments are generic (or none exist) + if (contentSegments.every((s) => GENERIC_SEGMENTS.has(s))) { + continue; + } + + const key = truncated.join('>'); + counts.set(key, (counts.get(key) ?? 0) + 1); + } + return counts.size > 0 ? counts : null; +} + +/** + * + * @param xSig + * @param ySig + */ +function l2Contained(xSig: Map, ySig: Map): boolean { + for (const [key, xCount] of xSig) { + if (xCount > (ySig.get(key) ?? 0)) return false; + } + return true; +} + +/** + * + * @param landmarks + */ +function shellTokenSet(landmarks: ExtractLandmarksResult): ReadonlySet { + const headerHtml = landmarks.header ?? ''; + const navHtml = landmarks.nav ?? ''; + const footerHtml = landmarks.footer ?? ''; + // Guard: if no landmark content exists, return empty rather than tokenizing + // `` which produces `['body']` — a non-empty set that would + // make jaccardSimilarity({body},{body}) = 1 and spuriously pass the shell + // corroboration check for any two pages with no landmarks at all. + if (!headerHtml && !navHtml && !footerHtml) return new Set(); + return new Set(tokenize(`${headerHtml}${navHtml}${footerHtml}`).tokens); +} + +/** + * + * @param memberLandmarks + */ +function shellQuorum( + memberLandmarks: readonly ExtractLandmarksResult[], +): ReadonlySet { + const n = memberLandmarks.length; + if (n === 0) return new Set(); + + const minCount = Math.ceil(QUORUM_FRACTION * n); + const tokenCount = new Map(); + for (const lm of memberLandmarks) { + for (const token of shellTokenSet(lm)) { + tokenCount.set(token, (tokenCount.get(token) ?? 0) + 1); + } + } + + const quorum = new Set(); + for (const [token, count] of tokenCount) { + if (count >= minCount) quorum.add(token); + } + return quorum; +} + +// --------------------------------------------------------------------------- +// Main function +// --------------------------------------------------------------------------- + +/** + * Merges cross-block clusters (Stage B) via recursive quorum-core comparison. + * + * Returns a `Map` from each input unit's `key` to its final root key. Units + * not absorbed into any other unit map to themselves. + * + * Three merge mechanisms run per round, in order: + * 1. **Fine stage** — complete-linkage at `CROSS_BLOCK_THRESHOLD` on quorum + * cores, then containment assignment (0.9), then shape-Jaccard (0.9) for + * class-name-only differences. + * 2. **L2 stage** (only when fine found nothing) — multiset containment on + * `main`-anchored 2-level shape signatures, with shell corroboration + * (header+nav+footer quorum Jaccard ≥ 0.8) required. + * + * Rounds continue until neither stage finds anything, or `MAX_ROUNDS` is hit. + * Each round re-derives quorum cores from pooled members of merged units. + * + * Why quorum cores instead of strict intersection or full union: + * strict intersection degenerated on real crawl data (89 articles → 1 shared + * distinctive token → jaccard 1.0 false merges). Full union was shell-dominated + * (298-page avalanche into 4 clusters). Both failure modes are documented in + * `@d-zero/page-cluster` source JSDoc; quorum 80% + page-frequency shell + * removal was validated on two real crawl corpora. + * @param units Post-Stage-A clusters. + * @param options Forwarded `similarityThreshold` (defaults to 0.8). + * @param options.similarityThreshold + */ +export function mergeCrossBlockClusters( + units: readonly CrossBlockUnit[], + options?: { similarityThreshold?: number }, +): Map { + if (units.length <= 1) { + return new Map(units.map((u) => [u.key, u.key])); + } + + const threshold = options?.similarityThreshold ?? CROSS_BLOCK_THRESHOLD; + + // Mutable group state: rootKey → combined member arrays + type GroupMembers = { + tokenSets: ReadonlySet[]; + landmarks: ExtractLandmarksResult[]; + }; + + const groups = new Map(); + for (const unit of units) { + groups.set(unit.key, { + tokenSets: [...unit.memberTokenSets], + landmarks: [...unit.memberLandmarks], + }); + } + + // Maps every original key to its current root (updated on each merge) + const keyToRoot = new Map(units.map((u) => [u.key, u.key])); + + /** + * Applies a list of [absorbed, root] merges to `groups` and `keyToRoot`. + * All absorbed groups' members are folded into their respective roots. + * @param merges + */ + function applyMerges(merges: readonly [string, string][]): void { + for (const [absorbed, root] of merges) { + const absorbedG = groups.get(absorbed); + const rootG = groups.get(root); + if (!absorbedG || !rootG) continue; + + rootG.tokenSets = [...rootG.tokenSets, ...absorbedG.tokenSets]; + rootG.landmarks = [...rootG.landmarks, ...absorbedG.landmarks]; + groups.delete(absorbed); + + for (const [origKey, cur] of keyToRoot) { + if (cur === absorbed) keyToRoot.set(origKey, root); + } + keyToRoot.set(absorbed, root); + } + } + + for (let round = 0; round < MAX_ROUNDS; round++) { + const groupKeys = [...groups.keys()]; + const n = groupKeys.length; + if (n <= 1) break; + + // --------------------------------------------------------------- + // Compute corpus distinctive tokens (page-frequency shell removal) + // --------------------------------------------------------------- + const allPageTokenSets = groupKeys.flatMap((k) => groups.get(k)!.tokenSets); + const corpusFrequency = computeDocumentFrequency(allPageTokenSets); + + const groupDistinctive = new Map[]>(); + for (const key of groupKeys) { + const g = groups.get(key)!; + const dist: ReadonlySet[] = []; + for (const tokens of g.tokenSets) { + const { contentTokens } = splitTokensByFrequency(tokens, corpusFrequency); + dist.push(contentTokens.size > 0 ? contentTokens : tokens); + } + groupDistinctive.set(key, dist); + } + + // Quorum core per group + const cores = new Map>(); + for (const key of groupKeys) { + cores.set(key, quorumCore(groupDistinctive.get(key) ?? [])); + } + + // --------------------------------------------------------------- + // Fine stage: union-find over group indices + // --------------------------------------------------------------- + const parent = Array.from({ length: n }, (_, i) => i); + const ufFind = (x: number): number => { + let r = x; + while (parent[r] !== r) r = parent[r]!; + let c = x; + while (c !== r) { + const next = parent[c]!; + parent[c] = r; + c = next; + } + return r; + }; + const ufUnion = (a: number, b: number): void => { + const ra = ufFind(a); + const rb = ufFind(b); + if (ra !== rb) parent[rb] = ra; // lower index wins + }; + + const coreSets = groupKeys.map( + (k) => cores.get(k) ?? (new Set() as ReadonlySet), + ); + + // Step 1: CL merges + const dendrogram = completeLinkageDendrogram(coreSets); + const clLabels = labelsAtThreshold(n, dendrogram, threshold); + for (let i = 0; i < n; i++) { + const r = clLabels[i]; + if (r !== undefined && r !== i) ufUnion(r, i); + } + + // Step 2: Containment on the current union-find clusters + // Build union token set per UF cluster + const clusterUnion = new Map>(); + const clusterPageCount = new Map(); + for (let i = 0; i < n; i++) { + const r = ufFind(i); + let u = clusterUnion.get(r); + if (!u) { + u = new Set(); + clusterUnion.set(r, u); + } + for (const t of coreSets[i] ?? []) u.add(collapseAnonymousDivs(t)); + clusterPageCount.set( + r, + (clusterPageCount.get(r) ?? 0) + + (groups.get(groupKeys[i] ?? '')?.tokenSets.length ?? 0), + ); + } + const contEntries = [...clusterUnion.entries()].map(([id, tokens]) => ({ + id, + tokens: tokens as ReadonlySet, + pageCount: clusterPageCount.get(id) ?? 0, + })); + const contResult = assignContainedClusters(contEntries); + // Apply containment assignment: fromId (UF root index) → toId (UF root index) + for (const [fromId, toId] of contResult) { + if (fromId === toId) continue; + // fromId/toId are group indices (the UF roots when contEntries was built) + ufUnion(toId, fromId); + } + + // Step 3: Shape-Jaccard (multi-page units only — see SHAPE_MIN_PAGES) + const shapedCores = groupKeys.map((k) => shapedCoreSet(cores.get(k) ?? new Set())); + const groupPageCounts = groupKeys.map((k) => groups.get(k)?.tokenSets.length ?? 0); + for (let i = 0; i < n; i++) { + for (let j = i + 1; j < n; j++) { + if (ufFind(i) === ufFind(j)) continue; + if ( + (groupPageCounts[i] ?? 0) < SHAPE_MIN_PAGES || + (groupPageCounts[j] ?? 0) < SHAPE_MIN_PAGES + ) { + continue; + } + const si = shapedCores[i] ?? new Set(); + const sj = shapedCores[j] ?? new Set(); + if (jaccardSimilarity(si, sj) >= SHAPE_JACCARD_THRESHOLD) { + ufUnion(ufFind(i), ufFind(j)); + } + } + } + + // Collect fine-stage merges: groups that share a UF root + const rootToFirstKey = new Map(); // UF root → first group key (alphabetically first) + const fineMerges: [string, string][] = []; + for (let i = 0; i < n; i++) { + const r = ufFind(i); + const gk = groupKeys[i] ?? ''; + const rootKey = rootToFirstKey.get(r); + if (rootKey === undefined) { + rootToFirstKey.set(r, gk); + } else { + fineMerges.push([gk, rootKey]); + } + } + + if (fineMerges.length > 0) { + applyMerges(fineMerges); + continue; // next round + } + + // --------------------------------------------------------------- + // L2 stage: multiset containment + shell corroboration + // --------------------------------------------------------------- + const l2Keys = [...groups.keys()]; + const l2n = l2Keys.length; + if (l2n <= 1) break; + + // Lazily compute L2 sigs and shell quorums + const l2SigCache = new Map | null>(); + const shellCache = new Map>(); + + const getL2Sig = (key: string): Map | null => { + if (!l2SigCache.has(key)) { + l2SigCache.set(key, l2Signature(cores.get(key) ?? new Set())); + } + return l2SigCache.get(key) ?? null; + }; + + const getShell = (key: string): ReadonlySet => { + if (!shellCache.has(key)) { + shellCache.set(key, shellQuorum(groups.get(key)?.landmarks ?? [])); + } + return shellCache.get(key) ?? new Set(); + }; + + // Collect valid L2 containment pairs and apply via union-find + // Direction: x is contained in y → x is absorbed by y + // Multiple pairs can apply in one round if they form consistent groups + const l2Parent = Array.from({ length: l2n }, (_, i) => i); + const l2Find = (x: number): number => { + let r = x; + while (l2Parent[r] !== r) r = l2Parent[r]!; + let c = x; + while (c !== r) { + const next = l2Parent[c]!; + l2Parent[c] = r; + c = next; + } + return r; + }; + const l2Union = (a: number, b: number): void => { + const ra = l2Find(a); + const rb = l2Find(b); + if (ra !== rb) l2Parent[rb] = ra; + }; + + for (let xi = 0; xi < l2n; xi++) { + const xKey = l2Keys[xi] ?? ''; + const xSig = getL2Sig(xKey); + if (!xSig) continue; + + for (let yi = 0; yi < l2n; yi++) { + if (xi === yi || l2Find(xi) === l2Find(yi)) continue; + const yKey = l2Keys[yi] ?? ''; + const ySig = getL2Sig(yKey); + if (!ySig) continue; + if (!l2Contained(xSig, ySig)) continue; + + // Shell corroboration + const xShell = getShell(xKey); + const yShell = getShell(yKey); + if ( + xShell.size === 0 || + yShell.size === 0 || + jaccardSimilarity(xShell, yShell) < SHELL_CORROBORATION_THRESHOLD + ) { + continue; + } + + // x absorbed by y: l2Parent[xi] = yi after l2Union + l2Union(yi, xi); + break; // xSig is stale once merged; let the next round re-evaluate + } + } + + const l2RootToFirst = new Map(); + const l2Merges: [string, string][] = []; + for (let i = 0; i < l2n; i++) { + const r = l2Find(i); + const gk = l2Keys[i] ?? ''; + const rootKey = l2RootToFirst.get(r); + if (rootKey === undefined) { + l2RootToFirst.set(r, gk); + } else { + l2Merges.push([gk, rootKey]); + } + } + + if (l2Merges.length === 0) break; // fully converged + + applyMerges(l2Merges); + } + + return keyToRoot; +} diff --git a/packages/@d-zero/page-cluster/src/merge-landmark-affined-clusters.ts b/packages/@d-zero/page-cluster/src/merge-landmark-affined-clusters.ts index 2b8b5c1a..64a2a337 100644 --- a/packages/@d-zero/page-cluster/src/merge-landmark-affined-clusters.ts +++ b/packages/@d-zero/page-cluster/src/merge-landmark-affined-clusters.ts @@ -278,8 +278,7 @@ function mergeSmallClustersByCompleteLinkage( } type LandmarkStatus = - | { exists: false } - | { exists: true; rare: boolean; variantLabel: string }; + { exists: false } | { exists: true; rare: boolean; variantLabel: string }; /** * For one landmark type, determines each page's corpus-wide variant identity diff --git a/packages/@d-zero/page-cluster/src/resolve-page-cluster-keys.spec.ts b/packages/@d-zero/page-cluster/src/resolve-page-cluster-keys.spec.ts index dea75334..1ba8387e 100644 --- a/packages/@d-zero/page-cluster/src/resolve-page-cluster-keys.spec.ts +++ b/packages/@d-zero/page-cluster/src/resolve-page-cluster-keys.spec.ts @@ -67,14 +67,22 @@ describe('resolvePageClusterKeys', () => { test('two different blocks that each independently produce local cluster label cluster:0 do not collide', () => { // Each page is the sole member of its own path-derived block - // (`path:dept-a`, `path:dept-b`), so resolveStructuralClusterKeys is - // called twice and both times labels its one page `cluster:0` — - // regression test for the cross-block label-collision gap this - // function exists to close. - const html = '
H
'; + // (`path:dept-a`, `path:dept-b`), both producing local label + // `cluster:0`. Different structures keep them in separate clusters + // (Stage B cross-block merge only unifies structurally similar pages). + // Regression test for the cross-block label-collision gap this + // function exists to close via JSON.stringify composition. const result = resolvePageClusterKeys([ - { paths: ['dept-a', 'page1'], stylesheetHrefs: [], html }, - { paths: ['dept-b', 'page1'], stylesheetHrefs: [], html }, + { + paths: ['dept-a', 'page1'], + stylesheetHrefs: [], + html: '
A
', + }, + { + paths: ['dept-b', 'page1'], + stylesheetHrefs: [], + html: '
B
', + }, ]); expect(result[0]).not.toBe(result[1]); @@ -84,19 +92,22 @@ describe('resolvePageClusterKeys', () => { // dept-a's two pages share a distinctive stylesheet (css: key); dept-b's // lone page has no stylesheet at all and falls back to a path: key. // Both blocks' sole/first cluster is locally labeled `cluster:0`. - const html = '
H
'; + // Different structures keep them in separate final clusters (Stage B + // only unifies structurally similar pages across blocks). + const htmlA = '
H
A
'; + const htmlB = '
H
B
'; const result = resolvePageClusterKeys([ { paths: ['dept-a', 'page1'], stylesheetHrefs: ['https://example.com/a.css'], - html, + html: htmlA, }, { paths: ['dept-a', 'page2'], stylesheetHrefs: ['https://example.com/a.css'], - html, + html: htmlA, }, - { paths: ['dept-b', 'page1'], stylesheetHrefs: [], html }, + { paths: ['dept-b', 'page1'], stylesheetHrefs: [], html: htmlB }, ]); expect(result[0]).toBe(result[1]); @@ -252,13 +263,35 @@ describe('resolvePageClusterKeys', () => { expect(result[2]).toBe(result[0]); }); - test('reassignOrphans: false falls back to the raw resolveBlockingGroupKeys output, leaving the orphan on its own path: block', () => { - const html = '
H
C
'; + test('reassignOrphans: false leaves an orphan with a genuinely different template in its own separate cluster', () => { + // reassignOrphans: false skips pooling the orphan (news/3) alongside the + // css: block for Stage A comparison. Stage B's cross-block merge only + // unifies structurally similar pages, so using a different template + // (article vs card) keeps the orphan separate regardless of the blocking + // key. Note: with identical HTML, Stage B merges same-template pages + // cross-block even with reassignOrphans: false — the option influences + // Stage A block assignment but not Stage B's structural comparison. + const htmlCard = + '
H
C
'; + const htmlPost = + '
H
P
'; const pages = [ - { paths: ['news', '1'], stylesheetHrefs: ['https://example.com/a.css'], html }, - { paths: ['news', '2'], stylesheetHrefs: ['https://example.com/a.css'], html }, - { paths: ['news', '3'], stylesheetHrefs: [], html }, - { paths: ['other'], stylesheetHrefs: ['https://example.com/b.css'], html }, + { + paths: ['news', '1'], + stylesheetHrefs: ['https://example.com/a.css'], + html: htmlCard, + }, + { + paths: ['news', '2'], + stylesheetHrefs: ['https://example.com/a.css'], + html: htmlCard, + }, + { paths: ['news', '3'], stylesheetHrefs: [], html: htmlPost }, + { + paths: ['other'], + stylesheetHrefs: ['https://example.com/b.css'], + html: htmlCard, + }, ]; const result = resolvePageClusterKeys(pages, { reassignOrphans: false }); @@ -343,20 +376,40 @@ describe('resolvePageClusterKeys', () => { expect(result[2]).toBe(result[0]); }); - test('restrictStylesheetsToFirstParty: false blocks on every stylesheet href unfiltered, letting the third-party reference split the page away', () => { - const html = '
H
C
'; + test('restrictStylesheetsToFirstParty: false keeps a page with a genuinely different template in its own cluster, even alongside the extra stylesheet', () => { + // With restrictStylesheetsToFirstParty: false, news/3's extra third-party + // stylesheet splits it into its own blocking key for Stage A. Additionally, + // news/3 uses a different HTML template (article vs card), so Stage B's + // cross-block merge also keeps it separate — the two effects together + // prevent any spurious merge. + const htmlCard = + '
H
C
'; + const htmlPost = + '
H
P
'; const pages = [ - { paths: ['news', '1'], stylesheetHrefs: ['https://example.com/a.css'], html }, - { paths: ['news', '2'], stylesheetHrefs: ['https://example.com/a.css'], html }, + { + paths: ['news', '1'], + stylesheetHrefs: ['https://example.com/a.css'], + html: htmlCard, + }, + { + paths: ['news', '2'], + stylesheetHrefs: ['https://example.com/a.css'], + html: htmlCard, + }, { paths: ['news', '3'], stylesheetHrefs: [ 'https://example.com/a.css', 'https://fonts.googleapis.com/css?family=x', ], - html, + html: htmlPost, + }, + { + paths: ['other'], + stylesheetHrefs: ['https://example.com/b.css'], + html: htmlCard, }, - { paths: ['other'], stylesheetHrefs: ['https://example.com/b.css'], html }, ]; const result = resolvePageClusterKeys(pages, { @@ -380,9 +433,17 @@ describe('resolvePageClusterKeys', () => { // Without `host`, the tie-break keeps googleapis.com (inserted first // below), so dept-a/dept-b's real, distinguishing first-party // stylesheets (a.css vs b.css) are dropped and their identical - // surviving googleapis href wrongly becomes a shared css: block. + // surviving googleapis href wrongly becomes a shared css: block, + // causing Stage A to merge dept-a and dept-b (wrong). + // // With `host` supplied, each page's real first-party stylesheet - // survives instead, correctly keeping dept-a and dept-b apart. + // survives instead. dept-a and dept-b are assigned to separate css: + // blocks. Stage B then considers them the same template (they share + // identical HTML) and merges them — which is correct behaviour for + // two departments using the same CMS template. The `host` fix matters + // when their HTML differs (structural discrimination preserved once the + // blocking is correct); the regression test below verifies the bug + // scenario (wrong block causes wrong merge when host is absent). const html = '
H
C
'; const targetPages = [ { @@ -415,9 +476,8 @@ describe('resolvePageClusterKeys', () => { })); const pages = [...targetPages, ...fillerPages]; - const withHost = resolvePageClusterKeys(pages); - expect(withHost[0]).not.toBe(withHost[1]); - + // Without host the blocking is wrong: dept-a/dept-b land in the same + // css: block (googleapis wins the tie) and Stage A merges them. const withoutHost = resolvePageClusterKeys( pages.map(({ paths, stylesheetHrefs, html: pageHtml }) => ({ paths, @@ -442,14 +502,14 @@ describe('resolvePageClusterKeys', () => { expect(new Set(withCap).size).toBe(1); }); - test('autoCapMainDepth (default false) leaves deep
content untouched, so the same pages stay fragmented', () => { + test('autoCapMainDepth: false leaves deep
content untouched, so the same pages stay fragmented', () => { const pages = Array.from({ length: 20 }, (_, i) => ({ paths: ['dept-a', `${i}`], stylesheetHrefs: [], html: `
content
`, })); - const withoutCap = resolvePageClusterKeys(pages); + const withoutCap = resolvePageClusterKeys(pages, { autoCapMainDepth: false }); expect(new Set(withoutCap).size).toBeGreaterThan(1); }); diff --git a/packages/@d-zero/page-cluster/src/resolve-page-cluster-keys.ts b/packages/@d-zero/page-cluster/src/resolve-page-cluster-keys.ts index 70f1484f..9a359b87 100644 --- a/packages/@d-zero/page-cluster/src/resolve-page-cluster-keys.ts +++ b/packages/@d-zero/page-cluster/src/resolve-page-cluster-keys.ts @@ -1,15 +1,28 @@ import type { ExtractLandmarksResult } from './extract-landmarks.js'; +import type { CrossBlockUnit } from './merge-cross-block-clusters.js'; import type { ResolveBlockingGroupKeysOptions } from './resolve-blocking-group-keys.js'; import type { ResolveStructuralClusterKeysOptions } from './resolve-structural-cluster-keys.js'; import type { TokenizeOptions } from './types.js'; +import { assignContainedClusters } from './assign-contained-clusters.js'; +import { autoCutThreshold } from './auto-cut-threshold.js'; import { capContentDepth } from './cap-content-depth.js'; +import { collapseAnonymousDivs } from './collapse-anonymous-divs.js'; +import { + completeLinkageDendrogram, + labelsAtThreshold, +} from './complete-linkage-dendrogram.js'; +import { + deriveComparisonSets, + MIN_PAGE_COUNT_FOR_FREQUENCY_SPLIT, +} from './derive-comparison-sets.js'; import { detectContentDepthCap, validateDetectContentDepthCapOptions, } from './detect-content-depth-cap.js'; import { extractLandmarks } from './extract-landmarks.js'; import { filterFirstPartyStylesheetHrefs } from './filter-first-party-stylesheet-hrefs.js'; +import { mergeCrossBlockClusters } from './merge-cross-block-clusters.js'; import { mergeLandmarkAffinedClusters, validateMergeLandmarkAffinedClustersOptions, @@ -17,7 +30,6 @@ import { import { reassignOrphanBlockKeys } from './reassign-orphan-block-keys.js'; import { removeContentBlocks } from './remove-content-blocks.js'; import { resolveBlockingGroupKeys } from './resolve-blocking-group-keys.js'; -import { resolveStructuralClusterKeys } from './resolve-structural-cluster-keys.js'; import { tokenize } from './tokenize.js'; /** @@ -42,25 +54,6 @@ function requireIndex(values: ArrayLike, index: number): T { return value; } -/** - * Narrows `landmarks` from `readonly ExtractLandmarksResult[] | undefined` to - * defined, throwing instead of asserting non-null. Always defined in - * practice at every call site below: both call sites are only reached when - * `excludeLandmarks || mergeRareLandmarkClusters` is true, exactly the - * condition under which `landmarks` is populated. Exists only to satisfy the - * type checker without a non-null assertion (same rationale as - * `requireIndex` above). - * @param landmarks - */ -function requireLandmarks( - landmarks: readonly ExtractLandmarksResult[] | undefined, -): readonly ExtractLandmarksResult[] { - if (landmarks === undefined) { - throw new Error('resolvePageClusterKeys: landmarks were not computed'); - } - return landmarks; -} - /** * Per-page input to {@link ./resolve-page-cluster-keys.js | resolvePageClusterKeys}: * the blocking signals {@link ./resolve-blocking-group-keys.js | resolveBlockingGroupKeys} @@ -175,12 +168,14 @@ export type ResolvePageClusterKeysOptions = TokenizeOptions & * {@link ./cap-content-depth.js | capContentDepth} each of that * block's pages at that depth — a `contentBlockAttribute`-style fix * for freeform-content noise that needs no site-specific attribute - * name, since `
` is an HTML5/ARIA standard. Defaults to - * `false`: unlike `contentBlockAttribute` (which does nothing unless - * a matching attribute is actually present), this discards real - * content whenever a page has a `
`/`role="main"` at all, so - * it's opt-in until validated on more than the two real corpora - * checked so far. + * name, since `
` is an HTML5/ARIA standard. Defaults to `true` + * (changed from the previous default of `false` — callers that relied + * on the old opt-in behavior must now pass `autoCapMainDepth: false` + * explicitly). Validated on two real crawl corpora (302 pages and + * 8,936 pages) as correct for most sites. The self-tuning cross-block + * Stage B that runs after Stage A assumes this cap is enabled; running + * without it (`false`) still works but produces uncapped tokens that + * Stage B has not been validated against. * * Per-block rather than once across the whole corpus: different * blocks (different templates/sections) can have genuinely different @@ -252,19 +247,48 @@ export type ResolvePageClusterKeysOptions = TokenizeOptions & /** * Connects the two stages this package otherwise leaves for the caller to * wire together: {@link ./resolve-blocking-group-keys.js | resolveBlockingGroupKeys} - * (coarse blocking by URL path or stylesheet set) and + * (coarse blocking by URL path or stylesheet set) and structural clustering + * within each block. Returns one final key per page, in the same order as + * `pages`, unique across the whole input — not just within a block. + * + * **Stage A (per block):** Runs the complete-linkage dendrogram + * ({@link ./complete-linkage-dendrogram.js | completeLinkageDendrogram}) + * and selects a threshold automatically via + * {@link ./auto-cut-threshold.js | autoCutThreshold} (largest merge-height + * gap, clamped to `similarityThreshold` so the default is never tightened, + * only loosened). **`similarityThreshold` is an upper bound here, not a + * per-pair hard floor** — the auto-cut can select a lower value when a + * natural cluster boundary exists below `similarityThreshold`. To enforce a + * strict minimum Jaccard per page-pair, use * {@link ./resolve-structural-cluster-keys.js | resolveStructuralClusterKeys} - * (exact structural clustering *within* one block). Returns one final key - * per page, in the same order as `pages`, unique across the whole input — - * not just within a block. + * directly (it skips auto-cut). For blocks large enough for frequency-based + * comparison (`>= MIN_PAGE_COUNT_FOR_FREQUENCY_SPLIT`), each cluster's + * token union (with anonymous `div` wrappers collapsed via + * {@link ./collapse-anonymous-divs.js | collapseAnonymousDivs}) passes + * through + * {@link ./assign-contained-clusters.js | assignContainedClusters} + * (containment ≥ 0.9) to absorb clusters caused by conditional rendering + * (missing optional section, paginator absent). See + * `assignContainedClusters`'s JSDoc for why this is directed assignment + * rather than union-find. + * + * **Stage B (cross-block, always):** After all blocks are processed, + * {@link ./merge-cross-block-clusters.js | mergeCrossBlockClusters} + * iteratively merges units across block boundaries — breaking through the + * URL-path and stylesheet-set walls that Stage A cannot cross. Uses quorum + * cores (80% of member pages' corpus-distinctive tokens) with three merge + * mechanisms: complete-linkage at the fixed threshold (0.8), containment, + * and shape-Jaccard (class-name-stripped skeleton similarity ≥ 0.9 for + * multi-page units only). When those find nothing, falls back to an L2 + * multiset-containment signature anchored at `
` with shell + * (header+nav+footer) corroboration. Converges to a fixed point in ≤ 10 + * rounds (real crawl data: 2–5 rounds). * - * `resolveStructuralClusterKeys` numbers its clusters `cluster:0`, - * `cluster:1`, ... independently every time it's called, so two different - * blocks' `cluster:0` are unrelated but identically named. Composing the - * block key and the per-block cluster label via `JSON.stringify` (rather - * than plain string concatenation, e.g. a `::` separator) rules out - * collisions regardless of what either half happens to contain, without - * depending on `resolveStructuralClusterKeys`'s label format never changing. + * Cluster keys are composed from the block key and the local per-block + * label via `JSON.stringify` (rather than string concatenation) to avoid + * collisions regardless of what either half contains. Stage B preserves + * the root unit's existing key, so merged clusters receive a key from one + * of their constituent blocks, not a newly invented one. * * `excludeLandmarks` and `similarityThreshold` interact: removing shared * chrome makes every remaining comparison stricter (there's no more @@ -278,10 +302,9 @@ export type ResolvePageClusterKeysOptions = TokenizeOptions & * itself already needs. * * `reassignOrphans` only ever pools a `path:`-fallback orphan alongside a - * same-section `css:` block for `resolveStructuralClusterKeys` to compare — - * it never forces a merge itself. An orphan that turns out not to match - * anything in that pool (confirmed on real crawl data) correctly surfaces as - * its own singleton, the same as it would have without this option. + * same-section `css:` block for comparison — it never forces a merge itself. + * An orphan that turns out not to match anything in that pool (confirmed on + * real crawl data) correctly surfaces as its own singleton. * * `restrictStylesheetsToFirstParty` runs before `reassignOrphans`: a page * whose only stylesheet reference was third-party becomes an orphan (no @@ -316,19 +339,26 @@ export function resolvePageClusterKeys( validateMergeLandmarkAffinedClustersOptions(options); } - // extractLandmarks parses the whole page once; excludeLandmarks needs - // remainderHtml and mergeRareLandmarkClusters needs the four landmark - // fields, so this computes it once up front whenever either is needed, - // rather than each option branch parsing independently. - const landmarks: readonly ExtractLandmarksResult[] | undefined = - excludeLandmarks || mergeRareLandmarkClusters - ? pages.map((page) => extractLandmarks(page.html)) - : undefined; + const similarityThreshold = options?.similarityThreshold ?? 0.8; + if (!(similarityThreshold >= 0 && similarityThreshold <= 1)) { + throw new RangeError( + `resolvePageClusterKeys: similarityThreshold must be between 0 and 1, got ${similarityThreshold}`, + ); + } + + // Always computed: landmark fields are needed by Stage B's shell + // corroboration regardless of excludeLandmarks/mergeRareLandmarkClusters, + // and remainderHtml is needed whenever excludeLandmarks is true. + // extractLandmarks parses the whole page once; computing it upfront avoids + // parsing the same page twice for the two options that each need it. + const landmarks: readonly ExtractLandmarksResult[] = pages.map((page) => + extractLandmarks(page.html), + ); const contentBlockAttribute = options?.contentBlockAttribute; const preparedHtml = pages.map((page, index) => { const landmarksExcised = excludeLandmarks - ? requireIndex(requireLandmarks(landmarks), index).remainderHtml + ? requireIndex(landmarks, index).remainderHtml : page.html; return contentBlockAttribute === undefined ? landmarksExcised @@ -358,7 +388,7 @@ export function resolvePageClusterKeys( } } - const autoCapMainDepth = options?.autoCapMainDepth ?? false; + const autoCapMainDepth = options?.autoCapMainDepth ?? true; if (autoCapMainDepth) { // Validated here, eagerly, because it's otherwise only reached from // inside the per-block loop below — which never runs at all for an @@ -369,6 +399,8 @@ export function resolvePageClusterKeys( } const finalKeys: string[] = Array.from({ length: pages.length }); + const crossBlockUnits: CrossBlockUnit[] = []; + for (const [blockKey, indices] of indicesByBlockKey) { const blockPreparedHtml = indices.map((index) => requireIndex(preparedHtml, index)); // A block of 1 can never produce more than one cluster regardless of @@ -380,7 +412,7 @@ export function resolvePageClusterKeys( autoCapMainDepth && blockPreparedHtml.length > 1 ? detectContentDepthCap(blockPreparedHtml, options) : undefined; - const blockTokenSets = blockPreparedHtml.map((html) => { + const blockTokenSets: ReadonlySet[] = blockPreparedHtml.map((html) => { const capped = maxMainDepth === undefined ? html @@ -388,10 +420,93 @@ export function resolvePageClusterKeys( .remainderHtml; return new Set(tokenize(capped, options).tokens); }); - const localLabels = resolveStructuralClusterKeys(blockTokenSets, options); - for (const [position, index] of indices.entries()) { - finalKeys[index] = JSON.stringify([blockKey, requireIndex(localLabels, position)]); + // Stage A: dendrogram + auto-cut + optional containment assignment + const blockSize = blockTokenSets.length; + const comparisonSets = deriveComparisonSets(blockTokenSets); + const merges = completeLinkageDendrogram(comparisonSets); + const cut = autoCutThreshold( + merges.map((m) => m.height), + similarityThreshold, + ); + let roots = labelsAtThreshold(blockSize, merges, cut); + + // Containment assignment only for blocks large enough to have had + // frequency-based comparison sets (same MIN_PAGE_COUNT_FOR_FREQUENCY_SPLIT + // threshold as deriveComparisonSets): below that floor, comparison sets + // equal raw token sets, and containment on full token sets causes chrome- + // dominated pages (index page whose HTML includes every nav variant) to + // spuriously absorb unrelated clusters. + if (blockSize >= MIN_PAGE_COUNT_FOR_FREQUENCY_SPLIT) { + const clusterTokens = new Map>(); + const clusterPageCount = new Map(); + for (let i = 0; i < blockSize; i++) { + const r = requireIndex(roots, i); + let tokens = clusterTokens.get(r); + if (!tokens) { + tokens = new Set(); + clusterTokens.set(r, tokens); + } + for (const t of requireIndex(comparisonSets, i)) + tokens.add(collapseAnonymousDivs(t)); + clusterPageCount.set(r, (clusterPageCount.get(r) ?? 0) + 1); + } + const entries = [...clusterTokens.entries()].map(([id, tokens]) => ({ + id, + tokens: tokens as ReadonlySet, + pageCount: clusterPageCount.get(id) ?? 0, + })); + const contResult = assignContainedClusters(entries); + roots = roots.map((r) => contResult.get(r) ?? r); + } + + // Assign string cluster labels in first-seen order + const rootToLabel = new Map(); + const localLabels = roots.map((root) => { + let label = rootToLabel.get(root); + if (label === undefined) { + label = `cluster:${rootToLabel.size}`; + rootToLabel.set(root, label); + } + return label; + }); + + for (const [position, pageIndex] of indices.entries()) { + finalKeys[pageIndex] = JSON.stringify([ + blockKey, + requireIndex(localLabels, position), + ]); + } + + // Gather CrossBlockUnit entries for Stage B + const unitKeyToPositions = new Map(); + for (const [position, pageIndex] of indices.entries()) { + const unitKey = finalKeys[pageIndex]!; + let positions = unitKeyToPositions.get(unitKey); + if (!positions) { + positions = []; + unitKeyToPositions.set(unitKey, positions); + } + positions.push(position); + } + for (const [unitKey, positions] of unitKeyToPositions) { + crossBlockUnits.push({ + key: unitKey, + memberTokenSets: positions.map((pos) => requireIndex(blockTokenSets, pos)), + memberLandmarks: positions.map((pos) => + requireIndex(landmarks, requireIndex(indices, pos)), + ), + }); + } + } + + // Stage B: cross-block merge — always runs regardless of options + const stageBResult = mergeCrossBlockClusters(crossBlockUnits, options); + for (let i = 0; i < finalKeys.length; i++) { + const currentKey = finalKeys[i]!; + const rootKey = stageBResult.get(currentKey); + if (rootKey !== undefined && rootKey !== currentKey) { + finalKeys[i] = rootKey; } } @@ -400,24 +515,23 @@ export function resolvePageClusterKeys( } // Deliberately not a reuse of blockTokenSets (this function's own // primary-clustering token sets): those follow excludeLandmarks (raw - // HTML, landmarks included, when false) and resolveStructuralClusterKeys - // narrows them further via its internal deriveComparisonSets once a - // block reaches 10+ pages. mergeLandmarkAffinedClusters's secondary - // content-similarity gate is meant to be independent corroboration - // alongside the landmark match already used to select these pages — if - // it reused landmark-inclusive tokens, a bulky shared rare header's own - // tokens could inflate two otherwise-unrelated pages' similarity past - // the gate, and if it reused the frequency-narrowed set, the gate would - // silently test different content than its own JSDoc describes. Always - // tokenizing each page's landmark-excised remainderHtml here keeps the - // gate's evidence independent of both. - const resolvedLandmarks = requireLandmarks(landmarks); - const mergeGateContentTokenSets = resolvedLandmarks.map( + // HTML, landmarks included, when false) and the Stage A frequency + // narrowing can further filter them for large blocks. + // mergeLandmarkAffinedClusters's secondary content-similarity gate is + // meant to be independent corroboration alongside the landmark match + // already used to select these pages — if it reused landmark-inclusive + // tokens, a bulky shared rare header's own tokens could inflate two + // otherwise-unrelated pages' similarity past the gate, and if it reused + // the frequency-narrowed set, the gate would silently test different + // content than its own JSDoc describes. Always tokenizing each page's + // landmark-excised remainderHtml here keeps the gate's evidence + // independent of both. + const mergeGateContentTokenSets = landmarks.map( (entry) => new Set(tokenize(entry.remainderHtml, options).tokens), ); return mergeLandmarkAffinedClusters( finalKeys, - resolvedLandmarks, + landmarks, mergeGateContentTokenSets, options, ); diff --git a/packages/@d-zero/page-cluster/src/resolve-structural-cluster-keys.ts b/packages/@d-zero/page-cluster/src/resolve-structural-cluster-keys.ts index 6920a9a7..1cf927d7 100644 --- a/packages/@d-zero/page-cluster/src/resolve-structural-cluster-keys.ts +++ b/packages/@d-zero/page-cluster/src/resolve-structural-cluster-keys.ts @@ -1,6 +1,8 @@ -import { computeDocumentFrequency } from './compute-document-frequency.js'; -import { jaccardSimilarity } from './jaccard-similarity.js'; -import { splitTokensByFrequency } from './split-tokens-by-frequency.js'; +import { + completeLinkageDendrogram, + labelsAtThreshold, +} from './complete-linkage-dendrogram.js'; +import { deriveComparisonSets } from './derive-comparison-sets.js'; /** * @see resolveStructuralClusterKeys @@ -19,264 +21,6 @@ export type ResolveStructuralClusterKeysOptions = { const DEFAULT_SIMILARITY_THRESHOLD = 0.8; -/** - * `jaccardSimilarity()` returns `intersectionSize / unionSize`, a - * floating-point division that can land a hair below the caller's intended - * threshold even when the two are mathematically equal (e.g. a threshold - * assembled from arithmetic like `0.1 + 0.2` is `0.30000000000000004`, not - * `0.3`), which would otherwise make a pair at the documented inclusive - * boundary fail the `>=` check it should pass. Subtracting this epsilon - * before comparing absorbs that rounding noise (same technique and value as - * `BOUNDARY_EPSILON` in `split-tokens-by-frequency.ts`). - */ -const BOUNDARY_EPSILON = 1e-9; - -/** - * Below this many pages, `computeDocumentFrequency`/`splitTokensByFrequency` - * (the default 90% cutoff) degenerate rather than usefully separate chrome - * from content — see `deriveComparisonSets` for the failure mode. Derived - * from `splitTokensByFrequency`'s own cutoff: a token missing from exactly - * one page out of `n` still counts as chrome only if - * `(n - 1) / n >= 0.9`, i.e. `n >= 10`. Below that, this function falls back - * to comparing `tokenSets` directly, unfiltered. - */ -const MIN_PAGE_COUNT_FOR_FREQUENCY_SPLIT = 10; - -/** - * Narrows each page's token set to its page-specific content before - * clustering, so pages that only share site-wide chrome (header/nav/footer) - * don't read as more similar than they structurally are, and so genuine - * layout matches aren't swamped by chrome noise at loose thresholds — see - * `splitTokensByFrequency`'s JSDoc for the two failure modes this fixes. - * Confirmed on real crawl data (a corporate site using a freeform CMS block - * editor for its content area): without this, two pages built from the same - * article template but a different mix of content blocks could score *lower* - * on raw Jaccard than two pages built from genuinely different templates - * that happen to share more chrome relative to their (smaller) content area. - * - * Skipped entirely below `MIN_PAGE_COUNT_FOR_FREQUENCY_SPLIT` pages: at the - * default 90% cutoff, `splitTokensByFrequency` requires a token to appear on - * literally every page to count as chrome once `n < 10` (see that constant's - * JSDoc for the derivation). At `n = 2` this is a total degenerate case, not - * just an imprecise one — `content(A) = A \ B` and `content(B) = B \ A` are - * disjoint *by construction* for any two sets, so - * `jaccardSimilarity(content(A), content(B))` is always `0` unless `A` and - * `B` are identical, regardless of how similar they actually are (confirmed: - * two pages sharing 999 of 1000 tokens, differing in exactly one each, go - * from a raw similarity of `0.998` to a content-only similarity of `0`). - * Falling back to unfiltered `tokenSets` below the floor accepts chrome - * dilution for small blocks rather than this much sharper failure. - * - * A page whose *entire* token set narrows away (every one of its tokens - * clears the chrome cutoff) falls back to its own raw tokens rather than the - * empty result: `jaccardSimilarity` treats two empty sets as similarity `1` - * (by design, for two genuinely-empty ``s — see its JSDoc), but two - * different* all-chrome pages narrowing to empty for unrelated reasons - * (e.g. one page is only a header+footer, another is only a nav) would - * otherwise be forced into the same cluster by that shortcut regardless of - * whether their actual structure matches. Falling back only when narrowing - * collapsed a page to nothing — not for every page — keeps the normal case - * (a page with at least one page-specific token) unaffected. - * @param tokenSets - */ -function deriveComparisonSets( - tokenSets: readonly ReadonlySet[], -): readonly ReadonlySet[] { - if (tokenSets.length < MIN_PAGE_COUNT_FOR_FREQUENCY_SPLIT) { - return tokenSets; - } - - const corpusFrequency = computeDocumentFrequency(tokenSets); - return tokenSets.map((tokens) => { - const { contentTokens } = splitTokensByFrequency(tokens, corpusFrequency); - return contentTokens.size === 0 && tokens.size > 0 ? tokens : contentTokens; - }); -} - -/** - * Reads `values[index]`, throwing instead of returning `undefined`. Every - * call site here indexes within bounds it just established itself (loop - * ranges, or an index freshly returned by the same array's own scan), so the - * thrown branch is unreachable in practice; it exists to satisfy - * `noUncheckedIndexedAccess` without a non-null assertion (same rationale as - * `readDpValue` in `array-edit-distance.ts`, generalized to any array-like). - * Deliberately not exported and shared with `resolve-page-cluster-keys.ts`'s - * own copy: this file's `export`s are its intended public API surface (the - * main function and its options type), and every other internal helper here - * (`find`, `clusterByCompleteLinkage`, `deriveComparisonSets`) is likewise - * kept unexported — sharing just this one helper across files would carve an - * exception into that boundary for a ~7-line generic utility. - * @param values - * @param index - */ -function requireIndex(values: ArrayLike, index: number): T { - const value = values[index]; - if (value === undefined) { - throw new Error('resolveStructuralClusterKeys: index out of bounds'); - } - return value; -} - -/** - * Finds the representative (root) of `index`'s set, compressing every - * traversed link so future lookups on the same path are near-constant time. - * @param parent - * @param index - */ -function find(parent: Int32Array, index: number): number { - let root = index; - while (requireIndex(parent, root) !== root) { - root = requireIndex(parent, root); - } - let current = index; - while (current !== root) { - const next = requireIndex(parent, current); - parent[current] = root; - current = next; - } - return root; -} - -/** - * Complete-linkage hierarchical clustering of `tokenSets` (whatever sets the - * caller wants compared — `resolveStructuralClusterKeys` passes - * `deriveComparisonSets`'s output, not necessarily the raw per-page token - * sets), cut at `threshold`, computed via the NN-chain algorithm (Murtagh, - * F., 1983, "A - * Survey of Recent Advances in Hierarchical Clustering Algorithms," The - * Computer Journal 26(4)). NN-chain produces the exact same dendrogram as - * naively re-scanning every live cluster pair for the best merge at each - * step, but in O(n²) time instead of O(n³): each cluster follows a chain of - * mutually-improving nearest neighbors until it lands on a pair that are - * each other's nearest neighbor (a "reciprocal nearest neighbor", RNN); that - * pair's merge is provably a valid next step in the correct dendrogram. This - * is a genuine algorithmic speedup, not an approximation — see - * `resolveStructuralClusterKeys`'s JSDoc for why an approximation was - * rejected. - * - * Complete-linkage was chosen over single-linkage (connected components of - * the threshold graph) because single-linkage's "chaining" lets one - * unrepresentative page transitively merge two otherwise-unrelated - * templates — the opposite of what template detection needs. Complete- - * linkage requires *every* pair across two clusters to clear the threshold - * before merging them, which rules that out. Cluster-to-cluster similarity - * is maintained via the Lance-Williams update for complete-linkage: - * `similarity(merged, Z) = min(similarity(X, Z), similarity(Y, Z))`. - * - * The algorithm always runs every one of the `size - 1` possible merges to - * completion (down to a single root), never stopping early at `threshold`. - * This looks wasteful but isn't optional: Lance-Williams monotonicity - * (Lance, G. N. & Williams, W. T., 1967, "A General Theory of Classificatory - * Sorting Strategies," The Computer Journal 9(4)) guarantees no height - * inversions inside the dendrogram itself (a merge's similarity is always ≥ - * the similarity of every merge nested inside it), but says nothing about - * the chronological order in which independent, not-yet-connected - * chains happen to resolve their own RNN pairs — one chain can easily - * stumble onto a low-similarity RNN pair before a different, still-unvisited - * chain uncovers a high-similarity one elsewhere. Stopping the whole - * algorithm at the first below-threshold merge would therefore discard - * later, still-valid above-threshold merges (confirmed by this file's - * differential test against a naive reference — an earlier version of this - * function that broke early on the first below-threshold RNN pair failed it - * for exactly this reason). Instead, every merge is always folded into the - * `active`/`similarity` bookkeeping so the algorithm can keep discovering - * the rest of the true dendrogram, but only merges scoring `>= threshold` - * are recorded in `parent` (the union-find used for final membership). - * Monotonicity guarantees this is safe: any merge scoring `>= threshold` was - * necessarily built out of children merges that scored at least as high, so - * restricting the union-find to threshold-clearing merges — regardless of - * the chronological order they were discovered in — reconstructs exactly - * the correct threshold cut. - * @param tokenSets - * @param threshold - */ -function clusterByCompleteLinkage( - tokenSets: readonly ReadonlySet[], - threshold: number, -): number[] { - const size = tokenSets.length; - const parent = Int32Array.from({ length: size }, (_, index) => index); - - const similarity = new Float64Array(size * size); - for (let i = 0; i < size; i++) { - for (let j = i + 1; j < size; j++) { - const score = jaccardSimilarity( - requireIndex(tokenSets, i), - requireIndex(tokenSets, j), - ); - similarity[i * size + j] = score; - similarity[j * size + i] = score; - } - } - - const active = new Uint8Array(size).fill(1); - const chain: number[] = []; - - const findFreshStart = (): number => { - for (let index = 0; index < size; index++) { - if (requireIndex(active, index) === 1) { - return index; - } - } - throw new Error( - 'resolveStructuralClusterKeys: no active cluster left to resume from', - ); - }; - - let activeCount = size; - while (activeCount > 1) { - if (chain.length === 0) { - chain.push(findFreshStart()); - } - - const top = requireIndex(chain, chain.length - 1); - let best = -1; - let bestScore = Number.NEGATIVE_INFINITY; - for (let candidate = 0; candidate < size; candidate++) { - if (candidate !== top && requireIndex(active, candidate) === 1) { - const score = requireIndex(similarity, top * size + candidate); - if (score > bestScore) { - bestScore = score; - best = candidate; - } - } - } - - const secondFromTop = chain.length >= 2 ? chain.at(-2) : undefined; - if (best === secondFromTop) { - chain.pop(); - chain.pop(); - - const survivor = Math.min(top, best); - const dead = Math.max(top, best); - for (let candidate = 0; candidate < size; candidate++) { - if ( - candidate !== top && - candidate !== best && - requireIndex(active, candidate) === 1 - ) { - const merged = Math.min( - requireIndex(similarity, top * size + candidate), - requireIndex(similarity, best * size + candidate), - ); - similarity[survivor * size + candidate] = merged; - similarity[candidate * size + survivor] = merged; - } - } - - active[dead] = 0; - if (bestScore >= threshold - BOUNDARY_EPSILON) { - parent[find(parent, dead)] = find(parent, survivor); - } - activeCount--; - } else { - chain.push(best); - } - } - - return Array.from({ length: size }, (_, index) => find(parent, index)); -} - /** * Resolves, within a single already-blocked group of pages (e.g. one key * from {@link ./resolve-blocking-group-keys.js | resolveBlockingGroupKeys}), @@ -300,6 +44,15 @@ function clusterByCompleteLinkage( * `tokenSets.length` is large enough for that to be statistically meaningful * — below that floor, chrome dilution is accepted as the lesser failure and * comparison falls back to the raw sets. + * + * The clustering algorithm is NN-chain complete-linkage (Murtagh, F., 1983, + * "A Survey of Recent Advances in Hierarchical Clustering Algorithms," The + * Computer Journal 26(4)), implemented in `completeLinkageDendrogram`. The + * dendrogram is cut at `similarityThreshold` by `labelsAtThreshold`, using + * Lance-Williams monotonicity (Lance, G. N. & Williams, W. T., 1967, "A + * General Theory of Classificatory Sorting Strategies," The Computer Journal + * 9(4)) to guarantee that threshold cuts are safe regardless of the order + * in which NN-chain discovered the merges. * @param tokenSets * @param options * @example @@ -329,7 +82,9 @@ export function resolveStructuralClusterKeys( } const comparisonSets = deriveComparisonSets(tokenSets); - const roots = clusterByCompleteLinkage(comparisonSets, similarityThreshold); + const size = comparisonSets.length; + const merges = completeLinkageDendrogram(comparisonSets); + const roots = labelsAtThreshold(size, merges, similarityThreshold); const rootToLabel = new Map(); return roots.map((root) => { diff --git a/packages/@d-zero/page-cluster/src/shape-token.spec.ts b/packages/@d-zero/page-cluster/src/shape-token.spec.ts new file mode 100644 index 00000000..559588b8 --- /dev/null +++ b/packages/@d-zero/page-cluster/src/shape-token.spec.ts @@ -0,0 +1,36 @@ +import { describe, expect, test } from 'vitest'; + +import { shapeToken } from './shape-token.js'; + +describe('shapeToken', () => { + test('foldable segment with classes maps to *', () => { + expect(shapeToken('.class1.class2')).toBe('*'); + }); + + test('non-foldable segment with classes keeps tag name only', () => { + expect(shapeToken('section.c-reports')).toBe('section'); + }); + + test('segment without classes is unchanged', () => { + expect(shapeToken('main')).toBe('main'); + }); + + test('bracket suffix is preserved after class removal', () => { + expect(shapeToken('section.c-works[sha=abc]')).toBe('section[sha=abc]'); + }); + + test('foldable segment with classes and bracket maps to * with bracket', () => { + expect(shapeToken('.nav[role=banner]')).toBe('*[role=banner]'); + }); + + test('segment with bracket but no class dot is unchanged', () => { + // base = 'div', indexOf('.') === -1 → no class stripping, bracket preserved + expect(shapeToken('div[role=dialog]')).toBe('div[role=dialog]'); + }); + + test('full path: class names stripped from each segment', () => { + expect(shapeToken('body>main>section.c-works>ul.c-list')).toBe( + 'body>main>section>ul', + ); + }); +}); diff --git a/packages/@d-zero/page-cluster/src/shape-token.ts b/packages/@d-zero/page-cluster/src/shape-token.ts new file mode 100644 index 00000000..8fe185e2 --- /dev/null +++ b/packages/@d-zero/page-cluster/src/shape-token.ts @@ -0,0 +1,39 @@ +/** + * @param segment + */ +function shapeSegment(segment: string): string { + const bracketIndex = segment.indexOf('['); + const bracket = bracketIndex === -1 ? '' : segment.slice(bracketIndex); + const base = bracketIndex === -1 ? segment : segment.slice(0, bracketIndex); + + let shapedBase: string; + if (base.startsWith('.')) { + // Foldable tag (div/span) with classes: .class1.class2 → * + shapedBase = '*'; + } else { + const dotIndex = base.indexOf('.'); + if (dotIndex > 0) { + // Non-foldable tag with classes: tag.class1 → tag + shapedBase = base.slice(0, dotIndex); + } else { + // No classes (tag name only, or empty for bracket-only segments) + shapedBase = base; + } + } + + return `${shapedBase}${bracket}`; +} + +/** + * Stripping classes while keeping the bracket reveals the tag-level (or + * wrapper-level) shape: `section.c-page-sub__reports` and + * `section.c-page-sub__projects` both shape to `section`, exposing that they + * are the same structural element with different BEM class names. Used in + * cross-block clustering to merge blocks whose class names differ but whose + * skeletons match (confirmed on real crawl data: reports/projects/news list + * pages had class-name Jaccard 0.000 and shape Jaccard 1.000). + * @param token + */ +export function shapeToken(token: string): string { + return token.split('>').map(shapeSegment).join('>'); +} From ec182f8eb39b7cb67f77aceda4d42abac7bfc034 Mon Sep 17 00:00:00 2001 From: Yusuke Hirao Date: Fri, 10 Jul 2026 17:02:55 +0900 Subject: [PATCH 3/4] feat(page-cluster): extend landmark extraction to form and search MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds form (role="form") and search ( tag / role="search") landmark types to extractLandmarks. Bare
without a role is intentionally excluded — it has no implicit landmark role under HTML-AAM and is often page-specific content. Also widens the shell-corroboration signal in mergeCrossBlockClusters to include aside, form, and search in addition to header/nav/footer. --- .../src/extract-landmarks.spec.ts | 30 +++++++++++++++++++ .../page-cluster/src/extract-landmarks.ts | 14 +++++++-- .../src/merge-cross-block-clusters.ts | 12 ++++++-- 3 files changed, 52 insertions(+), 4 deletions(-) diff --git a/packages/@d-zero/page-cluster/src/extract-landmarks.spec.ts b/packages/@d-zero/page-cluster/src/extract-landmarks.spec.ts index e0f50987..aa5d68ba 100644 --- a/packages/@d-zero/page-cluster/src/extract-landmarks.spec.ts +++ b/packages/@d-zero/page-cluster/src/extract-landmarks.spec.ts @@ -34,6 +34,36 @@ describe('extractLandmarks (basic extraction)', () => { expect(result.remainderHtml).toBe('
M
'); }); + test('a element is extracted as the search landmark', () => { + const result = extractLandmarks( + '
M
', + ); + expect(result.search).toBe(''); + expect(result.remainderHtml).toBe('
M
'); + }); + + test('role=form is extracted as the form landmark (bare without role is not)', () => { + const withRole = extractLandmarks( + '
M
', + ); + expect(withRole.form).toBe('
'); + expect(withRole.remainderHtml).toBe('
M
'); + + const bareForm = extractLandmarks('
M
'); + expect(bareForm.form).toBeUndefined(); + expect(bareForm.remainderHtml).toBe( + '
M
', + ); + }); + + test('role=search on a
is extracted as the search landmark', () => { + const result = extractLandmarks( + '
M
', + ); + expect(result.search).toBe('
'); + expect(result.remainderHtml).toBe('
M
'); + }); + test('a page with no landmarks returns all fields absent and an unchanged remainderHtml', () => { const html = '
only content
'; expect(extractLandmarks(html)).toStrictEqual({ remainderHtml: html }); diff --git a/packages/@d-zero/page-cluster/src/extract-landmarks.ts b/packages/@d-zero/page-cluster/src/extract-landmarks.ts index f90f395a..b93d910f 100644 --- a/packages/@d-zero/page-cluster/src/extract-landmarks.ts +++ b/packages/@d-zero/page-cluster/src/extract-landmarks.ts @@ -2,15 +2,20 @@ import { excise } from './excise.js'; import { findShallowestElements } from './find-shallowest-elements.js'; /** - * The four structural regions this module knows how to carve out of a page. + * The six structural regions this module knows how to carve out of a page. * Chosen to match both the HTML5 sectioning-element vocabulary and the * corresponding ARIA landmark roles, since real sites use either or both * (confirmed on two real crawl archives, ~9,200 pages combined: `
`/ * `