diff --git a/cspell.json b/cspell.json index bd69e85a..ddad4398 100644 --- a/cspell.json +++ b/cspell.json @@ -16,6 +16,7 @@ "medoids", "hrefs", "Murtagh", + "unioned", // "gaxios", diff --git a/packages/@d-zero/page-cluster/README.md b/packages/@d-zero/page-cluster/README.md index d9f10164..cfcad883 100644 --- a/packages/@d-zero/page-cluster/README.md +++ b/packages/@d-zero/page-cluster/README.md @@ -37,6 +37,8 @@ tokenize(html, { CMSのブロック属性名が分からない・サイトごとに違う場合は `autoCapMainDepth: true` を使う。`
`/`role="main"`という標準タグを起点に、構造クラスタ数が急増する直前の深さをブロックごとに実データから自動検出して打ち切るため、サイト固有の設定が一切不要(既定はfalse。実データ検証では`contentBlockAttribute`より良い結果になる場合もあった一方、計算コストが実測で数倍〜1桁台後半になる。倍率はコーパスのブロック構成に依存する)。詳細は `detectContentDepthCap()` のJSDocを参照。 +header/footer/nav/asideが一致するページ同士をさらに合流させたい場合は `mergeRareLandmarkClusters: true` を使う。ただし単純な一致判定は実データで過剰融合を招くことが分かっているため(header/footer/navは99%以上のページに存在し判別力を持たない)、コーパス全体で希少なランドマークバリアントが一致した場合に限り、より緩いコンテンツ類似度閾値(`landmarkGateSimilarityThreshold`)での合流を許可する(既定はfalse。実データでの検証は未実施で、合成フィクスチャでの単体・回帰テストのみ)。詳細・コスト特性は `mergeLandmarkAffinedClusters()` のJSDocを参照。 + ```ts import { resolvePageClusterKeys } from '@d-zero/page-cluster/resolve-page-cluster-keys'; diff --git a/packages/@d-zero/page-cluster/package.json b/packages/@d-zero/page-cluster/package.json index 4eebf01a..c0a0b2e2 100644 --- a/packages/@d-zero/page-cluster/package.json +++ b/packages/@d-zero/page-cluster/package.json @@ -49,6 +49,10 @@ "import": "./dist/jaccard-similarity.js", "types": "./dist/jaccard-similarity.d.ts" }, + "./merge-landmark-affined-clusters": { + "import": "./dist/merge-landmark-affined-clusters.js", + "types": "./dist/merge-landmark-affined-clusters.d.ts" + }, "./reassign-orphan-block-keys": { "import": "./dist/reassign-orphan-block-keys.js", "types": "./dist/reassign-orphan-block-keys.d.ts" diff --git a/packages/@d-zero/page-cluster/src/merge-landmark-affined-clusters.spec.ts b/packages/@d-zero/page-cluster/src/merge-landmark-affined-clusters.spec.ts new file mode 100644 index 00000000..2e1a69a2 --- /dev/null +++ b/packages/@d-zero/page-cluster/src/merge-landmark-affined-clusters.spec.ts @@ -0,0 +1,422 @@ +import { describe, expect, test } from 'vitest'; + +import { + mergeLandmarkAffinedClusters, + validateMergeLandmarkAffinedClustersOptions, +} from './merge-landmark-affined-clusters.js'; + +/** + * Builds a `
`/`
`, + }; + const fillers = Array.from({ length: 8 }, (_, i) => ({ + paths: ['dept-c', `${i}`], + stylesheetHrefs: [], + html: `${commonHeader}
`, + })); + + const result = resolvePageClusterKeys([pageA, pageB, ...fillers]); + + expect(result[0]).not.toBe(result[1]); + }); + + test('mergeRareLandmarkClusters: true bridges two different blocks once their shared header is rare and their content clears the looser gate threshold', () => { + // tokenize() discards visible text, so the two header variants below + // are distinguished by a child element's class, not by text. + const rareHeader = '
'; + const commonHeader = '
'; + const sharedMainOpen = + '
'; + const pageA = { + paths: ['dept-a', '1'], + stylesheetHrefs: [], + html: `${rareHeader}${sharedMainOpen}
`, + }; + const pageB = { + paths: ['dept-b', '1'], + stylesheetHrefs: [], + html: `${rareHeader}${sharedMainOpen}
`, + }; + const fillers = Array.from({ length: 8 }, (_, i) => ({ + paths: ['dept-c', `${i}`], + stylesheetHrefs: [], + html: `${commonHeader}
`, + })); + const pages = [pageA, pageB, ...fillers]; + + const withoutMerge = resolvePageClusterKeys(pages); + expect(withoutMerge[0]).not.toBe(withoutMerge[1]); + + const withMerge = resolvePageClusterKeys(pages, { + mergeRareLandmarkClusters: true, + landmarkRarityThreshold: 0.25, + landmarkGateSimilarityThreshold: 0.5, + }); + expect(withMerge[0]).toBe(withMerge[1]); + expect(withMerge[0]).toMatch(/^landmark-merge:/); + }); + + test('mergeRareLandmarkClusters validates its own options eagerly, even for an empty page list with no blocks to run the merge step on', () => { + expect(() => + resolvePageClusterKeys([], { + mergeRareLandmarkClusters: true, + landmarkRarityThreshold: -1, + }), + ).toThrow(RangeError); + }); + + test('mergeRareLandmarkClusters still works when excludeLandmarks: false (extractLandmarks is computed independently for the gate)', () => { + // tokenize() discards visible text, so the two header variants below + // are distinguished by a child element's class, not by text. + const rareHeader = '
'; + const commonHeader = '
'; + const pageA = { + paths: ['dept-a', '1'], + stylesheetHrefs: [], + html: `${rareHeader}
`, + }; + const pageB = { + paths: ['dept-b', '1'], + stylesheetHrefs: [], + html: `${rareHeader}
`, + }; + const fillers = Array.from({ length: 8 }, (_, i) => ({ + paths: ['dept-c', `${i}`], + stylesheetHrefs: [], + html: `${commonHeader}
`, + })); + + const result = resolvePageClusterKeys([pageA, pageB, ...fillers], { + excludeLandmarks: false, + mergeRareLandmarkClusters: true, + landmarkRarityThreshold: 0.25, + landmarkGateSimilarityThreshold: 0.5, + }); + + expect(result[0]).toBe(result[1]); + }); + + test("mergeRareLandmarkClusters does not let a shared landmark's own markup count as body-content evidence, even when excludeLandmarks: false (regression test for gate independence)", () => { + // A bulky 8-element rare header, identical on both pages, plus main + // content that is completely disjoint between the two pages. With + // excludeLandmarks: false, the *primary* clustering tokenizes raw + // HTML (header included) — but the merge gate must still judge + // similarity on landmark-excised content only. If the header's own + // tokens leaked into that judgment, shared header (8) vs disjoint + // content (2 + 2) would score 8/12 ≈ 0.667, clearing the 0.5 gate + // used here; on landmark-excised content alone the two pages share + // nothing (0/4 = 0), so they must not merge. + const rareHeader = `
${Array.from({ length: 8 }, (_, i) => ``).join('')}
`; + const commonHeader = '
'; + const pageA = { + paths: ['dept-a', '1'], + stylesheetHrefs: [], + html: `${rareHeader}
`, + }; + const pageB = { + paths: ['dept-b', '1'], + stylesheetHrefs: [], + html: `${rareHeader}
`, + }; + const fillers = Array.from({ length: 8 }, (_, i) => ({ + paths: ['dept-c', `${i}`], + stylesheetHrefs: [], + html: `${commonHeader}
`, + })); + + const result = resolvePageClusterKeys([pageA, pageB, ...fillers], { + excludeLandmarks: false, + mergeRareLandmarkClusters: true, + landmarkRarityThreshold: 0.25, + landmarkGateSimilarityThreshold: 0.5, + }); + + expect(result[0]).not.toBe(result[1]); + }); }); diff --git a/packages/@d-zero/page-cluster/src/resolve-page-cluster-keys.ts b/packages/@d-zero/page-cluster/src/resolve-page-cluster-keys.ts index 57a63264..746af3c6 100644 --- a/packages/@d-zero/page-cluster/src/resolve-page-cluster-keys.ts +++ b/packages/@d-zero/page-cluster/src/resolve-page-cluster-keys.ts @@ -1,3 +1,4 @@ +import type { ExtractLandmarksResult } from './extract-landmarks.js'; import type { ResolveBlockingGroupKeysOptions } from './resolve-blocking-group-keys.js'; import type { ResolveStructuralClusterKeysOptions } from './resolve-structural-cluster-keys.js'; import type { TokenizeOptions } from './types.js'; @@ -9,6 +10,10 @@ import { } from './detect-content-depth-cap.js'; import { extractLandmarks } from './extract-landmarks.js'; import { filterFirstPartyStylesheetHrefs } from './filter-first-party-stylesheet-hrefs.js'; +import { + mergeLandmarkAffinedClusters, + validateMergeLandmarkAffinedClustersOptions, +} from './merge-landmark-affined-clusters.js'; import { reassignOrphanBlockKeys } from './reassign-orphan-block-keys.js'; import { removeContentBlocks } from './remove-content-blocks.js'; import { resolveBlockingGroupKeys } from './resolve-blocking-group-keys.js'; @@ -37,6 +42,25 @@ function requireIndex(values: ArrayLike, index: number): T { return value; } +/** + * Narrows `landmarks` from `readonly ExtractLandmarksResult[] | undefined` to + * defined, throwing instead of asserting non-null. Always defined in + * practice at every call site below: both call sites are only reached when + * `excludeLandmarks || mergeRareLandmarkClusters` is true, exactly the + * condition under which `landmarks` is populated. Exists only to satisfy the + * type checker without a non-null assertion (same rationale as + * `requireIndex` above). + * @param landmarks + */ +function requireLandmarks( + landmarks: readonly ExtractLandmarksResult[] | undefined, +): readonly ExtractLandmarksResult[] { + if (landmarks === undefined) { + throw new Error('resolvePageClusterKeys: landmarks were not computed'); + } + return landmarks; +} + /** * Per-page input to {@link ./resolve-page-cluster-keys.js | resolvePageClusterKeys}: * the blocking signals {@link ./resolve-blocking-group-keys.js | resolveBlockingGroupKeys} @@ -184,6 +208,26 @@ export type ResolvePageClusterKeysOptions = TokenizeOptions & * measures on the same two corpora. */ autoCapMainDepth?: boolean; + /** + * Re-key two or more otherwise-distinct clusters onto one shared key + * when every landmark type present on their pages is both identical + * and rare corpus-wide — see + * {@link ./merge-landmark-affined-clusters.js | mergeLandmarkAffinedClusters}'s + * JSDoc for the exact rule, the withdrawn earlier prototype this + * reimplements, and why "rare" (not merely "identical") is required. + * Defaults to `false`. + * + * Kept `false` by default: unlike `autoCapMainDepth`/ + * `restrictStylesheetsToFirstParty`, this has not been run against + * real crawl data at all as of this change — only synthetic-fixture + * unit/regression tests. See `mergeLandmarkAffinedClusters`'s JSDoc + * for its cost profile before enabling this on a large corpus. + */ + mergeRareLandmarkClusters?: boolean; + /** Forwarded to {@link ./merge-landmark-affined-clusters.js | mergeLandmarkAffinedClusters} as-is. */ + landmarkRarityThreshold?: number; + /** Forwarded to {@link ./merge-landmark-affined-clusters.js | mergeLandmarkAffinedClusters} as-is. */ + landmarkGateSimilarityThreshold?: number; }; /** @@ -244,10 +288,28 @@ export function resolvePageClusterKeys( options?: ResolvePageClusterKeysOptions, ): string[] { const excludeLandmarks = options?.excludeLandmarks ?? true; + const mergeRareLandmarkClusters = options?.mergeRareLandmarkClusters ?? false; + if (mergeRareLandmarkClusters) { + // Eager, same rationale as autoCapMainDepth's own eager validation + // below: this option's own validation is otherwise only reached from + // mergeLandmarkAffinedClusters's call at the very end of this + // function, which never runs at all for an empty `pages`. + validateMergeLandmarkAffinedClustersOptions(options); + } + + // extractLandmarks parses the whole page once; excludeLandmarks needs + // remainderHtml and mergeRareLandmarkClusters needs the four landmark + // fields, so this computes it once up front whenever either is needed, + // rather than each option branch parsing independently. + const landmarks: readonly ExtractLandmarksResult[] | undefined = + excludeLandmarks || mergeRareLandmarkClusters + ? pages.map((page) => extractLandmarks(page.html)) + : undefined; + const contentBlockAttribute = options?.contentBlockAttribute; - const preparedHtml = pages.map((page) => { + const preparedHtml = pages.map((page, index) => { const landmarksExcised = excludeLandmarks - ? extractLandmarks(page.html).remainderHtml + ? requireIndex(requireLandmarks(landmarks), index).remainderHtml : page.html; return contentBlockAttribute === undefined ? landmarksExcised @@ -314,5 +376,30 @@ export function resolvePageClusterKeys( } } - return finalKeys; + if (!mergeRareLandmarkClusters) { + return finalKeys; + } + // Deliberately not a reuse of blockTokenSets (this function's own + // primary-clustering token sets): those follow excludeLandmarks (raw + // HTML, landmarks included, when false) and resolveStructuralClusterKeys + // narrows them further via its internal deriveComparisonSets once a + // block reaches 10+ pages. mergeLandmarkAffinedClusters's secondary + // content-similarity gate is meant to be independent corroboration + // alongside the landmark match already used to select these pages — if + // it reused landmark-inclusive tokens, a bulky shared rare header's own + // tokens could inflate two otherwise-unrelated pages' similarity past + // the gate, and if it reused the frequency-narrowed set, the gate would + // silently test different content than its own JSDoc describes. Always + // tokenizing each page's landmark-excised remainderHtml here keeps the + // gate's evidence independent of both. + const resolvedLandmarks = requireLandmarks(landmarks); + const mergeGateContentTokenSets = resolvedLandmarks.map( + (entry) => new Set(tokenize(entry.remainderHtml, options).tokens), + ); + return mergeLandmarkAffinedClusters( + finalKeys, + resolvedLandmarks, + mergeGateContentTokenSets, + options, + ); }