diff --git a/cspell.json b/cspell.json
index bd69e85a..ddad4398 100644
--- a/cspell.json
+++ b/cspell.json
@@ -16,6 +16,7 @@
"medoids",
"hrefs",
"Murtagh",
+ "unioned",
//
"gaxios",
diff --git a/packages/@d-zero/page-cluster/README.md b/packages/@d-zero/page-cluster/README.md
index d9f10164..cfcad883 100644
--- a/packages/@d-zero/page-cluster/README.md
+++ b/packages/@d-zero/page-cluster/README.md
@@ -37,6 +37,8 @@ tokenize(html, {
CMSのブロック属性名が分からない・サイトごとに違う場合は `autoCapMainDepth: true` を使う。``/`role="main"`という標準タグを起点に、構造クラスタ数が急増する直前の深さをブロックごとに実データから自動検出して打ち切るため、サイト固有の設定が一切不要(既定はfalse。実データ検証では`contentBlockAttribute`より良い結果になる場合もあった一方、計算コストが実測で数倍〜1桁台後半になる。倍率はコーパスのブロック構成に依存する)。詳細は `detectContentDepthCap()` のJSDocを参照。
+header/footer/nav/asideが一致するページ同士をさらに合流させたい場合は `mergeRareLandmarkClusters: true` を使う。ただし単純な一致判定は実データで過剰融合を招くことが分かっているため(header/footer/navは99%以上のページに存在し判別力を持たない)、コーパス全体で希少なランドマークバリアントが一致した場合に限り、より緩いコンテンツ類似度閾値(`landmarkGateSimilarityThreshold`)での合流を許可する(既定はfalse。実データでの検証は未実施で、合成フィクスチャでの単体・回帰テストのみ)。詳細・コスト特性は `mergeLandmarkAffinedClusters()` のJSDocを参照。
+
```ts
import { resolvePageClusterKeys } from '@d-zero/page-cluster/resolve-page-cluster-keys';
diff --git a/packages/@d-zero/page-cluster/package.json b/packages/@d-zero/page-cluster/package.json
index 4eebf01a..c0a0b2e2 100644
--- a/packages/@d-zero/page-cluster/package.json
+++ b/packages/@d-zero/page-cluster/package.json
@@ -49,6 +49,10 @@
"import": "./dist/jaccard-similarity.js",
"types": "./dist/jaccard-similarity.d.ts"
},
+ "./merge-landmark-affined-clusters": {
+ "import": "./dist/merge-landmark-affined-clusters.js",
+ "types": "./dist/merge-landmark-affined-clusters.d.ts"
+ },
"./reassign-orphan-block-keys": {
"import": "./dist/reassign-orphan-block-keys.js",
"types": "./dist/reassign-orphan-block-keys.d.ts"
diff --git a/packages/@d-zero/page-cluster/src/merge-landmark-affined-clusters.spec.ts b/packages/@d-zero/page-cluster/src/merge-landmark-affined-clusters.spec.ts
new file mode 100644
index 00000000..2e1a69a2
--- /dev/null
+++ b/packages/@d-zero/page-cluster/src/merge-landmark-affined-clusters.spec.ts
@@ -0,0 +1,422 @@
+import { describe, expect, test } from 'vitest';
+
+import {
+ mergeLandmarkAffinedClusters,
+ validateMergeLandmarkAffinedClustersOptions,
+} from './merge-landmark-affined-clusters.js';
+
+/**
+ * Builds a ``/``,
+ };
+ const fillers = Array.from({ length: 8 }, (_, i) => ({
+ paths: ['dept-c', `${i}`],
+ stylesheetHrefs: [],
+ html: `
${commonHeader}`,
+ }));
+
+ const result = resolvePageClusterKeys([pageA, pageB, ...fillers]);
+
+ expect(result[0]).not.toBe(result[1]);
+ });
+
+ test('mergeRareLandmarkClusters: true bridges two different blocks once their shared header is rare and their content clears the looser gate threshold', () => {
+ // tokenize() discards visible text, so the two header variants below
+ // are distinguished by a child element's class, not by text.
+ const rareHeader = '';
+ const commonHeader = '';
+ const sharedMainOpen =
+ '';
+ const pageA = {
+ paths: ['dept-a', '1'],
+ stylesheetHrefs: [],
+ html: `${rareHeader}${sharedMainOpen}`,
+ };
+ const pageB = {
+ paths: ['dept-b', '1'],
+ stylesheetHrefs: [],
+ html: `${rareHeader}${sharedMainOpen}`,
+ };
+ const fillers = Array.from({ length: 8 }, (_, i) => ({
+ paths: ['dept-c', `${i}`],
+ stylesheetHrefs: [],
+ html: `${commonHeader}`,
+ }));
+ const pages = [pageA, pageB, ...fillers];
+
+ const withoutMerge = resolvePageClusterKeys(pages);
+ expect(withoutMerge[0]).not.toBe(withoutMerge[1]);
+
+ const withMerge = resolvePageClusterKeys(pages, {
+ mergeRareLandmarkClusters: true,
+ landmarkRarityThreshold: 0.25,
+ landmarkGateSimilarityThreshold: 0.5,
+ });
+ expect(withMerge[0]).toBe(withMerge[1]);
+ expect(withMerge[0]).toMatch(/^landmark-merge:/);
+ });
+
+ test('mergeRareLandmarkClusters validates its own options eagerly, even for an empty page list with no blocks to run the merge step on', () => {
+ expect(() =>
+ resolvePageClusterKeys([], {
+ mergeRareLandmarkClusters: true,
+ landmarkRarityThreshold: -1,
+ }),
+ ).toThrow(RangeError);
+ });
+
+ test('mergeRareLandmarkClusters still works when excludeLandmarks: false (extractLandmarks is computed independently for the gate)', () => {
+ // tokenize() discards visible text, so the two header variants below
+ // are distinguished by a child element's class, not by text.
+ const rareHeader = '';
+ const commonHeader = '';
+ const pageA = {
+ paths: ['dept-a', '1'],
+ stylesheetHrefs: [],
+ html: `${rareHeader}`,
+ };
+ const pageB = {
+ paths: ['dept-b', '1'],
+ stylesheetHrefs: [],
+ html: `${rareHeader}`,
+ };
+ const fillers = Array.from({ length: 8 }, (_, i) => ({
+ paths: ['dept-c', `${i}`],
+ stylesheetHrefs: [],
+ html: `${commonHeader}`,
+ }));
+
+ const result = resolvePageClusterKeys([pageA, pageB, ...fillers], {
+ excludeLandmarks: false,
+ mergeRareLandmarkClusters: true,
+ landmarkRarityThreshold: 0.25,
+ landmarkGateSimilarityThreshold: 0.5,
+ });
+
+ expect(result[0]).toBe(result[1]);
+ });
+
+ test("mergeRareLandmarkClusters does not let a shared landmark's own markup count as body-content evidence, even when excludeLandmarks: false (regression test for gate independence)", () => {
+ // A bulky 8-element rare header, identical on both pages, plus main
+ // content that is completely disjoint between the two pages. With
+ // excludeLandmarks: false, the *primary* clustering tokenizes raw
+ // HTML (header included) — but the merge gate must still judge
+ // similarity on landmark-excised content only. If the header's own
+ // tokens leaked into that judgment, shared header (8) vs disjoint
+ // content (2 + 2) would score 8/12 ≈ 0.667, clearing the 0.5 gate
+ // used here; on landmark-excised content alone the two pages share
+ // nothing (0/4 = 0), so they must not merge.
+ const rareHeader = `${Array.from({ length: 8 }, (_, i) => ``).join('')}`;
+ const commonHeader = '';
+ const pageA = {
+ paths: ['dept-a', '1'],
+ stylesheetHrefs: [],
+ html: `${rareHeader}`,
+ };
+ const pageB = {
+ paths: ['dept-b', '1'],
+ stylesheetHrefs: [],
+ html: `${rareHeader}`,
+ };
+ const fillers = Array.from({ length: 8 }, (_, i) => ({
+ paths: ['dept-c', `${i}`],
+ stylesheetHrefs: [],
+ html: `${commonHeader}`,
+ }));
+
+ const result = resolvePageClusterKeys([pageA, pageB, ...fillers], {
+ excludeLandmarks: false,
+ mergeRareLandmarkClusters: true,
+ landmarkRarityThreshold: 0.25,
+ landmarkGateSimilarityThreshold: 0.5,
+ });
+
+ expect(result[0]).not.toBe(result[1]);
+ });
});
diff --git a/packages/@d-zero/page-cluster/src/resolve-page-cluster-keys.ts b/packages/@d-zero/page-cluster/src/resolve-page-cluster-keys.ts
index 57a63264..746af3c6 100644
--- a/packages/@d-zero/page-cluster/src/resolve-page-cluster-keys.ts
+++ b/packages/@d-zero/page-cluster/src/resolve-page-cluster-keys.ts
@@ -1,3 +1,4 @@
+import type { ExtractLandmarksResult } from './extract-landmarks.js';
import type { ResolveBlockingGroupKeysOptions } from './resolve-blocking-group-keys.js';
import type { ResolveStructuralClusterKeysOptions } from './resolve-structural-cluster-keys.js';
import type { TokenizeOptions } from './types.js';
@@ -9,6 +10,10 @@ import {
} from './detect-content-depth-cap.js';
import { extractLandmarks } from './extract-landmarks.js';
import { filterFirstPartyStylesheetHrefs } from './filter-first-party-stylesheet-hrefs.js';
+import {
+ mergeLandmarkAffinedClusters,
+ validateMergeLandmarkAffinedClustersOptions,
+} from './merge-landmark-affined-clusters.js';
import { reassignOrphanBlockKeys } from './reassign-orphan-block-keys.js';
import { removeContentBlocks } from './remove-content-blocks.js';
import { resolveBlockingGroupKeys } from './resolve-blocking-group-keys.js';
@@ -37,6 +42,25 @@ function requireIndex(values: ArrayLike, index: number): T {
return value;
}
+/**
+ * Narrows `landmarks` from `readonly ExtractLandmarksResult[] | undefined` to
+ * defined, throwing instead of asserting non-null. Always defined in
+ * practice at every call site below: both call sites are only reached when
+ * `excludeLandmarks || mergeRareLandmarkClusters` is true, exactly the
+ * condition under which `landmarks` is populated. Exists only to satisfy the
+ * type checker without a non-null assertion (same rationale as
+ * `requireIndex` above).
+ * @param landmarks
+ */
+function requireLandmarks(
+ landmarks: readonly ExtractLandmarksResult[] | undefined,
+): readonly ExtractLandmarksResult[] {
+ if (landmarks === undefined) {
+ throw new Error('resolvePageClusterKeys: landmarks were not computed');
+ }
+ return landmarks;
+}
+
/**
* Per-page input to {@link ./resolve-page-cluster-keys.js | resolvePageClusterKeys}:
* the blocking signals {@link ./resolve-blocking-group-keys.js | resolveBlockingGroupKeys}
@@ -184,6 +208,26 @@ export type ResolvePageClusterKeysOptions = TokenizeOptions &
* measures on the same two corpora.
*/
autoCapMainDepth?: boolean;
+ /**
+ * Re-key two or more otherwise-distinct clusters onto one shared key
+ * when every landmark type present on their pages is both identical
+ * and rare corpus-wide — see
+ * {@link ./merge-landmark-affined-clusters.js | mergeLandmarkAffinedClusters}'s
+ * JSDoc for the exact rule, the withdrawn earlier prototype this
+ * reimplements, and why "rare" (not merely "identical") is required.
+ * Defaults to `false`.
+ *
+ * Kept `false` by default: unlike `autoCapMainDepth`/
+ * `restrictStylesheetsToFirstParty`, this has not been run against
+ * real crawl data at all as of this change — only synthetic-fixture
+ * unit/regression tests. See `mergeLandmarkAffinedClusters`'s JSDoc
+ * for its cost profile before enabling this on a large corpus.
+ */
+ mergeRareLandmarkClusters?: boolean;
+ /** Forwarded to {@link ./merge-landmark-affined-clusters.js | mergeLandmarkAffinedClusters} as-is. */
+ landmarkRarityThreshold?: number;
+ /** Forwarded to {@link ./merge-landmark-affined-clusters.js | mergeLandmarkAffinedClusters} as-is. */
+ landmarkGateSimilarityThreshold?: number;
};
/**
@@ -244,10 +288,28 @@ export function resolvePageClusterKeys(
options?: ResolvePageClusterKeysOptions,
): string[] {
const excludeLandmarks = options?.excludeLandmarks ?? true;
+ const mergeRareLandmarkClusters = options?.mergeRareLandmarkClusters ?? false;
+ if (mergeRareLandmarkClusters) {
+ // Eager, same rationale as autoCapMainDepth's own eager validation
+ // below: this option's own validation is otherwise only reached from
+ // mergeLandmarkAffinedClusters's call at the very end of this
+ // function, which never runs at all for an empty `pages`.
+ validateMergeLandmarkAffinedClustersOptions(options);
+ }
+
+ // extractLandmarks parses the whole page once; excludeLandmarks needs
+ // remainderHtml and mergeRareLandmarkClusters needs the four landmark
+ // fields, so this computes it once up front whenever either is needed,
+ // rather than each option branch parsing independently.
+ const landmarks: readonly ExtractLandmarksResult[] | undefined =
+ excludeLandmarks || mergeRareLandmarkClusters
+ ? pages.map((page) => extractLandmarks(page.html))
+ : undefined;
+
const contentBlockAttribute = options?.contentBlockAttribute;
- const preparedHtml = pages.map((page) => {
+ const preparedHtml = pages.map((page, index) => {
const landmarksExcised = excludeLandmarks
- ? extractLandmarks(page.html).remainderHtml
+ ? requireIndex(requireLandmarks(landmarks), index).remainderHtml
: page.html;
return contentBlockAttribute === undefined
? landmarksExcised
@@ -314,5 +376,30 @@ export function resolvePageClusterKeys(
}
}
- return finalKeys;
+ if (!mergeRareLandmarkClusters) {
+ return finalKeys;
+ }
+ // Deliberately not a reuse of blockTokenSets (this function's own
+ // primary-clustering token sets): those follow excludeLandmarks (raw
+ // HTML, landmarks included, when false) and resolveStructuralClusterKeys
+ // narrows them further via its internal deriveComparisonSets once a
+ // block reaches 10+ pages. mergeLandmarkAffinedClusters's secondary
+ // content-similarity gate is meant to be independent corroboration
+ // alongside the landmark match already used to select these pages — if
+ // it reused landmark-inclusive tokens, a bulky shared rare header's own
+ // tokens could inflate two otherwise-unrelated pages' similarity past
+ // the gate, and if it reused the frequency-narrowed set, the gate would
+ // silently test different content than its own JSDoc describes. Always
+ // tokenizing each page's landmark-excised remainderHtml here keeps the
+ // gate's evidence independent of both.
+ const resolvedLandmarks = requireLandmarks(landmarks);
+ const mergeGateContentTokenSets = resolvedLandmarks.map(
+ (entry) => new Set(tokenize(entry.remainderHtml, options).tokens),
+ );
+ return mergeLandmarkAffinedClusters(
+ finalKeys,
+ resolvedLandmarks,
+ mergeGateContentTokenSets,
+ options,
+ );
}