feat(page-cluster): self-tuning recursive clustering with data-driven landmark scoping

.gitignore

@@ -81,6 +81,9 @@ test/fixture/.__*
 # Print Result Directory
 .print
 
+# page-cluster dogfooding scratch (results, throwaway scripts)
+.page-cluster
+
 # Secret Test Files
 ___*
 

packages/@d-zero/page-cluster/README.md

@@ -31,11 +31,13 @@ tokenize(html, {
 
 ### クラスタリング
 
-クロールしたページ群から最終的なクラスタキーを得るには `resolvePageClusterKeys()` を使う。ブロッキング（URLパス/スタイルシートによる粗い絞り込み）と構造クラスタリング（ブロック内でのcomplete-linkage階層的クラスタリング）を内部で連結し、ブロックを跨いで一意なキーを返す。既定で各ページの`<header>`/`<footer>`/`<nav>`/`<aside>`（タグ名またはARIAランドマークロール）を比較対象から除外し、共通chromeの影響を受けにくくする。また、スタイルシート参照が記録されていない「孤児」ページを、同一URLセクションに閉じたスタイルシート・ブロックへ再割当する処理（`reassignOrphanBlockKeys()`）や、埋め込みコンテンツが引き込むサードパーティCSS参照をブロッキング判定から除外する処理（`filterFirstPartyStylesheetHrefs()`）も既定で有効。挙動の詳細・トレードオフはそれぞれのJSDocを参照。
+クロールしたページ群から最終的なクラスタキーを得るには `resolvePageClusterKeys()` を使う。ブロッキング（URLパス/スタイルシートによる粗い絞り込み）と構造クラスタリング（ブロック内でのcomplete-linkage階層的クラスタリング）を内部で連結し、ブロックを跨いで一意なキーを返す。既定で各ページのランドマーク要素（`<header>`/`<footer>`/`<nav>`/`<aside>`/`<search>`および`role="form"`/`role="search"`。タグ名またはARIAランドマークロールで判定）を比較対象から除外し、共通chromeの影響を受けにくくする。また、スタイルシート参照が記録されていない「孤児」ページを、同一URLセクションに閉じたスタイルシート・ブロックへ再割当する処理（`reassignOrphanBlockKeys()`）や、埋め込みコンテンツが引き込むサードパーティCSS参照をブロッキング判定から除外する処理（`filterFirstPartyStylesheetHrefs()`）も既定で有効。挙動の詳細・トレードオフはそれぞれのJSDocを参照。
 
 自由編集ブロックエディタ（CMSが各コンテンツブロックに固有のdata属性を付与するタイプ）を使うサイトでは、`contentBlockAttribute` オプションでその属性名を指定すると、ページごとに異なるブロック構成が構造比較のノイズになるのを防げる（既定は未指定＝無効、サイトごとの属性名を推測できないため）。詳細は `removeContentBlocks()` のJSDocを参照。
 
-CMSのブロック属性名が分からない・サイトごとに違う場合は `autoCapMainDepth: true` を使う。`<main>`/`role="main"`という標準タグを起点に、構造クラスタ数が急増する直前の深さをブロックごとに実データから自動検出して打ち切るため、サイト固有の設定が一切不要（既定はfalse。実データ検証では`contentBlockAttribute`より良い結果になる場合もあった一方、計算コストが実測で数倍〜1桁台後半になる。倍率はコーパスのブロック構成に依存する）。詳細は `detectContentDepthCap()` のJSDocを参照。
+CMSのブロック属性名が分からない・サイトごとに違う場合、`autoCapMainDepth` を使う。`<main>`/`role="main"`という標準タグを起点に、構造クラスタ数が急増する直前の深さをブロックごとに実データから自動検出して打ち切るため、サイト固有の設定が一切不要（既定はtrue。無効にするには `autoCapMainDepth: false` を渡す。実データ検証では`contentBlockAttribute`より良い結果になる場合もあった。計算コストは実測で数倍〜1桁台後半。倍率はコーパスのブロック構成に依存する）。詳細は `detectContentDepthCap()` のJSDocを参照。
+
+ブロッキング後の各ブロックを個別に処理する Stage A に加え、`resolvePageClusterKeys()` はすべてのブロックを処理した後、ブロック境界を越えた Stage B（クロスブロック統合）を常時実行する。URLパス・スタイルシート集合が異なるだけで構造的に同一テンプレートと判定できるユニットを、クォーラムコア（メンバーページの80%以上が持つコーパス識別トークン）ベースの complete-linkage・包含・シェイプ類似度（クラス名剥きトークン）とL2シグネチャ比較（シェル裏付き）を組み合わせて不動点まで反復的に統合する（実コーパスで2〜5ラウンド収束）。詳細は `mergeCrossBlockClusters()` のJSDocを参照。
 
 header/footer/nav/asideが一致するページ同士をさらに合流させたい場合は `mergeRareLandmarkClusters: true` を使う。ただし単純な一致判定は実データで過剰融合を招くことが分かっているため（header/footer/navは99%以上のページに存在し判別力を持たない）、コーパス全体で希少なランドマークバリアントが一致した場合に限り、より緩いコンテンツ類似度閾値（`landmarkGateSimilarityThreshold`）での合流を許可する（既定はfalse。実データでの検証は未実施で、合成フィクスチャでの単体・回帰テストのみ）。詳細・コスト特性は `mergeLandmarkAffinedClusters()` のJSDocを参照。
 

packages/@d-zero/page-cluster/src/assign-contained-clusters.spec.ts

@@ -0,0 +1,116 @@
+import { describe, expect, test } from 'vitest';
+
+import { assignContainedClusters } from './assign-contained-clusters.js';
+
+describe('assignContainedClusters', () => {
+	test('single cluster maps to itself', () => {
+		const result = assignContainedClusters([
+			{ id: 0, tokens: new Set(['a', 'b', 'c']), pageCount: 3 },
+		]);
+		expect(result.get(0)).toBe(0);
+	});
+
+	test('empty cluster is never assigned (skipped in phase 1)', () => {
+		const result = assignContainedClusters([
+			{ id: 0, tokens: new Set<string>(), pageCount: 0 },
+			{ id: 1, tokens: new Set(['a', 'b']), pageCount: 2 },
+		]);
+		expect(result.get(0)).toBe(0);
+		expect(result.get(1)).toBe(1);
+	});
+
+	test('X wholly contained by Y: X → Y, Y → itself', () => {
+		const result = assignContainedClusters([
+			{ id: 0, tokens: new Set(['a', 'b']), pageCount: 1 },
+			{ id: 1, tokens: new Set(['a', 'b', 'c', 'd']), pageCount: 5 },
+		]);
+		expect(result.get(0)).toBe(1);
+		expect(result.get(1)).toBe(1);
+	});
+
+	test('assignment is one-directional: Y does not absorb into X', () => {
+		// Containment(Y → X) = 2/4 = 0.5 < 0.9 → Y is NOT absorbed by X
+		const result = assignContainedClusters([
+			{ id: 0, tokens: new Set(['a', 'b']), pageCount: 1 },
+			{ id: 1, tokens: new Set(['a', 'b', 'c', 'd']), pageCount: 5 },
+		]);
+		expect(result.get(1)).toBe(1); // Y is NOT absorbed into X
+	});
+
+	test('containment below 0.9 is ignored', () => {
+		// 8 tokens shared out of 10 = 0.8 containment < 0.9 → no assignment
+		const xTokens = new Set(['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j']);
+		const yTokens = new Set(['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'x', 'y']);
+		const result = assignContainedClusters([
+			{ id: 0, tokens: xTokens, pageCount: 1 },
+			{ id: 1, tokens: yTokens, pageCount: 5 },
+		]);
+		expect(result.get(0)).toBe(0);
+		expect(result.get(1)).toBe(1);
+	});
+
+	test('hub pattern: X and Z both absorbed by hub, but not merged with each other', () => {
+		// Hub has all tokens; X ⊂ Hub, Z ⊂ Hub, but X and Z are disjoint
+		const result = assignContainedClusters([
+			{ id: 0, tokens: new Set(['a', 'b']), pageCount: 1 },
+			{ id: 1, tokens: new Set(['a', 'b', 'c', 'd']), pageCount: 5 },
+			{ id: 2, tokens: new Set(['c', 'd']), pageCount: 1 },
+		]);
+		expect(result.get(0)).toBe(1); // X → hub
+		expect(result.get(2)).toBe(1); // Z → hub
+		expect(result.get(1)).toBe(1); // hub → itself
+		// X and Z map to the same hub but are not connected to each other
+	});
+
+	test('chain A → B → C: all resolve to C', () => {
+		// A ⊂ B ⊂ C
+		const result = assignContainedClusters([
+			{ id: 0, tokens: new Set(['a']), pageCount: 1 },
+			{ id: 1, tokens: new Set(['a', 'b']), pageCount: 2 },
+			{ id: 2, tokens: new Set(['a', 'b', 'c', 'd']), pageCount: 3 },
+		]);
+		expect(result.get(0)).toBe(2);
+		expect(result.get(1)).toBe(2);
+		expect(result.get(2)).toBe(2);
+	});
+
+	test('cycle: mutual containment ≥ 0.9 → larger token set is root', () => {
+		// A containment in B = 10/10 = 1.0 → A → B
+		// B containment in A = 10/11 ≈ 0.909 → B → A
+		// Cycle detected; B has more tokens → B is root
+		const aTokens = new Set(['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j']);
+		const bTokens = new Set(['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k']);
+		const result = assignContainedClusters([
+			{ id: 0, tokens: aTokens, pageCount: 1 },
+			{ id: 1, tokens: bTokens, pageCount: 1 },
+		]);
+		expect(result.get(0)).toBe(1);
+		expect(result.get(1)).toBe(1);
+	});
+
+	test('cycle tie-break uses pageCount when token sizes are equal', () => {
+		// Both share 9/10 tokens each → containment 0.9 → mutual assignment → cycle
+		// A has 10 tokens, B has 10 tokens (equal); B has more pages → B wins
+		const sharedTokens = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i'];
+		const aTokens = new Set([...sharedTokens, 'unique-a']);
+		const bTokens = new Set([...sharedTokens, 'unique-b']);
+		const result = assignContainedClusters([
+			{ id: 0, tokens: aTokens, pageCount: 1 },
+			{ id: 1, tokens: bTokens, pageCount: 5 },
+		]);
+		// Both have 10 tokens; B has more pages → B is root
+		expect(result.get(0)).toBe(1);
+		expect(result.get(1)).toBe(1);
+	});
+
+	test('every id has an entry in the result map', () => {
+		const result = assignContainedClusters([
+			{ id: 0, tokens: new Set(['a']), pageCount: 1 },
+			{ id: 1, tokens: new Set(['b']), pageCount: 1 },
+			{ id: 2, tokens: new Set(['c']), pageCount: 1 },
+		]);
+		expect(result.has(0)).toBe(true);
+		expect(result.has(1)).toBe(true);
+		expect(result.has(2)).toBe(true);
+	});
+});

packages/@d-zero/page-cluster/src/assign-contained-clusters.ts

@@ -0,0 +1,183 @@
+/**
+ * Minimum containment fraction for cluster X to be assigned into cluster Y.
+ * Containment is `|X ∩ Y| / |X|` — the fraction of X's tokens also present
+ * in Y's token set. At this threshold, X is "almost a subset" of Y.
+ *
+ * 0.9 was chosen because the observed containment scores for genuine
+ * template-subset relationships (e.g. a page missing one optional section)
+ * cluster tightly between 1.000 and 0.90–0.92 on real crawl data, while
+ * unrelated clusters score well below 0.8. A gap exists between ~0.92 and
+ * ~0.80 in practice, so the exact value within that gap is not sensitive.
+ */
+const CONTAINMENT_CUTOFF = 0.9;
+
+/**
+ * One cluster's entry for containment comparison. `tokens` is the union of
+ * comparison-set tokens for all pages in the cluster (Stage A uses
+ * frequency-narrowed comparison sets; Stage B uses quorum cores).
+ * `pageCount` is used only as a tiebreaker when containment and union size
+ * are tied between two candidate targets.
+ */
+export type ContainedClusterEntry = {
+	readonly id: number;
+	readonly tokens: ReadonlySet<string>;
+	readonly pageCount: number;
+};
+
+/**
+ * Assigns each cluster to the "best" cluster that contains it (i.e., whose
+ * token set subsumes the cluster's token union at `>= CONTAINMENT_CUTOFF`).
+ * Returns a `Map<id, rootId>` — clusters not assigned to anything map to
+ * themselves.
+ *
+ * This is a *directed* assignment (not union-find): X is absorbed by Y but Y
+ * is not absorbed by X, unless Y is independently assigned elsewhere too.
+ * This prevents hub-chaining: if /help/ is a structural superset of many
+ * clusters (because its HTML includes every nav variant), each of those
+ * clusters gets assigned to /help/, but they don't get merged with *each
+ * other* — only with /help/. Confirmed on real crawl data: a pure union-find
+ * approach produced a 9-cluster hub chain through one common-superset page.
+ *
+ * Principle: conditional rendering only *removes* elements from a template
+ * (an empty section, a missing paginator) — it never adds. So a
+ * conditionally-shorter page is always a structural subset of the
+ * full-featured template. Confirmed on real crawl data: a 43-page works
+ * cluster contained a 3-page outlier cluster at containment 1.000.
+ *
+ * Best target selection: highest containment → largest union size → most
+ * pages. Chain resolution and cycle breaking are applied after all raw
+ * assignments are computed (see the implementation).
+ *
+ * Cycles (mutual containment ≥ 0.9) mean the two clusters are practically
+ * identical token sets. The one with the larger token set (more pages as
+ * tiebreaker) becomes the root of the cycle.
+ * @param clusters
+ */
+export function assignContainedClusters(
+	clusters: readonly ContainedClusterEntry[],
+): Map<number, number> {
+	// Phase 1: find best raw assignment for each cluster
+	const raw = new Map<number, number>();
+
+	for (const x of clusters) {
+		if (x.tokens.size === 0) continue;
+
+		let bestTargetId = -1;
+		let bestContainment = CONTAINMENT_CUTOFF - 1e-9;
+		let bestUnionSize = 0;
+		let bestPageCount = 0;
+
+		for (const y of clusters) {
+			if (y.id === x.id) continue;
+
+			let intersection = 0;
+			for (const token of x.tokens) {
+				if (y.tokens.has(token)) intersection++;
+			}
+			const containment = intersection / x.tokens.size;
+			if (containment < CONTAINMENT_CUTOFF) continue;
+
+			const unionSize = x.tokens.size + y.tokens.size - intersection;
+			if (
+				containment > bestContainment ||
+				(containment === bestContainment && unionSize > bestUnionSize) ||
+				(containment === bestContainment &&
+					unionSize === bestUnionSize &&
+					y.pageCount > bestPageCount)
+			) {
+				bestContainment = containment;
+				bestUnionSize = unionSize;
+				bestPageCount = y.pageCount;
+				bestTargetId = y.id;
+			}
+		}
+
+		if (bestTargetId >= 0) {
+			raw.set(x.id, bestTargetId);
+		}
+	}
+
+	// Phase 2: resolve chains and cycles
+	// Walk the raw assignment chain from each node; detect cycles by tracking
+	// the path walked so far.
+	const resolved = new Map<number, number>();
+	const idToEntry = new Map(clusters.map((cl) => [cl.id, cl]));
+
+	for (const c of clusters) {
+		if (resolved.has(c.id)) continue;
+
+		const path: number[] = [];
+		const pathSet = new Set<number>();
+		let current = c.id;
+
+		// Walk until we reach a node with no further assignment or a cycle
+		while (!resolved.has(current)) {
+			const next = raw.get(current);
+			if (next === undefined) {
+				// No assignment → current is a root
+				resolved.set(current, current);
+				break;
+			}
+			if (pathSet.has(next)) {
+				// Cycle detected: find the cycle members and pick the root.
+				// Include `current` (the node that closed the back-edge) so it
+				// participates in root selection even if it has the largest token set.
+				const cycleStart = path.indexOf(next);
+				const cycleIds = [...path.slice(cycleStart), current];
+
+				// Root of the cycle: cluster with the largest token set
+				// (page count as tiebreaker)
+				let cycleRoot = cycleIds[0] ?? current;
+				for (const id of cycleIds) {
+					const bc = idToEntry.get(cycleRoot);
+					const cc = idToEntry.get(id);
+					const bcSize = bc?.tokens.size ?? 0;
+					const ccSize = cc?.tokens.size ?? 0;
+					if (ccSize > bcSize) {
+						cycleRoot = id;
+					} else if (ccSize === bcSize && (cc?.pageCount ?? 0) > (bc?.pageCount ?? 0)) {
+						cycleRoot = id;
+					}
+				}
+
+				for (const id of cycleIds) {
+					resolved.set(id, cycleRoot);
+				}
+				// Everything before the cycle resolves to the cycle root too
+				for (const id of path.slice(0, cycleStart)) {
+					resolved.set(id, cycleRoot);
+				}
+				break;
+			}
+			if (resolved.has(next)) {
+				// Already resolved — propagate to everything in path
+				const root = resolved.get(next) ?? next;
+				resolved.set(current, root);
+				for (const id of path) {
+					resolved.set(id, root);
+				}
+				break;
+			}
+			path.push(current);
+			pathSet.add(current);
+			current = next;
+		}
+
+		// If the path didn't resolve in the loop above, propagate what we know
+		if (!resolved.has(c.id) && resolved.has(current)) {
+			const root = resolved.get(current) ?? current;
+			for (const id of path) {
+				resolved.set(id, root);
+			}
+		}
+	}
+
+	// Ensure every cluster id has an entry (fallback to self)
+	for (const c of clusters) {
+		if (!resolved.has(c.id)) {
+			resolved.set(c.id, c.id);
+		}
+	}
+
+	return resolved;
+}

packages/@d-zero/page-cluster/src/auto-cut-threshold.spec.ts

@@ -0,0 +1,38 @@
+import { describe, expect, test } from 'vitest';
+
+import { autoCutThreshold } from './auto-cut-threshold.js';
+
+describe('autoCutThreshold', () => {
+	test('empty array returns upperBound', () => {
+		expect(autoCutThreshold([], 0.8)).toBe(0.8);
+	});
+
+	test('single-element array returns upperBound', () => {
+		expect(autoCutThreshold([0.5], 0.8)).toBe(0.8);
+	});
+
+	test('all-equal heights return upperBound (no gap to measure)', () => {
+		expect(autoCutThreshold([0.5, 0.5, 0.5], 0.8)).toBe(0.8);
+	});
+
+	test('returns the midpoint of the largest gap', () => {
+		// sorted [0.9, 0.1]: gap = 0.8, midpoint = (0.9 + 0.1) / 2 = 0.5
+		expect(autoCutThreshold([0.9, 0.1], 0.8)).toBe(0.5);
+	});
+
+	test('clamps midpoint to upperBound when midpoint exceeds it', () => {
+		// sorted [0.95, 0.85]: midpoint = 0.9, upperBound = 0.8 → clamped to 0.8
+		expect(autoCutThreshold([0.95, 0.85], 0.8)).toBe(0.8);
+	});
+
+	test('returns midpoint unchanged when it is below upperBound', () => {
+		// sorted [0.6, 0.2]: 0.6 + 0.2 === 0.8 (exact in IEEE 754), midpoint = 0.4
+		expect(autoCutThreshold([0.6, 0.2], 0.8)).toBe(0.4);
+	});
+
+	test('selects the largest gap, not the first', () => {
+		// sorted [0.9, 0.8, 0.2, 0.1]: gaps [0.1, 0.6, 0.1] — largest at index 1
+		// midpoint = (0.8 + 0.2) / 2 = 0.5
+		expect(autoCutThreshold([0.9, 0.8, 0.2, 0.1], 0.8)).toBe(0.5);
+	});
+});