feat(crawler): populate phase 6-d entity and edge tables

packages/@nitpicker/crawler/src/archive/phase6d/collapse-anchor-rows.spec.ts

@@ -0,0 +1,112 @@
+import { describe, it, expect } from 'vitest';
+
+import { collapseAnchorRows } from './collapse-anchor-rows.js';
+
+describe('collapseAnchorRows (anchor-edge-normalization)', () => {
+	it('collapses repeated (pageId, hrefId) pairs into a single edge with the first row winning', () => {
+		const edges = [
+			...collapseAnchorRows([
+				{ id: 1, pageId: 10, hrefId: 20, hash: 'aaa', textContent: 'first' },
+				{ id: 2, pageId: 10, hrefId: 20, hash: 'bbb', textContent: 'second' },
+				{ id: 3, pageId: 10, hrefId: 20, hash: 'ccc', textContent: 'third' },
+			]),
+		];
+		expect(edges).toHaveLength(1);
+		expect(edges[0]!).toEqual({
+			page_id: 10,
+			href_page_id: 20,
+			count: 3,
+			first_hash: 'aaa',
+			first_textContent: 'first',
+		});
+	});
+
+	it('emits distinct edges per (pageId, hrefId) pair in traversal order', () => {
+		const edges = [
+			...collapseAnchorRows([
+				{ id: 1, pageId: 10, hrefId: 20, hash: 'a', textContent: 'a' },
+				{ id: 2, pageId: 10, hrefId: 30, hash: 'b', textContent: 'b' },
+				{ id: 3, pageId: 11, hrefId: 20, hash: 'c', textContent: 'c' },
+			]),
+		];
+		expect(edges).toEqual([
+			{
+				page_id: 10,
+				href_page_id: 20,
+				count: 1,
+				first_hash: 'a',
+				first_textContent: 'a',
+			},
+			{
+				page_id: 10,
+				href_page_id: 30,
+				count: 1,
+				first_hash: 'b',
+				first_textContent: 'b',
+			},
+			{
+				page_id: 11,
+				href_page_id: 20,
+				count: 1,
+				first_hash: 'c',
+				first_textContent: 'c',
+			},
+		]);
+	});
+
+	it('preserves null hash / textContent from the first instance', () => {
+		const edges = [
+			...collapseAnchorRows([
+				{ id: 1, pageId: 5, hrefId: 6, hash: null, textContent: null },
+				{ id: 2, pageId: 5, hrefId: 6, hash: 'later', textContent: 'later' },
+			]),
+		];
+		expect(edges).toEqual([
+			{
+				page_id: 5,
+				href_page_id: 6,
+				count: 2,
+				first_hash: null,
+				first_textContent: null,
+			},
+		]);
+	});
+
+	it('yields nothing for an empty input', () => {
+		expect([...collapseAnchorRows([])]).toEqual([]);
+	});
+
+	it('throws when input is not sorted by id within a (pageId, hrefId) group', () => {
+		// Regression guard: a caller that drops the `id` tie-breaker from
+		// `ORDER BY pageId, hrefId, id` silently loses the "first
+		// instance wins" contract. The count-based acceptance check
+		// wouldn't catch it (SUM(count) is order-independent), so the
+		// collapser defends its own precondition.
+		expect(() =>
+			[
+				...collapseAnchorRows([
+					{ id: 5, pageId: 10, hrefId: 20, hash: 'later', textContent: 'later' },
+					{ id: 2, pageId: 10, hrefId: 20, hash: 'earlier', textContent: 'earlier' },
+				]),
+			]
+				// force generator evaluation
+				.at(0),
+		).toThrow(/not sorted by id/);
+	});
+
+	it('does not conflate different hrefIds within the same page', () => {
+		// Regression guard: a naive "current pair" comparator that only
+		// checked pageId would collapse pageId=10 hrefId=20 with the
+		// following pageId=10 hrefId=30 into one edge.
+		const edges = [
+			...collapseAnchorRows([
+				{ id: 1, pageId: 10, hrefId: 20, hash: 'x', textContent: 'x' },
+				{ id: 2, pageId: 10, hrefId: 30, hash: 'y', textContent: 'y' },
+				{ id: 3, pageId: 10, hrefId: 30, hash: 'z', textContent: 'z' },
+			]),
+		];
+		expect(edges).toHaveLength(2);
+		expect(edges[0]!.count).toBe(1);
+		expect(edges[1]!.count).toBe(2);
+	});
+});

packages/@nitpicker/crawler/src/archive/phase6d/collapse-anchor-rows.ts

@@ -0,0 +1,94 @@
+import type { AnchorEdgeRowInProgress, AnchorInputRow } from './types.js';
+
+/**
+ * Collapses consecutive `anchors`-shaped rows into `anchor_edges` rows by
+ * `(pageId, hrefId)` (issue #193 step 6-D-4).
+ *
+ * **Input contract**: `rows` MUST be sorted by `(pageId, hrefId, id)`
+ * ascending. The caller (`populate-anchor-edges.ts`) achieves this via a
+ * single keyset scan `ORDER BY pageId, hrefId, id` over `anchors`.
+ *
+ * The `id`-then-key sort matters because the plan's dedup rule is
+ * "first instance wins": the smallest `anchors.id` for a given
+ * `(pageId, hrefId)` pair contributes its `hash` and `textContent` to
+ * the edge; every subsequent duplicate is counted but its body is
+ * discarded. A naive `min(hash) GROUP BY pageId, hrefId` would pick the
+ * lexicographically smallest hex string (uniformly distributed) rather
+ * than the earliest occurrence, so the pass has to happen in JS.
+ *
+ * The generator yields one {@link AnchorEdgeRowInProgress} per distinct
+ * pair as soon as the next pair boundary is observed. Emitting eagerly
+ * keeps peak memory bounded to O(1) — the collapser holds at most one
+ * open pair state at a time.
+ *
+ * `first_text_id` is intentionally left `null` here; the caller resolves
+ * it in a second pass after all edges are known (see
+ * `populate-anchor-edges.ts`). The `first_hash` field is set to the
+ * first row's `hash` verbatim; a first row with `hash === null` results
+ * in `first_hash = null` on the edge (rare but possible on legacy
+ * archives that failed to compute a hash).
+ * @param rows - Anchor input rows sorted by `(pageId, hrefId, id)`.
+ * @yields {AnchorEdgeRowInProgress} One entry per distinct
+ *   `(pageId, hrefId)` pair in the input.
+ * @example
+ * const edges = [...collapseAnchorRows([
+ *   { id: 1, pageId: 10, hrefId: 20, hash: 'a', textContent: 'first' },
+ *   { id: 2, pageId: 10, hrefId: 20, hash: 'b', textContent: 'second' },
+ *   { id: 3, pageId: 10, hrefId: 30, hash: 'c', textContent: 'x' },
+ * ])];
+ * // edges[0] = { page_id: 10, href_page_id: 20, count: 2, first_hash: 'a', ... }
+ * // edges[1] = { page_id: 10, href_page_id: 30, count: 1, first_hash: 'c', ... }
+ */
+export function* collapseAnchorRows(
+	rows: Iterable<AnchorInputRow>,
+): Generator<AnchorEdgeRowInProgress> {
+	let openPageId: number | null = null;
+	let openHrefId: number | null = null;
+	let openCount = 0;
+	let openFirstHash: string | null = null;
+	let openFirstTextContent: string | null = null;
+	let openLastId: number | null = null;
+
+	for (const row of rows) {
+		if (openPageId === row.pageId && openHrefId === row.hrefId) {
+			// Same pair — assert monotonically-increasing `id` so callers
+			// that regress the `ORDER BY id` fail loudly instead of
+			// silently producing the wrong "first instance". The count-
+			// based acceptance check does not catch this (it only sums
+			// counts, not first_hash values), so the collapser defends
+			// its own precondition.
+			if (openLastId !== null && row.id <= openLastId) {
+				throw new Error(
+					`collapseAnchorRows: input not sorted by id within (pageId=${row.pageId}, hrefId=${row.hrefId}) — saw id=${row.id} after id=${openLastId}. Caller must ORDER BY pageId, hrefId, id ASC.`,
+				);
+			}
+			openLastId = row.id;
+			openCount += 1;
+			continue;
+		}
+		if (openPageId !== null && openHrefId !== null) {
+			yield {
+				page_id: openPageId,
+				href_page_id: openHrefId,
+				count: openCount,
+				first_hash: openFirstHash,
+				first_textContent: openFirstTextContent,
+			};
+		}
+		openPageId = row.pageId;
+		openHrefId = row.hrefId;
+		openCount = 1;
+		openFirstHash = row.hash;
+		openFirstTextContent = row.textContent;
+		openLastId = row.id;
+	}
+	if (openPageId !== null && openHrefId !== null) {
+		yield {
+			page_id: openPageId,
+			href_page_id: openHrefId,
+			count: openCount,
+			first_hash: openFirstHash,
+			first_textContent: openFirstTextContent,
+		};
+	}
+}

packages/@nitpicker/crawler/src/archive/phase6d/derive-dom-path.spec.ts

@@ -0,0 +1,58 @@
+import { JSDOM } from 'jsdom';
+import { describe, it, expect } from 'vitest';
+
+import { deriveDomPath } from './derive-dom-path.js';
+
+/**
+ * Parses `html` and returns the first `<img>` element for path testing.
+ * Kept as a helper so each `it` block reads as one assertion pair.
+ * @param html - HTML source to parse.
+ */
+function firstImg(html: string): Element {
+	const dom = new JSDOM(html);
+	const img = dom.window.document.querySelector('img');
+	if (img === null) {
+		throw new Error(`fixture has no <img>: ${html}`);
+	}
+	return img;
+}
+
+describe('deriveDomPath (dom-path-derivation)', () => {
+	it('produces a plain html/body[1]/img[1] path for a top-level image', () => {
+		const img = firstImg('<!doctype html><html><body><img></body></html>');
+		expect(deriveDomPath(img)).toBe('html/body[1]/img[1]');
+	});
+
+	it('counts same-tag siblings with a 1-based ordinal', () => {
+		const dom = new JSDOM(
+			'<!doctype html><html><body><img><img><img id="target"></body></html>',
+		);
+		const target = dom.window.document.querySelector('#target')!;
+		expect(deriveDomPath(target)).toBe('html/body[1]/img[3]');
+	});
+
+	it('nests ancestor tags with per-tag sibling ordinals', () => {
+		const dom = new JSDOM(
+			'<!doctype html><html><body><main><section></section><section><picture><img></picture></section></main></body></html>',
+		);
+		const img = dom.window.document.querySelector('img')!;
+		expect(deriveDomPath(img)).toBe('html/body[1]/main[1]/section[2]/picture[1]/img[1]');
+	});
+
+	it('ignores non-Element siblings when counting ordinals', () => {
+		// Text nodes are Nodes but not Elements. The walk uses
+		// `previousElementSibling`, so a text-node between two `<img>`
+		// tags must NOT reset the ordinal.
+		const dom = new JSDOM(
+			'<!doctype html><html><body><img>text between<img id="target"></body></html>',
+		);
+		const target = dom.window.document.querySelector('#target')!;
+		expect(deriveDomPath(target)).toBe('html/body[1]/img[2]');
+	});
+
+	it('handles a detached element by falling back to the traversed prefix', () => {
+		const dom = new JSDOM('<!doctype html><html><body></body></html>');
+		const detached = dom.window.document.createElement('img');
+		expect(deriveDomPath(detached)).toBe('img[1]');
+	});
+});

packages/@nitpicker/crawler/src/archive/phase6d/derive-dom-path.ts

@@ -0,0 +1,74 @@
+/**
+ * Computes the `dom_path` string for one DOM element as required by
+ * `image_items.dom_path_text_id` (issue #193 step 6-D-6).
+ *
+ * Format: slash-joined ancestor tags starting at `<html>`, each tag
+ * followed by a 1-based sibling ordinal counting **same-tag** siblings
+ * only (identical to the spec's example
+ * `html/body[1]/main[1]/section[2]/picture[1]/img[1]`). The `<html>`
+ * root has no bracketed ordinal because there is only one document
+ * root per page.
+ *
+ * The walk uses `parentElement` + `previousElementSibling` — no
+ * document mutation, no reliance on `id` / `class` — so a re-render
+ * that only reshuffles class names or ids still produces the same path.
+ * The chain of tag names alone is stable for a given DOM shape.
+ *
+ * The function operates on the generic `Element` interface from DOM lib
+ * (available in the crawler's TS lib config) so it does not force
+ * jsdom into the crawler runtime — the migration script and unit tests
+ * inject jsdom-backed elements, while a future crawler-side wrapper
+ * (Phase 6-G) can pass puppeteer's DOM handles through the same code.
+ *
+ * Detached elements (`element.parentElement === null` before reaching
+ * `<html>`) return the tag chain from wherever the walk terminates.
+ * This matches the plan's "walk the DOM ancestor chain from the
+ * `<img>` element to `<html>`" — a legally detached image (rare, but
+ * technically possible during scripted mutations) yields e.g.
+ * `img[1]` alone instead of throwing.
+ * @param element - The DOM element to derive the path for. Typically an
+ *   `<img>` in the archived HTML, but the function is not tag-specific.
+ * @returns The dom_path string.
+ * @example
+ * // In an HTML snapshot `<html><body><main><img></main></body></html>`
+ * deriveDomPath(imgElement); // 'html/body[1]/main[1]/img[1]'
+ */
+export function deriveDomPath(element: Element): string {
+	const segments: string[] = [];
+	let current: Element | null = element;
+	while (current !== null) {
+		const tag = current.tagName.toLowerCase();
+		if (tag === 'html') {
+			segments.unshift('html');
+			break;
+		}
+		segments.unshift(`${tag}[${computeSiblingOrdinal(current, tag)}]`);
+		current = current.parentElement;
+	}
+	return segments.join('/');
+}
+
+/**
+ * Counts how many earlier siblings share the same tag name as `element`,
+ * returning `count + 1` as the 1-based ordinal.
+ *
+ * Iterating `previousElementSibling` is O(k) where k is the sibling
+ * position; for typical DOM depth and sibling counts this is negligible
+ * next to the jsdom parse cost of the enclosing page.
+ * @param element - The element whose ordinal is being computed.
+ * @param tag - The lower-cased tag name to match on. Passed in by the
+ *   caller so `element.tagName.toLowerCase()` is not recomputed inside
+ *   the loop for every sibling comparison.
+ * @returns 1-based ordinal among same-tag siblings.
+ */
+function computeSiblingOrdinal(element: Element, tag: string): number {
+	let count = 1;
+	let sibling: Element | null = element.previousElementSibling;
+	while (sibling !== null) {
+		if (sibling.tagName.toLowerCase() === tag) {
+			count += 1;
+		}
+		sibling = sibling.previousElementSibling;
+	}
+	return count;
+}