From 4b11ca85e7f2c947c8e370d25dc69067c595abde Mon Sep 17 00:00:00 2001 From: Yusuke Hirao Date: Mon, 13 Jul 2026 23:02:14 +0900 Subject: [PATCH] feat(crawler): populate phase 6-d entity and edge tables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the Phase 6-D migration helpers that populate content_items, page_meta, resource_items, anchor_edges, resource_ref_edges, and image_items from the legacy write model, plus a standalone migration script `scripts/migrate-to-phase6.mjs` that runs the whole 6-A → 6-D sequence on an existing archive. Every populate helper is chunked keyset-paginated and idempotent via `INSERT OR IGNORE` on the natural key. `anchor_edges` collapses duplicate `(pageId, hrefId)` rows via a single-pass JS scan whose "first row wins" contract is guarded by a sort-order assertion. `image_items.dom_path_text_id` is derived through an injected callback so the crawler runtime stays jsdom-free; the migration script wires the jsdom-backed implementation. Correctness safeguards applied in response to review findings: - `resolve-blob-refs` maps `hash → [raw values]` so two distinct data-URI strings that decode to the same payload both get the shared blob_refs id. - `resolve-header-sets` falls back from `raw_json_hash` to `raw_hash` so a JSON variant with different key ordering still resolves to the shared header_sets row. - Populate helpers throw when a non-null source value has no matching ref, instead of silently writing NULL — the count-based acceptance check would otherwise pass while individual rows lost data. - `matchImagesToDomPaths` overflow returns `unknown/` instead of mapping every extra row to the last DOM candidate. - `populateImageItems` iterates by pageId, resolving each page's HTML and dom paths exactly once so a page whose images straddle a chunk cannot re-parse HTML or reset the ordinal cursor. Co-Authored-By: Claude Opus 4.7 --- .../phase6d/collapse-anchor-rows.spec.ts | 112 +++++ .../archive/phase6d/collapse-anchor-rows.ts | 94 +++++ .../archive/phase6d/derive-dom-path.spec.ts | 58 +++ .../src/archive/phase6d/derive-dom-path.ts | 74 ++++ .../phase6d/match-images-to-dom-paths.spec.ts | 120 ++++++ .../phase6d/match-images-to-dom-paths.ts | 141 +++++++ .../phase6d/populate-anchor-edges.spec.ts | 146 +++++++ .../archive/phase6d/populate-anchor-edges.ts | 149 +++++++ .../phase6d/populate-content-items.spec.ts | 141 +++++++ .../archive/phase6d/populate-content-items.ts | 205 +++++++++ .../phase6d/populate-image-items.spec.ts | 266 ++++++++++++ .../archive/phase6d/populate-image-items.ts | 319 ++++++++++++++ .../phase6d/populate-page-meta.spec.ts | 126 ++++++ .../src/archive/phase6d/populate-page-meta.ts | 319 ++++++++++++++ .../phase6d/populate-phase6d-entities.spec.ts | 82 ++++ .../phase6d/populate-phase6d-entities.ts | 84 ++++ .../phase6d/populate-resource-items.spec.ts | 65 +++ .../phase6d/populate-resource-items.ts | 149 +++++++ .../populate-resource-ref-edges.spec.ts | 59 +++ .../phase6d/populate-resource-ref-edges.ts | 36 ++ .../archive/phase6d/resolve-blob-refs.spec.ts | 87 ++++ .../src/archive/phase6d/resolve-blob-refs.ts | 107 +++++ .../phase6d/resolve-content-type-refs.spec.ts | 37 ++ .../phase6d/resolve-content-type-refs.ts | 33 ++ .../phase6d/resolve-header-sets.spec.ts | 73 ++++ .../archive/phase6d/resolve-header-sets.ts | 131 ++++++ .../archive/phase6d/resolve-json-refs.spec.ts | 34 ++ .../src/archive/phase6d/resolve-json-refs.ts | 74 ++++ .../archive/phase6d/resolve-text-refs.spec.ts | 38 ++ .../src/archive/phase6d/resolve-text-refs.ts | 69 +++ .../archive/phase6d/resolve-url-refs.spec.ts | 59 +++ .../src/archive/phase6d/resolve-url-refs.ts | 67 +++ .../archive/phase6d/test-utils/count-rows.ts | 26 ++ .../phase6d/test-utils/setup-phase6d-db.ts | 146 +++++++ .../crawler/src/archive/phase6d/types.ts | 152 +++++++ .../archive/phase6d/upsert-text-refs.spec.ts | 38 ++ .../src/archive/phase6d/upsert-text-refs.ts | 85 ++++ scripts/migrate-to-phase6.mjs | 396 ++++++++++++++++++ 38 files changed, 4397 insertions(+) create mode 100644 packages/@nitpicker/crawler/src/archive/phase6d/collapse-anchor-rows.spec.ts create mode 100644 packages/@nitpicker/crawler/src/archive/phase6d/collapse-anchor-rows.ts create mode 100644 packages/@nitpicker/crawler/src/archive/phase6d/derive-dom-path.spec.ts create mode 100644 packages/@nitpicker/crawler/src/archive/phase6d/derive-dom-path.ts create mode 100644 packages/@nitpicker/crawler/src/archive/phase6d/match-images-to-dom-paths.spec.ts create mode 100644 packages/@nitpicker/crawler/src/archive/phase6d/match-images-to-dom-paths.ts create mode 100644 packages/@nitpicker/crawler/src/archive/phase6d/populate-anchor-edges.spec.ts create mode 100644 packages/@nitpicker/crawler/src/archive/phase6d/populate-anchor-edges.ts create mode 100644 packages/@nitpicker/crawler/src/archive/phase6d/populate-content-items.spec.ts create mode 100644 packages/@nitpicker/crawler/src/archive/phase6d/populate-content-items.ts create mode 100644 packages/@nitpicker/crawler/src/archive/phase6d/populate-image-items.spec.ts create mode 100644 packages/@nitpicker/crawler/src/archive/phase6d/populate-image-items.ts create mode 100644 packages/@nitpicker/crawler/src/archive/phase6d/populate-page-meta.spec.ts create mode 100644 packages/@nitpicker/crawler/src/archive/phase6d/populate-page-meta.ts create mode 100644 packages/@nitpicker/crawler/src/archive/phase6d/populate-phase6d-entities.spec.ts create mode 100644 packages/@nitpicker/crawler/src/archive/phase6d/populate-phase6d-entities.ts create mode 100644 packages/@nitpicker/crawler/src/archive/phase6d/populate-resource-items.spec.ts create mode 100644 packages/@nitpicker/crawler/src/archive/phase6d/populate-resource-items.ts create mode 100644 packages/@nitpicker/crawler/src/archive/phase6d/populate-resource-ref-edges.spec.ts create mode 100644 packages/@nitpicker/crawler/src/archive/phase6d/populate-resource-ref-edges.ts create mode 100644 packages/@nitpicker/crawler/src/archive/phase6d/resolve-blob-refs.spec.ts create mode 100644 packages/@nitpicker/crawler/src/archive/phase6d/resolve-blob-refs.ts create mode 100644 packages/@nitpicker/crawler/src/archive/phase6d/resolve-content-type-refs.spec.ts create mode 100644 packages/@nitpicker/crawler/src/archive/phase6d/resolve-content-type-refs.ts create mode 100644 packages/@nitpicker/crawler/src/archive/phase6d/resolve-header-sets.spec.ts create mode 100644 packages/@nitpicker/crawler/src/archive/phase6d/resolve-header-sets.ts create mode 100644 packages/@nitpicker/crawler/src/archive/phase6d/resolve-json-refs.spec.ts create mode 100644 packages/@nitpicker/crawler/src/archive/phase6d/resolve-json-refs.ts create mode 100644 packages/@nitpicker/crawler/src/archive/phase6d/resolve-text-refs.spec.ts create mode 100644 packages/@nitpicker/crawler/src/archive/phase6d/resolve-text-refs.ts create mode 100644 packages/@nitpicker/crawler/src/archive/phase6d/resolve-url-refs.spec.ts create mode 100644 packages/@nitpicker/crawler/src/archive/phase6d/resolve-url-refs.ts create mode 100644 packages/@nitpicker/crawler/src/archive/phase6d/test-utils/count-rows.ts create mode 100644 packages/@nitpicker/crawler/src/archive/phase6d/test-utils/setup-phase6d-db.ts create mode 100644 packages/@nitpicker/crawler/src/archive/phase6d/types.ts create mode 100644 packages/@nitpicker/crawler/src/archive/phase6d/upsert-text-refs.spec.ts create mode 100644 packages/@nitpicker/crawler/src/archive/phase6d/upsert-text-refs.ts create mode 100644 scripts/migrate-to-phase6.mjs diff --git a/packages/@nitpicker/crawler/src/archive/phase6d/collapse-anchor-rows.spec.ts b/packages/@nitpicker/crawler/src/archive/phase6d/collapse-anchor-rows.spec.ts new file mode 100644 index 0000000..4ae71a5 --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6d/collapse-anchor-rows.spec.ts @@ -0,0 +1,112 @@ +import { describe, it, expect } from 'vitest'; + +import { collapseAnchorRows } from './collapse-anchor-rows.js'; + +describe('collapseAnchorRows (anchor-edge-normalization)', () => { + it('collapses repeated (pageId, hrefId) pairs into a single edge with the first row winning', () => { + const edges = [ + ...collapseAnchorRows([ + { id: 1, pageId: 10, hrefId: 20, hash: 'aaa', textContent: 'first' }, + { id: 2, pageId: 10, hrefId: 20, hash: 'bbb', textContent: 'second' }, + { id: 3, pageId: 10, hrefId: 20, hash: 'ccc', textContent: 'third' }, + ]), + ]; + expect(edges).toHaveLength(1); + expect(edges[0]!).toEqual({ + page_id: 10, + href_page_id: 20, + count: 3, + first_hash: 'aaa', + first_textContent: 'first', + }); + }); + + it('emits distinct edges per (pageId, hrefId) pair in traversal order', () => { + const edges = [ + ...collapseAnchorRows([ + { id: 1, pageId: 10, hrefId: 20, hash: 'a', textContent: 'a' }, + { id: 2, pageId: 10, hrefId: 30, hash: 'b', textContent: 'b' }, + { id: 3, pageId: 11, hrefId: 20, hash: 'c', textContent: 'c' }, + ]), + ]; + expect(edges).toEqual([ + { + page_id: 10, + href_page_id: 20, + count: 1, + first_hash: 'a', + first_textContent: 'a', + }, + { + page_id: 10, + href_page_id: 30, + count: 1, + first_hash: 'b', + first_textContent: 'b', + }, + { + page_id: 11, + href_page_id: 20, + count: 1, + first_hash: 'c', + first_textContent: 'c', + }, + ]); + }); + + it('preserves null hash / textContent from the first instance', () => { + const edges = [ + ...collapseAnchorRows([ + { id: 1, pageId: 5, hrefId: 6, hash: null, textContent: null }, + { id: 2, pageId: 5, hrefId: 6, hash: 'later', textContent: 'later' }, + ]), + ]; + expect(edges).toEqual([ + { + page_id: 5, + href_page_id: 6, + count: 2, + first_hash: null, + first_textContent: null, + }, + ]); + }); + + it('yields nothing for an empty input', () => { + expect([...collapseAnchorRows([])]).toEqual([]); + }); + + it('throws when input is not sorted by id within a (pageId, hrefId) group', () => { + // Regression guard: a caller that drops the `id` tie-breaker from + // `ORDER BY pageId, hrefId, id` silently loses the "first + // instance wins" contract. The count-based acceptance check + // wouldn't catch it (SUM(count) is order-independent), so the + // collapser defends its own precondition. + expect(() => + [ + ...collapseAnchorRows([ + { id: 5, pageId: 10, hrefId: 20, hash: 'later', textContent: 'later' }, + { id: 2, pageId: 10, hrefId: 20, hash: 'earlier', textContent: 'earlier' }, + ]), + ] + // force generator evaluation + .at(0), + ).toThrow(/not sorted by id/); + }); + + it('does not conflate different hrefIds within the same page', () => { + // Regression guard: a naive "current pair" comparator that only + // checked pageId would collapse pageId=10 hrefId=20 with the + // following pageId=10 hrefId=30 into one edge. + const edges = [ + ...collapseAnchorRows([ + { id: 1, pageId: 10, hrefId: 20, hash: 'x', textContent: 'x' }, + { id: 2, pageId: 10, hrefId: 30, hash: 'y', textContent: 'y' }, + { id: 3, pageId: 10, hrefId: 30, hash: 'z', textContent: 'z' }, + ]), + ]; + expect(edges).toHaveLength(2); + expect(edges[0]!.count).toBe(1); + expect(edges[1]!.count).toBe(2); + }); +}); diff --git a/packages/@nitpicker/crawler/src/archive/phase6d/collapse-anchor-rows.ts b/packages/@nitpicker/crawler/src/archive/phase6d/collapse-anchor-rows.ts new file mode 100644 index 0000000..0800ed1 --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6d/collapse-anchor-rows.ts @@ -0,0 +1,94 @@ +import type { AnchorEdgeRowInProgress, AnchorInputRow } from './types.js'; + +/** + * Collapses consecutive `anchors`-shaped rows into `anchor_edges` rows by + * `(pageId, hrefId)` (issue #193 step 6-D-4). + * + * **Input contract**: `rows` MUST be sorted by `(pageId, hrefId, id)` + * ascending. The caller (`populate-anchor-edges.ts`) achieves this via a + * single keyset scan `ORDER BY pageId, hrefId, id` over `anchors`. + * + * The `id`-then-key sort matters because the plan's dedup rule is + * "first instance wins": the smallest `anchors.id` for a given + * `(pageId, hrefId)` pair contributes its `hash` and `textContent` to + * the edge; every subsequent duplicate is counted but its body is + * discarded. A naive `min(hash) GROUP BY pageId, hrefId` would pick the + * lexicographically smallest hex string (uniformly distributed) rather + * than the earliest occurrence, so the pass has to happen in JS. + * + * The generator yields one {@link AnchorEdgeRowInProgress} per distinct + * pair as soon as the next pair boundary is observed. Emitting eagerly + * keeps peak memory bounded to O(1) — the collapser holds at most one + * open pair state at a time. + * + * `first_text_id` is intentionally left `null` here; the caller resolves + * it in a second pass after all edges are known (see + * `populate-anchor-edges.ts`). The `first_hash` field is set to the + * first row's `hash` verbatim; a first row with `hash === null` results + * in `first_hash = null` on the edge (rare but possible on legacy + * archives that failed to compute a hash). + * @param rows - Anchor input rows sorted by `(pageId, hrefId, id)`. + * @yields {AnchorEdgeRowInProgress} One entry per distinct + * `(pageId, hrefId)` pair in the input. + * @example + * const edges = [...collapseAnchorRows([ + * { id: 1, pageId: 10, hrefId: 20, hash: 'a', textContent: 'first' }, + * { id: 2, pageId: 10, hrefId: 20, hash: 'b', textContent: 'second' }, + * { id: 3, pageId: 10, hrefId: 30, hash: 'c', textContent: 'x' }, + * ])]; + * // edges[0] = { page_id: 10, href_page_id: 20, count: 2, first_hash: 'a', ... } + * // edges[1] = { page_id: 10, href_page_id: 30, count: 1, first_hash: 'c', ... } + */ +export function* collapseAnchorRows( + rows: Iterable, +): Generator { + let openPageId: number | null = null; + let openHrefId: number | null = null; + let openCount = 0; + let openFirstHash: string | null = null; + let openFirstTextContent: string | null = null; + let openLastId: number | null = null; + + for (const row of rows) { + if (openPageId === row.pageId && openHrefId === row.hrefId) { + // Same pair — assert monotonically-increasing `id` so callers + // that regress the `ORDER BY id` fail loudly instead of + // silently producing the wrong "first instance". The count- + // based acceptance check does not catch this (it only sums + // counts, not first_hash values), so the collapser defends + // its own precondition. + if (openLastId !== null && row.id <= openLastId) { + throw new Error( + `collapseAnchorRows: input not sorted by id within (pageId=${row.pageId}, hrefId=${row.hrefId}) — saw id=${row.id} after id=${openLastId}. Caller must ORDER BY pageId, hrefId, id ASC.`, + ); + } + openLastId = row.id; + openCount += 1; + continue; + } + if (openPageId !== null && openHrefId !== null) { + yield { + page_id: openPageId, + href_page_id: openHrefId, + count: openCount, + first_hash: openFirstHash, + first_textContent: openFirstTextContent, + }; + } + openPageId = row.pageId; + openHrefId = row.hrefId; + openCount = 1; + openFirstHash = row.hash; + openFirstTextContent = row.textContent; + openLastId = row.id; + } + if (openPageId !== null && openHrefId !== null) { + yield { + page_id: openPageId, + href_page_id: openHrefId, + count: openCount, + first_hash: openFirstHash, + first_textContent: openFirstTextContent, + }; + } +} diff --git a/packages/@nitpicker/crawler/src/archive/phase6d/derive-dom-path.spec.ts b/packages/@nitpicker/crawler/src/archive/phase6d/derive-dom-path.spec.ts new file mode 100644 index 0000000..300bdff --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6d/derive-dom-path.spec.ts @@ -0,0 +1,58 @@ +import { JSDOM } from 'jsdom'; +import { describe, it, expect } from 'vitest'; + +import { deriveDomPath } from './derive-dom-path.js'; + +/** + * Parses `html` and returns the first `` element for path testing. + * Kept as a helper so each `it` block reads as one assertion pair. + * @param html - HTML source to parse. + */ +function firstImg(html: string): Element { + const dom = new JSDOM(html); + const img = dom.window.document.querySelector('img'); + if (img === null) { + throw new Error(`fixture has no : ${html}`); + } + return img; +} + +describe('deriveDomPath (dom-path-derivation)', () => { + it('produces a plain html/body[1]/img[1] path for a top-level image', () => { + const img = firstImg(''); + expect(deriveDomPath(img)).toBe('html/body[1]/img[1]'); + }); + + it('counts same-tag siblings with a 1-based ordinal', () => { + const dom = new JSDOM( + '', + ); + const target = dom.window.document.querySelector('#target')!; + expect(deriveDomPath(target)).toBe('html/body[1]/img[3]'); + }); + + it('nests ancestor tags with per-tag sibling ordinals', () => { + const dom = new JSDOM( + '
', + ); + const img = dom.window.document.querySelector('img')!; + expect(deriveDomPath(img)).toBe('html/body[1]/main[1]/section[2]/picture[1]/img[1]'); + }); + + it('ignores non-Element siblings when counting ordinals', () => { + // Text nodes are Nodes but not Elements. The walk uses + // `previousElementSibling`, so a text-node between two `` + // tags must NOT reset the ordinal. + const dom = new JSDOM( + 'text between', + ); + const target = dom.window.document.querySelector('#target')!; + expect(deriveDomPath(target)).toBe('html/body[1]/img[2]'); + }); + + it('handles a detached element by falling back to the traversed prefix', () => { + const dom = new JSDOM(''); + const detached = dom.window.document.createElement('img'); + expect(deriveDomPath(detached)).toBe('img[1]'); + }); +}); diff --git a/packages/@nitpicker/crawler/src/archive/phase6d/derive-dom-path.ts b/packages/@nitpicker/crawler/src/archive/phase6d/derive-dom-path.ts new file mode 100644 index 0000000..177a6da --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6d/derive-dom-path.ts @@ -0,0 +1,74 @@ +/** + * Computes the `dom_path` string for one DOM element as required by + * `image_items.dom_path_text_id` (issue #193 step 6-D-6). + * + * Format: slash-joined ancestor tags starting at ``, each tag + * followed by a 1-based sibling ordinal counting **same-tag** siblings + * only (identical to the spec's example + * `html/body[1]/main[1]/section[2]/picture[1]/img[1]`). The `` + * root has no bracketed ordinal because there is only one document + * root per page. + * + * The walk uses `parentElement` + `previousElementSibling` — no + * document mutation, no reliance on `id` / `class` — so a re-render + * that only reshuffles class names or ids still produces the same path. + * The chain of tag names alone is stable for a given DOM shape. + * + * The function operates on the generic `Element` interface from DOM lib + * (available in the crawler's TS lib config) so it does not force + * jsdom into the crawler runtime — the migration script and unit tests + * inject jsdom-backed elements, while a future crawler-side wrapper + * (Phase 6-G) can pass puppeteer's DOM handles through the same code. + * + * Detached elements (`element.parentElement === null` before reaching + * ``) return the tag chain from wherever the walk terminates. + * This matches the plan's "walk the DOM ancestor chain from the + * `` element to ``" — a legally detached image (rare, but + * technically possible during scripted mutations) yields e.g. + * `img[1]` alone instead of throwing. + * @param element - The DOM element to derive the path for. Typically an + * `` in the archived HTML, but the function is not tag-specific. + * @returns The dom_path string. + * @example + * // In an HTML snapshot `
` + * deriveDomPath(imgElement); // 'html/body[1]/main[1]/img[1]' + */ +export function deriveDomPath(element: Element): string { + const segments: string[] = []; + let current: Element | null = element; + while (current !== null) { + const tag = current.tagName.toLowerCase(); + if (tag === 'html') { + segments.unshift('html'); + break; + } + segments.unshift(`${tag}[${computeSiblingOrdinal(current, tag)}]`); + current = current.parentElement; + } + return segments.join('/'); +} + +/** + * Counts how many earlier siblings share the same tag name as `element`, + * returning `count + 1` as the 1-based ordinal. + * + * Iterating `previousElementSibling` is O(k) where k is the sibling + * position; for typical DOM depth and sibling counts this is negligible + * next to the jsdom parse cost of the enclosing page. + * @param element - The element whose ordinal is being computed. + * @param tag - The lower-cased tag name to match on. Passed in by the + * caller so `element.tagName.toLowerCase()` is not recomputed inside + * the loop for every sibling comparison. + * @returns 1-based ordinal among same-tag siblings. + */ +function computeSiblingOrdinal(element: Element, tag: string): number { + let count = 1; + let sibling: Element | null = element.previousElementSibling; + while (sibling !== null) { + if (sibling.tagName.toLowerCase() === tag) { + count += 1; + } + sibling = sibling.previousElementSibling; + } + return count; +} diff --git a/packages/@nitpicker/crawler/src/archive/phase6d/match-images-to-dom-paths.spec.ts b/packages/@nitpicker/crawler/src/archive/phase6d/match-images-to-dom-paths.spec.ts new file mode 100644 index 0000000..c86b932 --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6d/match-images-to-dom-paths.spec.ts @@ -0,0 +1,120 @@ +import { JSDOM } from 'jsdom'; +import { describe, it, expect } from 'vitest'; + +import { matchImagesToDomPaths } from './match-images-to-dom-paths.js'; + +/** + * Parses `html`, returns the `` list in document order for use with + * `matchImagesToDomPaths`. + * @param html - Fixture HTML. + */ +function imgs(html: string): Element[] { + const dom = new JSDOM(html); + return [...dom.window.document.querySelectorAll('img')]; +} + +describe('matchImagesToDomPaths (dom-path-derivation, all 3 cases)', () => { + it('single-match: assigns the unique DOM path for exact outerHTML match', () => { + const elements = imgs( + '
', + ); + const result = matchImagesToDomPaths( + [{ id: 42, sourceCode: '' }], + elements, + ); + expect(result.get(42)).toEqual({ + path: 'html/body[1]/main[1]/img[1]', + case: 'single-match', + }); + }); + + it('ordinal-match: assigns identical-outerHTML matches in images.id order', () => { + // Two identical `` tags on the same page — differentiated only + // by their DOM position. `images.id` ordering (10 then 20) + // corresponds to document order (first `` then second). + const elements = imgs( + '
', + ); + const result = matchImagesToDomPaths( + [ + { id: 10, sourceCode: '' }, + { id: 20, sourceCode: '' }, + ], + elements, + ); + expect(result.get(10)).toEqual({ + path: 'html/body[1]/main[1]/img[1]', + case: 'ordinal-match', + }); + expect(result.get(20)).toEqual({ + path: 'html/body[1]/main[1]/img[2]', + case: 'ordinal-match', + }); + }); + + it('unknown: falls back to synthetic path when sourceCode is null', () => { + const elements = imgs(''); + const result = matchImagesToDomPaths([{ id: 7, sourceCode: null }], elements); + expect(result.get(7)).toEqual({ path: 'unknown/7', case: 'unknown' }); + }); + + it('unknown: falls back when no DOM element matches the outerHTML', () => { + const elements = imgs(''); + const result = matchImagesToDomPaths( + [{ id: 99, sourceCode: '' }], + elements, + ); + expect(result.get(99)).toEqual({ path: 'unknown/99', case: 'unknown' }); + }); + + it('unknown: falls back when the archived HTML has zero elements', () => { + const elements = imgs(''); + const result = matchImagesToDomPaths( + [{ id: 1, sourceCode: '' }], + elements, + ); + expect(result.get(1)).toEqual({ path: 'unknown/1', case: 'unknown' }); + }); + + it('falls back to unknown when more images share sourceCode than DOM candidates', () => { + // Regression guard: an earlier fallback (`candidates.at(-1)`) + // silently mapped every overflow row to the same last DOM + // element. Overflow rows must each keep a unique `unknown/` + // marker so they stay distinguishable in the archive. + const elements = imgs( + '', + ); + const result = matchImagesToDomPaths( + [ + { id: 10, sourceCode: '' }, + { id: 20, sourceCode: '' }, + { id: 30, sourceCode: '' }, + ], + elements, + ); + expect(result.get(10)?.case).toBe('ordinal-match'); + expect(result.get(20)?.case).toBe('ordinal-match'); + expect(result.get(30)).toEqual({ path: 'unknown/30', case: 'unknown' }); + }); + + it('handles mixed single-match / ordinal-match / unknown in one call', () => { + const elements = imgs( + '', + ); + const result = matchImagesToDomPaths( + [ + { id: 1, sourceCode: '' }, + { id: 2, sourceCode: '' }, + { id: 3, sourceCode: '' }, + { id: 4, sourceCode: null }, + { id: 5, sourceCode: '' }, + ], + elements, + ); + expect(result.get(1)?.case).toBe('single-match'); + expect(result.get(2)?.case).toBe('ordinal-match'); + expect(result.get(3)?.case).toBe('ordinal-match'); + expect(result.get(4)?.case).toBe('unknown'); + expect(result.get(5)?.case).toBe('unknown'); + }); +}); diff --git a/packages/@nitpicker/crawler/src/archive/phase6d/match-images-to-dom-paths.ts b/packages/@nitpicker/crawler/src/archive/phase6d/match-images-to-dom-paths.ts new file mode 100644 index 0000000..88d8058 --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6d/match-images-to-dom-paths.ts @@ -0,0 +1,141 @@ +import type { DomPathResult } from './types.js'; + +import { deriveDomPath } from './derive-dom-path.js'; + +/** + * One legacy `images` row projected onto the fields the matcher needs. + * Restricted to `id` (used to build the returned Map) and `sourceCode` + * (the outerHTML string to match against DOM candidates). + */ +interface ImageRowForMatching { + /** Legacy `images.id`. */ + id: number; + /** + * Legacy `images.sourceCode` — the `` element's `outerHTML` as + * stored by the crawler. `null` when the crawler failed to capture + * the source (a null blob or a puppeteer serialisation edge case). + */ + sourceCode: string | null; +} + +/** + * Matches a page's legacy `images` rows to their DOM-path strings using + * the plan's 3-case algorithm (issue #193 step 6-D-6). + * + * The three cases: + * + * 1. **Single match** — `image.sourceCode` matches exactly one `` + * outerHTML in the document. Assign that element's `dom_path`. + * 2. **Ordinal match** — multiple `` elements share the same + * outerHTML (identical `src` / `alt` / attributes on the same page). + * Assign paths in `images.id` order, matching the crawler's + * insertion order which is expected to correspond to DOM order. + * 3. **Unknown** — `sourceCode` is null OR no matching `` exists + * in the document (crawler-side rewriting drift). Fall back to the + * synthetic `unknown/` marker so the `dom_path_text_id NOT NULL` + * constraint is still satisfied. + * + * The function is pure and does not touch the DB. Callers pre-parse the + * page's HTML with jsdom (or any DOM implementation), pass the resulting + * `` NodeList / array plus the `images` rows filtered to that page, + * and receive one `DomPathResult` per input row. + * + * `imgElementsInDocumentOrder` MUST already be in document order — the + * matcher does not sort. jsdom's `document.querySelectorAll('img')` + * satisfies this contract; ordering after the call is the caller's + * responsibility. + * + * `images` MUST already be sorted by `id` ascending for the ordinal- + * match case to reproduce the crawler's insertion order deterministically. + * A minor deviation from this ordering only affects the ordinal-match + * branch — single-match and unknown branches are order-independent. + * @param images - The page's `images` rows in `id` order. + * @param imgElementsInDocumentOrder - `` elements from the parsed + * HTML, in document order. + * @returns Map keyed by `images.id`; every input row gets one entry. + * @example + * const jsdom = new JSDOM(html); + * const imgs = [...jsdom.window.document.querySelectorAll('img')]; + * const result = matchImagesToDomPaths( + * [{ id: 1, sourceCode: '' }], + * imgs, + * ); + * result.get(1); // { path: 'html/body[1]/img[1]', case: 'single-match' } + */ +export function matchImagesToDomPaths( + images: readonly ImageRowForMatching[], + imgElementsInDocumentOrder: readonly Element[], +): ReadonlyMap { + const byOuterHtml = new Map(); + const domPathByElement = new WeakMap(); + for (const img of imgElementsInDocumentOrder) { + const outerHtml = img.outerHTML; + const bucket = byOuterHtml.get(outerHtml); + if (bucket === undefined) { + byOuterHtml.set(outerHtml, [img]); + } else { + bucket.push(img); + } + } + + const cursorByOuterHtml = new Map(); + const result = new Map(); + for (const image of images) { + if (image.sourceCode === null || image.sourceCode === '') { + result.set(image.id, { path: `unknown/${image.id}`, case: 'unknown' }); + continue; + } + const candidates = byOuterHtml.get(image.sourceCode); + if (candidates === undefined || candidates.length === 0) { + result.set(image.id, { path: `unknown/${image.id}`, case: 'unknown' }); + continue; + } + if (candidates.length === 1) { + result.set(image.id, { + path: getOrDeriveDomPath(candidates[0]!, domPathByElement), + case: 'single-match', + }); + continue; + } + const cursor = cursorByOuterHtml.get(image.sourceCode) ?? 0; + cursorByOuterHtml.set(image.sourceCode, cursor + 1); + const chosen = candidates[cursor]; + if (chosen === undefined) { + // More `images` rows share this outerHTML than the archived HTML + // has matching `` elements — the earlier fallback + // (`candidates.at(-1)`) silently mapped every overflow row to + // the last matched element, producing multiple `image_items` + // rows with the same `dom_path_text_id`. Falling back to + // `unknown/` keeps overflow rows individually + // distinguishable in the archive. + result.set(image.id, { path: `unknown/${image.id}`, case: 'unknown' }); + continue; + } + result.set(image.id, { + path: getOrDeriveDomPath(chosen, domPathByElement), + case: 'ordinal-match', + }); + } + return result; +} + +/** + * Caches {@link deriveDomPath} results against `element` identity. Duplicate + * `` outerHTML values in the same document can point the ordinal + * cursor at the same element more than once (when the ordinal exceeds + * the candidate list length — a defensive fallback via `candidates.at(-1)`); + * caching keeps the DOM walk cost O(unique elements) rather than + * O(matches). + * @param element - The DOM element to derive against. + * @param cache - The memo cache used across one page's match pass. + * @returns Derived `dom_path` string. + */ +function getOrDeriveDomPath(element: Element, cache: WeakMap): string { + const cached = cache.get(element); + if (cached !== undefined) { + return cached; + } + const path = deriveDomPath(element); + cache.set(element, path); + return path; +} diff --git a/packages/@nitpicker/crawler/src/archive/phase6d/populate-anchor-edges.spec.ts b/packages/@nitpicker/crawler/src/archive/phase6d/populate-anchor-edges.spec.ts new file mode 100644 index 0000000..d00090f --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6d/populate-anchor-edges.spec.ts @@ -0,0 +1,146 @@ +import type knex from 'knex'; + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { populatePhase6BRefs } from '../phase6b/populate-phase6b-refs.js'; + +import { populateAnchorEdges } from './populate-anchor-edges.js'; +import { populateContentItems } from './populate-content-items.js'; +import { countRows } from './test-utils/count-rows.js'; +import { setupPhase6DDb } from './test-utils/setup-phase6d-db.js'; + +/** + * Inserts N pages so `anchor_edges` FKs resolve. `populatePhase6BRefs` + * is intentionally NOT called here — the caller inserts anchors first + * and then runs the phase-6-B populate, matching the real migration + * ordering where anchor textContent has already been ingested into + * `text_refs` by the time `populateAnchorEdges` runs. + * @param db - Test DB instance. + * @param count - Number of pages to insert (id 1..count). + */ +async function seedPages(db: ReturnType, count: number): Promise { + const rows = Array.from({ length: count }, (_, index) => ({ + id: index + 1, + url: `https://example.com/${index + 1}`, + scraped: 1, + isTarget: 1, + })); + await db('pages').insert(rows); +} + +/** + * Runs Phase 6-B populate + Phase 6-D content_items so anchor_edges' + * `text_refs` lookup and `content_items(id)` FKs both succeed at + * insert time. Called after every test's anchor seed data so the + * required refs land in text_refs. + * @param db - Test DB instance. + */ +async function completePhase6Prerequisites(db: ReturnType): Promise { + await populatePhase6BRefs(db); + await populateContentItems(db); +} + +describe('populateAnchorEdges (anchor-edge-normalization)', () => { + let db: ReturnType; + + beforeEach(async () => { + db = await setupPhase6DDb(); + }); + afterEach(async () => { + await db.destroy(); + }); + + it('collapses duplicate anchors sharing (pageId, hrefId) with first-wins semantics', async () => { + await seedPages(db, 3); + await db('anchors').insert([ + { pageId: 1, hrefId: 2, hash: 'a1', textContent: 'first anchor text' }, + { pageId: 1, hrefId: 2, hash: 'a2', textContent: 'second anchor text' }, + { pageId: 1, hrefId: 2, hash: 'a3', textContent: 'third anchor text' }, + { pageId: 1, hrefId: 3, hash: 'b1', textContent: 'other target' }, + ]); + await completePhase6Prerequisites(db); + await populateAnchorEdges(db); + const edges = await db('anchor_edges').select().orderBy(['page_id', 'href_page_id']); + expect(edges).toHaveLength(2); + expect(edges[0]!.page_id).toBe(1); + expect(edges[0]!.href_page_id).toBe(2); + expect(edges[0]!.count).toBe(3); + expect(edges[0]!.first_hash).toBe('a1'); + expect(edges[0]!.first_text_id).not.toBeNull(); + expect(edges[1]!.href_page_id).toBe(3); + expect(edges[1]!.count).toBe(1); + }); + + it('acceptance: SUM(anchor_edges.count) equals count(anchors)', async () => { + await seedPages(db, 4); + await db('anchors').insert([ + { pageId: 1, hrefId: 2, hash: 'a', textContent: 'x' }, + { pageId: 1, hrefId: 2, hash: 'b', textContent: 'y' }, + { pageId: 1, hrefId: 3, hash: 'c', textContent: 'z' }, + { pageId: 2, hrefId: 4, hash: 'd', textContent: 'q' }, + { pageId: 2, hrefId: 4, hash: 'e', textContent: 'r' }, + { pageId: 3, hrefId: 4, hash: 'f', textContent: 's' }, + ]); + await completePhase6Prerequisites(db); + await populateAnchorEdges(db); + const anchorsCount = await countRows(db, 'anchors'); + const edgesSumRows = await db('anchor_edges').sum<{ n: number | null }[]>({ + n: 'count', + }); + const edgesSum = Number(edgesSumRows[0]!.n ?? 0); + expect(edgesSum).toBe(anchorsCount); + }); + + it('handles null hash / textContent on the first instance', async () => { + await seedPages(db, 2); + await db('anchors').insert([ + { pageId: 1, hrefId: 2, hash: null, textContent: null }, + { pageId: 1, hrefId: 2, hash: 'later', textContent: 'later text' }, + ]); + await completePhase6Prerequisites(db); + await populateAnchorEdges(db); + const edge = await db('anchor_edges').where('page_id', 1).first(); + expect(edge.first_hash).toBeNull(); + expect(edge.first_text_id).toBeNull(); + expect(edge.count).toBe(2); + }); + + it('collapses correctly when a (pageId, hrefId) pair straddles two READ_CHUNK_SIZE boundaries', async () => { + // Regression guard for the internal `carryOver` state — this is + // the ONLY streaming state that spans chunk boundaries. + // READ_CHUNK_SIZE is 5000, so 5001 anchors on the same + // (pageId=1, hrefId=2) pair force the collapse across the + // boundary. The final edge must have `count = 5001` and + // `first_hash = 'first'`. + await seedPages(db, 2); + const bulk = Array.from({ length: 5001 }, (_, index) => ({ + pageId: 1, + hrefId: 2, + hash: index === 0 ? 'first' : `later-${index}`, + textContent: index === 0 ? 'first anchor' : `anchor-${index}`, + })); + // Batch-insert in small chunks — knex serialises `INSERT ... + // VALUES` for libsql as a big `SELECT ... UNION ALL` and the + // generated SQL text hits the SQL length ceiling well before the + // variable-count ceiling. 200 rows keeps each statement under + // the ceiling. + for (let index = 0; index < bulk.length; index += 200) { + await db('anchors').insert(bulk.slice(index, index + 200)); + } + await completePhase6Prerequisites(db); + await populateAnchorEdges(db); + const edges = await db('anchor_edges').select(); + expect(edges).toHaveLength(1); + expect(edges[0]!.count).toBe(5001); + expect(edges[0]!.first_hash).toBe('first'); + }, 30_000); + + it('is idempotent (upsert on unique(page_id, href_page_id))', async () => { + await seedPages(db, 2); + await db('anchors').insert([{ pageId: 1, hrefId: 2, hash: 'x', textContent: 'x' }]); + await completePhase6Prerequisites(db); + await populateAnchorEdges(db); + await populateAnchorEdges(db); + expect(await countRows(db, 'anchor_edges')).toBe(1); + }); +}); diff --git a/packages/@nitpicker/crawler/src/archive/phase6d/populate-anchor-edges.ts b/packages/@nitpicker/crawler/src/archive/phase6d/populate-anchor-edges.ts new file mode 100644 index 0000000..28e35ba --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6d/populate-anchor-edges.ts @@ -0,0 +1,149 @@ +import type { AnchorEdgeRowInProgress, AnchorInputRow } from './types.js'; +import type { Knex } from 'knex'; + +import { collapseAnchorRows } from './collapse-anchor-rows.js'; +import { resolveTextRefs } from './resolve-text-refs.js'; + +/** + * Rows scanned per keyset-paginated `SELECT` chunk against `anchors`. + * `anchors` is the largest table in the archive (≈ 13 M rows on the + * reference archive) so the chunk size trades off memory against + * round-trip overhead. 5 000 rows per SELECT keeps peak chunk memory + * ≈ 5 MB (per-row ≈ 1 KB with URL / textContent stored elsewhere) and + * amortises the per-query round-trip cost across many collapses. + */ +const READ_CHUNK_SIZE = 5000; + +/** + * Edges buffered before a bulk `INSERT INTO anchor_edges ... VALUES + * (...)`. Each edge binds 5 params so 1 000 rows = 5 000 params — well + * inside the SQLite variable limit. Emitting on this cadence bounds + * peak buffered-edge memory to ≈ 100 KB. + */ +const INSERT_CHUNK_SIZE = 1000; + +/** + * Populates `anchor_edges` from `anchors` (issue #193 step 6-D-4). + * + * The algorithm is a **single keyset-paginated scan** over `anchors` + * ordered by `(pageId, hrefId, id)`. Each chunk feeds + * {@link ./collapse-anchor-rows.ts}, which yields one edge per distinct + * `(pageId, hrefId)` pair as soon as the pair boundary is observed. + * + * A boundary can straddle two chunks (the last row of chunk N shares a + * pair with the first row of chunk N+1), so the outer loop keeps an + * "open edge" state that flushes only when the *next* pair actually + * starts. On end-of-stream the last open edge is emitted. + * + * `first_text_id` is left unresolved during the streaming pass — the + * text is buffered on each pending edge and resolved in bulk by + * {@link ./resolve-text-refs.ts} at INSERT time. Every INSERT batch + * therefore issues one text-refs lookup + one INSERT, keeping DB + * round-trips per edge low. + * + * `INSERT OR IGNORE` on the `(page_id, href_page_id)` UNIQUE composite + * makes the step idempotent: a re-run after partial failure re-emits the + * same edges but the writes no-op. + * @param trx - Knex instance or transaction connected to the archive DB. + * @example + * await knex.transaction(async (trx) => { + * await populateAnchorEdges(trx); + * }); + */ +export async function populateAnchorEdges(trx: Knex): Promise { + let cursorPageId = 0; + let cursorHrefId = 0; + let cursorId = 0; + let carryOver: AnchorEdgeRowInProgress | null = null; + const pending: AnchorEdgeRowInProgress[] = []; + + while (true) { + const rows: AnchorInputRow[] = await trx('anchors') + .select('id', 'pageId', 'hrefId', 'hash', 'textContent') + .where(function () { + this.where('pageId', '>', cursorPageId) + .orWhere(function () { + this.where('pageId', cursorPageId).andWhere('hrefId', '>', cursorHrefId); + }) + .orWhere(function () { + this.where('pageId', cursorPageId) + .andWhere('hrefId', cursorHrefId) + .andWhere('id', '>', cursorId); + }); + }) + .orderBy([ + { column: 'pageId', order: 'asc' }, + { column: 'hrefId', order: 'asc' }, + { column: 'id', order: 'asc' }, + ]) + .limit(READ_CHUNK_SIZE); + if (rows.length === 0) { + break; + } + const lastRow = rows.at(-1)!; + cursorPageId = lastRow.pageId; + cursorHrefId = lastRow.hrefId; + cursorId = lastRow.id; + + const chunkEdges = [...collapseAnchorRows(rows)]; + for (const edge of chunkEdges) { + if ( + carryOver !== null && + carryOver.page_id === edge.page_id && + carryOver.href_page_id === edge.href_page_id + ) { + carryOver.count += edge.count; + continue; + } + if (carryOver !== null) { + pending.push(carryOver); + } + carryOver = edge; + if (pending.length >= INSERT_CHUNK_SIZE) { + await flush(trx, pending); + } + } + } + if (carryOver !== null) { + pending.push(carryOver); + } + if (pending.length > 0) { + await flush(trx, pending); + } +} + +/** + * Resolves `first_text_id` for every buffered edge in `pending`, then + * bulk-inserts the batch into `anchor_edges` and clears the buffer. + * + * Extracted from the main loop so the "resolve + insert" pair happens in + * exactly one place — the streaming loop, the carry-over flush at + * end-of-stream, and any future re-order boundary all land here. + * @param trx - Knex instance or transaction. + * @param pending - Buffered edges awaiting `first_text_id` resolution + * and INSERT; mutated in place (cleared on return). + */ +async function flush(trx: Knex, pending: AnchorEdgeRowInProgress[]): Promise { + const texts = new Set(); + for (const edge of pending) { + if (edge.first_textContent != null && edge.first_textContent !== '') { + texts.add(edge.first_textContent); + } + } + const textIds = await resolveTextRefs(trx, texts); + const inserts = pending.map((edge) => ({ + page_id: edge.page_id, + href_page_id: edge.href_page_id, + count: edge.count, + first_hash: edge.first_hash, + first_text_id: + edge.first_textContent != null && edge.first_textContent !== '' + ? (textIds.get(edge.first_textContent) ?? null) + : null, + })); + await trx('anchor_edges') + .insert(inserts) + .onConflict(['page_id', 'href_page_id']) + .ignore(); + pending.length = 0; +} diff --git a/packages/@nitpicker/crawler/src/archive/phase6d/populate-content-items.spec.ts b/packages/@nitpicker/crawler/src/archive/phase6d/populate-content-items.spec.ts new file mode 100644 index 0000000..8ad704c --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6d/populate-content-items.spec.ts @@ -0,0 +1,141 @@ +import type knex from 'knex'; + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { populatePhase6BRefs } from '../phase6b/populate-phase6b-refs.js'; + +import { populateContentItems } from './populate-content-items.js'; +import { countRows } from './test-utils/count-rows.js'; +import { setupPhase6DDb } from './test-utils/setup-phase6d-db.js'; + +describe('populateContentItems', () => { + let db: ReturnType; + + beforeEach(async () => { + db = await setupPhase6DDb(); + }); + afterEach(async () => { + await db.destroy(); + }); + + it('copies pages 1:1 into content_items with resolved ref ids', async () => { + await db('pages').insert([ + { + url: 'https://example.com/a', + scraped: 1, + isTarget: 1, + isExternal: 0, + status: 200, + statusText: 'OK', + contentType: 'text/html; charset=utf-8', + contentLength: 1024, + responseHeaders: '{"content-type":"text/html"}', + source: 'crawled', + order: 1, + }, + { + url: 'https://example.com/b', + scraped: 0, + isTarget: 0, + isExternal: 1, + status: null, + statusText: null, + contentType: null, + contentLength: null, + responseHeaders: null, + source: 'inventory-seed', + order: 2, + }, + ]); + await populatePhase6BRefs(db); + await populateContentItems(db); + const rows = await db('content_items').select().orderBy('id'); + expect(rows).toHaveLength(2); + expect(rows[0]!.id).toBe(1); + expect(rows[0]!.url_id).not.toBeNull(); + expect(rows[0]!.content_type_id).not.toBeNull(); + expect(rows[0]!.header_set_id).not.toBeNull(); + expect(rows[0]!.status).toBe(200); + expect(rows[0]!.source).toBe('crawled'); + expect(rows[0]!.crawl_order).toBe(1); + expect(rows[1]!.is_external).toBe(1); + expect(rows[1]!.header_set_id).toBeNull(); + expect(rows[1]!.content_type_id).toBeNull(); + }); + + it('preserves the exact pages.id as content_items.id', async () => { + await db('pages').insert([ + { id: 100, url: 'https://example.com/a', scraped: 1, isTarget: 1 }, + { id: 250, url: 'https://example.com/b', scraped: 1, isTarget: 1 }, + ]); + await populatePhase6BRefs(db); + await populateContentItems(db); + const idRows = await db('content_items').select('id').orderBy('id'); + expect(idRows.map((r) => r.id)).toEqual([100, 250]); + }); + + it('propagates redirectDestId as-is (deferred FK)', async () => { + await db('pages').insert([ + { id: 10, url: 'https://example.com/redirect', scraped: 1, isTarget: 0 }, + { + id: 20, + url: 'https://example.com/dest', + scraped: 1, + isTarget: 1, + redirectDestId: null, + }, + ]); + // Redirect from id=10 to id=20 — with content_items DEFERRABLE FK + // the source row can be inserted before the destination. + await db('pages').where('id', 10).update({ redirectDestId: 20 }); + await populatePhase6BRefs(db); + await db.transaction(async (trx) => { + await populateContentItems(trx); + }); + const redirect = await db('content_items').where('id', 10).first(); + expect(redirect.redirect_dest_id).toBe(20); + }); + + it('is idempotent (upsert on id)', async () => { + await db('pages').insert([{ url: 'https://example.com/a', scraped: 1, isTarget: 1 }]); + await populatePhase6BRefs(db); + await populateContentItems(db); + await populateContentItems(db); + expect(await countRows(db, 'content_items')).toBe(1); + }); + + it('throws when url_refs.id is not resolvable (Phase 6-B-1 not run)', async () => { + await db('pages').insert([{ url: 'https://example.com/a', scraped: 1, isTarget: 1 }]); + // Deliberately skip populatePhase6BRefs — url_refs stays empty. + await expect(populateContentItems(db)).rejects.toThrow(/url_refs\.id not resolved/); + }); + + it('throws when content_type_refs.id is not resolvable', async () => { + await db('pages').insert([ + { + url: 'https://example.com/a', + scraped: 1, + isTarget: 1, + contentType: 'text/html', + }, + ]); + // Only populate url_refs — content_type_refs stays empty. + await db('url_refs').insert({ url: 'https://example.com/a' }); + await expect(populateContentItems(db)).rejects.toThrow( + /content_type_refs\.id not resolved/, + ); + }); + + it('acceptance: count(content_items) equals count(pages)', async () => { + await db('pages').insert([ + { url: 'https://example.com/a', scraped: 1, isTarget: 1 }, + { url: 'https://example.com/b', scraped: 1, isTarget: 1 }, + { url: 'https://example.com/c', scraped: 0, isTarget: 0 }, + ]); + await populatePhase6BRefs(db); + await populateContentItems(db); + const pagesCount = await countRows(db, 'pages'); + const contentItemsCount = await countRows(db, 'content_items'); + expect(contentItemsCount).toBe(pagesCount); + }); +}); diff --git a/packages/@nitpicker/crawler/src/archive/phase6d/populate-content-items.ts b/packages/@nitpicker/crawler/src/archive/phase6d/populate-content-items.ts new file mode 100644 index 0000000..1829da8 --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6d/populate-content-items.ts @@ -0,0 +1,205 @@ +import type { Knex } from 'knex'; + +import { loadContentTypeRefs } from './resolve-content-type-refs.js'; +import { resolveHeaderSets } from './resolve-header-sets.js'; +import { resolveUrlRefs } from './resolve-url-refs.js'; + +/** + * Rows scanned per keyset-paginated `SELECT` chunk against `pages`. The + * migration reads every page column that maps to `content_items` in one + * SELECT per chunk, so scaling this too high would blow up the result-set + * size for long meta text columns. 500 keeps peak memory bounded at + * ≈ 30 MB for a chunk of `pages` with populated meta and header sets. + */ +const READ_CHUNK_SIZE = 500; + +/** + * Rows sent per `INSERT INTO content_items ... VALUES (...)` statement. + * Each row binds 16 params (id + 15 core columns), so 300 rows = + * 4 800 params — safely under SQLite's default variable limit of 32 766. + */ +const INSERT_CHUNK_SIZE = 300; + +/** + * Populates `content_items` from `pages` (issue #193 step 6-D-1). + * + * Strategy per chunk: + * + * 1. **Keyset-paginate** `pages` in id order so partial-failure re-runs + * resume from the last committed row without reading the full table. + * Same pattern as {@link ../phase6b/populate-url-refs.ts}. + * 2. **Batch-resolve** `url_refs.id`, `content_type_refs.id`, and + * `header_sets.id` for every distinct value in the chunk — one round + * trip per ref table instead of N round trips per row. + * 3. **Bulk INSERT** with explicit `id = pages.id`. Reusing the legacy + * PK preserves every FK reference in `page_errors` / `page_tags` / + * `page_jsonld` / `page_html_ref` without a per-row UPDATE. + * + * `redirect_dest_id` is copied verbatim from `redirectDestId` — the FK + * self-reference is `DEFERRABLE INITIALLY DEFERRED` (see + * {@link ../create-phase6c-entity-tables.ts}) so a redirect source + * inserted before its destination is validated only at COMMIT time. + * + * `INSERT OR IGNORE` on the natural PK makes the step idempotent — a + * re-run after a partial failure only inserts the rows that are still + * missing. + * + * `content_type_refs` is preloaded once at the start because its + * cardinality is small (see {@link ./resolve-content-type-refs.ts}); the + * URL and header-set resolvers are called per chunk because their + * per-archive cardinality can be very large (one row per distinct URL / + * response JSON). + * @param trx - Knex instance or transaction connected to the archive DB. + * @example + * await knex.transaction(async (trx) => { + * await populateContentItems(trx); + * }); + */ +export async function populateContentItems(trx: Knex): Promise { + const contentTypeIds = await loadContentTypeRefs(trx); + let cursor = 0; + while (true) { + const rows: PageRow[] = await trx('pages') + .select( + 'id', + 'url', + 'redirectDestId', + 'scraped', + 'isTarget', + 'isExternal', + 'status', + 'statusText', + 'contentType', + 'contentLength', + 'responseHeaders', + 'firstCrawledAt', + 'lastCrawledAt', + 'order', + 'isSkipped', + 'skipReason', + 'source', + ) + .where('id', '>', cursor) + .orderBy('id', 'asc') + .limit(READ_CHUNK_SIZE); + if (rows.length === 0) { + break; + } + cursor = rows.at(-1)!.id; + + const urls = new Set(); + const headerJsonStrings = new Set(); + for (const row of rows) { + urls.add(row.url); + if (typeof row.responseHeaders === 'string' && row.responseHeaders !== '') { + headerJsonStrings.add(row.responseHeaders); + } + } + const urlIds = await resolveUrlRefs(trx, urls); + const headerSetIds = await resolveHeaderSets(trx, headerJsonStrings); + + const inserts = rows.map((row) => { + const urlId = urlIds.get(row.url) ?? null; + if (urlId === null) { + throw new Error( + `populateContentItems: url_refs.id not resolved for page id=${row.id} url=${row.url} — Phase 6-B-1 populate must run first`, + ); + } + let contentTypeId: number | null = null; + if (row.contentType != null && row.contentType !== '') { + const resolved = contentTypeIds.get(row.contentType); + if (resolved === undefined) { + throw new Error( + `populateContentItems: content_type_refs.id not resolved for page id=${row.id} contentType=${row.contentType} — Phase 6-B-0 populate must run first`, + ); + } + contentTypeId = resolved; + } + let headerSetId: number | null = null; + if (typeof row.responseHeaders === 'string' && row.responseHeaders !== '') { + const resolved = headerSetIds.get(row.responseHeaders); + if (resolved === undefined) { + // `resolveHeaderSets` skips sentinel values ('{}' / 'null') + // AND payloads that `decomposeHeaderSet` cannot parse into + // any entries — in both cases the source row genuinely had + // no header set to point at, so `header_set_id = null` is + // the correct final state. `resolveHeaderSets` already ran + // the raw_hash fallback, so this branch only fires when the + // row has no persistable header set (parse error, or every + // entry stripped for non-string values). + headerSetId = null; + } else { + headerSetId = resolved; + } + } + return { + id: row.id, + url_id: urlId, + is_external: row.isExternal == null ? 0 : row.isExternal ? 1 : 0, + scraped: row.scraped ? 1 : 0, + is_target: row.isTarget ? 1 : 0, + status: row.status ?? null, + status_text: row.statusText ?? null, + content_type_id: contentTypeId, + content_length: row.contentLength ?? null, + header_set_id: headerSetId, + redirect_dest_id: row.redirectDestId ?? null, + source: row.source, + first_crawled_at: row.firstCrawledAt ?? null, + last_crawled_at: row.lastCrawledAt ?? null, + crawl_order: row.order ?? null, + is_skipped: row.isSkipped == null ? null : row.isSkipped ? 1 : 0, + skip_reason: row.skipReason ?? null, + }; + }); + + for (let index = 0; index < inserts.length; index += INSERT_CHUNK_SIZE) { + const chunk = inserts.slice(index, index + INSERT_CHUNK_SIZE); + await trx('content_items').insert(chunk).onConflict('id').ignore(); + } + } +} + +/** + * Shape of one row read from the legacy `pages` table by + * {@link populateContentItems}. Every column mapped to `content_items` + * or used to decide ref lookups is declared here — meta-only columns + * (title / description / etc.) live in a separate SELECT run by + * {@link ./populate-page-meta.ts}. + */ +interface PageRow { + /** Legacy `pages.id`, reused verbatim as `content_items.id`. */ + id: number; + /** Legacy `pages.url` — resolved to `content_items.url_id` via `url_refs`. */ + url: string; + /** Legacy `pages.redirectDestId` — copied to `content_items.redirect_dest_id`. */ + redirectDestId: number | null; + /** Legacy `pages.scraped` — copied to `content_items.scraped`. */ + scraped: boolean | number; + /** Legacy `pages.isTarget` — copied to `content_items.is_target`. */ + isTarget: boolean | number; + /** Legacy `pages.isExternal` — copied to `content_items.is_external`. */ + isExternal: boolean | number | null; + /** Legacy `pages.status` — copied verbatim. */ + status: number | null; + /** Legacy `pages.statusText` — copied verbatim. */ + statusText: string | null; + /** Legacy `pages.contentType` — resolved to `content_type_id` via `content_type_refs`. */ + contentType: string | null; + /** Legacy `pages.contentLength` — copied verbatim. */ + contentLength: number | null; + /** Legacy `pages.responseHeaders` — resolved to `header_set_id` via `header_sets.raw_json_hash`. */ + responseHeaders: string | null; + /** Legacy `pages.firstCrawledAt` — copied to `content_items.first_crawled_at`. */ + firstCrawledAt: number | null; + /** Legacy `pages.lastCrawledAt` — copied to `content_items.last_crawled_at`. */ + lastCrawledAt: number | null; + /** Legacy `pages.order` — copied to `content_items.crawl_order`. */ + order: number | null; + /** Legacy `pages.isSkipped` — copied to `content_items.is_skipped`. */ + isSkipped: boolean | number | null; + /** Legacy `pages.skipReason` — copied to `content_items.skip_reason`. */ + skipReason: string | null; + /** Legacy `pages.source` — copied verbatim to `content_items.source`. */ + source: string; +} diff --git a/packages/@nitpicker/crawler/src/archive/phase6d/populate-image-items.spec.ts b/packages/@nitpicker/crawler/src/archive/phase6d/populate-image-items.spec.ts new file mode 100644 index 0000000..e540cb7 --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6d/populate-image-items.spec.ts @@ -0,0 +1,266 @@ +import type { DomPathResult } from './types.js'; +import type knex from 'knex'; + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { populatePhase6BRefs } from '../phase6b/populate-phase6b-refs.js'; + +import { populateContentItems } from './populate-content-items.js'; +import { populateImageItems } from './populate-image-items.js'; +import { countRows } from './test-utils/count-rows.js'; +import { setupPhase6DDb } from './test-utils/setup-phase6d-db.js'; + +/** + * Builds an `unknown/` resolver — every image gets the synthetic + * fallback path. Kept non-async because the eslint + * `@typescript-eslint/require-await` rule bans `async` bodies with no + * `await`; `Promise.resolve` produces the same `Promise<...>` return + * type the `PageDomPathResolver` contract in `populate-image-items.ts` + * requires. + * @param images - The image rows passed by the caller. + */ +function unknownResolver( + images: readonly { id: number }[], +): Promise> { + const map = new Map(); + for (const image of images) { + map.set(image.id, { path: `unknown/${image.id}`, case: 'unknown' }); + } + return Promise.resolve(map); +} + +describe('populateImageItems', () => { + let db: ReturnType; + + beforeEach(async () => { + db = await setupPhase6DDb(); + }); + afterEach(async () => { + await db.destroy(); + }); + + it('routes plain-URL src to src_url_id and data-URI to src_blob_id', async () => { + const longDataUri = 'data:image/png;base64,' + 'A'.repeat(600); + await db('pages').insert([ + { id: 1, url: 'https://example.com/a', scraped: 1, isTarget: 1 }, + ]); + await db('images').insert([ + { + id: 10, + pageId: 1, + src: 'https://example.com/img.png', + currentSrc: 'https://example.com/img.png', + alt: 'alt text', + width: 100, + height: 50, + naturalWidth: 200, + naturalHeight: 100, + isLazy: 0, + viewportWidth: 1280, + sourceCode: '', + }, + { + id: 20, + pageId: 1, + src: longDataUri, + currentSrc: longDataUri, + alt: null, + width: 1, + height: 1, + naturalWidth: 1, + naturalHeight: 1, + isLazy: 0, + viewportWidth: 1280, + sourceCode: null, + }, + ]); + await populatePhase6BRefs(db); + await populateContentItems(db); + await populateImageItems( + db, + (_pageId, _html, images) => unknownResolver(images), + () => Promise.resolve(null), + ); + const rows = await db('image_items').select().orderBy('id'); + expect(rows).toHaveLength(2); + expect(rows[0]!.id).toBe(10); + expect(rows[0]!.src_url_id).not.toBeNull(); + expect(rows[0]!.src_blob_id).toBeNull(); + expect(rows[0]!.alt_text_id).not.toBeNull(); + expect(rows[1]!.id).toBe(20); + expect(rows[1]!.src_url_id).toBeNull(); + expect(rows[1]!.src_blob_id).not.toBeNull(); + }); + + it('inserts dom_path strings into text_refs and links image_items.dom_path_text_id', async () => { + await db('pages').insert([ + { id: 1, url: 'https://example.com/a', scraped: 1, isTarget: 1 }, + ]); + await db('images').insert([ + { + id: 5, + pageId: 1, + src: 'https://example.com/img.png', + alt: null, + width: 1, + height: 1, + naturalWidth: 1, + naturalHeight: 1, + isLazy: 0, + viewportWidth: 1, + sourceCode: '', + }, + ]); + await populatePhase6BRefs(db); + await populateContentItems(db); + const singleMatchResolver = ( + _pageId: number, + _html: string | null, + images: readonly { id: number }[], + ): Promise> => { + const map = new Map(); + for (const image of images) { + map.set(image.id, { + path: 'html/body[1]/main[1]/img[1]', + case: 'single-match', + }); + } + return Promise.resolve(map); + }; + await populateImageItems(db, singleMatchResolver, () => Promise.resolve(null)); + const row = await db('image_items').where('id', 5).first(); + const domPathText = await db('text_refs').where('id', row.dom_path_text_id).first(); + expect(domPathText.text).toBe('html/body[1]/main[1]/img[1]'); + }); + + it('processes each page as one unit — dom-path resolver is called once per page', async () => { + // Regression guard for the earlier image-chunked implementation + // that reset `matchImagesToDomPaths`' ordinal cursor at each + // READ_CHUNK_SIZE boundary and duplicated dom_paths. + // Iterating by pageId means the resolver sees every image of a + // page in one call regardless of how many rows the page has. + await db('pages').insert([ + { id: 1, url: 'https://example.com/a', scraped: 1, isTarget: 1 }, + { id: 2, url: 'https://example.com/b', scraped: 1, isTarget: 1 }, + ]); + const bulk = Array.from({ length: 20 }, (_, index) => ({ + pageId: 1, + src: 'https://example.com/logo.png', + alt: null, + width: 1, + height: 1, + naturalWidth: 1, + naturalHeight: 1, + isLazy: 0, + viewportWidth: 1, + sourceCode: ``, + })); + await db('images').insert([ + ...bulk, + { + pageId: 2, + src: 'https://example.com/other.png', + alt: null, + width: 1, + height: 1, + naturalWidth: 1, + naturalHeight: 1, + isLazy: 0, + viewportWidth: 1, + sourceCode: '', + }, + ]); + await populatePhase6BRefs(db); + await populateContentItems(db); + const resolverCalls: number[] = []; + const resolver = ( + pageId: number, + _html: string | null, + images: readonly { id: number }[], + ): Promise> => { + resolverCalls.push(pageId); + const map = new Map(); + for (const image of images) { + map.set(image.id, { path: `html/body[1]/img[${image.id}]`, case: 'unknown' }); + } + return Promise.resolve(map); + }; + await populateImageItems(db, resolver, () => Promise.resolve(null)); + // Each pageId visited exactly once. + expect(resolverCalls.toSorted()).toEqual([1, 2]); + expect(await countRows(db, 'image_items')).toBe(21); + }); + + it('does not send data URIs into url_refs lookups (routing partition)', async () => { + // Regression guard for the earlier bug that added every value to + // both `urls` and `dataUris` sets — large data URIs must never + // reach `resolveUrlRefs`' `WHERE url IN (?)` because they can + // blow past SQLITE_MAX_SQL_LENGTH. + const longDataUri = 'data:image/png;base64,' + 'A'.repeat(600); + await db('pages').insert([ + { id: 1, url: 'https://example.com/a', scraped: 1, isTarget: 1 }, + ]); + await db('images').insert([ + { + id: 1, + pageId: 1, + src: longDataUri, + alt: null, + width: 1, + height: 1, + naturalWidth: 1, + naturalHeight: 1, + isLazy: 0, + viewportWidth: 1, + sourceCode: null, + }, + ]); + await populatePhase6BRefs(db); + await populateContentItems(db); + await populateImageItems( + db, + (_pageId, _html, images) => unknownResolver(images), + () => Promise.resolve(null), + ); + // The data URI must land in blob_refs, not url_refs. `url_refs` + // should not contain the data URI (populateUrlRefs already + // filtered it out; this just guards against a future regression + // where populateImageItems tries to insert it opportunistically). + const urlRefRow = await db('url_refs').where('url', longDataUri).first(); + expect(urlRefRow).toBeUndefined(); + const image = await db('image_items').where('id', 1).first(); + expect(image.src_url_id).toBeNull(); + expect(image.src_blob_id).not.toBeNull(); + }); + + it('is idempotent', async () => { + await db('pages').insert([ + { id: 1, url: 'https://example.com/a', scraped: 1, isTarget: 1 }, + ]); + await db('images').insert([ + { + id: 1, + pageId: 1, + src: 'https://example.com/x.png', + alt: null, + width: 1, + height: 1, + naturalWidth: 1, + naturalHeight: 1, + isLazy: 0, + viewportWidth: 1, + sourceCode: '', + }, + ]); + await populatePhase6BRefs(db); + await populateContentItems(db); + const resolver = ( + _pageId: number, + _html: string | null, + images: readonly { id: number }[], + ): Promise> => unknownResolver(images); + await populateImageItems(db, resolver, () => Promise.resolve(null)); + await populateImageItems(db, resolver, () => Promise.resolve(null)); + expect(await countRows(db, 'image_items')).toBe(1); + }); +}); diff --git a/packages/@nitpicker/crawler/src/archive/phase6d/populate-image-items.ts b/packages/@nitpicker/crawler/src/archive/phase6d/populate-image-items.ts new file mode 100644 index 0000000..82b830b --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6d/populate-image-items.ts @@ -0,0 +1,319 @@ +import type { DomPathResult } from './types.js'; +import type { Knex } from 'knex'; + +import { DATA_URI_URL_REFS_LIMIT } from '../phase6b/data-uri-url-refs-limit.js'; + +import { resolveBlobRefs } from './resolve-blob-refs.js'; +import { resolveTextRefs } from './resolve-text-refs.js'; +import { resolveUrlRefs } from './resolve-url-refs.js'; +import { upsertTextRefs } from './upsert-text-refs.js'; + +/** + * Page ids scanned per keyset-paginated `SELECT DISTINCT pageId` chunk. + * Batches the outer loop so 470 K pages worth of `images` rows are + * traversed in ~1 K round trips instead of one round-trip per page, + * without holding the entire distinct-pageId list in memory at once. + */ +const PAGE_ID_CHUNK_SIZE = 500; + +/** + * Rows sent per `INSERT INTO image_items ... VALUES (...)` statement. + * Each row binds 13 params, so 300 rows = 3 900 params — safely under + * the SQLite variable limit. + */ +const INSERT_CHUNK_SIZE = 300; + +/** + * Callback that resolves DOM-path strings for one page's images. The + * callback receives the page's HTML string (or `null` when no snapshot + * is stored) plus the page's `images` rows, and must return one + * {@link DomPathResult} per row. + * + * Injected rather than hard-coded so `@nitpicker/crawler` does not + * become a jsdom consumer at runtime — the migration script + * (`scripts/migrate-to-phase6.mjs`) wires a jsdom-backed + * implementation, while unit tests inject stubs. A future crawler-time + * DOM-path capture (Phase 6-G) can use the same signature with + * puppeteer-backed elements. + */ +export type PageDomPathResolver = ( + pageId: number, + htmlString: string | null, + images: readonly ImageRowForResolver[], +) => Promise>; + +/** + * The projection of `images` rows passed to a + * {@link PageDomPathResolver}. Kept as a narrow shape (only the fields + * the resolver needs) so callers of the resolver do not depend on the + * full row width. + */ +export interface ImageRowForResolver { + /** Legacy `images.id`. */ + id: number; + /** Legacy `images.sourceCode`. */ + sourceCode: string | null; +} + +/** + * Populates `image_items` from `images` (issue #193 step 6-D-6). + * + * The outer loop iterates **pages**, not images. Each page's images are + * processed as one whole unit — dom-path derivation, HTML BLOB fetch, + * and ref lookups all happen once per page. This design closes three + * separate correctness / efficiency traps that a purely image-chunked + * loop would expose: + * + * 1. **Ordinal-cursor bleed across chunks** — `matchImagesToDomPaths` + * tracks per-outerHTML cursor state, so a page with 1 000 identical + * `` tags spanning two image-chunks would reset the cursor at + * the boundary and duplicate dom_paths already assigned in the + * previous chunk. Iterating by page keeps every page's cursor state + * self-contained. + * 2. **Repeated `getPageHtml` + jsdom parse** — a page whose images + * straddle N image-chunks would fetch and re-parse the same multi-MB + * HTML snapshot N times. Iterating by page makes this exactly once + * per page. + * 3. **Data-URI routing gap** — legacy `images.src` can be either a + * plain URL (goes to `url_refs`) or a large `data:` URI (goes to + * `blob_refs`, per the 512-byte threshold). The lookups partition + * by that rule, so a data URI is never bound into + * `resolveUrlRefs`' `WHERE url IN (?)` — which could otherwise + * exceed SQLite's `SQLITE_MAX_SQL_LENGTH` for multi-KB URIs. + * + * Per outer iteration: + * + * 1. **Keyset-paginate** distinct `pageId` values from `images`. Peak + * memory ≈ 500 integers. + * 2. **Fetch all images** for those page ids in one query, ordered by + * `(pageId, id)` so JS-side grouping is a linear pass. + * 3. **Resolve dom paths per page** — one `getPageHtml` + one resolver + * call each. `dom_path` texts are collected across the whole batch + * so `upsertTextRefs` runs once per batch. + * 4. **Partition src / currentSrc** by the data-URI routing rule and + * batch-resolve url refs, blob refs, and alt text refs across the + * whole batch. + * 5. **Bulk INSERT** with explicit `id = images.id`. + * + * `INSERT OR IGNORE` on the natural PK makes the step idempotent. + * @param trx - Knex instance or transaction connected to the archive DB. + * @param resolvePageDomPaths - Callback that returns dom_path strings + * for one page's images (see {@link PageDomPathResolver}). + * @param getPageHtml - Callback returning the HTML string for one + * `pages.id` (typically wraps `getHtmlOfPageById`). Returns `null` + * when the page has no stored snapshot. + * @example + * const jsdomResolver = createJsdomResolver(); + * await knex.transaction(async (trx) => { + * await populateImageItems(trx, jsdomResolver, (pid) => db.getHtmlOfPageById(pid)); + * }); + */ +export async function populateImageItems( + trx: Knex, + resolvePageDomPaths: PageDomPathResolver, + getPageHtml: (pageId: number) => Promise, +): Promise { + let cursorPageId = 0; + while (true) { + const pageIdRows: { pageId: number }[] = await trx('images') + .distinct('pageId') + .where('pageId', '>', cursorPageId) + .orderBy('pageId', 'asc') + .limit(PAGE_ID_CHUNK_SIZE); + if (pageIdRows.length === 0) { + break; + } + const pageIds = pageIdRows.map((r) => r.pageId); + cursorPageId = pageIds.at(-1)!; + + const rows: ImageRow[] = await trx('images') + .select( + 'id', + 'pageId', + 'src', + 'currentSrc', + 'alt', + 'width', + 'height', + 'naturalWidth', + 'naturalHeight', + 'isLazy', + 'viewportWidth', + 'sourceCode', + ) + .whereIn('pageId', pageIds) + .orderBy([ + { column: 'pageId', order: 'asc' }, + { column: 'id', order: 'asc' }, + ]); + if (rows.length === 0) { + continue; + } + + const byPage = new Map(); + for (const row of rows) { + const bucket = byPage.get(row.pageId); + if (bucket === undefined) { + byPage.set(row.pageId, [row]); + } else { + bucket.push(row); + } + } + + const domPaths = new Map(); + for (const [pageId, pageImages] of byPage) { + const html = await getPageHtml(pageId); + const resolved = await resolvePageDomPaths(pageId, html, pageImages); + for (const [imageId, entry] of resolved) { + domPaths.set(imageId, entry); + } + } + + const urls = new Set(); + const dataUris = new Set(); + const alts = new Set(); + const domPathTexts = new Set(); + for (const row of rows) { + for (const value of [row.src, row.currentSrc]) { + if (typeof value !== 'string' || value === '') { + continue; + } + if (isBlobRefValue(value)) { + dataUris.add(value); + } else { + urls.add(value); + } + } + if (typeof row.alt === 'string' && row.alt !== '') { + alts.add(row.alt); + } + const domPath = domPaths.get(row.id); + if (domPath !== undefined) { + domPathTexts.add(domPath.path); + } + } + const urlIds = await resolveUrlRefs(trx, urls); + const blobIds = await resolveBlobRefs(trx, dataUris); + const altIds = await resolveTextRefs(trx, alts); + // `dom_path` strings are synthesised at Phase 6-D-6 and were not + // among the Phase 6-B-2 text sources — insert every distinct + // derived path (including `unknown/` fallbacks) before + // resolving its id. The upsert is idempotent so re-runs across + // partial failures do not duplicate rows. + const domPathIds = await upsertTextRefs(trx, domPathTexts); + + const inserts: Record[] = []; + for (const row of rows) { + const domPath = domPaths.get(row.id); + if (domPath === undefined) { + throw new Error( + `populateImageItems: dom_path not resolved for image id=${row.id}`, + ); + } + const domPathId = domPathIds.get(domPath.path); + if (domPathId === undefined) { + throw new Error( + `populateImageItems: text_refs.id not resolved for dom_path=${domPath.path} — the upsertTextRefs above must have inserted it`, + ); + } + const srcSlot = resolveUrlOrBlob(row.src, urlIds, blobIds); + const currentSrcSlot = resolveUrlOrBlob(row.currentSrc, urlIds, blobIds); + inserts.push({ + id: row.id, + page_id: row.pageId, + src_url_id: srcSlot.url, + current_src_url_id: currentSrcSlot.url, + src_blob_id: srcSlot.blob, + current_src_blob_id: currentSrcSlot.blob, + alt_text_id: + typeof row.alt === 'string' && row.alt !== '' + ? (altIds.get(row.alt) ?? null) + : null, + width: row.width, + height: row.height, + natural_width: row.naturalWidth, + natural_height: row.naturalHeight, + is_lazy: row.isLazy == null ? null : row.isLazy ? 1 : 0, + viewport_width: row.viewportWidth, + dom_path_text_id: domPathId, + }); + } + + for (let index = 0; index < inserts.length; index += INSERT_CHUNK_SIZE) { + const chunk = inserts.slice(index, index + INSERT_CHUNK_SIZE); + await trx('image_items').insert(chunk).onConflict('id').ignore(); + } + } +} + +/** + * Predicate matching Phase 6-B-1's routing rule: a `data:` URI whose + * length exceeds {@link DATA_URI_URL_REFS_LIMIT} lands in `blob_refs`; + * anything else (regular URL or short data URI) lands in `url_refs`. + * @param value - Raw `src` / `currentSrc` column value. + */ +function isBlobRefValue(value: string): boolean { + return value.startsWith('data:') && value.length > DATA_URI_URL_REFS_LIMIT; +} + +/** + * Routes one legacy `src` / `currentSrc` value to either `url_refs` or + * `blob_refs` per the plan's data-URI threshold rule. Exactly one of + * `url` / `blob` is non-null; both may be `null` when the value is null + * or fails to resolve (e.g. the `blob_refs` row is missing because the + * data URI was malformed and skipped by `populateBlobRefs`). + * + * Colocated with {@link populateImageItems} rather than exported as its + * own file because it exists solely to serve this one call site and + * captures three parameters (value + both id maps) that are not + * meaningful to any other caller. + * @param value - Raw `src` / `currentSrc` column value. + * @param urlIds - Map of URL string → `url_refs.id`. + * @param blobIds - Map of data-URI string → `blob_refs.id`. + * @returns `{ url, blob }` pair with at most one non-null field. + */ +function resolveUrlOrBlob( + value: string | null, + urlIds: ReadonlyMap, + blobIds: ReadonlyMap, +): { url: number | null; blob: number | null } { + if (typeof value !== 'string' || value === '') { + return { url: null, blob: null }; + } + if (isBlobRefValue(value)) { + return { url: null, blob: blobIds.get(value) ?? null }; + } + return { url: urlIds.get(value) ?? null, blob: null }; +} + +/** + * One row read from the legacy `images` table by + * {@link populateImageItems}. Every column mapped to `image_items` is + * declared; internal-only helper columns are absent. + */ +interface ImageRow { + /** Legacy `images.id`, reused verbatim as `image_items.id`. */ + id: number; + /** Legacy `images.pageId` — `content_items.id` of the owning page. */ + pageId: number; + /** Legacy `images.src` — routed to `src_url_id` or `src_blob_id`. */ + src: string | null; + /** Legacy `images.currentSrc` — routed to `current_src_url_id` or `current_src_blob_id`. */ + currentSrc: string | null; + /** Legacy `images.alt` — resolved to `alt_text_id` via `text_refs`. */ + alt: string | null; + /** Legacy `images.width` — copied verbatim. */ + width: number; + /** Legacy `images.height` — copied verbatim. */ + height: number; + /** Legacy `images.naturalWidth` — copied to `natural_width`. */ + naturalWidth: number; + /** Legacy `images.naturalHeight` — copied to `natural_height`. */ + naturalHeight: number; + /** Legacy `images.isLazy` — copied to `is_lazy`. */ + isLazy: boolean | number | null; + /** Legacy `images.viewportWidth` — copied to `viewport_width`. */ + viewportWidth: number; + /** Legacy `images.sourceCode` — the outerHTML fed to the dom-path resolver. */ + sourceCode: string | null; +} diff --git a/packages/@nitpicker/crawler/src/archive/phase6d/populate-page-meta.spec.ts b/packages/@nitpicker/crawler/src/archive/phase6d/populate-page-meta.spec.ts new file mode 100644 index 0000000..40ab63b --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6d/populate-page-meta.spec.ts @@ -0,0 +1,126 @@ +import type knex from 'knex'; + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { populatePhase6BRefs } from '../phase6b/populate-phase6b-refs.js'; + +import { populateContentItems } from './populate-content-items.js'; +import { populatePageMeta } from './populate-page-meta.js'; +import { countRows } from './test-utils/count-rows.js'; +import { setupPhase6DDb } from './test-utils/setup-phase6d-db.js'; + +describe('populatePageMeta', () => { + let db: ReturnType; + + beforeEach(async () => { + db = await setupPhase6DDb(); + }); + afterEach(async () => { + await db.destroy(); + }); + + it('inserts one row per scraped page with resolved text / url ids', async () => { + await db('pages').insert([ + { + url: 'https://example.com/a', + scraped: 1, + isTarget: 1, + title: 'Page A Title', + description: 'A description', + canonical: 'https://example.com/a', + og_title: 'OG A', + og_url: 'https://example.com/og-a', + meta_extras: '{"foo":"bar"}', + }, + ]); + await populatePhase6BRefs(db); + await populateContentItems(db); + await populatePageMeta(db); + const row = await db('page_meta').where('page_id', 1).first(); + expect(row).toBeDefined(); + expect(row.title_text_id).not.toBeNull(); + expect(row.description_text_id).not.toBeNull(); + expect(row.canonical_url_id).not.toBeNull(); + expect(row.og_url_id).not.toBeNull(); + expect(row.meta_extras_json_id).not.toBeNull(); + }); + + it('skips pages where scraped = 0 (acceptance count invariant)', async () => { + await db('pages').insert([ + { url: 'https://example.com/a', scraped: 1, isTarget: 1, title: 'A' }, + { url: 'https://example.com/b', scraped: 0, isTarget: 0, title: 'B' }, + { url: 'https://example.com/c', scraped: 1, isTarget: 1, title: 'C' }, + ]); + await populatePhase6BRefs(db); + await populateContentItems(db); + await populatePageMeta(db); + const metaCount = await countRows(db, 'page_meta', 'page_id'); + const scrapedRows = await db('pages') + .where('scraped', true) + .count<{ n: number }[]>({ n: 'id' }); + const scrapedCount = Number(scrapedRows[0]!.n); + expect(metaCount).toBe(scrapedCount); + expect(metaCount).toBe(2); + }); + + it('is idempotent (upsert on page_id)', async () => { + await db('pages').insert([ + { url: 'https://example.com/a', scraped: 1, isTarget: 1, title: 'A' }, + ]); + await populatePhase6BRefs(db); + await populateContentItems(db); + await populatePageMeta(db); + await populatePageMeta(db); + expect(await countRows(db, 'page_meta', 'page_id')).toBe(1); + }); + + it('throws when a text_refs.id is not resolvable (Phase 6-B-2 not run)', async () => { + await db('pages').insert([ + { url: 'https://example.com/a', scraped: 1, isTarget: 1, title: 'Missing text' }, + ]); + // Populate only URL refs; text_refs is left empty so `title` cannot resolve. + await db('url_refs').insert({ url: 'https://example.com/a' }); + await populateContentItems(db); + await expect(populatePageMeta(db)).rejects.toThrow(/text_refs\.id not resolved/); + }); + + it('drops large data URIs from URL columns with a warning', async () => { + const longDataUri = 'data:image/png;base64,' + 'A'.repeat(600); + await db('pages').insert([ + { + url: 'https://example.com/a', + scraped: 1, + isTarget: 1, + og_image: longDataUri, + }, + ]); + await populatePhase6BRefs(db); + await populateContentItems(db); + await populatePageMeta(db); + const row = await db('page_meta').where('page_id', 1).first(); + // og_image points at a large data URI that Phase 6-B-1 routed to + // blob_refs, which page_meta cannot reference (no *_blob_id + // column). The URL is dropped with a warning; the row still exists. + expect(row.og_image_url_id).toBeNull(); + }); + + it('preserves denormalised aggregates (tag_count, tags_providers_csv)', async () => { + await db('pages').insert([ + { + url: 'https://example.com/a', + scraped: 1, + isTarget: 1, + tag_count: 5, + jsonld_count: 2, + tags_providers_csv: 'wordpress,gtm', + }, + ]); + await populatePhase6BRefs(db); + await populateContentItems(db); + await populatePageMeta(db); + const row = await db('page_meta').where('page_id', 1).first(); + expect(row.tag_count).toBe(5); + expect(row.jsonld_count).toBe(2); + expect(row.tags_providers_csv).toBe('wordpress,gtm'); + }); +}); diff --git a/packages/@nitpicker/crawler/src/archive/phase6d/populate-page-meta.ts b/packages/@nitpicker/crawler/src/archive/phase6d/populate-page-meta.ts new file mode 100644 index 0000000..9e359f6 --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6d/populate-page-meta.ts @@ -0,0 +1,319 @@ +import type { Knex } from 'knex'; + +import { DATA_URI_URL_REFS_LIMIT } from '../phase6b/data-uri-url-refs-limit.js'; + +import { resolveJsonRefs } from './resolve-json-refs.js'; +import { resolveTextRefs } from './resolve-text-refs.js'; +import { resolveUrlRefs } from './resolve-url-refs.js'; + +/** + * Rows scanned per keyset-paginated `SELECT` chunk against + * `pages WHERE scraped = 1`. The row width here is dominated by long + * text columns (`meta_extras` JSON, `description`, `twitter_description`) + * so 400 rows caps peak chunk memory at ≈ 40 MB on the reference + * archive. + */ +const READ_CHUNK_SIZE = 400; + +/** + * Rows sent per `INSERT INTO page_meta ... VALUES (...)` statement. Each + * row binds 47 params (see the column list below), so 200 rows = + * 9 400 params — safely under SQLite's default variable limit. + */ +const INSERT_CHUNK_SIZE = 200; + +/** + * Text-shaped columns on `pages` mapped 1:1 to `_text_id` FKs on + * `page_meta`. `robots_raw` is grouped here despite the different suffix + * — it maps to `robots_raw_text_id`, still a `text_refs` FK. + */ +const TEXT_COLUMN_MAP: readonly { source: string; target: string }[] = [ + { source: 'title', target: 'title_text_id' }, + { source: 'description', target: 'description_text_id' }, + { source: 'keywords', target: 'keywords_text_id' }, + { source: 'robots_raw', target: 'robots_raw_text_id' }, + { source: 'og_title', target: 'og_title_text_id' }, + { source: 'og_description', target: 'og_description_text_id' }, + { source: 'twitter_title', target: 'twitter_title_text_id' }, + { source: 'twitter_description', target: 'twitter_description_text_id' }, +]; + +/** + * URL-shaped columns on `pages` mapped 1:1 to `_url_id` FKs on + * `page_meta`. + */ +const URL_COLUMN_MAP: readonly { source: string; target: string }[] = [ + { source: 'canonical', target: 'canonical_url_id' }, + { source: 'amphtml', target: 'amphtml_url_id' }, + { source: 'manifest', target: 'manifest_url_id' }, + { source: 'icon_href', target: 'icon_url_id' }, + { source: 'appleTouchIcon_href', target: 'apple_touch_icon_url_id' }, + { source: 'og_url', target: 'og_url_id' }, + { source: 'og_image', target: 'og_image_url_id' }, + { source: 'twitter_image', target: 'twitter_image_url_id' }, +]; + +/** + * Populates `page_meta` from `pages WHERE scraped = 1` (issue #193 step 6-D-2). + * + * Pages that were never scraped contribute no meta rows — they have no rendered title, + * description, or JSON-LD to preserve, so filtering at the SELECT keeps + * the reader working set narrow and matches the Phase 6-E check + * `count(*) FROM page_meta = count(*) FROM pages WHERE scraped = 1`. + * + * Per chunk: + * + * 1. **Keyset-paginate** `pages.id > cursor AND scraped = 1`. + * 2. **Batch-resolve** all text / URL / JSON ref ids for every value in + * the chunk — three resolvers, one round-trip per ref table. + * 3. **Bulk INSERT** with `page_id = pages.id`. `page_id` is both PK and + * FK on `content_items(id)`; Phase 6-D-1 must have already inserted + * the corresponding `content_items` row for the FK to be valid at + * COMMIT time. + * + * `INSERT OR IGNORE` on the `page_id` PK makes the step idempotent so + * partial-failure re-runs are safe. + * @param trx - Knex instance or transaction connected to the archive DB. + * @example + * await knex.transaction(async (trx) => { + * await populateContentItems(trx); + * await populatePageMeta(trx); + * }); + */ +export async function populatePageMeta(trx: Knex): Promise { + const textColumns = TEXT_COLUMN_MAP.map((entry) => entry.source); + const urlColumns = URL_COLUMN_MAP.map((entry) => entry.source); + const plainColumns = [ + 'id', + 'lang', + 'dir', + 'charset', + 'baseHref', + 'viewport_raw', + 'themeColor', + 'applicationName', + 'author', + 'generator', + 'publisher', + 'robots_noindex', + 'robots_nofollow', + 'robots_noarchive', + 'robots_noimageindex', + 'googlebot', + 'og_type', + 'og_site_name', + 'og_image_alt', + 'og_image_width', + 'og_image_height', + 'og_locale', + 'og_article_published_time', + 'og_article_modified_time', + 'twitter_card', + 'twitter_site', + 'twitter_creator', + 'fb_app_id', + 'verification_google', + 'formatDetection_telephone', + 'tag_count', + 'jsonld_count', + 'tags_providers_csv', + 'meta_extras', + ]; + + let cursor = 0; + while (true) { + const rows: Record[] = await trx('pages') + .select(...plainColumns, ...textColumns, ...urlColumns) + .where('id', '>', cursor) + .andWhere('scraped', true) + .orderBy('id', 'asc') + .limit(READ_CHUNK_SIZE); + if (rows.length === 0) { + break; + } + cursor = rows.at(-1)!.id as number; + + const texts = new Set(); + const urls = new Set(); + const jsonStrings = new Set(); + for (const row of rows) { + for (const column of textColumns) { + const value = row[column]; + if (typeof value === 'string' && value !== '') { + texts.add(value); + } + } + for (const column of urlColumns) { + const value = row[column]; + if (typeof value === 'string' && value !== '' && !isLargeDataUri(value)) { + urls.add(value); + } + } + const metaExtras = row.meta_extras; + if (typeof metaExtras === 'string' && metaExtras !== '') { + jsonStrings.add(metaExtras); + } + } + const textIds = await resolveTextRefs(trx, texts); + const urlIds = await resolveUrlRefs(trx, urls); + const jsonIds = await resolveJsonRefs(trx, jsonStrings); + + const inserts = rows.map((row) => { + const rowId = row.id as number; + const insert: Record = { + page_id: rowId, + lang: row.lang ?? null, + dir: row.dir ?? null, + charset: row.charset ?? null, + base_href: row.baseHref ?? null, + viewport_raw: row.viewport_raw ?? null, + theme_color: row.themeColor ?? null, + application_name: row.applicationName ?? null, + author: row.author ?? null, + generator: row.generator ?? null, + publisher: row.publisher ?? null, + robots_noindex: row.robots_noindex ?? null, + robots_nofollow: row.robots_nofollow ?? null, + robots_noarchive: row.robots_noarchive ?? null, + robots_noimageindex: row.robots_noimageindex ?? null, + googlebot: row.googlebot ?? null, + og_type: row.og_type ?? null, + og_site_name: row.og_site_name ?? null, + og_image_alt: row.og_image_alt ?? null, + og_image_width: row.og_image_width ?? null, + og_image_height: row.og_image_height ?? null, + og_locale: row.og_locale ?? null, + og_article_published_time: row.og_article_published_time ?? null, + og_article_modified_time: row.og_article_modified_time ?? null, + twitter_card: row.twitter_card ?? null, + twitter_site: row.twitter_site ?? null, + twitter_creator: row.twitter_creator ?? null, + fb_app_id: row.fb_app_id ?? null, + verification_google: row.verification_google ?? null, + format_detection_telephone: row.formatDetection_telephone ?? null, + tag_count: row.tag_count ?? null, + jsonld_count: row.jsonld_count ?? null, + tags_providers_csv: row.tags_providers_csv ?? null, + meta_extras_json_id: resolveOptionalJsonId(row.meta_extras, jsonIds, rowId), + }; + for (const { source, target } of TEXT_COLUMN_MAP) { + insert[target] = resolveOptionalTextId(row[source], textIds, rowId, source); + } + for (const { source, target } of URL_COLUMN_MAP) { + insert[target] = resolveOptionalUrlId(row[source], urlIds, rowId, source); + } + return insert; + }); + + for (let index = 0; index < inserts.length; index += INSERT_CHUNK_SIZE) { + const chunk = inserts.slice(index, index + INSERT_CHUNK_SIZE); + await trx('page_meta').insert(chunk).onConflict('page_id').ignore(); + } + } +} + +/** + * Returns `true` when `value` is a `data:` URI larger than the routing + * threshold — i.e. one that Phase 6-B-1 sent to `blob_refs` instead of + * `url_refs`. `page_meta` has no `*_blob_id` companion columns for its + * URL slots (see plan §page_meta schema), so we skip these values from + * the `url_refs` lookup rather than issue a doomed `WHERE url IN (?)` + * that would burn a chunk slot per multi-KB URI. The resulting + * `*_url_id` is null with a `console.warn` breadcrumb; the operator + * can then decide whether the archive needs a schema extension or the + * raw data warrants normalisation. + * @param value - Raw URL column value. + */ +function isLargeDataUri(value: string): boolean { + return value.startsWith('data:') && value.length > DATA_URI_URL_REFS_LIMIT; +} + +/** + * Resolves `text_refs.id` for a `page_meta._text_id` slot or + * throws when a non-empty source value has no matching `text_refs` row + * (parity with `populateContentItems`' `url_id` throw — silent NULL + * corruption would still let the count-based acceptance invariant pass + * while individual `page_meta` rows lose their meta text). + * @param value - Raw column value from the source `pages` row. + * @param textIds - Map from Phase 6-B-2's populate step. + * @param pageId - Owning page id (for the error message). + * @param column - Source column name (for the error message). + */ +function resolveOptionalTextId( + value: unknown, + textIds: ReadonlyMap, + pageId: number, + column: string, +): number | null { + if (typeof value !== 'string' || value === '') { + return null; + } + const id = textIds.get(value); + if (id === undefined) { + throw new Error( + `populatePageMeta: text_refs.id not resolved for page id=${pageId} column=${column} — Phase 6-B-2 populate must run first`, + ); + } + return id; +} + +/** + * Resolves `url_refs.id` for a `page_meta._url_id` slot, throws + * on a missing lookup for a regular URL, and returns `null` (with a + * warning) for large data URIs that Phase 6-B-1 routed to `blob_refs` + * — those cannot land in `page_meta` because the schema has no + * `*_blob_id` companion columns. + * @param value - Raw column value from the source `pages` row. + * @param urlIds - Map from Phase 6-B-1's populate step. + * @param pageId - Owning page id (for the error message). + * @param column - Source column name (for the error message). + */ +function resolveOptionalUrlId( + value: unknown, + urlIds: ReadonlyMap, + pageId: number, + column: string, +): number | null { + if (typeof value !== 'string' || value === '') { + return null; + } + if (isLargeDataUri(value)) { + // eslint-disable-next-line no-console + console.warn( + `populatePageMeta: dropping large data URI in page id=${pageId} column=${column} (page_meta has no *_blob_id companion for URL columns)`, + ); + return null; + } + const id = urlIds.get(value); + if (id === undefined) { + throw new Error( + `populatePageMeta: url_refs.id not resolved for page id=${pageId} column=${column} — Phase 6-B-1 populate must run first`, + ); + } + return id; +} + +/** + * Resolves `json_refs.id` for `page_meta.meta_extras_json_id` or throws + * when the source `meta_extras` string has no matching `json_refs` row + * (parity with the text/url branches above). + * @param value - Raw `meta_extras` value from the source `pages` row. + * @param jsonIds - Map from Phase 6-B-3's populate step. + * @param pageId - Owning page id (for the error message). + */ +function resolveOptionalJsonId( + value: unknown, + jsonIds: ReadonlyMap, + pageId: number, +): number | null { + if (typeof value !== 'string' || value === '') { + return null; + } + const id = jsonIds.get(value); + if (id === undefined) { + throw new Error( + `populatePageMeta: json_refs.id not resolved for page id=${pageId} column=meta_extras — Phase 6-B-3 populate must run first`, + ); + } + return id; +} diff --git a/packages/@nitpicker/crawler/src/archive/phase6d/populate-phase6d-entities.spec.ts b/packages/@nitpicker/crawler/src/archive/phase6d/populate-phase6d-entities.spec.ts new file mode 100644 index 0000000..6d9c63a --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6d/populate-phase6d-entities.spec.ts @@ -0,0 +1,82 @@ +import type { DomPathResult } from './types.js'; +import type knex from 'knex'; + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { populatePhase6BRefs } from '../phase6b/populate-phase6b-refs.js'; + +import { populatePhase6DEntities } from './populate-phase6d-entities.js'; +import { countRows } from './test-utils/count-rows.js'; +import { setupPhase6DDb } from './test-utils/setup-phase6d-db.js'; + +describe('populatePhase6DEntities (orchestrator)', () => { + let db: ReturnType; + + beforeEach(async () => { + db = await setupPhase6DDb(); + }); + afterEach(async () => { + await db.destroy(); + }); + + it('runs every sub-step and satisfies all four acceptance count invariants', async () => { + await db('pages').insert([ + { id: 1, url: 'https://example.com/a', scraped: 1, isTarget: 1 }, + { id: 2, url: 'https://example.com/b', scraped: 1, isTarget: 1 }, + { id: 3, url: 'https://example.com/c', scraped: 0, isTarget: 0 }, + ]); + await db('resources').insert([{ id: 10, url: 'https://cdn.example.com/x.js' }]); + await db('resources-referrers').insert([{ resourceId: 10, pageId: 1 }]); + await db('anchors').insert([ + { pageId: 1, hrefId: 2, hash: 'a', textContent: 'link' }, + { pageId: 1, hrefId: 2, hash: 'b', textContent: 'link' }, + ]); + await db('images').insert([ + { + id: 5, + pageId: 1, + src: 'https://example.com/img.png', + alt: null, + width: 1, + height: 1, + naturalWidth: 1, + naturalHeight: 1, + isLazy: 0, + viewportWidth: 1, + sourceCode: '', + }, + ]); + await populatePhase6BRefs(db); + + const resolver = ( + _pageId: number, + _html: string | null, + images: readonly { id: number }[], + ): Promise> => { + const map = new Map(); + for (const image of images) { + map.set(image.id, { path: `unknown/${image.id}`, case: 'unknown' }); + } + return Promise.resolve(map); + }; + await db.transaction(async (trx) => { + await populatePhase6DEntities(trx, resolver, () => Promise.resolve(null)); + }); + + const contentItemsCount = await countRows(db, 'content_items'); + const pagesCount = await countRows(db, 'pages'); + expect(contentItemsCount).toBe(pagesCount); + const pageMetaCount = await countRows(db, 'page_meta', 'page_id'); + const scrapedRows = await db('pages') + .where('scraped', true) + .count<{ n: number }[]>({ n: 'id' }); + const scrapedCount = Number(scrapedRows[0]!.n); + expect(pageMetaCount).toBe(scrapedCount); + const anchorSumRows = await db('anchor_edges').sum<{ n: number | null }[]>({ + n: 'count', + }); + const anchorSum = Number(anchorSumRows[0]!.n ?? 0); + expect(anchorSum).toBe(await countRows(db, 'anchors')); + expect(await countRows(db, 'image_items')).toBe(await countRows(db, 'images')); + }); +}); diff --git a/packages/@nitpicker/crawler/src/archive/phase6d/populate-phase6d-entities.ts b/packages/@nitpicker/crawler/src/archive/phase6d/populate-phase6d-entities.ts new file mode 100644 index 0000000..4253184 --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6d/populate-phase6d-entities.ts @@ -0,0 +1,84 @@ +import type { PageDomPathResolver } from './populate-image-items.js'; +import type { Knex } from 'knex'; + +import { populateAnchorEdges } from './populate-anchor-edges.js'; +import { populateContentItems } from './populate-content-items.js'; +import { populateImageItems } from './populate-image-items.js'; +import { populatePageMeta } from './populate-page-meta.js'; +import { populateResourceItems } from './populate-resource-items.js'; +import { populateResourceRefEdges } from './populate-resource-ref-edges.js'; + +/** + * Runs the six Phase 6-D sub-steps (issue #193) in the plan-specified + * order against an already-connected archive. + * + * Order rationale: + * + * 1. **`content_items`** (6-D-1) — every downstream step's FKs reference + * `content_items(id)`. Must land first so the FKs are valid at + * COMMIT time. `redirect_dest_id` is `DEFERRABLE INITIALLY DEFERRED` + * (see {@link ../create-phase6c-entity-tables.ts}) so a redirect + * source inserted before its destination is validated only at + * COMMIT — the intra-step insert order within this step is not + * load-bearing. + * 2. **`page_meta`** (6-D-2) — FK to `content_items(id)`. + * 3. **`resource_items`** (6-D-3) — required before `resource_ref_edges`. + * 4. **`anchor_edges`** (6-D-4) — FKs to `content_items(id)` on both + * sides. + * 5. **`resource_ref_edges`** (6-D-5) — FKs to `resource_items(id)` and + * `content_items(id)`. + * 6. **`image_items`** (6-D-6) — FK to `content_items(id)`; the + * dom-path text_refs upsert is scoped to this step so re-runs are + * self-contained. + * + * Every sub-step is independently idempotent via `INSERT OR IGNORE` on + * its natural key or PK. Running this orchestrator twice on the same + * archive produces the same rows — no phase marker table is used. + * + * The whole invocation is expected to run inside one writer transaction + * with `.bak` protection at the caller level (matching the plan's + * "All steps run inside a single WAL transaction with `.bak` rollback + * on failure"); this function does not open its own transaction so the + * caller controls the boundary — see {@link + * ../phase6b/populate-phase6b-refs.ts} for the same convention. + * + * **`PRAGMA foreign_keys = ON` is REQUIRED** on the underlying + * connection before this function is called. `content_items.redirect_dest_id` + * is `DEFERRABLE INITIALLY DEFERRED` (see + * {@link ../create-phase6c-entity-tables.ts}); without foreign-key + * enforcement libsql commits `content_items` rows whose `redirect_dest_id` + * points at a non-existent page and the invariant the plan calls out + * (deferrable FK enforcement at COMMIT) is silently broken. The + * migration script sets the pragma explicitly; any other caller must do + * the same. + * @param trx - Knex instance or transaction connected to the archive DB. + * @param resolvePageDomPaths - Callback that returns dom_path strings + * for one page's images. Injected rather than hard-coded so + * `@nitpicker/crawler` does not become a jsdom consumer at runtime. + * @param getPageHtml - Callback returning the HTML string for one + * `pages.id`. Typically wraps `Database.getHtmlOfPageById`. + * @example + * const archive = await Archive.open(archivePath); + * const knex = archive.getKnex(); + * const db = archive.getDatabase(); + * await knex.transaction(async (trx) => { + * await populatePhase6DEntities( + * trx, + * jsdomResolver, + * (pageId) => db.getHtmlOfPageById(pageId), + * ); + * }); + * await archive.write(); + */ +export async function populatePhase6DEntities( + trx: Knex, + resolvePageDomPaths: PageDomPathResolver, + getPageHtml: (pageId: number) => Promise, +): Promise { + await populateContentItems(trx); + await populatePageMeta(trx); + await populateResourceItems(trx); + await populateAnchorEdges(trx); + await populateResourceRefEdges(trx); + await populateImageItems(trx, resolvePageDomPaths, getPageHtml); +} diff --git a/packages/@nitpicker/crawler/src/archive/phase6d/populate-resource-items.spec.ts b/packages/@nitpicker/crawler/src/archive/phase6d/populate-resource-items.spec.ts new file mode 100644 index 0000000..c9967e3 --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6d/populate-resource-items.spec.ts @@ -0,0 +1,65 @@ +import type knex from 'knex'; + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { populatePhase6BRefs } from '../phase6b/populate-phase6b-refs.js'; + +import { populateResourceItems } from './populate-resource-items.js'; +import { countRows } from './test-utils/count-rows.js'; +import { setupPhase6DDb } from './test-utils/setup-phase6d-db.js'; + +describe('populateResourceItems', () => { + let db: ReturnType; + + beforeEach(async () => { + db = await setupPhase6DDb(); + }); + afterEach(async () => { + await db.destroy(); + }); + + it('copies resources 1:1 into resource_items with resolved ref ids', async () => { + await db('resources').insert([ + { + url: 'https://cdn.example.com/one.js', + isExternal: 1, + status: 200, + statusText: 'OK', + contentType: 'application/javascript', + contentLength: 512, + responseHeaders: '{"cache-control":"public"}', + compress: 'gzip', + cdn: 'cloudflare', + source: 'crawled', + }, + ]); + await populatePhase6BRefs(db); + await populateResourceItems(db); + const row = await db('resource_items').first(); + expect(row.id).toBe(1); + expect(row.url_id).not.toBeNull(); + expect(row.content_type_id).not.toBeNull(); + expect(row.header_set_id).not.toBeNull(); + expect(row.compress).toBe('gzip'); + expect(row.cdn).toBe('cloudflare'); + }); + + it('preserves resources.id verbatim', async () => { + await db('resources').insert([ + { id: 42, url: 'https://cdn.example.com/x.js' }, + { id: 88, url: 'https://cdn.example.com/y.css' }, + ]); + await populatePhase6BRefs(db); + await populateResourceItems(db); + const idRows = await db('resource_items').select('id').orderBy('id'); + expect(idRows.map((r) => r.id)).toEqual([42, 88]); + }); + + it('is idempotent', async () => { + await db('resources').insert([{ url: 'https://cdn.example.com/x.js' }]); + await populatePhase6BRefs(db); + await populateResourceItems(db); + await populateResourceItems(db); + expect(await countRows(db, 'resource_items')).toBe(1); + }); +}); diff --git a/packages/@nitpicker/crawler/src/archive/phase6d/populate-resource-items.ts b/packages/@nitpicker/crawler/src/archive/phase6d/populate-resource-items.ts new file mode 100644 index 0000000..185703c --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6d/populate-resource-items.ts @@ -0,0 +1,149 @@ +import type { Knex } from 'knex'; + +import { loadContentTypeRefs } from './resolve-content-type-refs.js'; +import { resolveHeaderSets } from './resolve-header-sets.js'; +import { resolveUrlRefs } from './resolve-url-refs.js'; + +/** + * Rows scanned per keyset-paginated `SELECT` chunk against `resources`. + * `resources` rows are narrow (no meta columns) so 800 rows fits + * comfortably in a chunk without stressing memory. Bumping this higher + * would help throughput but risks pushing large `responseHeaders` JSON + * blobs past a healthy per-result-set size on archives with per-row + * multi-KB header payloads. + */ +const READ_CHUNK_SIZE = 800; + +/** + * Rows sent per `INSERT INTO resource_items ... VALUES (...)` statement. + * Each row binds 11 params (id + 10 columns), so 500 rows = 5 500 params + * — well under SQLite's default variable limit. + */ +const INSERT_CHUNK_SIZE = 500; + +/** + * Populates `resource_items` from `resources` (issue #193 step 6-D-3). + * + * Structurally analogous to {@link ./populate-content-items.ts}: for each + * `resources` chunk, batch-resolve `url_refs.id`, `content_type_refs.id`, + * and `header_sets.id`, then bulk-INSERT with explicit `id = resources.id`. + * Reusing the legacy PK preserves the `resources-referrers.resourceId` + * FK reference (which becomes `resource_ref_edges.resource_id` in Phase + * 6-D-5) without any per-row UPDATE. + * + * `INSERT OR IGNORE` on `id` makes the step idempotent. + * @param trx - Knex instance or transaction connected to the archive DB. + * @example + * await knex.transaction(async (trx) => { + * await populateResourceItems(trx); + * }); + */ +export async function populateResourceItems(trx: Knex): Promise { + const contentTypeIds = await loadContentTypeRefs(trx); + let cursor = 0; + while (true) { + const rows: ResourceRow[] = await trx('resources') + .select( + 'id', + 'url', + 'isExternal', + 'status', + 'statusText', + 'contentType', + 'contentLength', + 'responseHeaders', + 'compress', + 'cdn', + 'source', + ) + .where('id', '>', cursor) + .orderBy('id', 'asc') + .limit(READ_CHUNK_SIZE); + if (rows.length === 0) { + break; + } + cursor = rows.at(-1)!.id; + + const urls = new Set(); + const headerJsonStrings = new Set(); + for (const row of rows) { + urls.add(row.url); + if (typeof row.responseHeaders === 'string' && row.responseHeaders !== '') { + headerJsonStrings.add(row.responseHeaders); + } + } + const urlIds = await resolveUrlRefs(trx, urls); + const headerSetIds = await resolveHeaderSets(trx, headerJsonStrings); + + const inserts = rows.map((row) => { + const urlId = urlIds.get(row.url) ?? null; + if (urlId === null) { + throw new Error( + `populateResourceItems: url_refs.id not resolved for resource id=${row.id} url=${row.url} — Phase 6-B-1 populate must run first`, + ); + } + let contentTypeId: number | null = null; + if (row.contentType != null && row.contentType !== '') { + const resolved = contentTypeIds.get(row.contentType); + if (resolved === undefined) { + throw new Error( + `populateResourceItems: content_type_refs.id not resolved for resource id=${row.id} contentType=${row.contentType} — Phase 6-B-0 populate must run first`, + ); + } + contentTypeId = resolved; + } + const headerSetId = + typeof row.responseHeaders === 'string' && row.responseHeaders !== '' + ? (headerSetIds.get(row.responseHeaders) ?? null) + : null; + return { + id: row.id, + url_id: urlId, + is_external: row.isExternal == null ? 0 : row.isExternal ? 1 : 0, + status: row.status ?? null, + status_text: row.statusText ?? null, + content_type_id: contentTypeId, + content_length: row.contentLength ?? null, + header_set_id: headerSetId, + compress: row.compress ?? null, + cdn: row.cdn ?? null, + source: row.source, + }; + }); + + for (let index = 0; index < inserts.length; index += INSERT_CHUNK_SIZE) { + const chunk = inserts.slice(index, index + INSERT_CHUNK_SIZE); + await trx('resource_items').insert(chunk).onConflict('id').ignore(); + } + } +} + +/** + * Shape of one row read from the legacy `resources` table by + * {@link populateResourceItems}. Restricted to the columns that map to + * `resource_items` — resources have no meta split. + */ +interface ResourceRow { + /** Legacy `resources.id`, reused verbatim as `resource_items.id`. */ + id: number; + /** Legacy `resources.url` — resolved to `resource_items.url_id`. */ + url: string; + /** Legacy `resources.isExternal` — copied to `resource_items.is_external`. */ + isExternal: boolean | number | null; + /** Legacy `resources.status` — copied verbatim. */ + status: number | null; + /** Legacy `resources.statusText` — copied verbatim. */ + statusText: string | null; + /** Legacy `resources.contentType` — resolved via `content_type_refs`. */ + contentType: string | null; + /** Legacy `resources.contentLength` — copied verbatim. */ + contentLength: number | null; + /** Legacy `resources.responseHeaders` — resolved via `header_sets.raw_json_hash`. */ + responseHeaders: string | null; + /** Legacy `resources.compress` — copied verbatim. */ + compress: string | null; + /** Legacy `resources.cdn` — copied verbatim. */ + cdn: string | null; + /** Legacy `resources.source` — copied verbatim. */ + source: string; +} diff --git a/packages/@nitpicker/crawler/src/archive/phase6d/populate-resource-ref-edges.spec.ts b/packages/@nitpicker/crawler/src/archive/phase6d/populate-resource-ref-edges.spec.ts new file mode 100644 index 0000000..9a10a8b --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6d/populate-resource-ref-edges.spec.ts @@ -0,0 +1,59 @@ +import type knex from 'knex'; + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { populatePhase6BRefs } from '../phase6b/populate-phase6b-refs.js'; + +import { populateContentItems } from './populate-content-items.js'; +import { populateResourceItems } from './populate-resource-items.js'; +import { populateResourceRefEdges } from './populate-resource-ref-edges.js'; +import { countRows } from './test-utils/count-rows.js'; +import { setupPhase6DDb } from './test-utils/setup-phase6d-db.js'; + +describe('populateResourceRefEdges', () => { + let db: ReturnType; + + beforeEach(async () => { + db = await setupPhase6DDb(); + }); + afterEach(async () => { + await db.destroy(); + }); + + it('copies resources-referrers into resource_ref_edges with count=1 each', async () => { + await db('pages').insert([ + { id: 10, url: 'https://example.com/a', scraped: 1, isTarget: 1 }, + { id: 20, url: 'https://example.com/b', scraped: 1, isTarget: 1 }, + ]); + await db('resources').insert([{ id: 100, url: 'https://cdn.example.com/x.js' }]); + await db('resources-referrers').insert([ + { resourceId: 100, pageId: 10 }, + { resourceId: 100, pageId: 20 }, + ]); + await populatePhase6BRefs(db); + await populateContentItems(db); + await populateResourceItems(db); + await populateResourceRefEdges(db); + const rows = await db('resource_ref_edges') + .select() + .orderBy(['resource_id', 'page_id']); + expect(rows).toEqual([ + { resource_id: 100, page_id: 10, count: 1 }, + { resource_id: 100, page_id: 20, count: 1 }, + ]); + }); + + it('is idempotent', async () => { + await db('pages').insert([ + { id: 10, url: 'https://example.com/a', scraped: 1, isTarget: 1 }, + ]); + await db('resources').insert([{ id: 100, url: 'https://cdn.example.com/x.js' }]); + await db('resources-referrers').insert([{ resourceId: 100, pageId: 10 }]); + await populatePhase6BRefs(db); + await populateContentItems(db); + await populateResourceItems(db); + await populateResourceRefEdges(db); + await populateResourceRefEdges(db); + expect(await countRows(db, 'resource_ref_edges', 'page_id')).toBe(1); + }); +}); diff --git a/packages/@nitpicker/crawler/src/archive/phase6d/populate-resource-ref-edges.ts b/packages/@nitpicker/crawler/src/archive/phase6d/populate-resource-ref-edges.ts new file mode 100644 index 0000000..5155c4e --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6d/populate-resource-ref-edges.ts @@ -0,0 +1,36 @@ +import type { Knex } from 'knex'; + +/** + * Populates `resource_ref_edges` from `resources-referrers` + * (issue #193 step 6-D-5). + * + * The transformation is a **direct SQL `INSERT ... SELECT`** — no ref + * lookups, no per-row JS work. Legacy `resources-referrers` already + * enforces `UNIQUE(resourceId, pageId)` (see `init-schema.ts`), so every + * source row maps 1:1 to a distinct `resource_ref_edges` PK; every + * `count` starts at `1` because the legacy shape observed each + * (resource, page) pair exactly once. + * + * `INSERT OR IGNORE` on the `(resource_id, page_id)` PK is redundant + * with the legacy unique constraint on the source but present anyway to + * make partial-failure re-runs safe. + * + * The PKs on both `resource_items.id` and `content_items.id` were + * preserved from `resources.id` / `pages.id`, so no per-row translation + * is required — the FKs `resource_items(id)` and `content_items(id)` + * declared in `resource_ref_edges` are already satisfied for every + * source row by the earlier Phase 6-D steps. + * @param trx - Knex instance or transaction connected to the archive DB. + * @example + * await knex.transaction(async (trx) => { + * await populateResourceItems(trx); + * await populateContentItems(trx); + * await populateResourceRefEdges(trx); + * }); + */ +export async function populateResourceRefEdges(trx: Knex): Promise { + await trx.raw( + `INSERT OR IGNORE INTO resource_ref_edges (resource_id, page_id, count) + SELECT resourceId, pageId, 1 FROM "resources-referrers"`, + ); +} diff --git a/packages/@nitpicker/crawler/src/archive/phase6d/resolve-blob-refs.spec.ts b/packages/@nitpicker/crawler/src/archive/phase6d/resolve-blob-refs.spec.ts new file mode 100644 index 0000000..aee7833 --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6d/resolve-blob-refs.spec.ts @@ -0,0 +1,87 @@ +import type knex from 'knex'; + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { populateBlobRefs } from '../phase6b/populate-blob-refs.js'; + +import { resolveBlobRefs } from './resolve-blob-refs.js'; +import { setupPhase6DDb } from './test-utils/setup-phase6d-db.js'; + +describe('resolveBlobRefs', () => { + let db: ReturnType; + + beforeEach(async () => { + db = await setupPhase6DDb(); + }); + afterEach(async () => { + await db.destroy(); + }); + + it('resolves blob_refs.id for large data URIs present in the dictionary', async () => { + const longDataUri = 'data:image/png;base64,' + 'A'.repeat(600); + await db('images').insert([ + { + pageId: 1, + src: longDataUri, + alt: null, + width: 1, + height: 1, + naturalWidth: 1, + naturalHeight: 1, + isLazy: 0, + viewportWidth: 1, + sourceCode: null, + }, + ]); + await populateBlobRefs(db); + const map = await resolveBlobRefs(db, [longDataUri]); + expect(map.get(longDataUri)).toBeTypeOf('number'); + }); + + it('skips values that fail the data-URI routing rule', async () => { + const shortDataUri = 'data:image/svg+xml;base64,PHN2Zy8+'; + const plainUrl = 'https://example.com/img.png'; + const map = await resolveBlobRefs(db, [shortDataUri, plainUrl]); + expect(map.size).toBe(0); + }); + + it('resolves both raw URIs when two distinct data-URI headers decode to the same payload', async () => { + // Regression guard: a last-wins hash → value map would drop the + // earlier raw URI and leave that image row with a NULL + // src_blob_id. Both raw URIs must resolve to the shared + // blob_refs.id. + const payloadPadding = 'A'.repeat(600); + const uriPng = 'data:image/png;base64,' + payloadPadding; + const uriSvg = 'data:image/svg+xml;base64,' + payloadPadding; + await db('images').insert([ + { + pageId: 1, + src: uriPng, + alt: null, + width: 1, + height: 1, + naturalWidth: 1, + naturalHeight: 1, + isLazy: 0, + viewportWidth: 1, + sourceCode: null, + }, + { + pageId: 1, + src: uriSvg, + alt: null, + width: 1, + height: 1, + naturalWidth: 1, + naturalHeight: 1, + isLazy: 0, + viewportWidth: 1, + sourceCode: null, + }, + ]); + await populateBlobRefs(db); + const map = await resolveBlobRefs(db, [uriPng, uriSvg]); + expect(map.get(uriPng)).toBeTypeOf('number'); + expect(map.get(uriSvg)).toBe(map.get(uriPng)); + }); +}); diff --git a/packages/@nitpicker/crawler/src/archive/phase6d/resolve-blob-refs.ts b/packages/@nitpicker/crawler/src/archive/phase6d/resolve-blob-refs.ts new file mode 100644 index 0000000..a50af78 --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6d/resolve-blob-refs.ts @@ -0,0 +1,107 @@ +import type { BlobRefIdMap } from './types.js'; +import type { Knex } from 'knex'; + +import { computeContentHash } from '../phase6b/compute-content-hash.js'; +import { DATA_URI_URL_REFS_LIMIT } from '../phase6b/data-uri-url-refs-limit.js'; +import { decodeDataUri } from '../phase6b/decode-data-uri.js'; + +/** + * Rows sent per `SELECT ... WHERE hash IN (?, ...)` chunk. `blob_refs` + * cardinality is tiny in the reference archive (≈ 429 rows) so 200 per + * chunk covers most realistic uses in a single query. + */ +const LOOKUP_CHUNK_SIZE = 200; + +/** + * Batch-resolves `blob_refs.id` for a set of large data-URI values + * (issue #193). + * + * The routing rule matches {@link ../phase6b/populate-blob-refs.ts}: + * only `data:` URIs longer than {@link DATA_URI_URL_REFS_LIMIT} bytes + * land in `blob_refs`; smaller data URIs (and every regular URL) live + * in `url_refs` instead. Callers pass URL-column values verbatim; this + * function filters out anything that does not match the routing rule + * before hashing. + * + * Malformed data URIs that fail {@link ../phase6b/decode-data-uri.ts} + * are skipped silently — `populateBlobRefs` also skipped them (with a + * warning), so no matching row exists to resolve. + * + * Duplicate values in `values` are deduped internally so an image whose + * `src` and `currentSrc` share the same data URI only counts once. + * @param trx - Knex instance or transaction connected to the archive DB. + * @param values - Iterable of raw URL-column values that may or may not + * be large data URIs. + * @returns Map keyed by the raw URI string; missing entries indicate + * the value is not stored in `blob_refs` (either not a data URI at + * all, below the size threshold, malformed, or absent from the + * dictionary). + * @example + * const idMap = await resolveBlobRefs(trx, [longDataUri, 'https://…']); + * const blobId = idMap.get(longDataUri); // number | undefined + */ +export async function resolveBlobRefs( + trx: Knex, + values: Iterable, +): Promise { + const distinct = new Set(); + for (const value of values) { + if (typeof value !== 'string' || value === '') { + continue; + } + if (value.length <= DATA_URI_URL_REFS_LIMIT || !value.startsWith('data:')) { + continue; + } + distinct.add(value); + } + if (distinct.size === 0) { + return new Map(); + } + const rawValues = [...distinct]; + const hashByValue = new Map(); + // `hashHexToValues` maps `hex(hash) → [raw URI strings that decoded to + // those bytes]`. Two distinct raw data-URI strings can decode to the + // same payload bytes (e.g. `data:image/png;base64,XXXX…` and + // `data:image/svg+xml;base64,XXXX…` when the underlying base64 is + // identical). `populateBlobRefs` dedups them into one row keyed by the + // payload hash, so both raw URIs must resolve to that shared id — a + // last-wins Map keyed by hash would drop the earlier variant, and + // downstream `image_items` writes would lose the blob reference. + const hashHexToValues = new Map(); + for (const value of rawValues) { + const decoded = decodeDataUri(value); + if (decoded === null) { + continue; + } + const hash = computeContentHash(decoded.bytes); + hashByValue.set(value, hash); + const hex = hash.toString('hex'); + const bucket = hashHexToValues.get(hex); + if (bucket === undefined) { + hashHexToValues.set(hex, [value]); + } else { + bucket.push(value); + } + } + if (hashByValue.size === 0) { + return new Map(); + } + const hashes = [...hashByValue.values()]; + const result = new Map(); + for (let index = 0; index < hashes.length; index += LOOKUP_CHUNK_SIZE) { + const chunkHashes = hashes.slice(index, index + LOOKUP_CHUNK_SIZE); + const rows: { id: number; hash: Uint8Array }[] = await trx('blob_refs') + .select('id', 'hash') + .whereIn('hash', chunkHashes); + for (const row of rows) { + const hex = Buffer.from(row.hash).toString('hex'); + const values = hashHexToValues.get(hex); + if (values !== undefined) { + for (const value of values) { + result.set(value, row.id); + } + } + } + } + return result; +} diff --git a/packages/@nitpicker/crawler/src/archive/phase6d/resolve-content-type-refs.spec.ts b/packages/@nitpicker/crawler/src/archive/phase6d/resolve-content-type-refs.spec.ts new file mode 100644 index 0000000..4ebac9f --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6d/resolve-content-type-refs.spec.ts @@ -0,0 +1,37 @@ +import type knex from 'knex'; + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { populateContentTypeRefs } from '../phase6b/populate-content-type-refs.js'; + +import { loadContentTypeRefs } from './resolve-content-type-refs.js'; +import { setupPhase6DDb } from './test-utils/setup-phase6d-db.js'; + +describe('loadContentTypeRefs', () => { + let db: ReturnType; + + beforeEach(async () => { + db = await setupPhase6DDb(); + }); + afterEach(async () => { + await db.destroy(); + }); + + it('returns every content_type_refs row as raw → id', async () => { + await db('pages').insert([ + { url: 'https://example.com/a', scraped: 1, isTarget: 1, contentType: 'text/html' }, + ]); + await db('resources').insert([ + { url: 'https://cdn.example.com/x.css', contentType: 'text/css' }, + ]); + await populateContentTypeRefs(db); + const map = await loadContentTypeRefs(db); + expect(map.get('text/html')).toBeTypeOf('number'); + expect(map.get('text/css')).toBeTypeOf('number'); + }); + + it('returns an empty map when the dictionary is empty', async () => { + const map = await loadContentTypeRefs(db); + expect(map.size).toBe(0); + }); +}); diff --git a/packages/@nitpicker/crawler/src/archive/phase6d/resolve-content-type-refs.ts b/packages/@nitpicker/crawler/src/archive/phase6d/resolve-content-type-refs.ts new file mode 100644 index 0000000..287119a --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6d/resolve-content-type-refs.ts @@ -0,0 +1,33 @@ +import type { ContentTypeRefIdMap } from './types.js'; +import type { Knex } from 'knex'; + +/** + * Loads the entire `content_type_refs` dictionary into an in-process map + * keyed by the raw wire value (issue #193). + * + * Content-type cardinality is small in practice — the reference archive + * carries ≈ 400 distinct raw values across `pages` + `resources`, so + * loading everything into a `Map` costs ≈ 40 KB and eliminates every + * per-chunk round-trip. This is the same trade-off {@link + * ../phase6b/populate-header-tables.ts} makes for `header_name_refs`. + * + * Callers should invoke this once at the start of a Phase 6-D populate + * step (e.g. `populateContentItems`, `populateResourceItems`) and reuse + * the map for every chunk. + * @param trx - Knex instance or transaction connected to the archive DB. + * @returns Map keyed by `content_type_refs.raw`. + * @example + * const contentTypeIds = await loadContentTypeRefs(trx); + * const id = contentTypeIds.get('text/html; charset=utf-8'); // number | undefined + */ +export async function loadContentTypeRefs(trx: Knex): Promise { + const rows: { id: number; raw: string }[] = await trx('content_type_refs').select( + 'id', + 'raw', + ); + const map = new Map(); + for (const row of rows) { + map.set(row.raw, row.id); + } + return map; +} diff --git a/packages/@nitpicker/crawler/src/archive/phase6d/resolve-header-sets.spec.ts b/packages/@nitpicker/crawler/src/archive/phase6d/resolve-header-sets.spec.ts new file mode 100644 index 0000000..ecbfc99 --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6d/resolve-header-sets.spec.ts @@ -0,0 +1,73 @@ +import type knex from 'knex'; + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { populateHeaderTables } from '../phase6b/populate-header-tables.js'; + +import { resolveHeaderSets } from './resolve-header-sets.js'; +import { setupPhase6DDb } from './test-utils/setup-phase6d-db.js'; + +describe('resolveHeaderSets', () => { + let db: ReturnType; + + beforeEach(async () => { + db = await setupPhase6DDb(); + }); + afterEach(async () => { + await db.destroy(); + }); + + it('resolves header_sets.id from the raw responseHeaders JSON', async () => { + const raw = '{"content-type":"text/html"}'; + await db('pages').insert([ + { + url: 'https://example.com/a', + scraped: 1, + isTarget: 1, + responseHeaders: raw, + }, + ]); + await populateHeaderTables(db); + const map = await resolveHeaderSets(db, [raw]); + expect(map.get(raw)).toBeTypeOf('number'); + }); + + it('skips empty / sentinel strings', async () => { + const map = await resolveHeaderSets(db, ['', 'null', '{}']); + expect(map.size).toBe(0); + }); + + it('returns misses as absent keys', async () => { + const map = await resolveHeaderSets(db, ['{"x-never":"seen"}']); + expect(map.size).toBe(0); + }); + + it('falls back to raw_hash lookup when JSON key ordering differs but decoded content matches', async () => { + // Regression guard for the PdM finding: Phase 6-B-5 stores exactly + // one `raw_json_hash` per `raw_hash` equivalence class. A second + // variant with identical decoded content but different key + // ordering never has its `raw_json_hash` persisted — the primary + // lookup misses, but the `raw_hash` fallback still points at the + // shared header_sets row. + const variantOne = '{"content-type":"text/html","cache-control":"no-store"}'; + const variantTwo = '{"cache-control":"no-store","content-type":"text/html"}'; + await db('pages').insert([ + { + url: 'https://example.com/one', + scraped: 1, + isTarget: 1, + responseHeaders: variantOne, + }, + { + url: 'https://example.com/two', + scraped: 1, + isTarget: 1, + responseHeaders: variantTwo, + }, + ]); + await populateHeaderTables(db); + const map = await resolveHeaderSets(db, [variantOne, variantTwo]); + expect(map.get(variantOne)).toBeTypeOf('number'); + expect(map.get(variantTwo)).toBe(map.get(variantOne)); + }); +}); diff --git a/packages/@nitpicker/crawler/src/archive/phase6d/resolve-header-sets.ts b/packages/@nitpicker/crawler/src/archive/phase6d/resolve-header-sets.ts new file mode 100644 index 0000000..94267dc --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6d/resolve-header-sets.ts @@ -0,0 +1,131 @@ +import type { HeaderSetIdMap } from './types.js'; +import type { Knex } from 'knex'; + +import { computeContentHash } from '../phase6b/compute-content-hash.js'; +import { decomposeHeaderSet } from '../phase6b/decompose-header-set.js'; + +/** + * Rows sent per `SELECT ... WHERE raw_json_hash IN (?, ...)` chunk. Same + * rationale as {@link ./resolve-url-refs.ts}: 500 rows × 1 param each is + * well under the SQLite variable limit and gives good round-trip + * amortisation. + */ +const LOOKUP_CHUNK_SIZE = 500; + +/** + * Batch-resolves `header_sets.id` for a set of raw `responseHeaders` + * JSON strings (issue #193). + * + * The lookup is a two-stage cascade: + * + * 1. **Primary lookup** by `header_sets.raw_json_hash` — SHA-256 of the + * raw JSON string exactly as stored on the source row. This hits when + * the row's JSON key ordering matches whichever ordering populated + * `header_sets` first for that stable-set equivalence class (see + * {@link ../phase6b/populate-header-tables.ts}). + * 2. **Fallback lookup** by `header_sets.raw_hash` for the misses — SHA- + * 256 of the sorted `name=value` pairs. `populate-header-tables.ts` + * stores exactly one `raw_json_hash` per `raw_hash` equivalence class + * (the first JSON variant encountered), so a second variant with + * identical decoded content but different key ordering never had its + * `raw_json_hash` persisted — but its `raw_hash` still points at the + * shared `header_sets` row. Without the fallback these rows would + * silently get `header_set_id = NULL` on `content_items` / + * `resource_items` and the count-based acceptance check would still + * pass; the fallback closes that gap. + * + * Hashes are computed in JS because SQLite has no built-in BLAKE3 / + * SHA-256 (see {@link ../phase6b/compute-content-hash.ts} for the + * algorithm choice). The `raw_hash` fallback re-decomposes each miss via + * {@link ../phase6b/decompose-header-set.ts} so the canonicalisation + * matches the one Phase 6-B-5 used at insert time. + * + * Empty / null values and the sentinel `'{}'` / `'null'` strings are + * skipped — those responses were decomposed to `null` by + * `decomposeHeaderSet` and never produced a `header_sets` row, so the + * caller writes `header_set_id = null` for them. + * + * Duplicate JSON strings are deduped internally so a page and a + * resource sharing the same raw responseHeaders JSON only contribute + * one lookup slot. + * @param trx - Knex instance or transaction connected to the archive DB. + * @param rawJsonStrings - Iterable of raw responseHeaders JSON strings. + * @returns Map keyed by the raw JSON string; missing entries indicate + * no matching `header_sets` row exists (parse failure or empty set). + * @example + * const idMap = await resolveHeaderSets(trx, [ + * '{"content-type":"text/html"}', + * ]); + * const setId = idMap.get('{"content-type":"text/html"}'); // number | undefined + */ +export async function resolveHeaderSets( + trx: Knex, + rawJsonStrings: Iterable, +): Promise { + const distinct = new Set(); + for (const raw of rawJsonStrings) { + if (typeof raw !== 'string' || raw === '' || raw === 'null' || raw === '{}') { + continue; + } + distinct.add(raw); + } + if (distinct.size === 0) { + return new Map(); + } + const values = [...distinct]; + const hashByValue = new Map(); + const hashHexToValue = new Map(); + for (const value of values) { + const hash = computeContentHash(value); + hashByValue.set(value, hash); + hashHexToValue.set(hash.toString('hex'), value); + } + const hashes = [...hashByValue.values()]; + const result = new Map(); + for (let index = 0; index < hashes.length; index += LOOKUP_CHUNK_SIZE) { + const chunkHashes = hashes.slice(index, index + LOOKUP_CHUNK_SIZE); + const rows: { id: number; raw_json_hash: Uint8Array }[] = await trx('header_sets') + .select('id', 'raw_json_hash') + .whereIn('raw_json_hash', chunkHashes); + for (const row of rows) { + const hex = Buffer.from(row.raw_json_hash).toString('hex'); + const value = hashHexToValue.get(hex); + if (value !== undefined) { + result.set(value, row.id); + } + } + } + + // Fallback pass: any value we could not resolve by `raw_json_hash` + // might still map to an existing `header_sets` row via `raw_hash` + // (identical decoded content, different JSON key ordering). + const misses = values.filter((value) => !result.has(value)); + if (misses.length === 0) { + return result; + } + const rawHashByValue = new Map(); + const rawHashHexToValue = new Map(); + for (const value of misses) { + const decomposed = decomposeHeaderSet(value); + if (decomposed === null) { + continue; + } + rawHashByValue.set(value, decomposed.rawHash); + rawHashHexToValue.set(decomposed.rawHash.toString('hex'), value); + } + const rawHashes = [...rawHashByValue.values()]; + for (let index = 0; index < rawHashes.length; index += LOOKUP_CHUNK_SIZE) { + const chunkHashes = rawHashes.slice(index, index + LOOKUP_CHUNK_SIZE); + const rows: { id: number; raw_hash: Uint8Array }[] = await trx('header_sets') + .select('id', 'raw_hash') + .whereIn('raw_hash', chunkHashes); + for (const row of rows) { + const hex = Buffer.from(row.raw_hash).toString('hex'); + const value = rawHashHexToValue.get(hex); + if (value !== undefined) { + result.set(value, row.id); + } + } + } + return result; +} diff --git a/packages/@nitpicker/crawler/src/archive/phase6d/resolve-json-refs.spec.ts b/packages/@nitpicker/crawler/src/archive/phase6d/resolve-json-refs.spec.ts new file mode 100644 index 0000000..168deb6 --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6d/resolve-json-refs.spec.ts @@ -0,0 +1,34 @@ +import type knex from 'knex'; + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { populateJsonRefs } from '../phase6b/populate-json-refs.js'; + +import { resolveJsonRefs } from './resolve-json-refs.js'; +import { setupPhase6DDb } from './test-utils/setup-phase6d-db.js'; + +describe('resolveJsonRefs', () => { + let db: ReturnType; + + beforeEach(async () => { + db = await setupPhase6DDb(); + }); + afterEach(async () => { + await db.destroy(); + }); + + it('resolves json_refs.id for present meta_extras JSON strings', async () => { + const raw = '{"custom":"payload"}'; + await db('pages').insert([ + { url: 'https://example.com/a', scraped: 1, isTarget: 1, meta_extras: raw }, + ]); + await populateJsonRefs(db); + const map = await resolveJsonRefs(db, [raw]); + expect(map.get(raw)).toBeTypeOf('number'); + }); + + it('skips empty / null inputs', async () => { + const map = await resolveJsonRefs(db, ['']); + expect(map.size).toBe(0); + }); +}); diff --git a/packages/@nitpicker/crawler/src/archive/phase6d/resolve-json-refs.ts b/packages/@nitpicker/crawler/src/archive/phase6d/resolve-json-refs.ts new file mode 100644 index 0000000..cbadc64 --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6d/resolve-json-refs.ts @@ -0,0 +1,74 @@ +import type { Knex } from 'knex'; + +import { computeContentHash } from '../phase6b/compute-content-hash.js'; + +/** + * Rows sent per `SELECT ... WHERE hash IN (?, ...)` chunk. `json_refs` + * cardinality is bounded by the count of distinct `pages.meta_extras` + * JSON strings — typically low thousands on a reference archive — so + * 500 per chunk keeps round-trips low without straining the SQLite + * variable limit. + */ +const LOOKUP_CHUNK_SIZE = 500; + +/** + * Batch-resolves `json_refs.id` for a set of raw `meta_extras` JSON + * strings (issue #193). + * + * The lookup goes through `json_refs.hash`, the SHA-256 hash of the raw + * JSON string stored by Phase 6-B-3 (see + * {@link ../phase6b/populate-json-refs.ts}). Hashes are computed in JS + * for the same reason as {@link ./resolve-header-sets.ts}: SQLite has no + * built-in hash function. + * + * Empty / null strings are skipped — those pages had no `meta_extras` + * and never produced a `json_refs` row, so the caller writes + * `meta_extras_json_id = null` for them. + * + * Duplicate strings in `rawJsonStrings` are deduped internally. + * @param trx - Knex instance or transaction connected to the archive DB. + * @param rawJsonStrings - Iterable of raw `meta_extras` JSON strings. + * @returns Map keyed by the raw JSON string; missing entries indicate + * no matching `json_refs` row exists. + * @example + * const idMap = await resolveJsonRefs(trx, ['{"customField":"x"}']); + * const jsonId = idMap.get('{"customField":"x"}'); // number | undefined + */ +export async function resolveJsonRefs( + trx: Knex, + rawJsonStrings: Iterable, +): Promise> { + const distinct = new Set(); + for (const raw of rawJsonStrings) { + if (typeof raw === 'string' && raw !== '') { + distinct.add(raw); + } + } + if (distinct.size === 0) { + return new Map(); + } + const values = [...distinct]; + const hashByValue = new Map(); + const hashHexToValue = new Map(); + for (const value of values) { + const hash = computeContentHash(value); + hashByValue.set(value, hash); + hashHexToValue.set(hash.toString('hex'), value); + } + const hashes = [...hashByValue.values()]; + const result = new Map(); + for (let index = 0; index < hashes.length; index += LOOKUP_CHUNK_SIZE) { + const chunkHashes = hashes.slice(index, index + LOOKUP_CHUNK_SIZE); + const rows: { id: number; hash: Uint8Array }[] = await trx('json_refs') + .select('id', 'hash') + .whereIn('hash', chunkHashes); + for (const row of rows) { + const hex = Buffer.from(row.hash).toString('hex'); + const value = hashHexToValue.get(hex); + if (value !== undefined) { + result.set(value, row.id); + } + } + } + return result; +} diff --git a/packages/@nitpicker/crawler/src/archive/phase6d/resolve-text-refs.spec.ts b/packages/@nitpicker/crawler/src/archive/phase6d/resolve-text-refs.spec.ts new file mode 100644 index 0000000..8f751b6 --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6d/resolve-text-refs.spec.ts @@ -0,0 +1,38 @@ +import type knex from 'knex'; + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { populateTextRefs } from '../phase6b/populate-text-refs.js'; + +import { resolveTextRefs } from './resolve-text-refs.js'; +import { setupPhase6DDb } from './test-utils/setup-phase6d-db.js'; + +describe('resolveTextRefs', () => { + let db: ReturnType; + + beforeEach(async () => { + db = await setupPhase6DDb(); + }); + afterEach(async () => { + await db.destroy(); + }); + + it('returns text_refs.id for every present text', async () => { + await db('pages').insert([ + { url: 'https://example.com/a', scraped: 1, isTarget: 1, title: 'Page A' }, + ]); + await populateTextRefs(db); + const map = await resolveTextRefs(db, ['Page A']); + expect(map.get('Page A')).toBeTypeOf('number'); + }); + + it('omits texts absent from text_refs', async () => { + const map = await resolveTextRefs(db, ['Never inserted']); + expect(map.size).toBe(0); + }); + + it('is a no-op for empty / null-like input', async () => { + const map = await resolveTextRefs(db, ['']); + expect(map.size).toBe(0); + }); +}); diff --git a/packages/@nitpicker/crawler/src/archive/phase6d/resolve-text-refs.ts b/packages/@nitpicker/crawler/src/archive/phase6d/resolve-text-refs.ts new file mode 100644 index 0000000..689588b --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6d/resolve-text-refs.ts @@ -0,0 +1,69 @@ +import type { TextRefIdMap } from './types.js'; +import type { Knex } from 'knex'; + +import { computeContentHash } from '../phase6b/compute-content-hash.js'; + +/** + * Rows sent per `SELECT ... WHERE hash IN (?, ...)` chunk. Same rationale + * as {@link ./resolve-url-refs.ts}: 800 rows keeps the parameter count + * well under the SQLite variable limit even in the worst case where every + * hash is bound as its own parameter. + */ +const LOOKUP_CHUNK_SIZE = 800; + +/** + * Batch-resolves `text_refs.id` for a set of raw text strings (issue #193). + * + * The `text_refs` UNIQUE constraint is on `(hash, text)` so lookups + * prefix-seek on `hash`. Each caller-supplied text is hashed in JS via + * {@link ../phase6b/compute-content-hash.ts} (32-byte SHA-256, matching + * how `populateTextRefs` inserted the rows in Phase 6-B-2) before the + * SQL query; providing the hash lets SQLite's index seek in O(log n) + * without a full table scan. + * + * The `(hash, text)` UNIQUE composite makes the trailing text column + * theoretically necessary to disambiguate the astronomically improbable + * hash collision — we narrow on both by including `text` in the WHERE + * clause, then filter the map keys by the exact text on read-back. + * + * Empty / null texts are ignored. Duplicate strings in `texts` are + * deduped internally. + * @param trx - Knex instance or transaction connected to the archive DB. + * @param texts - Iterable of raw text strings to resolve. + * @returns Map keyed by the raw text; missing entries indicate the text + * is not present in `text_refs` (which should not happen after + * Phase 6-B-2 completes but does happen mid-migration when a caller + * passes texts that were never inserted). + * @example + * const idMap = await resolveTextRefs(trx, ['My Page Title', 'Alt text']); + * const titleId = idMap.get('My Page Title'); // number | undefined + */ +export async function resolveTextRefs( + trx: Knex, + texts: Iterable, +): Promise { + const distinct = new Set(); + for (const text of texts) { + if (typeof text === 'string' && text !== '') { + distinct.add(text); + } + } + if (distinct.size === 0) { + return new Map(); + } + const values = [...distinct]; + const hashes = values.map((text) => computeContentHash(text)); + const result = new Map(); + for (let index = 0; index < values.length; index += LOOKUP_CHUNK_SIZE) { + const chunkValues = values.slice(index, index + LOOKUP_CHUNK_SIZE); + const chunkHashes = hashes.slice(index, index + LOOKUP_CHUNK_SIZE); + const rows: { id: number; hash: Uint8Array; text: string }[] = await trx('text_refs') + .select('id', 'hash', 'text') + .whereIn('hash', chunkHashes) + .whereIn('text', chunkValues); + for (const row of rows) { + result.set(row.text, row.id); + } + } + return result; +} diff --git a/packages/@nitpicker/crawler/src/archive/phase6d/resolve-url-refs.spec.ts b/packages/@nitpicker/crawler/src/archive/phase6d/resolve-url-refs.spec.ts new file mode 100644 index 0000000..cba30f1 --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6d/resolve-url-refs.spec.ts @@ -0,0 +1,59 @@ +import type knex from 'knex'; + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { populatePhase6BRefs } from '../phase6b/populate-phase6b-refs.js'; + +import { resolveUrlRefs } from './resolve-url-refs.js'; +import { setupPhase6DDb } from './test-utils/setup-phase6d-db.js'; + +describe('resolveUrlRefs', () => { + let db: ReturnType; + + beforeEach(async () => { + db = await setupPhase6DDb(); + }); + afterEach(async () => { + await db.destroy(); + }); + + it('returns a map keyed by URL for every present url', async () => { + await db('pages').insert([ + { url: 'https://example.com/a', scraped: 1, isTarget: 1 }, + { url: 'https://example.com/b', scraped: 1, isTarget: 1 }, + ]); + await populatePhase6BRefs(db); + const map = await resolveUrlRefs(db, [ + 'https://example.com/a', + 'https://example.com/b', + ]); + expect(map.get('https://example.com/a')).toBeTypeOf('number'); + expect(map.get('https://example.com/b')).toBeTypeOf('number'); + }); + + it('omits missing URLs from the returned map', async () => { + await db('pages').insert([{ url: 'https://example.com/a', scraped: 1, isTarget: 1 }]); + await populatePhase6BRefs(db); + const map = await resolveUrlRefs(db, [ + 'https://example.com/a', + 'https://example.com/missing', + ]); + expect(map.has('https://example.com/a')).toBe(true); + expect(map.has('https://example.com/missing')).toBe(false); + }); + + it('is a no-op for empty input', async () => { + const map = await resolveUrlRefs(db, []); + expect(map.size).toBe(0); + }); + + it('dedupes duplicate URLs in the request set', async () => { + await db('pages').insert([{ url: 'https://example.com/a', scraped: 1, isTarget: 1 }]); + await populatePhase6BRefs(db); + const map = await resolveUrlRefs(db, [ + 'https://example.com/a', + 'https://example.com/a', + ]); + expect(map.size).toBe(1); + }); +}); diff --git a/packages/@nitpicker/crawler/src/archive/phase6d/resolve-url-refs.ts b/packages/@nitpicker/crawler/src/archive/phase6d/resolve-url-refs.ts new file mode 100644 index 0000000..2704ba0 --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6d/resolve-url-refs.ts @@ -0,0 +1,67 @@ +import type { UrlRefIdMap } from './types.js'; +import type { Knex } from 'knex'; + +/** + * Rows sent per `SELECT ... WHERE url IN (?, ...)` chunk. SQLite's default + * `SQLITE_MAX_VARIABLE_NUMBER` is 32766 on modern builds — 800 URLs per + * chunk stays well under that while amortising the round-trip cost across + * many rows. + */ +const LOOKUP_CHUNK_SIZE = 800; + +/** + * Batch-resolves `url_refs.id` for a set of URL strings (issue #193). + * + * Called by every Phase 6-D populate step that needs to translate a + * legacy URL string column into a `url_id` FK. Rather than open a + * separate SELECT per row (which multiplies migration wall-clock by + * chunk size × URL column count) the resolver collects the distinct + * strings the caller cares about, splits them into chunks of + * {@link LOOKUP_CHUNK_SIZE}, issues one `WHERE url IN (?, ...)` per + * chunk, and returns a single map. + * + * Missing entries fall out of the map — callers distinguish "URL is a + * large data URI routed to blob_refs" from "URL is absent from the + * dictionary" via a separate {@link ./resolve-blob-refs.ts} probe. + * + * Duplicates in `urls` are deduped internally so an image whose `src` + * and `currentSrc` are the same value only counts once against + * `LOOKUP_CHUNK_SIZE`. + * @param trx - Knex instance or transaction connected to the archive DB. + * @param urls - Iterable of URL strings to resolve. Empty and null-like + * values are ignored. + * @returns Map keyed by the raw URL string; missing entries indicate + * the URL is not present in `url_refs`. + * @example + * const idMap = await resolveUrlRefs(trx, [ + * 'https://example.com/a', + * 'https://example.com/b', + * ]); + * const id = idMap.get('https://example.com/a'); // number | undefined + */ +export async function resolveUrlRefs( + trx: Knex, + urls: Iterable, +): Promise { + const distinct = new Set(); + for (const url of urls) { + if (typeof url === 'string' && url !== '') { + distinct.add(url); + } + } + if (distinct.size === 0) { + return new Map(); + } + const values = [...distinct]; + const result = new Map(); + for (let index = 0; index < values.length; index += LOOKUP_CHUNK_SIZE) { + const chunk = values.slice(index, index + LOOKUP_CHUNK_SIZE); + const rows: { id: number; url: string }[] = await trx('url_refs') + .select('id', 'url') + .whereIn('url', chunk); + for (const row of rows) { + result.set(row.url, row.id); + } + } + return result; +} diff --git a/packages/@nitpicker/crawler/src/archive/phase6d/test-utils/count-rows.ts b/packages/@nitpicker/crawler/src/archive/phase6d/test-utils/count-rows.ts new file mode 100644 index 0000000..301eb01 --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6d/test-utils/count-rows.ts @@ -0,0 +1,26 @@ +import type { Knex } from 'knex'; + +/** + * Runs `SELECT count() FROM ` and returns the count + * as a plain JS number. Kept in this package instead of importing from + * `phase6b/test-utils/count-rows.ts` because a spec in phase6d importing + * from phase6b test-utils would establish a cross-directory test-utils + * dependency; the phase6b copy has the same shape but its default + * countColumn is scoped to that phase's tables. Duplication is minimal + * and keeps each phase's test scaffolding self-contained. + * @param db - Knex instance (typically a spec-local in-memory DB). + * @param table - Table name. + * @param countColumn - Column to count; defaults to `'*'` so this helper + * works uniformly across `content_items` / `page_meta` / + * `resource_ref_edges` (some of which do not have an `id` column). + * @returns Row count. + */ +export async function countRows( + db: Knex, + table: string, + countColumn: string = '*', +): Promise { + const rows = await db(table).count<{ n: number }[]>({ n: countColumn }); + const first = rows[0]!; + return Number(first.n); +} diff --git a/packages/@nitpicker/crawler/src/archive/phase6d/test-utils/setup-phase6d-db.ts b/packages/@nitpicker/crawler/src/archive/phase6d/test-utils/setup-phase6d-db.ts new file mode 100644 index 0000000..52529d2 --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6d/test-utils/setup-phase6d-db.ts @@ -0,0 +1,146 @@ +import knex from 'knex'; + +import { createPhase6ARefTables } from '../../create-phase6a-ref-tables.js'; +import { createPhase6CEntityTables } from '../../create-phase6c-entity-tables.js'; +import { LibsqlDialect } from '../../libsql-dialect.js'; + +/** + * Provisions an in-memory SQLite database with: + * + * - A minimal subset of the legacy write model (`pages`, `resources`, + * `anchors`, `images`, `resources-referrers`, `page_html_ref`, + * `page_html_blobs`) — only the columns the Phase 6-D populate steps + * actually read. + * - The Phase 6-A ref / header tables (via {@link createPhase6ARefTables}). + * - The Phase 6-C entity tables (via {@link createPhase6CEntityTables}). + * + * Every Phase 6-D populate spec calls this to obtain a fresh DB. The + * caller is responsible for `db.destroy()` (spec `afterEach`). + * + * `PRAGMA foreign_keys = ON` is enabled at setup time so specs mirror + * the migration script's execution mode. `content_items.redirect_dest_id`'s + * DEFERRABLE FK is enforced at COMMIT, so specs that exercise the + * self-reference must run their inserts inside a `db.transaction` + * boundary (see `populate-content-items.spec.ts`'s redirect test) — + * bare inserts would trip the FK immediately. + * @returns Connected Knex instance. + */ +export async function setupPhase6DDb(): Promise> { + const db = knex({ + client: LibsqlDialect, + connection: { filename: ':memory:' }, + useNullAsDefault: true, + }); + await db.raw('PRAGMA foreign_keys = ON'); + await db.schema.createTable('pages', (t) => { + t.increments('id'); + t.string('url').notNullable(); + t.integer('redirectDestId').nullable(); + t.boolean('scraped').notNullable(); + t.boolean('isTarget').notNullable(); + t.boolean('isExternal'); + t.integer('status'); + t.string('statusText'); + t.string('contentType').nullable(); + t.integer('contentLength').nullable(); + t.json('responseHeaders').nullable(); + t.string('lang'); + t.string('dir'); + t.string('charset'); + t.string('baseHref'); + t.text('viewport_raw'); + t.string('themeColor'); + t.string('applicationName'); + t.string('author'); + t.string('generator'); + t.string('publisher'); + t.string('title'); + t.text('description'); + t.text('keywords'); + t.text('robots_raw'); + t.integer('robots_noindex'); + t.integer('robots_nofollow'); + t.integer('robots_noarchive'); + t.integer('robots_noimageindex'); + t.string('googlebot'); + t.string('canonical'); + t.string('amphtml'); + t.string('manifest'); + t.string('icon_href'); + t.string('appleTouchIcon_href'); + t.string('og_type'); + t.string('og_title'); + t.string('og_url'); + t.string('og_site_name'); + t.text('og_description'); + t.string('og_image'); + t.string('og_image_alt'); + t.string('og_image_width'); + t.string('og_image_height'); + t.string('og_locale'); + t.string('og_article_published_time'); + t.string('og_article_modified_time'); + t.string('twitter_card'); + t.string('twitter_site'); + t.string('twitter_creator'); + t.string('twitter_title'); + t.text('twitter_description'); + t.string('twitter_image'); + t.string('fb_app_id'); + t.string('verification_google'); + t.integer('formatDetection_telephone'); + t.integer('firstCrawledAt'); + t.integer('lastCrawledAt'); + t.integer('tag_count'); + t.integer('jsonld_count'); + t.text('tags_providers_csv'); + t.json('meta_extras'); + t.boolean('isSkipped'); + t.string('skipReason'); + t.integer('order'); + t.string('source').notNullable().defaultTo('crawled'); + }); + await db.schema.createTable('resources', (t) => { + t.increments('id'); + t.string('url').notNullable(); + t.boolean('isExternal'); + t.integer('status'); + t.string('statusText'); + t.string('contentType').nullable(); + t.integer('contentLength').nullable(); + t.string('compress').nullable(); + t.string('cdn').nullable(); + t.json('responseHeaders').nullable(); + t.string('source').notNullable().defaultTo('crawled'); + }); + await db.schema.createTable('anchors', (t) => { + t.increments('id'); + t.integer('pageId').notNullable(); + t.integer('hrefId').notNullable(); + t.string('hash'); + t.string('textContent').nullable(); + }); + await db.schema.createTable('images', (t) => { + t.increments('id'); + t.integer('pageId').notNullable(); + t.string('src').nullable(); + t.string('currentSrc').nullable(); + t.string('alt'); + t.float('width'); + t.float('height'); + t.integer('naturalWidth'); + t.integer('naturalHeight'); + t.boolean('isLazy'); + t.integer('viewportWidth'); + t.string('sourceCode'); + }); + await db.schema.createTable('resources-referrers', (t) => { + t.increments('id'); + t.integer('resourceId').notNullable(); + t.integer('pageId').notNullable(); + t.unique(['resourceId', 'pageId']); + }); + await createPhase6ARefTables(db); + await createPhase6CEntityTables(db); + return db; +} diff --git a/packages/@nitpicker/crawler/src/archive/phase6d/types.ts b/packages/@nitpicker/crawler/src/archive/phase6d/types.ts new file mode 100644 index 0000000..f397f93 --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6d/types.ts @@ -0,0 +1,152 @@ +/** + * Domain types for the Phase 6-D entity/edge population step (issue #193). + * + * The Phase 6-D populate helpers translate rows from the legacy write + * model (`pages`, `resources`, `anchors`, `images`, `resources-referrers`) + * into the normalised entity + edge tables (`content_items`, `page_meta`, + * `resource_items`, `anchor_edges`, `resource_ref_edges`, `image_items`) + * that Phase 6-C created. Every ref-id lookup goes through a small set of + * shared resolvers whose interfaces live here per the repo-wide + * `型は types.ts に集約` rule. + */ + +/** + * Mapping from `url_refs.url` (the natural key) to `url_refs.id`. Populated + * on demand from the DB by {@link ./resolve-url-refs.ts}. `null` on values + * that are stored in {@link ./resolve-blob-refs.ts} instead (large data + * URIs, see §image `src` / `currentSrc` — URL vs `blob_refs` Routing in + * the plan) or on lookup miss. + */ +export interface UrlRefResolution { + /** `url_refs.id` when the URL string is present in the dictionary. */ + readonly urlId: number | null; + /** `blob_refs.id` when the value is a large data URI routed to blob_refs. */ + readonly blobId: number | null; +} + +/** + * Batch-resolved `text_refs.id` for a set of raw text strings. Keyed by + * the raw text verbatim so the caller does not need to re-hash for + * cache probes. + */ +export type TextRefIdMap = ReadonlyMap; + +/** + * Batch-resolved `url_refs.id` for a set of URL strings. Keyed by the + * raw URL verbatim; missing entries mean the URL is either a large data + * URI (routed to `blob_refs`) or absent from the dictionary — the caller + * distinguishes via a separate `blob_refs` probe. + */ +export type UrlRefIdMap = ReadonlyMap; + +/** + * Batch-resolved `blob_refs.id` for a set of large data URI values. + * Keyed by the raw value verbatim (the data URI string), consistent with + * how `populateBlobRefs` inserts one row per distinct data URI payload + * hash — the resolver looks up by SHA-256 of the decoded payload but + * returns the map keyed by the raw URI so callers do not have to keep + * the decoded bytes around. + */ +export type BlobRefIdMap = ReadonlyMap; + +/** + * Batch-resolved `content_type_refs.id` for a set of raw content-type + * header values. Keyed by the raw value exactly as stored in + * `pages.contentType` / `resources.contentType`; missing entries mean + * the content type is null or absent from the dictionary (which should + * not happen after Phase 6-B-0 populates every distinct value). + */ +export type ContentTypeRefIdMap = ReadonlyMap; + +/** + * Batch-resolved `header_sets.id` for a set of raw `responseHeaders` + * JSON strings. Keyed by the raw JSON string; the resolver hashes each + * value in JS and looks up `header_sets.raw_json_hash` because SQLite + * has no built-in BLAKE3 / SHA-256 (see + * {@link ../phase6b/compute-content-hash.ts} for the algorithm choice). + */ +export type HeaderSetIdMap = ReadonlyMap; + +/** + * One collapsed anchor edge produced by the single-pass keyset scan in + * {@link ./populate-anchor-edges.ts}. Rows are emitted in `(page_id, + * href_page_id)` order; `first_text_id` is resolved in a second pass + * after `text_refs` lookups. + */ +export interface AnchorEdgeRow { + /** `content_items.id` of the page containing the anchor. */ + page_id: number; + /** `content_items.id` of the anchor target. */ + href_page_id: number; + /** Number of `anchors` rows collapsed into this edge. */ + count: number; + /** `first_hash` — the `anchors.hash` of the first (lowest-id) instance. */ + first_hash: string | null; + /** + * `first_text_id` — resolved from `first_textContent` against + * `text_refs`. `null` when the first instance had no textContent (an + * empty `` tag) or when the text failed to resolve. + */ + first_text_id: number | null; +} + +/** + * One input row for {@link ./collapse-anchor-rows.ts}. Every field + * mirrors the legacy `anchors` schema; the collapser does not depend on + * knex or the DB so it can be unit-tested with plain arrays. + */ +export interface AnchorInputRow { + /** Legacy `anchors.id` — used only to sort input; not stored on the edge. */ + id: number; + /** Legacy `anchors.pageId` — `content_items.id` of the source page. */ + pageId: number; + /** Legacy `anchors.hrefId` — `content_items.id` of the anchor target. */ + hrefId: number; + /** Legacy `anchors.hash` — verbatim SHA-256 hex from beholder. */ + hash: string | null; + /** Legacy `anchors.textContent` — anchor visible text (empty for `` with no content). */ + textContent: string | null; +} + +/** + * Intermediate shape yielded by {@link ./collapse-anchor-rows.ts}. Differs + * from the final {@link AnchorEdgeRow} in that `first_text_id` has not + * been resolved yet — the raw text is carried through as + * `first_textContent` so the second pass can batch-look up every + * distinct text against `text_refs`. + */ +export interface AnchorEdgeRowInProgress { + /** `content_items.id` of the page. */ + page_id: number; + /** `content_items.id` of the anchor target. */ + href_page_id: number; + /** Number of collapsed `anchors` rows. */ + count: number; + /** Verbatim `anchors.hash` of the first (lowest-id) instance. */ + first_hash: string | null; + /** + * Verbatim `anchors.textContent` of the first instance — carried + * unresolved so the caller can batch-look up ids across every edge. + */ + first_textContent: string | null; +} + +/** + * Case discriminator for {@link ./derive-dom-path.ts}: which of the three + * resolution paths produced the returned `dom_path` string. Used by the + * populate step to emit a diagnostic warning log for `unknown` fallbacks + * so operators can audit reconstruction fidelity. + */ +export type DomPathDerivationCase = 'single-match' | 'ordinal-match' | 'unknown'; + +/** + * Result of the DOM-path derivation for one legacy `images` row. + * `path` is the string to insert into `text_refs.text`; `case` records + * which branch produced it. + */ +export interface DomPathResult { + /** The `dom_path` string, e.g. `html/body[1]/main[1]/img[3]`. */ + path: string; + /** Which branch produced the string. */ + case: DomPathDerivationCase; +} diff --git a/packages/@nitpicker/crawler/src/archive/phase6d/upsert-text-refs.spec.ts b/packages/@nitpicker/crawler/src/archive/phase6d/upsert-text-refs.spec.ts new file mode 100644 index 0000000..87cac67 --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6d/upsert-text-refs.spec.ts @@ -0,0 +1,38 @@ +import type knex from 'knex'; + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { countRows } from './test-utils/count-rows.js'; +import { setupPhase6DDb } from './test-utils/setup-phase6d-db.js'; +import { upsertTextRefs } from './upsert-text-refs.js'; + +describe('upsertTextRefs', () => { + let db: ReturnType; + + beforeEach(async () => { + db = await setupPhase6DDb(); + }); + afterEach(async () => { + await db.destroy(); + }); + + it('inserts previously-missing rows and returns their new ids', async () => { + const map = await upsertTextRefs(db, ['html/body[1]/img[1]', 'unknown/42']); + expect(map.get('html/body[1]/img[1]')).toBeTypeOf('number'); + expect(map.get('unknown/42')).toBeTypeOf('number'); + const rows = await db('text_refs').select('text').orderBy('text'); + expect(rows.map((r) => r.text)).toEqual(['html/body[1]/img[1]', 'unknown/42']); + }); + + it('is idempotent — re-upserts return the same ids without duplicating rows', async () => { + const first = await upsertTextRefs(db, ['unique/1']); + const second = await upsertTextRefs(db, ['unique/1']); + expect(second.get('unique/1')).toBe(first.get('unique/1')); + expect(await countRows(db, 'text_refs')).toBe(1); + }); + + it('ignores empty / null-like inputs', async () => { + const map = await upsertTextRefs(db, ['', '']); + expect(map.size).toBe(0); + }); +}); diff --git a/packages/@nitpicker/crawler/src/archive/phase6d/upsert-text-refs.ts b/packages/@nitpicker/crawler/src/archive/phase6d/upsert-text-refs.ts new file mode 100644 index 0000000..83fe052 --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6d/upsert-text-refs.ts @@ -0,0 +1,85 @@ +import type { TextRefIdMap } from './types.js'; +import type { Knex } from 'knex'; + +import { computeContentHash } from '../phase6b/compute-content-hash.js'; + +/** + * Rows sent per `INSERT INTO text_refs ... VALUES (...)` statement. Each + * row binds 2 params (hash + text) so 500 rows = 1 000 params — well + * under the SQLite variable limit. + */ +const INSERT_CHUNK_SIZE = 500; + +/** + * Rows sent per `SELECT ... WHERE hash IN (?, ...)` chunk on the + * post-insert read-back. Same reasoning as {@link ./resolve-text-refs.ts}. + */ +const LOOKUP_CHUNK_SIZE = 800; + +/** + * Upserts a set of text strings into `text_refs` and returns their ids + * (issue #193 step 6-D-6). + * + * Unlike {@link ./resolve-text-refs.ts}, which only reads, this helper + * inserts every missing string first. It exists specifically for + * `dom_path` strings, which are synthesised during Phase 6-D-6 and were + * not among the Phase 6-B-2 sources — the caller cannot rely on the + * dictionary already containing them. + * + * Strategy: + * + * 1. **Hash and INSERT OR IGNORE** every input string in bulk. New rows + * materialise; conflicts are silently skipped. + * 2. **Read back** by hash + text (the `(hash, text)` UNIQUE composite) + * to pick up both freshly-inserted and pre-existing ids. + * + * The read-back narrows on both hash and text (same defensive shape as + * {@link ../phase6b/populate-header-tables.ts}'s `resolveValueIds`) so + * an astronomically improbable hash collision cannot leak the wrong + * id back to the caller. + * + * Empty / null strings are ignored; duplicates in `texts` are deduped + * internally. + * @param trx - Knex instance or transaction connected to the archive DB. + * @param texts - Iterable of text strings to upsert. + * @returns Map keyed by the raw text string. + * @example + * const idMap = await upsertTextRefs(trx, [ + * 'html/body[1]/img[1]', + * 'unknown/42', + * ]); + */ +export async function upsertTextRefs( + trx: Knex, + texts: Iterable, +): Promise { + const distinct = new Set(); + for (const text of texts) { + if (typeof text === 'string' && text !== '') { + distinct.add(text); + } + } + if (distinct.size === 0) { + return new Map(); + } + const values = [...distinct]; + const hashes = values.map((text) => computeContentHash(text)); + const inserts = values.map((text, index) => ({ hash: hashes[index]!, text })); + for (let index = 0; index < inserts.length; index += INSERT_CHUNK_SIZE) { + const chunk = inserts.slice(index, index + INSERT_CHUNK_SIZE); + await trx('text_refs').insert(chunk).onConflict(['hash', 'text']).ignore(); + } + const result = new Map(); + for (let index = 0; index < values.length; index += LOOKUP_CHUNK_SIZE) { + const chunkValues = values.slice(index, index + LOOKUP_CHUNK_SIZE); + const chunkHashes = hashes.slice(index, index + LOOKUP_CHUNK_SIZE); + const rows: { id: number; hash: Uint8Array; text: string }[] = await trx('text_refs') + .select('id', 'hash', 'text') + .whereIn('hash', chunkHashes) + .whereIn('text', chunkValues); + for (const row of rows) { + result.set(row.text, row.id); + } + } + return result; +} diff --git a/scripts/migrate-to-phase6.mjs b/scripts/migrate-to-phase6.mjs new file mode 100644 index 0000000..bc5a9ff --- /dev/null +++ b/scripts/migrate-to-phase6.mjs @@ -0,0 +1,396 @@ +#!/usr/bin/env node +/** + * Upgrades a 0.10-format `.nitpicker` archive to the Phase 6 write model + * (issue #103 epic, sub-issue #193). The migration script mirrors the + * shape of `scripts/migrate-to-0.10.mjs`: extract input → mutate + * `db.sqlite` inside a work dir → re-tar to a new output path. The + * original input file is never touched. + * + * USAGE + * ----- + * + * node scripts/migrate-to-phase6.mjs [] + * + * If is omitted, writes to `.phase6.nitpicker` next + * to the input. + * + * WHAT IT DOES + * ------------ + * + * 1. **Phase 6-A/6-C schema catch-up** — creates the ref / header / + * entity tables if they are absent. Idempotent via `CREATE TABLE IF + * NOT EXISTS`. + * 2. **Phase 6-B populate** — populates every ref table (url_refs, + * text_refs, json_refs, blob_refs, content_type_refs, header_*). + * 3. **Phase 6-D populate** — populates the six entity / edge tables + * (content_items, page_meta, resource_items, anchor_edges, + * resource_ref_edges, image_items). All six run inside a single + * knex transaction so a failure aborts the whole step; SQLite's WAL + * rollback returns the DB to the pre-migration state. + * 4. **Acceptance verification** — asserts the four row-count invariants + * from issue #193's acceptance criteria: + * - `count(content_items) == count(pages)` + * - `count(page_meta) == count(pages WHERE scraped=1)` + * - `sum(anchor_edges.count) == count(anchors)` + * - `count(image_items) == count(images)` + * 5. **Re-tar** the work dir to the output path. + * + * DOM-PATH DERIVATION + * ------------------- + * + * `image_items.dom_path_text_id` is derived from `images.sourceCode` + * against the archived HTML snapshot by parsing each page's HTML with + * jsdom and applying the 3-case match algorithm from the plan: + * + * - single match — exact `outerHTML` match in the DOM. + * - ordinal match — multiple identical `` outerHTML on one page, + * assigned in `images.id` order. + * - unknown — no `sourceCode` or no DOM match; falls back to the + * synthetic `unknown/` marker. A warning is logged for + * every unknown fallback so operators can audit fidelity. + * + * The jsdom dependency lives only in this script (not in the crawler + * runtime); `populate-image-items.ts` accepts an injected resolver so + * the crawler package stays jsdom-free at runtime. + * + * NOT SHIPPED IN NPM + * ------------------ + * + * This script is not part of the `@nitpicker/*` npm bundles; use it via + * `git clone` + `yarn build`. + */ + +/* eslint-disable no-console, import-x/no-extraneous-dependencies */ + +import { existsSync, mkdirSync, readdirSync, rmSync, statSync } from 'node:fs'; +import path from 'node:path'; +import process from 'node:process'; +import { zstdDecompressSync } from 'node:zlib'; + +import { JSDOM, VirtualConsole } from 'jsdom'; +import knex from 'knex'; +import * as tar from 'tar'; + +import { LibsqlDialect } from '../packages/@nitpicker/crawler/lib/archive/libsql-dialect.js'; +import { migratePhase6ARefTables } from '../packages/@nitpicker/crawler/lib/archive/migrate-phase6a-ref-tables.js'; +import { migratePhase6CEntityTables } from '../packages/@nitpicker/crawler/lib/archive/migrate-phase6c-entity-tables.js'; +import { populatePhase6BRefs } from '../packages/@nitpicker/crawler/lib/archive/phase6b/populate-phase6b-refs.js'; +import { matchImagesToDomPaths } from '../packages/@nitpicker/crawler/lib/archive/phase6d/match-images-to-dom-paths.js'; +import { populatePhase6DEntities } from '../packages/@nitpicker/crawler/lib/archive/phase6d/populate-phase6d-entities.js'; + +const SQLITE_DB_FILE_NAME = 'db.sqlite'; + +/** + * Entry point. + */ +async function main() { + const [inputArg, outputArg] = process.argv.slice(2); + if (!inputArg) { + console.error( + 'Usage: node scripts/migrate-to-phase6.mjs []', + ); + process.exit(1); + } + const inputPath = path.resolve(inputArg); + const outputPath = path.resolve( + outputArg ?? + path.join( + path.dirname(inputPath), + `${path.basename(inputPath, path.extname(inputPath))}.phase6.nitpicker`, + ), + ); + if (!existsSync(inputPath)) { + console.error(`Input not found: ${inputPath}`); + process.exit(1); + } + if (existsSync(outputPath)) { + console.error(`Output already exists: ${outputPath} — remove it first`); + process.exit(1); + } + + const workDir = path.resolve( + path.dirname(outputPath), + `._migrate-phase6-${process.pid}-${path.basename(inputPath, path.extname(inputPath))}`, + ); + if (existsSync(workDir)) { + console.error(`Stale work dir present: ${workDir} — remove it first`); + process.exit(1); + } + mkdirSync(workDir, { recursive: true }); + + try { + console.log(`[1/3] untar ${inputPath} -> ${workDir}`); + await tar.x({ file: inputPath, cwd: workDir }); + + const innerDirName = findInnerDir(workDir); + const extractedDir = path.join(workDir, innerDirName); + const dbPath = path.join(extractedDir, SQLITE_DB_FILE_NAME); + if (!existsSync(dbPath)) { + throw new Error(`Missing ${SQLITE_DB_FILE_NAME} in input archive`); + } + + console.log('[2/3] apply Phase 6 migrations'); + await applyPhase6Migrations(dbPath); + + // libsql leaves -wal / -shm sidecars after close — remove them so + // the re-tar contains a clean single-file db.sqlite. + for (const sidecar of [`${dbPath}-wal`, `${dbPath}-shm`]) { + if (existsSync(sidecar)) rmSync(sidecar, { force: true }); + } + + console.log(`[3/3] tar -> ${outputPath}`); + await tar.c({ file: outputPath, cwd: workDir, portable: true }, [innerDirName]); + + console.log('Done.'); + } catch (error) { + if (existsSync(outputPath)) { + rmSync(outputPath, { force: true }); + } + throw error; + } finally { + rmSync(workDir, { recursive: true, force: true }); + } +} + +/** + * Finds the single top-level directory created by untar inside `workDir`. + * Copied from `migrate-to-0.10.mjs` because that script is a stand-alone + * .mjs and does not export helpers; keeping the helper local avoids a + * cross-script coupling for a 20-line predicate. + * + * Skips macOS AppleDouble (`._*`) sidecar files that BSD tar embeds and + * Node's `tar` library surfaces verbatim. + * @param {string} workDir + * @returns {string} Inner directory basename. + */ +function findInnerDir(workDir) { + const candidates = readdirSync(workDir).filter((name) => { + if (name.startsWith('._')) return false; + const stat = statSync(path.join(workDir, name)); + return stat.isDirectory(); + }); + if (candidates.length === 0) { + throw new Error( + `Untar did not produce a top-level directory inside ${workDir}. ` + + `Input may be a non-Nitpicker tar or corrupted.`, + ); + } + if (candidates.length > 1) { + throw new Error( + `Untar produced multiple top-level directories inside ${workDir}: ${candidates.join(', ')}.`, + ); + } + return candidates[0]; +} + +/** + * Opens the extracted `db.sqlite` and applies every Phase 6 step: + * schema catch-up (6-A, 6-C), populate refs (6-B), populate entities + * (6-D). Phase 6-D runs inside a single knex transaction so the whole + * batch either commits or rolls back. + * @param {string} dbPath - Path to the extracted `db.sqlite`. + */ +async function applyPhase6Migrations(dbPath) { + const db = knex({ + client: LibsqlDialect, + connection: { filename: dbPath }, + useNullAsDefault: true, + }); + try { + await db.raw('PRAGMA journal_mode = WAL'); + await db.raw('PRAGMA foreign_keys = ON'); + + console.log(' [6-A] ensure ref tables'); + await migratePhase6ARefTables(db); + console.log(' [6-C] ensure entity tables'); + await migratePhase6CEntityTables(db); + + console.log(' [6-B] populate ref tables'); + await db.transaction(async (trx) => { + await populatePhase6BRefs(trx); + }); + + console.log(' [6-D] populate entity tables'); + const domPathResolver = createJsdomDomPathResolver(); + const getPageHtml = createHtmlGetter(db); + await db.transaction(async (trx) => { + await populatePhase6DEntities(trx, domPathResolver, getPageHtml); + }); + + console.log(' [verify] acceptance counts'); + await verifyAcceptanceCounts(db); + + await db.raw('PRAGMA wal_checkpoint(TRUNCATE)'); + } finally { + await db.destroy(); + } +} + +/** + * Returns a `PageDomPathResolver` (see + * `packages/@nitpicker/crawler/src/archive/phase6d/populate-image-items.ts`) + * that parses HTML with jsdom and applies the 3-case match algorithm from + * `packages/@nitpicker/crawler/src/archive/phase6d/match-images-to-dom-paths.ts`. + * + * jsdom's `VirtualConsole` is silenced because production HTML crashes + * emit a torrent of `unhandled` warnings (unrecognised CSS at-rules, + * scripts that reference `window`) that are irrelevant to `` DOM + * position derivation. + */ +function createJsdomDomPathResolver() { + const virtualConsole = new VirtualConsole(); + virtualConsole.on('jsdomError', () => {}); + return async (pageId, htmlString, images) => { + if (htmlString === null) { + // Every image on this page falls back — the plan requires a + // per-image warning so operators can audit reconstruction + // fidelity, not just a single "no HTML for page X" line. + return fallbackAllUnknown(images, pageId, 'no HTML snapshot stored'); + } + let dom; + try { + dom = new JSDOM(htmlString, { virtualConsole }); + } catch (error) { + return fallbackAllUnknown( + images, + pageId, + `jsdom parse failed: ${error?.message ?? error}`, + ); + } + try { + const imgElements = [...dom.window.document.querySelectorAll('img')]; + const result = matchImagesToDomPaths(images, imgElements); + for (const [imageId, entry] of result) { + if (entry.case === 'unknown') { + console.warn( + `[dom-path] unknown fallback for image id=${imageId} (page ${pageId})`, + ); + } + } + return result; + } finally { + dom.window.close(); + } + }; +} + +/** + * Returns a `Map` where every image resolves to + * the `unknown/` fallback and emits one warning line per image so + * the audit log satisfies the plan's "record a warning log for every + * unknown/* fallback" contract. Used when jsdom cannot parse the page + * OR when the page has no stored HTML snapshot at all. + * @param {readonly { id: number }[]} images + * @param {number} pageId + * @param {string} reason - Human-readable reason for the whole-page fallback. + */ +function fallbackAllUnknown(images, pageId, reason) { + const map = new Map(); + for (const image of images) { + map.set(image.id, { path: `unknown/${image.id}`, case: 'unknown' }); + console.warn( + `[dom-path] unknown fallback for image id=${image.id} (page ${pageId}): ${reason}`, + ); + } + return map; +} + +/** + * Builds a `(pageId) => Promise` getter that reads the + * archived HTML BLOB via the `page_html_ref` + `page_html_blobs` join + * and decompresses via zstd. Analogous to + * `Database.getHtmlOfPageById` but implemented directly against knex + * so the migration script does not need to instantiate the full + * `Database` class (which is a private constructor). + * @param {import('knex').Knex} db + */ +function createHtmlGetter(db) { + return async (pageId) => { + const row = await db('page_html_ref') + .join('page_html_blobs', 'page_html_ref.hash', '=', 'page_html_blobs.hash') + .select('page_html_blobs.body as body', 'page_html_blobs.codec as codec') + .where('page_html_ref.page_id', pageId) + .first(); + if (!row) return null; + return decodeStoredBlob(row.body, row.codec); + }; +} + +/** + * Decompresses one `page_html_blobs.body` value into UTF-8 text. + * Mirrors the `codec` union from `init-schema.ts` — only `zstd` and + * `none` are valid; anything else is a corrupt archive. + * @param {Uint8Array} body + * @param {string} codec + * @returns {string} + */ +function decodeStoredBlob(body, codec) { + const buffer = Buffer.isBuffer(body) ? body : Buffer.from(body); + if (codec === 'zstd') { + return zstdDecompressSync(buffer).toString('utf8'); + } + if (codec === 'none') { + return buffer.toString('utf8'); + } + throw new Error(`Unknown page_html_blobs.codec: ${codec}`); +} + +/** + * Asserts the four row-count invariants from issue #193's acceptance + * criteria. Throws on any mismatch so the caller can surface the failure + * to the operator. + * @param {import('knex').Knex} db + */ +async function verifyAcceptanceCounts(db) { + const contentItemsCount = await countRows(db, 'content_items'); + const pagesCount = await countRows(db, 'pages'); + if (contentItemsCount !== pagesCount) { + throw new Error( + `Acceptance failed: count(content_items)=${contentItemsCount} != count(pages)=${pagesCount}`, + ); + } + const pageMetaCount = await countRows(db, 'page_meta'); + const scrapedRows = await db('pages').where('scraped', true).count({ n: 'id' }); + const scrapedPagesCount = Number(scrapedRows[0].n); + if (pageMetaCount !== scrapedPagesCount) { + throw new Error( + `Acceptance failed: count(page_meta)=${pageMetaCount} != count(pages WHERE scraped=1)=${scrapedPagesCount}`, + ); + } + const anchorEdgesSumRows = await db('anchor_edges').sum({ n: 'count' }); + const anchorEdgesSum = Number(anchorEdgesSumRows[0].n ?? 0); + const anchorsCount = await countRows(db, 'anchors'); + if (anchorEdgesSum !== anchorsCount) { + throw new Error( + `Acceptance failed: SUM(anchor_edges.count)=${anchorEdgesSum} != count(anchors)=${anchorsCount}`, + ); + } + const imageItemsCount = await countRows(db, 'image_items'); + const imagesCount = await countRows(db, 'images'); + if (imageItemsCount !== imagesCount) { + throw new Error( + `Acceptance failed: count(image_items)=${imageItemsCount} != count(images)=${imagesCount}`, + ); + } + console.log( + ` [verify] OK — content_items=${contentItemsCount}, page_meta=${pageMetaCount}, anchor_edges(sum count)=${anchorEdgesSum}, image_items=${imageItemsCount}`, + ); +} + +/** + * Runs `SELECT count(*) FROM
` via knex and returns the count as + * a plain JS number. + * @param {import('knex').Knex} db + * @param {string} table + */ +async function countRows(db, table) { + const rows = await db(table).count({ n: '*' }); + return Number(rows[0].n); +} + +try { + await main(); +} catch (error) { + console.error(error); + process.exit(1); +}