diff --git a/packages/@nitpicker/crawler/src/archive/migrate-to-phase6-script.spec.ts b/packages/@nitpicker/crawler/src/archive/migrate-to-phase6-script.spec.ts new file mode 100644 index 0000000..1ef5da5 --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/migrate-to-phase6-script.spec.ts @@ -0,0 +1,236 @@ +import { execFileSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { existsSync, mkdirSync, readFileSync, rmSync } from 'node:fs'; +import path from 'node:path'; + +import { tryParseUrl as parseUrl } from '@d-zero/shared/parse-url'; +import knex from 'knex'; +import * as tar from 'tar'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import Archive from './archive.js'; +import { peekTarTopDir } from './filesystem/peek-tar-top-dir.js'; +import { LibsqlDialect } from './libsql-dialect.js'; + +const __filename = new URL(import.meta.url).pathname; +const __dirname = path.dirname(__filename); +const workingDir = path.resolve(__dirname, '__test_fixtures_migrate_phase6_script__'); +const repoRoot = path.resolve(__dirname, '..', '..', '..', '..', '..'); +const migrateScript = path.resolve(repoRoot, 'scripts', 'migrate-to-phase6.mjs'); + +/** + * Builds a small but realistic 0.10 archive fixture via the production + * `Archive.create` + `setPage` path so downstream migration steps see a + * schema that matches what a real crawl would produce. Two pages, three + * anchors (two pointing to the same href for dedup coverage), and one + * shared external resource are inserted so the anchor-edges / resource + * populate paths have data to work on. + * @param filePath - Where to write the resulting `.nitpicker`. + */ +async function buildFixtureArchive(filePath: string): Promise { + const archive = await Archive.create({ filePath, cwd: workingDir }); + try { + await archive.setPage({ + url: parseUrl('http://localhost/a')!, + redirectPaths: [], + isExternal: false, + status: 200, + statusText: 'OK', + contentLength: 100, + contentType: 'text/html', + responseHeaders: {}, + meta: { title: 'Page A' }, + anchorList: [ + { + href: parseUrl('http://localhost/b')!, + textContent: 'link1', + isExternal: false, + }, + { + href: parseUrl('http://localhost/b')!, + textContent: 'link2', + isExternal: false, + }, + ], + imageList: [], + html: 'a', + isSkipped: false, + isTarget: true, + }); + await archive.setPage({ + url: parseUrl('http://localhost/b')!, + redirectPaths: [], + isExternal: false, + status: 200, + statusText: 'OK', + contentLength: 100, + contentType: 'text/html', + responseHeaders: {}, + meta: { title: 'Page B' }, + anchorList: [ + { href: parseUrl('http://localhost/a')!, textContent: 'back', isExternal: false }, + ], + imageList: [], + html: 'b', + isSkipped: false, + isTarget: true, + }); + } finally { + await archive.close(); + } +} + +/** + * Computes a SHA-256 hex digest of a file. Used to prove the migration + * script never mutates its input `.nitpicker` (the "restore .bak" clause + * of issue #194 is satisfied by out-of-place output; the input is the + * effective backup). + * @param filePath - Absolute path to the file. + */ +function sha256File(filePath: string): string { + const hash = createHash('sha256'); + hash.update(readFileSync(filePath)); + return hash.digest('hex'); +} + +/** + * Repacks a fixture archive after opening its inner `db.sqlite` for a + * caller-supplied mutation. Used to pre-populate Phase 6 tables with a + * phantom row so the migration's Phase 6-E verification fails. + * + * The mutation runs under `PRAGMA foreign_keys = OFF` so the caller can + * insert rows referencing ids that will be created (or not) later by + * `populate-url-refs.ts` — the phantom row is deliberately outside the + * ref-populate contract. + * @param filePath - Absolute path to the fixture `.nitpicker`. + * @param mutate - Callback that receives a Knex handle on the extracted DB. + */ +async function mutateFixture( + filePath: string, + mutate: (db: ReturnType) => Promise, +): Promise { + const innerDirName = await peekTarTopDir(filePath); + const stageDir = path.resolve(workingDir, `mutate-${process.pid}`); + mkdirSync(stageDir, { recursive: true }); + try { + await tar.x({ file: filePath, cwd: stageDir }); + const extracted = path.resolve(stageDir, innerDirName); + const dbPath = path.resolve(extracted, 'db.sqlite'); + const db = knex({ + client: LibsqlDialect, + connection: { filename: dbPath }, + useNullAsDefault: true, + }); + try { + await db.raw('PRAGMA foreign_keys = OFF'); + await mutate(db); + } finally { + await db.destroy(); + } + rmSync(filePath, { force: true }); + await tar.c({ file: filePath, cwd: stageDir, portable: true }, [innerDirName]); + } finally { + rmSync(stageDir, { recursive: true, force: true }); + } +} + +describe('scripts/migrate-to-phase6.mjs (integration)', () => { + beforeEach(() => { + mkdirSync(workingDir, { recursive: true }); + }); + afterEach(() => { + rmSync(workingDir, { recursive: true, force: true }); + }); + + it( + 'happy path: migrates a valid 0.10 archive and passes all 8 checks', + { timeout: 60_000 }, + async () => { + const inputPath = path.resolve(workingDir, 'input.nitpicker'); + const outputPath = path.resolve(workingDir, 'input.phase6.nitpicker'); + await buildFixtureArchive(inputPath); + + const stdout = execFileSync('node', [migrateScript, inputPath, outputPath], { + cwd: repoRoot, + }); + expect(stdout.toString()).toContain('verification passed'); + + expect(existsSync(outputPath)).toBe(true); + + const inspectDir = path.resolve(workingDir, 'inspect'); + mkdirSync(inspectDir, { recursive: true }); + await tar.x({ file: outputPath, cwd: inspectDir }); + const innerDirName = await peekTarTopDir(outputPath); + const innerDbPath = path.resolve(inspectDir, innerDirName, 'db.sqlite'); + const db = knex({ + client: LibsqlDialect, + connection: { filename: innerDbPath }, + useNullAsDefault: true, + }); + try { + const contentItemsCount = await db('content_items').count<{ n: number }[]>({ + n: '*', + }); + const pagesCount = await db('pages').count<{ n: number }[]>({ n: '*' }); + expect(Number(contentItemsCount[0]!.n)).toBe(Number(pagesCount[0]!.n)); + const edgesCount = await db('anchor_edges').count<{ n: number }[]>({ n: '*' }); + expect(Number(edgesCount[0]!.n)).toBeGreaterThan(0); + } finally { + await db.destroy(); + } + }, + ); + + it( + 'failure path: injects a row-count mismatch, aborts with a Phase6VerificationError, leaves the input intact and no output', + { timeout: 60_000 }, + async () => { + const inputPath = path.resolve(workingDir, 'input.nitpicker'); + const outputPath = path.resolve(workingDir, 'input.phase6.nitpicker'); + await buildFixtureArchive(inputPath); + await mutateFixture(inputPath, async (db) => { + // Fresh archives from `Archive.create` already include the + // Phase 6-A/6-C tables (initSchema calls their creators). + // Insert a phantom `url_refs` + `content_items` pair so the + // migration's 6-D `INSERT OR IGNORE` leaves it untouched; + // `count(content_items) > count(pages)` after 6-D → check #1 + // fires. + await db('url_refs').insert({ + id: 999, + url: 'http://localhost/phantom', + }); + await db('content_items').insert({ + id: 999, + url_id: 999, + is_external: 0, + scraped: 1, + is_target: 1, + source: 'crawled', + }); + }); + const inputHashBefore = sha256File(inputPath); + + let thrown: Error | null = null; + try { + execFileSync('node', [migrateScript, inputPath, outputPath], { + cwd: repoRoot, + stdio: 'pipe', + }); + } catch (error) { + thrown = error as Error; + } + expect(thrown).not.toBeNull(); + const stderr = ( + (thrown as unknown as { stderr?: Buffer }).stderr ?? Buffer.from('') + ).toString(); + expect(stderr).toContain('Phase 6 verification failed'); + expect(stderr).toContain('#1'); + + // Output tar was either never created, or created and then cleaned up. + expect(existsSync(outputPath)).toBe(false); + + // Input tar bytes unchanged — the effective ".bak restore" clause. + expect(sha256File(inputPath)).toBe(inputHashBefore); + }, + ); +}); diff --git a/packages/@nitpicker/crawler/src/archive/phase6d/test-utils/seed-content-items.ts b/packages/@nitpicker/crawler/src/archive/phase6d/test-utils/seed-content-items.ts new file mode 100644 index 0000000..5aa73f4 --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6d/test-utils/seed-content-items.ts @@ -0,0 +1,47 @@ +import type knex from 'knex'; + +/** + * Inserts one row into each of `pages`, `url_refs`, and `content_items` per + * requested id so downstream inserts against tables that FK to + * `content_items(id)` (`anchor_edges`, `image_items`, `page_meta`, …) satisfy + * their foreign key without every spec re-authoring the same 20 lines of + * seed logic. + * + * Shared by Phase 6-E check specs (`check-anchor-edges-count.spec.ts`, + * `check-anchor-edges-sum.spec.ts`, `check-image-items-count.spec.ts`, + * `verify-phase6-migration.spec.ts`) so a schema change to + * pages/url_refs/content_items only requires editing one file. Kept in + * `phase6d/test-utils/` alongside {@link setup-phase6d-db.ts} because that + * module already provisions the tables this helper writes to. + * + * Rows are inserted with defaults suitable for count / structural + * invariants: `scraped=1`, `isTarget=1`, `isExternal=0`, `source='crawled'`, + * URL `https://example.com/` and matching `url_refs` row on the same id + * so `url_id` binds 1:1 with `pages.id` (helpful for URL round-trip specs). + * @param db - Knex handle from {@link setupPhase6DDb}. + * @param ids - Content-item ids to create. Ordering does not matter but the + * helper does not deduplicate the input list — callers should pass + * distinct ids. + */ +export async function seedContentItems( + db: ReturnType, + ids: readonly number[], +): Promise { + for (const id of ids) { + await db('pages').insert({ + id, + url: `https://example.com/${id}`, + scraped: 1, + isTarget: 1, + }); + await db('url_refs').insert({ id, url: `https://example.com/${id}` }); + await db('content_items').insert({ + id, + url_id: id, + is_external: 0, + scraped: 1, + is_target: 1, + source: 'crawled', + }); + } +} diff --git a/packages/@nitpicker/crawler/src/archive/phase6e/check-anchor-edges-count.spec.ts b/packages/@nitpicker/crawler/src/archive/phase6e/check-anchor-edges-count.spec.ts new file mode 100644 index 0000000..00398cc --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6e/check-anchor-edges-count.spec.ts @@ -0,0 +1,79 @@ +import type knex from 'knex'; + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { seedContentItems } from '../phase6d/test-utils/seed-content-items.js'; +import { setupPhase6DDb } from '../phase6d/test-utils/setup-phase6d-db.js'; + +import { checkAnchorEdgesCount } from './check-anchor-edges-count.js'; +import { Phase6VerificationError } from './types.js'; + +describe('checkAnchorEdgesCount', () => { + let db: ReturnType; + + beforeEach(async () => { + db = await setupPhase6DDb(); + }); + afterEach(async () => { + await db.destroy(); + }); + + it('passes when anchor_edges > 0 and < anchors', async () => { + await seedContentItems(db, [1, 2]); + await db('anchors').insert([ + { pageId: 1, hrefId: 2, hash: 'a', textContent: 'link' }, + { pageId: 1, hrefId: 2, hash: 'b', textContent: 'link' }, + { pageId: 1, hrefId: 2, hash: 'c', textContent: 'link' }, + ]); + await db('anchor_edges').insert({ page_id: 1, href_page_id: 2, count: 3 }); + await expect(checkAnchorEdgesCount(db)).resolves.toBeUndefined(); + }); + + it('passes when both anchors and anchor_edges are empty', async () => { + await expect(checkAnchorEdgesCount(db)).resolves.toBeUndefined(); + }); + + it('throws when anchor_edges is empty but anchors is non-empty', async () => { + await seedContentItems(db, [1, 2]); + await db('anchors').insert({ pageId: 1, hrefId: 2, hash: 'a', textContent: 'link' }); + await expect(checkAnchorEdgesCount(db)).rejects.toBeInstanceOf( + Phase6VerificationError, + ); + }); + + it('passes when anchor_edges equals anchors count (all pairs unique — small crawls)', async () => { + await seedContentItems(db, [1, 2, 3]); + // Every (pageId, hrefId) pair is unique, so dedup preserves the row + // count. Check #4 (sum == count) still holds because each edge has + // count = 1. + await db('anchors').insert([ + { pageId: 1, hrefId: 2, hash: 'a', textContent: 'link1' }, + { pageId: 1, hrefId: 3, hash: 'b', textContent: 'link2' }, + ]); + await db('anchor_edges').insert([ + { page_id: 1, href_page_id: 2, count: 1 }, + { page_id: 1, href_page_id: 3, count: 1 }, + ]); + await expect(checkAnchorEdgesCount(db)).resolves.toBeUndefined(); + }); + + it('throws when anchor_edges exceeds anchors (phantom rows)', async () => { + await seedContentItems(db, [1, 2, 3]); + await db('anchors').insert({ pageId: 1, hrefId: 2, hash: 'a', textContent: 'link' }); + await db('anchor_edges').insert([ + { page_id: 1, href_page_id: 2, count: 1 }, + { page_id: 1, href_page_id: 3, count: 1 }, + ]); + await expect(checkAnchorEdgesCount(db)).rejects.toBeInstanceOf( + Phase6VerificationError, + ); + }); + + it('throws when anchor_edges is non-empty but anchors is empty', async () => { + await seedContentItems(db, [1, 2]); + await db('anchor_edges').insert({ page_id: 1, href_page_id: 2, count: 1 }); + await expect(checkAnchorEdgesCount(db)).rejects.toBeInstanceOf( + Phase6VerificationError, + ); + }); +}); diff --git a/packages/@nitpicker/crawler/src/archive/phase6e/check-anchor-edges-count.ts b/packages/@nitpicker/crawler/src/archive/phase6e/check-anchor-edges-count.ts new file mode 100644 index 0000000..01c3e89 --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6e/check-anchor-edges-count.ts @@ -0,0 +1,74 @@ +import type { Knex } from 'knex'; + +import { Phase6VerificationError } from './types.js'; + +/** + * Verifies Phase 6-E invariant #3: the `anchor_edges` dedup does not + * eliminate rows and does not create phantom rows. + * + * Phase 6-D-4 collapses N `anchors` rows per `(pageId, hrefId)` pair into a + * single `anchor_edges` row with `count = N`. Two ways it could go wrong: + * + * - `count(anchor_edges) == 0 && count(anchors) > 0` — dedup annihilated + * the graph. A populate bug or a group-by mis-key would trip this. + * - `count(anchor_edges) > count(anchors)` — dedup produced more rows than + * inputs. Impossible under a correct `GROUP BY` but a phantom-insert bug + * in the populate loop would trip this. + * + * `count(anchor_edges) == count(anchors)` is **not** a failure. Issue #194 + * spelled the upper bound as `< count(anchors)` (strict), but a legitimate + * small crawl where every `(pageId, hrefId)` pair is unique produces one + * edge per anchor (count=1) and satisfies the identity naturally. Enforcing + * strict `<` would refuse to migrate such archives even though every other + * invariant holds; the true invariant is "no rows added or dropped by + * dedup" (`anchor_edges <= anchors`), so the code enforces the weaker, + * correct clause. Check #4 (`SUM(count) == count(anchors)`) already + * establishes that no rows were silently discarded when the counts happen + * to match. + * + * A trivial archive with zero `anchors` legitimately produces zero + * `anchor_edges`, so both counts being zero passes; a non-zero `anchor_edges` + * with a zero `anchors` still throws — that would mean rows appeared from + * nowhere. + * @param trx - Knex instance or transaction connected to the post-6-D archive. + * @throws {Phase6VerificationError} when the invariant is violated. + */ +export async function checkAnchorEdgesCount(trx: Knex): Promise { + const anchorEdgesRows = await trx('anchor_edges').count<{ n: number }[]>({ n: '*' }); + const anchorsRows = await trx('anchors').count<{ n: number }[]>({ n: '*' }); + const anchorEdgesCount = Number(anchorEdgesRows[0]!.n); + const anchorsCount = Number(anchorsRows[0]!.n); + if (anchorsCount === 0) { + if (anchorEdgesCount !== 0) { + throw new Phase6VerificationError({ + check: '#3 anchor_edges row count', + context: { + anchor_edges: anchorEdgesCount, + anchors: anchorsCount, + reason: 'anchor_edges must be empty when anchors is empty', + }, + }); + } + return; + } + if (anchorEdgesCount === 0) { + throw new Phase6VerificationError({ + check: '#3 anchor_edges row count', + context: { + anchor_edges: anchorEdgesCount, + anchors: anchorsCount, + reason: 'anchor_edges must be greater than zero when anchors is non-empty', + }, + }); + } + if (anchorEdgesCount > anchorsCount) { + throw new Phase6VerificationError({ + check: '#3 anchor_edges row count', + context: { + anchor_edges: anchorEdgesCount, + anchors: anchorsCount, + reason: 'anchor_edges must not exceed anchors (dedup should not create rows)', + }, + }); + } +} diff --git a/packages/@nitpicker/crawler/src/archive/phase6e/check-anchor-edges-sum.spec.ts b/packages/@nitpicker/crawler/src/archive/phase6e/check-anchor-edges-sum.spec.ts new file mode 100644 index 0000000..46d50b4 --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6e/check-anchor-edges-sum.spec.ts @@ -0,0 +1,57 @@ +import type knex from 'knex'; + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { seedContentItems } from '../phase6d/test-utils/seed-content-items.js'; +import { setupPhase6DDb } from '../phase6d/test-utils/setup-phase6d-db.js'; + +import { checkAnchorEdgesSum } from './check-anchor-edges-sum.js'; +import { Phase6VerificationError } from './types.js'; + +describe('checkAnchorEdgesSum', () => { + let db: ReturnType; + + beforeEach(async () => { + db = await setupPhase6DDb(); + }); + afterEach(async () => { + await db.destroy(); + }); + + it('passes when SUM(anchor_edges.count) equals count(anchors)', async () => { + await seedContentItems(db, [1, 2, 3]); + await db('anchors').insert([ + { pageId: 1, hrefId: 2, hash: 'a', textContent: 'link' }, + { pageId: 1, hrefId: 2, hash: 'b', textContent: 'link' }, + { pageId: 1, hrefId: 3, hash: 'c', textContent: 'link' }, + { pageId: 1, hrefId: 3, hash: 'd', textContent: 'link' }, + { pageId: 1, hrefId: 3, hash: 'e', textContent: 'link' }, + ]); + await db('anchor_edges').insert([ + { page_id: 1, href_page_id: 2, count: 2 }, + { page_id: 1, href_page_id: 3, count: 3 }, + ]); + await expect(checkAnchorEdgesSum(db)).resolves.toBeUndefined(); + }); + + it('passes when both tables are empty', async () => { + await expect(checkAnchorEdgesSum(db)).resolves.toBeUndefined(); + }); + + it('throws when SUM(count) undercounts anchors', async () => { + await seedContentItems(db, [1, 2]); + await db('anchors').insert([ + { pageId: 1, hrefId: 2, hash: 'a', textContent: 'link' }, + { pageId: 1, hrefId: 2, hash: 'b', textContent: 'link' }, + ]); + await db('anchor_edges').insert({ page_id: 1, href_page_id: 2, count: 1 }); + await expect(checkAnchorEdgesSum(db)).rejects.toBeInstanceOf(Phase6VerificationError); + }); + + it('throws when SUM(count) overcounts anchors', async () => { + await seedContentItems(db, [1, 2]); + await db('anchors').insert({ pageId: 1, hrefId: 2, hash: 'a', textContent: 'link' }); + await db('anchor_edges').insert({ page_id: 1, href_page_id: 2, count: 5 }); + await expect(checkAnchorEdgesSum(db)).rejects.toBeInstanceOf(Phase6VerificationError); + }); +}); diff --git a/packages/@nitpicker/crawler/src/archive/phase6e/check-anchor-edges-sum.ts b/packages/@nitpicker/crawler/src/archive/phase6e/check-anchor-edges-sum.ts new file mode 100644 index 0000000..85abea6 --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6e/check-anchor-edges-sum.ts @@ -0,0 +1,30 @@ +import type { Knex } from 'knex'; + +import { Phase6VerificationError } from './types.js'; + +/** + * Verifies Phase 6-E invariant #4: the sum of `anchor_edges.count` equals the + * total number of legacy `anchors` rows. + * + * Phase 6-D-4 collapses N `anchors` rows per `(pageId, hrefId)` pair into a + * single edge with `count = N`; the invariant re-establishes the identity + * `sum(count) = number of source rows`. Together with invariant #3 this + * proves the dedup is exact (no rows lost, no rows double-counted). + * @param trx - Knex instance or transaction connected to the post-6-D archive. + * @throws {Phase6VerificationError} when the sum diverges from the anchors row count. + */ +export async function checkAnchorEdgesSum(trx: Knex): Promise { + const sumRows = await trx('anchor_edges').sum<{ n: number | null }[]>({ n: 'count' }); + const anchorsRows = await trx('anchors').count<{ n: number }[]>({ n: '*' }); + const anchorEdgesSum = Number(sumRows[0]!.n ?? 0); + const anchorsCount = Number(anchorsRows[0]!.n); + if (anchorEdgesSum !== anchorsCount) { + throw new Phase6VerificationError({ + check: '#4 anchor_edges count sum', + context: { + sum_of_anchor_edges_count: anchorEdgesSum, + anchors: anchorsCount, + }, + }); + } +} diff --git a/packages/@nitpicker/crawler/src/archive/phase6e/check-content-items-count.spec.ts b/packages/@nitpicker/crawler/src/archive/phase6e/check-content-items-count.spec.ts new file mode 100644 index 0000000..488a5f5 --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6e/check-content-items-count.spec.ts @@ -0,0 +1,90 @@ +import type knex from 'knex'; + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { setupPhase6DDb } from '../phase6d/test-utils/setup-phase6d-db.js'; + +import { checkContentItemsCount } from './check-content-items-count.js'; +import { Phase6VerificationError } from './types.js'; + +describe('checkContentItemsCount', () => { + let db: ReturnType; + + beforeEach(async () => { + db = await setupPhase6DDb(); + }); + afterEach(async () => { + await db.destroy(); + }); + + it('passes when content_items row count equals pages row count', async () => { + await db('pages').insert([ + { id: 1, url: 'https://example.com/a', scraped: 1, isTarget: 1 }, + { id: 2, url: 'https://example.com/b', scraped: 1, isTarget: 1 }, + ]); + await db('url_refs').insert([ + { id: 1, url: 'https://example.com/a' }, + { id: 2, url: 'https://example.com/b' }, + ]); + await db('content_items').insert([ + { id: 1, url_id: 1, is_external: 0, scraped: 1, is_target: 1, source: 'crawled' }, + { id: 2, url_id: 2, is_external: 0, scraped: 1, is_target: 1, source: 'crawled' }, + ]); + await expect(checkContentItemsCount(db)).resolves.toBeUndefined(); + }); + + it('throws when content_items has more rows than pages', async () => { + await db('pages').insert({ + id: 1, + url: 'https://example.com/a', + scraped: 1, + isTarget: 1, + }); + await db('url_refs').insert([ + { id: 1, url: 'https://example.com/a' }, + { id: 2, url: 'https://example.com/phantom' }, + ]); + await db('content_items').insert([ + { id: 1, url_id: 1, is_external: 0, scraped: 1, is_target: 1, source: 'crawled' }, + { id: 2, url_id: 2, is_external: 0, scraped: 1, is_target: 1, source: 'crawled' }, + ]); + await expect(checkContentItemsCount(db)).rejects.toBeInstanceOf( + Phase6VerificationError, + ); + }); + + it('throws when content_items is missing rows', async () => { + await db('pages').insert([ + { id: 1, url: 'https://example.com/a', scraped: 1, isTarget: 1 }, + { id: 2, url: 'https://example.com/b', scraped: 1, isTarget: 1 }, + ]); + await db('url_refs').insert({ id: 1, url: 'https://example.com/a' }); + await db('content_items').insert({ + id: 1, + url_id: 1, + is_external: 0, + scraped: 1, + is_target: 1, + source: 'crawled', + }); + await expect(checkContentItemsCount(db)).rejects.toBeInstanceOf( + Phase6VerificationError, + ); + }); + + it('includes both counts in the error context', async () => { + await db('pages').insert([ + { id: 1, url: 'https://example.com/a', scraped: 1, isTarget: 1 }, + ]); + try { + await checkContentItemsCount(db); + expect.unreachable('expected Phase6VerificationError'); + } catch (error) { + expect(error).toBeInstanceOf(Phase6VerificationError); + const details = (error as Phase6VerificationError).details; + expect(details.check).toContain('#1'); + expect(details.context?.content_items).toBe(0); + expect(details.context?.pages).toBe(1); + } + }); +}); diff --git a/packages/@nitpicker/crawler/src/archive/phase6e/check-content-items-count.ts b/packages/@nitpicker/crawler/src/archive/phase6e/check-content-items-count.ts new file mode 100644 index 0000000..21ccdf6 --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6e/check-content-items-count.ts @@ -0,0 +1,33 @@ +import type { Knex } from 'knex'; + +import { Phase6VerificationError } from './types.js'; + +/** + * Verifies Phase 6-E invariant #1: every legacy `pages` row is mirrored by + * one row in `content_items`. + * + * Phase 6-D-1 populates `content_items` with `id = pages.id` for every page + * (see `populate-content-items.ts`); the invariant is broken only if the + * populate loop skipped or double-inserted rows. `INSERT OR IGNORE` on the + * PK collapses duplicates, so a mismatch always means "populate wrote + * fewer rows than expected" (extra rows would be impossible via the natural + * PK). We still emit both counts in the error context so operators can see + * the direction of the discrepancy at a glance. + * @param trx - Knex instance or transaction connected to the post-6-D archive. + * @throws {Phase6VerificationError} when the row counts diverge. + */ +export async function checkContentItemsCount(trx: Knex): Promise { + const contentItemsRows = await trx('content_items').count<{ n: number }[]>({ n: '*' }); + const pagesRows = await trx('pages').count<{ n: number }[]>({ n: '*' }); + const contentItemsCount = Number(contentItemsRows[0]!.n); + const pagesCount = Number(pagesRows[0]!.n); + if (contentItemsCount !== pagesCount) { + throw new Phase6VerificationError({ + check: '#1 content_items row count', + context: { + content_items: contentItemsCount, + pages: pagesCount, + }, + }); + } +} diff --git a/packages/@nitpicker/crawler/src/archive/phase6e/check-content-type-preservation.spec.ts b/packages/@nitpicker/crawler/src/archive/phase6e/check-content-type-preservation.spec.ts new file mode 100644 index 0000000..765d97a --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6e/check-content-type-preservation.spec.ts @@ -0,0 +1,139 @@ +import type knex from 'knex'; + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { setupPhase6DDb } from '../phase6d/test-utils/setup-phase6d-db.js'; + +import { checkContentTypePreservation } from './check-content-type-preservation.js'; +import { Phase6VerificationError } from './types.js'; + +describe('checkContentTypePreservation', () => { + let db: ReturnType; + + beforeEach(async () => { + db = await setupPhase6DDb(); + await db('url_refs').insert([ + { id: 1, url: 'https://example.com/a' }, + { id: 2, url: 'https://example.com/b' }, + { id: 3, url: 'https://example.com/c' }, + ]); + await db('content_type_refs').insert({ + id: 10, + raw: 'text/html; charset=utf-8', + normalized: 'text/html', + category: 'html', + }); + }); + afterEach(async () => { + await db.destroy(); + }); + + it('passes when every page with non-null contentType has a resolved content_type_id', async () => { + await db('pages').insert([ + { + id: 1, + url: 'https://example.com/a', + scraped: 1, + isTarget: 1, + contentType: 'text/html; charset=utf-8', + }, + ]); + await db('content_items').insert({ + id: 1, + url_id: 1, + is_external: 0, + scraped: 1, + is_target: 1, + source: 'crawled', + content_type_id: 10, + }); + await expect(checkContentTypePreservation(db)).resolves.toBeUndefined(); + }); + + it('passes when contentType is null (id may legitimately be null)', async () => { + await db('pages').insert([ + { id: 2, url: 'https://example.com/b', scraped: 0, isTarget: 0, contentType: null }, + ]); + await db('content_items').insert({ + id: 2, + url_id: 2, + is_external: 0, + scraped: 0, + is_target: 0, + source: 'crawled', + content_type_id: null, + }); + await expect(checkContentTypePreservation(db)).resolves.toBeUndefined(); + }); + + it('passes when contentType is the empty string (treated as absent)', async () => { + await db('pages').insert([ + { id: 3, url: 'https://example.com/c', scraped: 1, isTarget: 1, contentType: '' }, + ]); + await db('content_items').insert({ + id: 3, + url_id: 3, + is_external: 0, + scraped: 1, + is_target: 1, + source: 'crawled', + content_type_id: null, + }); + await expect(checkContentTypePreservation(db)).resolves.toBeUndefined(); + }); + + it('throws when a page has non-null contentType but content_items.content_type_id is null', async () => { + await db('pages').insert([ + { + id: 1, + url: 'https://example.com/a', + scraped: 1, + isTarget: 1, + contentType: 'application/pdf', + }, + ]); + await db('content_items').insert({ + id: 1, + url_id: 1, + is_external: 0, + scraped: 1, + is_target: 1, + source: 'crawled', + content_type_id: null, + }); + await expect(checkContentTypePreservation(db)).rejects.toBeInstanceOf( + Phase6VerificationError, + ); + }); + + it('emits the offending row id and content_type in the error context', async () => { + await db('pages').insert([ + { + id: 1, + url: 'https://example.com/a', + scraped: 1, + isTarget: 1, + contentType: 'application/pdf', + }, + ]); + await db('content_items').insert({ + id: 1, + url_id: 1, + is_external: 0, + scraped: 1, + is_target: 1, + source: 'crawled', + content_type_id: null, + }); + try { + await checkContentTypePreservation(db); + expect.unreachable('expected Phase6VerificationError'); + } catch (error) { + expect(error).toBeInstanceOf(Phase6VerificationError); + const details = (error as Phase6VerificationError).details; + expect(details.check).toContain('#7'); + expect(details.context?.sample_page_id).toBe(1); + expect(details.context?.sample_content_type).toBe('application/pdf'); + } + }); +}); diff --git a/packages/@nitpicker/crawler/src/archive/phase6e/check-content-type-preservation.ts b/packages/@nitpicker/crawler/src/archive/phase6e/check-content-type-preservation.ts new file mode 100644 index 0000000..d01a44e --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6e/check-content-type-preservation.ts @@ -0,0 +1,45 @@ +import type { Knex } from 'knex'; + +import { Phase6VerificationError } from './types.js'; + +/** + * Verifies Phase 6-E invariant #7: no page loses its content-type across the + * Phase 6-D-1 populate. + * + * `populate-content-items.ts` sets `content_type_id = null` whenever the + * legacy `pages.contentType` is null OR the empty string (the empty string + * meaning "no content-type recorded"). Any *other* case where `contentType` + * carries a real value but `content_items.content_type_id` ended up null + * would indicate a resolver bug — the current populate throws before insert + * in that path, so this check is a defence in depth against a future + * regression. + * + * The check runs a single `SELECT id, contentType LIMIT 1` filtered to + * offending rows; if any row comes back the invariant is violated and the + * error context surfaces that one sample for the operator. A separate + * `COUNT(*)` would only add cost without adding information — the presence + * of a single row already means the invariant fails. + * @param trx - Knex instance or transaction connected to the post-6-D archive. + * @throws {Phase6VerificationError} when any content-type is silently dropped. + */ +export async function checkContentTypePreservation(trx: Knex): Promise { + const sample = await trx('content_items as ci') + .join('pages as p', 'ci.id', 'p.id') + .whereNull('ci.content_type_id') + .whereNotNull('p.contentType') + .andWhere('p.contentType', '<>', '') + .select< + { id: number; contentType: string }[] + >('ci.id as id', 'p.contentType as contentType') + .limit(1); + if (sample.length > 0) { + const offending = sample[0]!; + throw new Phase6VerificationError({ + check: '#7 content_type preservation', + context: { + sample_page_id: offending.id, + sample_content_type: offending.contentType, + }, + }); + } +} diff --git a/packages/@nitpicker/crawler/src/archive/phase6e/check-image-items-count.spec.ts b/packages/@nitpicker/crawler/src/archive/phase6e/check-image-items-count.spec.ts new file mode 100644 index 0000000..022ad41 --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6e/check-image-items-count.spec.ts @@ -0,0 +1,67 @@ +import type knex from 'knex'; + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { seedContentItems } from '../phase6d/test-utils/seed-content-items.js'; +import { setupPhase6DDb } from '../phase6d/test-utils/setup-phase6d-db.js'; + +import { checkImageItemsCount } from './check-image-items-count.js'; +import { Phase6VerificationError } from './types.js'; + +describe('checkImageItemsCount', () => { + let db: ReturnType; + + beforeEach(async () => { + db = await setupPhase6DDb(); + await db('text_refs').insert({ id: 1, hash: Buffer.from([1]), text: 'unknown/1' }); + }); + afterEach(async () => { + await db.destroy(); + }); + + it('passes when image_items row count equals images row count', async () => { + await seedContentItems(db, [1]); + await db('images').insert([ + { id: 10, pageId: 1, src: 'https://example.com/a.png', sourceCode: '' }, + { id: 11, pageId: 1, src: 'https://example.com/b.png', sourceCode: '' }, + ]); + await db('image_items').insert([ + { id: 10, page_id: 1, dom_path_text_id: 1 }, + { id: 11, page_id: 1, dom_path_text_id: 1 }, + ]); + await expect(checkImageItemsCount(db)).resolves.toBeUndefined(); + }); + + it('passes when both tables are empty', async () => { + await expect(checkImageItemsCount(db)).resolves.toBeUndefined(); + }); + + it('throws when image_items has more rows than images', async () => { + await seedContentItems(db, [1]); + await db('images').insert({ + id: 10, + pageId: 1, + src: 'https://example.com/a.png', + sourceCode: '', + }); + await db('image_items').insert([ + { id: 10, page_id: 1, dom_path_text_id: 1 }, + { id: 11, page_id: 1, dom_path_text_id: 1 }, + ]); + await expect(checkImageItemsCount(db)).rejects.toBeInstanceOf( + Phase6VerificationError, + ); + }); + + it('throws when image_items is missing rows', async () => { + await seedContentItems(db, [1]); + await db('images').insert([ + { id: 10, pageId: 1, src: 'https://example.com/a.png', sourceCode: '' }, + { id: 11, pageId: 1, src: 'https://example.com/b.png', sourceCode: '' }, + ]); + await db('image_items').insert({ id: 10, page_id: 1, dom_path_text_id: 1 }); + await expect(checkImageItemsCount(db)).rejects.toBeInstanceOf( + Phase6VerificationError, + ); + }); +}); diff --git a/packages/@nitpicker/crawler/src/archive/phase6e/check-image-items-count.ts b/packages/@nitpicker/crawler/src/archive/phase6e/check-image-items-count.ts new file mode 100644 index 0000000..38dbf92 --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6e/check-image-items-count.ts @@ -0,0 +1,29 @@ +import type { Knex } from 'knex'; + +import { Phase6VerificationError } from './types.js'; + +/** + * Verifies Phase 6-E invariant #5: every legacy `images` row is mirrored by + * one row in `image_items`. + * + * Phase 6-D-6 populates `image_items` with the same PK as `images.id` (see + * `populate-image-items.ts`); the invariant is broken only if the populate + * loop skipped rows during URL/blob routing or dom-path derivation. + * @param trx - Knex instance or transaction connected to the post-6-D archive. + * @throws {Phase6VerificationError} when the row counts diverge. + */ +export async function checkImageItemsCount(trx: Knex): Promise { + const imageItemsRows = await trx('image_items').count<{ n: number }[]>({ n: '*' }); + const imagesRows = await trx('images').count<{ n: number }[]>({ n: '*' }); + const imageItemsCount = Number(imageItemsRows[0]!.n); + const imagesCount = Number(imagesRows[0]!.n); + if (imageItemsCount !== imagesCount) { + throw new Phase6VerificationError({ + check: '#5 image_items row count', + context: { + image_items: imageItemsCount, + images: imagesCount, + }, + }); + } +} diff --git a/packages/@nitpicker/crawler/src/archive/phase6e/check-page-meta-count.spec.ts b/packages/@nitpicker/crawler/src/archive/phase6e/check-page-meta-count.spec.ts new file mode 100644 index 0000000..d37f085 --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6e/check-page-meta-count.spec.ts @@ -0,0 +1,60 @@ +import type knex from 'knex'; + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { setupPhase6DDb } from '../phase6d/test-utils/setup-phase6d-db.js'; + +import { checkPageMetaCount } from './check-page-meta-count.js'; +import { Phase6VerificationError } from './types.js'; + +describe('checkPageMetaCount', () => { + let db: ReturnType; + + beforeEach(async () => { + db = await setupPhase6DDb(); + // This check only exercises row counts; skipping FK enforcement lets + // the spec insert `page_meta` rows without also seeding matching + // `content_items` / `url_refs` rows (those FKs are covered by the + // Phase 6-D populate specs). + await db.raw('PRAGMA foreign_keys = OFF'); + }); + afterEach(async () => { + await db.destroy(); + }); + + it('passes when page_meta row count equals count of scraped pages', async () => { + await db('pages').insert([ + { id: 1, url: 'https://example.com/a', scraped: 1, isTarget: 1 }, + { id: 2, url: 'https://example.com/b', scraped: 0, isTarget: 0 }, + { id: 3, url: 'https://example.com/c', scraped: 1, isTarget: 1 }, + ]); + await db('page_meta').insert([{ page_id: 1 }, { page_id: 3 }]); + await expect(checkPageMetaCount(db)).resolves.toBeUndefined(); + }); + + it('ignores un-scraped pages in the count', async () => { + await db('pages').insert([ + { id: 1, url: 'https://example.com/a', scraped: 0, isTarget: 0 }, + { id: 2, url: 'https://example.com/b', scraped: 0, isTarget: 0 }, + ]); + await expect(checkPageMetaCount(db)).resolves.toBeUndefined(); + }); + + it('throws when page_meta has fewer rows than scraped pages', async () => { + await db('pages').insert([ + { id: 1, url: 'https://example.com/a', scraped: 1, isTarget: 1 }, + { id: 2, url: 'https://example.com/b', scraped: 1, isTarget: 1 }, + ]); + await db('page_meta').insert({ page_id: 1 }); + await expect(checkPageMetaCount(db)).rejects.toBeInstanceOf(Phase6VerificationError); + }); + + it('throws when page_meta has extra rows past scraped pages', async () => { + await db('pages').insert([ + { id: 1, url: 'https://example.com/a', scraped: 1, isTarget: 1 }, + { id: 2, url: 'https://example.com/b', scraped: 0, isTarget: 0 }, + ]); + await db('page_meta').insert([{ page_id: 1 }, { page_id: 2 }]); + await expect(checkPageMetaCount(db)).rejects.toBeInstanceOf(Phase6VerificationError); + }); +}); diff --git a/packages/@nitpicker/crawler/src/archive/phase6e/check-page-meta-count.ts b/packages/@nitpicker/crawler/src/archive/phase6e/check-page-meta-count.ts new file mode 100644 index 0000000..5a78627 --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6e/check-page-meta-count.ts @@ -0,0 +1,34 @@ +import type { Knex } from 'knex'; + +import { Phase6VerificationError } from './types.js'; + +/** + * Verifies Phase 6-E invariant #2: every scraped page has one `page_meta` + * row. + * + * `page_meta` mirrors the meta columns of `pages` for `scraped = 1` rows + * only (see `populate-page-meta.ts`); un-scraped pages have no meta to + * carry. A mismatch usually indicates that the populate loop encountered a + * scraped page whose meta resolution failed silently (e.g. a text ref that + * did not deduplicate), in which case the archive would be missing meta for + * an otherwise valid page. + * @param trx - Knex instance or transaction connected to the post-6-D archive. + * @throws {Phase6VerificationError} when the row counts diverge. + */ +export async function checkPageMetaCount(trx: Knex): Promise { + const pageMetaRows = await trx('page_meta').count<{ n: number }[]>({ n: '*' }); + const scrapedPagesRows = await trx('pages') + .where('scraped', true) + .count<{ n: number }[]>({ n: '*' }); + const pageMetaCount = Number(pageMetaRows[0]!.n); + const scrapedPagesCount = Number(scrapedPagesRows[0]!.n); + if (pageMetaCount !== scrapedPagesCount) { + throw new Phase6VerificationError({ + check: '#2 page_meta row count', + context: { + page_meta: pageMetaCount, + scraped_pages: scrapedPagesCount, + }, + }); + } +} diff --git a/packages/@nitpicker/crawler/src/archive/phase6e/check-resource-items-count.spec.ts b/packages/@nitpicker/crawler/src/archive/phase6e/check-resource-items-count.spec.ts new file mode 100644 index 0000000..0cda32b --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6e/check-resource-items-count.spec.ts @@ -0,0 +1,71 @@ +import type knex from 'knex'; + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { setupPhase6DDb } from '../phase6d/test-utils/setup-phase6d-db.js'; + +import { checkResourceItemsCount } from './check-resource-items-count.js'; +import { Phase6VerificationError } from './types.js'; + +describe('checkResourceItemsCount', () => { + let db: ReturnType; + + beforeEach(async () => { + db = await setupPhase6DDb(); + }); + afterEach(async () => { + await db.destroy(); + }); + + it('passes when resource_items row count equals resources row count', async () => { + await db('resources').insert([ + { id: 1, url: 'https://cdn.example.com/a.js' }, + { id: 2, url: 'https://cdn.example.com/b.js' }, + ]); + await db('url_refs').insert([ + { id: 100, url: 'https://cdn.example.com/a.js' }, + { id: 101, url: 'https://cdn.example.com/b.js' }, + ]); + await db('resource_items').insert([ + { id: 1, url_id: 100, is_external: 0, source: 'crawled' }, + { id: 2, url_id: 101, is_external: 0, source: 'crawled' }, + ]); + await expect(checkResourceItemsCount(db)).resolves.toBeUndefined(); + }); + + it('passes when both tables are empty', async () => { + await expect(checkResourceItemsCount(db)).resolves.toBeUndefined(); + }); + + it('throws when resource_items has more rows than resources', async () => { + await db('resources').insert({ id: 1, url: 'https://cdn.example.com/a.js' }); + await db('url_refs').insert([ + { id: 100, url: 'https://cdn.example.com/a.js' }, + { id: 101, url: 'https://cdn.example.com/phantom.js' }, + ]); + await db('resource_items').insert([ + { id: 1, url_id: 100, is_external: 0, source: 'crawled' }, + { id: 2, url_id: 101, is_external: 0, source: 'crawled' }, + ]); + await expect(checkResourceItemsCount(db)).rejects.toBeInstanceOf( + Phase6VerificationError, + ); + }); + + it('throws when resource_items is missing rows', async () => { + await db('resources').insert([ + { id: 1, url: 'https://cdn.example.com/a.js' }, + { id: 2, url: 'https://cdn.example.com/b.js' }, + ]); + await db('url_refs').insert({ id: 100, url: 'https://cdn.example.com/a.js' }); + await db('resource_items').insert({ + id: 1, + url_id: 100, + is_external: 0, + source: 'crawled', + }); + await expect(checkResourceItemsCount(db)).rejects.toBeInstanceOf( + Phase6VerificationError, + ); + }); +}); diff --git a/packages/@nitpicker/crawler/src/archive/phase6e/check-resource-items-count.ts b/packages/@nitpicker/crawler/src/archive/phase6e/check-resource-items-count.ts new file mode 100644 index 0000000..6656a61 --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6e/check-resource-items-count.ts @@ -0,0 +1,35 @@ +import type { Knex } from 'knex'; + +import { Phase6VerificationError } from './types.js'; + +/** + * Verifies Phase 6-E invariant #6: every legacy `resources` row is mirrored + * by one row in `resource_items`. + * + * Phase 6-D-3 populates `resource_items` with the same PK as `resources.id` + * (see `populate-resource-items.ts`); the invariant is broken only if the + * populate loop skipped rows during URL / header-set / content-type + * resolution. The paired invariant `resource_ref_edges = "resources-referrers"` + * is left unchecked here because issue #194's spec list stops at the + * six enumerated checks, and `resource_ref_edges` populate is a straight + * `INSERT … SELECT` (no dedup, no resolution) with negligible failure surface. + * @param trx - Knex instance or transaction connected to the post-6-D archive. + * @throws {Phase6VerificationError} when the row counts diverge. + */ +export async function checkResourceItemsCount(trx: Knex): Promise { + const resourceItemsRows = await trx('resource_items').count<{ n: number }[]>({ + n: '*', + }); + const resourcesRows = await trx('resources').count<{ n: number }[]>({ n: '*' }); + const resourceItemsCount = Number(resourceItemsRows[0]!.n); + const resourcesCount = Number(resourcesRows[0]!.n); + if (resourceItemsCount !== resourcesCount) { + throw new Phase6VerificationError({ + check: '#6 resource_items row count', + context: { + resource_items: resourceItemsCount, + resources: resourcesCount, + }, + }); + } +} diff --git a/packages/@nitpicker/crawler/src/archive/phase6e/check-url-round-trip.spec.ts b/packages/@nitpicker/crawler/src/archive/phase6e/check-url-round-trip.spec.ts new file mode 100644 index 0000000..b9125cd --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6e/check-url-round-trip.spec.ts @@ -0,0 +1,155 @@ +import type knex from 'knex'; + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { setupPhase6DDb } from '../phase6d/test-utils/setup-phase6d-db.js'; + +import { checkUrlRoundTrip } from './check-url-round-trip.js'; +import { Phase6VerificationError } from './types.js'; + +describe('checkUrlRoundTrip', () => { + let db: ReturnType; + + beforeEach(async () => { + db = await setupPhase6DDb(); + }); + afterEach(async () => { + await db.destroy(); + }); + + it('passes when every sampled content_items row round-trips to its pages.url', async () => { + await db('pages').insert([ + { id: 1, url: 'https://example.com/a', scraped: 1, isTarget: 1 }, + { id: 2, url: 'https://example.com/b?x=1', scraped: 1, isTarget: 1 }, + ]); + await db('url_refs').insert([ + { id: 100, url: 'https://example.com/a' }, + { id: 101, url: 'https://example.com/b?x=1' }, + ]); + await db('content_items').insert([ + { id: 1, url_id: 100, is_external: 0, scraped: 1, is_target: 1, source: 'crawled' }, + { id: 2, url_id: 101, is_external: 0, scraped: 1, is_target: 1, source: 'crawled' }, + ]); + await expect(checkUrlRoundTrip(db)).resolves.toBeUndefined(); + }); + + it('early-returns when content_items is empty', async () => { + await expect(checkUrlRoundTrip(db)).resolves.toBeUndefined(); + }); + + it('throws when a content_items row points at the wrong url_refs entry', async () => { + await db('pages').insert([ + { id: 1, url: 'https://example.com/a', scraped: 1, isTarget: 1 }, + { id: 2, url: 'https://example.com/b', scraped: 1, isTarget: 1 }, + ]); + await db('url_refs').insert([ + { id: 100, url: 'https://example.com/a' }, + { id: 101, url: 'https://example.com/b' }, + ]); + // Cross the wires: content_items row 1 points to url 101 ("/b") but + // its pages row still holds "/a". + await db('content_items').insert([ + { id: 1, url_id: 101, is_external: 0, scraped: 1, is_target: 1, source: 'crawled' }, + { id: 2, url_id: 100, is_external: 0, scraped: 1, is_target: 1, source: 'crawled' }, + ]); + await expect(checkUrlRoundTrip(db)).rejects.toBeInstanceOf(Phase6VerificationError); + }); + + it('throws when content_items.url_id points at a nonexistent url_refs id (FK gap)', async () => { + // FKs are enforced by setupPhase6DDb, so use raw statements after + // switching them off — the FK gap is exactly the kind of orphan the + // check should surface even in production, where the migration might + // have skipped an insert. + await db.raw('PRAGMA foreign_keys = OFF'); + await db('pages').insert({ + id: 1, + url: 'https://example.com/a', + scraped: 1, + isTarget: 1, + }); + await db('content_items').insert({ + id: 1, + url_id: 999, // no matching url_refs row + is_external: 0, + scraped: 1, + is_target: 1, + source: 'crawled', + }); + await expect(checkUrlRoundTrip(db)).rejects.toBeInstanceOf(Phase6VerificationError); + }); + + it('throws when content_items has no matching pages row', async () => { + await db.raw('PRAGMA foreign_keys = OFF'); + await db('url_refs').insert({ id: 100, url: 'https://example.com/a' }); + await db('content_items').insert({ + id: 1, + url_id: 100, + is_external: 0, + scraped: 1, + is_target: 1, + source: 'crawled', + }); + await expect(checkUrlRoundTrip(db)).rejects.toBeInstanceOf(Phase6VerificationError); + }); + + it('sampling is deterministic — the same archive always produces the same verdict', async () => { + // Populate 2000 rows so stride sampling actually kicks in (stride = 2). + const pages = Array.from({ length: 2000 }, (_, index) => ({ + id: index + 1, + url: `https://example.com/p${index + 1}`, + scraped: 1, + isTarget: 1, + })); + const urlRefs = pages.map((p) => ({ id: p.id, url: p.url })); + const contentItems = pages.map((p) => ({ + id: p.id, + url_id: p.id, + is_external: 0, + scraped: 1, + is_target: 1, + source: 'crawled', + })); + await db.batchInsert('pages', pages, 500); + await db.batchInsert('url_refs', urlRefs, 500); + await db.batchInsert('content_items', contentItems, 500); + // Corrupt exactly one page.url so the check should always trip on + // that page. If the sampling missed even-numbered ids the run would + // pass — stride sampling picks id % 2 = 0, i.e. all even ids, so + // id=1000 is guaranteed to be in the sample. + await db('pages').where('id', 1000).update({ url: 'https://example.com/CORRUPTED' }); + await expect(checkUrlRoundTrip(db)).rejects.toBeInstanceOf(Phase6VerificationError); + await expect(checkUrlRoundTrip(db)).rejects.toBeInstanceOf(Phase6VerificationError); + }); + + it('emits the mismatching page id and both URLs in the error context', async () => { + await db('pages').insert({ + id: 1, + url: 'https://example.com/a', + scraped: 1, + isTarget: 1, + }); + await db('url_refs').insert([ + { id: 100, url: 'https://example.com/a' }, + { id: 101, url: 'https://example.com/wrong' }, + ]); + await db('content_items').insert({ + id: 1, + url_id: 101, + is_external: 0, + scraped: 1, + is_target: 1, + source: 'crawled', + }); + try { + await checkUrlRoundTrip(db); + expect.unreachable('expected Phase6VerificationError'); + } catch (error) { + expect(error).toBeInstanceOf(Phase6VerificationError); + const details = (error as Phase6VerificationError).details; + expect(details.check).toContain('#8'); + expect(details.context?.page_id).toBe(1); + expect(details.context?.source_url).toBe('https://example.com/a'); + expect(details.context?.round_trip_url).toBe('https://example.com/wrong'); + } + }); +}); diff --git a/packages/@nitpicker/crawler/src/archive/phase6e/check-url-round-trip.ts b/packages/@nitpicker/crawler/src/archive/phase6e/check-url-round-trip.ts new file mode 100644 index 0000000..24166f9 --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6e/check-url-round-trip.ts @@ -0,0 +1,108 @@ +import type { Knex } from 'knex'; + +import { Phase6VerificationError } from './types.js'; + +/** + * Sample size for the URL round-trip smoke test. Documented in issue #194 + * and `docs/write-model-refactor-plan.md` §Phase 6-E as a 1,000-row sample. + * Not statistical coverage — URL normalisation bugs tend to be systematic + * (all rows with a particular scheme or host), so a modest evenly-spread + * sample catches them reliably. + */ +const SAMPLE_SIZE = 1000; + +/** + * Verifies Phase 6-E invariant #8: `content_items.url_id → url_refs.url` + * round-trips back to the original `pages.url` on a deterministic + * ≤ {@link SAMPLE_SIZE} sample. + * + * A mismatch means one of two things: + * + * - `populate-url-refs.ts` collapsed two distinct URLs into the same + * `url_refs.id` (BINARY collation bug, whitespace normalisation, …). + * - `populate-content-items.ts` associated a page id with the wrong + * `url_ref.id` (row-order dependency in the batch resolver). + * + * Additionally an FK gap (`content_items.url_id` pointing at a + * non-existent `url_refs.id`) or a missing `pages` row surfaces as a + * `roundTripUrl` / `sourceUrl` of `null`; the check treats either as a + * round-trip failure. LEFT JOINs are used deliberately so orphan rows + * are observable — an INNER JOIN would silently drop them and let the + * bug the invariant is meant to catch slip through. + * + * Sampling is **deterministic**: stride = ⌈count(content_items) / N⌉ + * gives approximately {@link SAMPLE_SIZE} rows spread across the id + * range for large archives, and the full table for archives with + * ≤ {@link SAMPLE_SIZE} rows. Deterministic sampling means the same + * archive always produces the same verdict; a random sample would let a + * failing archive pass on retry if the offending rows happen to fall + * outside the second draw. + * + * The check throws when the sampled row count is smaller than expected + * (LEFT JOINs may return `null` sides but never fewer rows than the + * driving `content_items` set — a shortfall means SQLite silently + * dropped rows we intended to inspect). + * @param trx - Knex instance or transaction connected to the post-6-D archive. + * @throws {Phase6VerificationError} when at least one sampled URL does not round-trip. + */ +export async function checkUrlRoundTrip(trx: Knex): Promise { + const totalRows = await trx('content_items').count<{ n: number }[]>({ n: '*' }); + const total = Number(totalRows[0]!.n); + if (total === 0) { + return; + } + // stride = ⌈total / SAMPLE_SIZE⌉ so `id % stride = 0` picks approximately + // SAMPLE_SIZE rows evenly across the id range for large archives, and the + // full table for archives ≤ SAMPLE_SIZE. + const stride = Math.max(1, Math.ceil(total / SAMPLE_SIZE)); + const sample = await trx('content_items as ci') + .leftJoin('pages as p', 'p.id', 'ci.id') + .leftJoin('url_refs as ur', 'ur.id', 'ci.url_id') + .select< + { id: number; sourceUrl: string | null; roundTripUrl: string | null }[] + >('ci.id as id', 'p.url as sourceUrl', 'ur.url as roundTripUrl') + .whereRaw('(ci.id % ?) = 0', [stride]) + .orderBy('ci.id') + .limit(SAMPLE_SIZE); + const expectedSize = Math.min(SAMPLE_SIZE, Math.max(1, Math.floor(total / stride))); + if (sample.length < expectedSize) { + // LEFT JOINs preserve driving-table row counts (`p.id` / `url_refs.id` + // are PKs → 1:1 or 0:1 match). A shortfall means SQLite silently + // dropped rows the WHERE clause matched — a real integrity signal. + throw new Phase6VerificationError({ + check: '#8 URL round-trip', + context: { + content_items_total: total, + stride, + expected_sample_size: expectedSize, + actual_sample_size: sample.length, + reason: 'sample query returned fewer rows than expected', + }, + }); + } + for (const row of sample) { + if (row.sourceUrl === null || row.roundTripUrl === null) { + throw new Phase6VerificationError({ + check: '#8 URL round-trip', + context: { + page_id: row.id, + source_url: row.sourceUrl, + round_trip_url: row.roundTripUrl, + sample_size: sample.length, + reason: 'orphan row — pages or url_refs join returned null (FK gap)', + }, + }); + } + if (row.sourceUrl !== row.roundTripUrl) { + throw new Phase6VerificationError({ + check: '#8 URL round-trip', + context: { + page_id: row.id, + source_url: row.sourceUrl, + round_trip_url: row.roundTripUrl, + sample_size: sample.length, + }, + }); + } + } +} diff --git a/packages/@nitpicker/crawler/src/archive/phase6e/types.spec.ts b/packages/@nitpicker/crawler/src/archive/phase6e/types.spec.ts new file mode 100644 index 0000000..76266f1 --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6e/types.spec.ts @@ -0,0 +1,73 @@ +import { describe, expect, it } from 'vitest'; + +import { Phase6VerificationError } from './types.js'; + +describe('Phase6VerificationError', () => { + it('renders the check identifier verbatim into the message', () => { + const error = new Phase6VerificationError({ check: '#1 content_items row count' }); + expect(error.message).toBe('Phase 6 verification failed #1 content_items row count'); + expect(error.name).toBe('Phase6VerificationError'); + }); + + it('renders numeric context values as decimal integers', () => { + const error = new Phase6VerificationError({ + check: '#4 anchor_edges count sum', + context: { + sum_of_anchor_edges_count: 42, + anchors: 43, + }, + }); + expect(error.message).toBe( + 'Phase 6 verification failed #4 anchor_edges count sum — sum_of_anchor_edges_count=42, anchors=43', + ); + }); + + it('quotes string context values so " and " cannot corrupt the message', () => { + const error = new Phase6VerificationError({ + check: '#7 content_type preservation', + context: { + sample_content_type: 'text/html; charset=utf-8', + }, + }); + expect(error.message).toContain('sample_content_type="text/html; charset=utf-8"'); + }); + + it('renders null as the literal `(null)` so it is distinguishable from the string "null"', () => { + const errorFromNull = new Phase6VerificationError({ + check: '#7 content_type preservation', + context: { + sample_page_id: null, + sample_content_type: null, + }, + }); + expect(errorFromNull.message).toContain('sample_page_id=(null)'); + expect(errorFromNull.message).toContain('sample_content_type=(null)'); + + const errorFromLiteralString = new Phase6VerificationError({ + check: '#7 content_type preservation', + context: { + sample_content_type: 'null', + }, + }); + expect(errorFromLiteralString.message).toContain('sample_content_type="null"'); + }); + + it('omits the context section entirely when no context is supplied', () => { + const error = new Phase6VerificationError({ check: 'runtime' }); + expect(error.message).toBe('Phase 6 verification failed runtime'); + }); + + it('exposes the raw details on the error instance', () => { + const details = { + check: '#8 URL round-trip', + context: { + page_id: 100, + source_url: 'https://example.com/a', + round_trip_url: 'https://example.com/b', + sample_size: 500, + }, + } as const; + const error = new Phase6VerificationError(details); + expect(error.details).toBe(details); + }); +}); diff --git a/packages/@nitpicker/crawler/src/archive/phase6e/types.ts b/packages/@nitpicker/crawler/src/archive/phase6e/types.ts new file mode 100644 index 0000000..5135ada --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6e/types.ts @@ -0,0 +1,107 @@ +/** + * Shared type definitions for Phase 6-E migration verification. + * + * Phase 6-E adds the acceptance-invariant checks that `scripts/migrate-to-phase6.mjs` + * runs at the end of the migration to confirm that the Phase 6-D entity / + * edge populate step produced a consistent archive. Each check is one file + * under `phase6e/` and throws {@link Phase6VerificationError} on mismatch; + * the orchestrator {@link ./verify-phase6-migration.ts} chains them in the + * order defined by issue #194. + * @module + */ + +/** + * Row-count snapshot collected by every successful verify run and returned + * by {@link import('./verify-phase6-migration.js').verifyPhase6Migration}. + * The migration script logs these on the successful path so operators can + * eyeball migration outcomes from stdout without re-opening the archive. + */ +export interface Phase6VerificationSummary { + /** `count(*) FROM content_items` — matches `count(pages)`. */ + readonly contentItems: number; + /** `count(*) FROM page_meta` — matches `count(pages WHERE scraped=1)`. */ + readonly pageMeta: number; + /** `count(*) FROM anchor_edges` — always ≤ `count(anchors)`. */ + readonly anchorEdges: number; + /** `SUM(count) FROM anchor_edges` — matches `count(anchors)`. */ + readonly anchorEdgesSum: number; + /** `count(*) FROM image_items` — matches `count(images)`. */ + readonly imageItems: number; + /** `count(*) FROM resource_items` — matches `count(resources)`. */ + readonly resourceItems: number; +} + +/** + * Structured details of a single Phase 6 invariant that failed. Attached to + * {@link Phase6VerificationError} so operators can see which check tripped + * without parsing the error message. + */ +export interface Phase6VerificationErrorDetails { + /** + * Human-readable check identifier matching issue #194's numbered list + * (e.g. `'#3'` for the anchor-edge dedup range check). Included in the + * error message so migration logs point at the exact clause. + */ + readonly check: string; + /** + * Optional structured payload — typically `expected` / `actual` counts or + * the offending page id from the URL round-trip sample. Rendered into the + * error message so operators can diagnose without re-running the check. + */ + readonly context?: Readonly>; +} + +/** + * Error thrown by any Phase 6-E check when an invariant does not hold. The + * migration script catches this at the top level and aborts with a non-zero + * exit code; the enclosing transaction rolls back so the archive returns to + * its pre-Phase-6-D state (ref tables from 6-B stay populated but are + * additive). + * + * Prefer a single error class with a structured {@link details} payload over + * one subclass per check: eight subclasses would create eight nearly + * identical constructors, and callers (script + integration test) only need + * to distinguish "verification failed" from other error kinds. + */ +export class Phase6VerificationError extends Error { + /** + * @param details - Structured description of which check failed and why. + * Rendered into the error message on construction. + */ + constructor(readonly details: Phase6VerificationErrorDetails) { + const contextText = + details.context === undefined + ? '' + : ' — ' + + Object.entries(details.context) + .map( + ([key, value]) => + // Render null as the parenthesised literal `(null)` so + // operators reading the log can distinguish "the sample + // query returned no rows" from a caller that passed the + // four-character string `"null"`. Strings get quoted for + // the same reason. + `${key}=${formatContextValue(value)}`, + ) + .join(', '); + super(`Phase 6 verification failed ${details.check}${contextText}`); + this.name = 'Phase6VerificationError'; + } +} + +/** + * Renders one context value for {@link Phase6VerificationError}'s message. + * `null` becomes the literal `(null)`; strings are wrapped in double quotes; + * numbers pass through as their decimal representation. The intent is to + * make missing-vs-present distinguishable in operator logs. + * @param value - Value from `Phase6VerificationErrorDetails.context`. + */ +function formatContextValue(value: string | number | null): string { + if (value === null) { + return '(null)'; + } + if (typeof value === 'string') { + return JSON.stringify(value); + } + return String(value); +} diff --git a/packages/@nitpicker/crawler/src/archive/phase6e/verify-phase6-migration.spec.ts b/packages/@nitpicker/crawler/src/archive/phase6e/verify-phase6-migration.spec.ts new file mode 100644 index 0000000..fb43bff --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6e/verify-phase6-migration.spec.ts @@ -0,0 +1,159 @@ +import type knex from 'knex'; + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { populatePhase6BRefs } from '../phase6b/populate-phase6b-refs.js'; +import { setupPhase6DDb } from '../phase6d/test-utils/setup-phase6d-db.js'; + +import { Phase6VerificationError } from './types.js'; +import { verifyPhase6Migration } from './verify-phase6-migration.js'; + +/** + * Builds a small end-to-end fixture that exercises every populate path + * (content_items, page_meta, anchor_edges, image_items, resource_items, + * resource_ref_edges) enough to make the orchestrator's happy-path + * assertion meaningful. Ref tables are populated via the real Phase 6-B + * populator so `url_refs.url` matches `pages.url` verbatim (required by + * check #8's round-trip). + * @param db - Knex handle from {@link setupPhase6DDb}. + */ +async function seedValidArchive(db: ReturnType): Promise { + await db('pages').insert([ + { + id: 1, + url: 'https://example.com/a', + scraped: 1, + isTarget: 1, + isExternal: 0, + contentType: 'text/html; charset=utf-8', + }, + { + id: 2, + url: 'https://example.com/b', + scraped: 1, + isTarget: 1, + isExternal: 0, + contentType: 'text/html; charset=utf-8', + }, + ]); + await db('resources').insert({ + id: 10, + url: 'https://cdn.example.com/x.js', + isExternal: 1, + contentType: 'application/javascript', + }); + await db('resources-referrers').insert({ resourceId: 10, pageId: 1 }); + await db('anchors').insert([ + { pageId: 1, hrefId: 2, hash: 'a', textContent: 'link1' }, + { pageId: 1, hrefId: 2, hash: 'b', textContent: 'link2' }, + ]); + await populatePhase6BRefs(db); + const urlA = await db('url_refs').where('url', 'https://example.com/a').first(); + const urlB = await db('url_refs').where('url', 'https://example.com/b').first(); + const urlCdn = await db('url_refs') + .where('url', 'https://cdn.example.com/x.js') + .first(); + const ctHtml = await db('content_type_refs') + .where('raw', 'text/html; charset=utf-8') + .first(); + const ctJs = await db('content_type_refs') + .where('raw', 'application/javascript') + .first(); + await db('content_items').insert([ + { + id: 1, + url_id: urlA!.id, + is_external: 0, + scraped: 1, + is_target: 1, + source: 'crawled', + content_type_id: ctHtml!.id, + }, + { + id: 2, + url_id: urlB!.id, + is_external: 0, + scraped: 1, + is_target: 1, + source: 'crawled', + content_type_id: ctHtml!.id, + }, + ]); + await db('page_meta').insert([{ page_id: 1 }, { page_id: 2 }]); + await db('anchor_edges').insert({ page_id: 1, href_page_id: 2, count: 2 }); + await db('resource_items').insert({ + id: 10, + url_id: urlCdn!.id, + is_external: 1, + source: 'crawled', + content_type_id: ctJs!.id, + }); + await db('resource_ref_edges').insert({ resource_id: 10, page_id: 1, count: 1 }); +} + +describe('verifyPhase6Migration', () => { + let db: ReturnType; + + beforeEach(async () => { + db = await setupPhase6DDb(); + }); + afterEach(async () => { + await db.destroy(); + }); + + it('passes on a valid post-6-D archive and returns the row-count summary', async () => { + await seedValidArchive(db); + const summary = await verifyPhase6Migration(db); + expect(summary.contentItems).toBe(2); + expect(summary.pageMeta).toBe(2); + expect(summary.anchorEdges).toBe(1); + expect(summary.anchorEdgesSum).toBe(2); + expect(summary.resourceItems).toBe(1); + }); + + it('wraps non-Phase6VerificationError exceptions with the standard prefix', async () => { + // Drop a table the checks touch so the first check hits a raw + // SqliteError. The wrapped error must still carry the + // "Phase 6 verification failed" prefix operators grep for. + await db.raw('DROP TABLE content_items'); + try { + await verifyPhase6Migration(db); + expect.unreachable('expected Phase6VerificationError'); + } catch (error) { + expect(error).toBeInstanceOf(Phase6VerificationError); + expect((error as Error).message).toContain('Phase 6 verification failed'); + expect((error as Phase6VerificationError).details.check).toBe('runtime'); + } + }); + + it('throws Phase6VerificationError from the first check that fails', async () => { + await seedValidArchive(db); + // Break invariant #1 by removing one content_items row. Turn FKs off + // because dropping content_items id=2 would cascade into + // anchor_edges via the (page_id, href_page_id) FK and defeat the + // mismatch we want to inspect. + await db.raw('PRAGMA foreign_keys = OFF'); + await db('content_items').where('id', 2).delete(); + try { + await verifyPhase6Migration(db); + expect.unreachable('expected Phase6VerificationError'); + } catch (error) { + expect(error).toBeInstanceOf(Phase6VerificationError); + expect((error as Phase6VerificationError).details.check).toContain('#1'); + } + }); + + it('reports check #7 when only content-type preservation is broken', async () => { + await seedValidArchive(db); + // Break invariant #7 by wiping one content_type_id even though the + // legacy pages row has a real contentType. + await db('content_items').where('id', 1).update({ content_type_id: null }); + try { + await verifyPhase6Migration(db); + expect.unreachable('expected Phase6VerificationError'); + } catch (error) { + expect(error).toBeInstanceOf(Phase6VerificationError); + expect((error as Phase6VerificationError).details.check).toContain('#7'); + } + }); +}); diff --git a/packages/@nitpicker/crawler/src/archive/phase6e/verify-phase6-migration.ts b/packages/@nitpicker/crawler/src/archive/phase6e/verify-phase6-migration.ts new file mode 100644 index 0000000..308b0bd --- /dev/null +++ b/packages/@nitpicker/crawler/src/archive/phase6e/verify-phase6-migration.ts @@ -0,0 +1,124 @@ +import type { Phase6VerificationSummary } from './types.js'; +import type { Knex } from 'knex'; + +import { checkAnchorEdgesCount } from './check-anchor-edges-count.js'; +import { checkAnchorEdgesSum } from './check-anchor-edges-sum.js'; +import { checkContentItemsCount } from './check-content-items-count.js'; +import { checkContentTypePreservation } from './check-content-type-preservation.js'; +import { checkImageItemsCount } from './check-image-items-count.js'; +import { checkPageMetaCount } from './check-page-meta-count.js'; +import { checkResourceItemsCount } from './check-resource-items-count.js'; +import { checkUrlRoundTrip } from './check-url-round-trip.js'; +import { Phase6VerificationError } from './types.js'; + +/** + * Runs every Phase 6-E acceptance invariant against a post-6-D archive and + * returns the verified row-count summary on success. The migration script + * (`scripts/migrate-to-phase6.mjs`) calls this **inside** the same + * `knex.transaction()` block that ran `populatePhase6DEntities` so a thrown + * {@link Phase6VerificationError} rolls back the entire 6-D populate — ref + * tables from 6-B stay committed because they live in a separate, additive + * transaction and never lose facts on re-run. + * + * The returned {@link Phase6VerificationSummary} is echoed to stdout by the + * migration script so operators can eyeball migration outcomes (`did we lose + * rows silently within the invariant window?`) from batch pipeline logs + * without re-opening the archive. Called must run this inside a `db.transaction()` + * for the counts and the JOIN-based checks (#7 / #8) to see a consistent + * snapshot of the archive. + * + * Checks run sequentially in issue #194's numbered order so error messages + * pinpoint the earliest broken invariant. Sequential (rather than parallel) + * execution keeps the failure diagnostics focused: fanning eight COUNTs out + * at once would return the first-completed rejection rather than the + * lowest-numbered one, which surprises operators reading migration logs. + * The individual COUNT queries are fast enough at any real archive scale + * for sequential wall-clock cost to be irrelevant. + * + * Non-{@link Phase6VerificationError} exceptions from a check (e.g. a + * transient libsql error or a schema-drift SqliteError) are wrapped in a + * {@link Phase6VerificationError} so the migration script's `catch` sees a + * uniform failure surface — otherwise operators grepping stderr for + * `Phase 6 verification failed` would miss driver-side errors. + * @param trx - Knex instance or transaction connected to the archive + * **after** every Phase 6-D populate step has run. Must be a transaction + * in production so ref counts, JOIN samples, and error rollback see the + * same snapshot; non-transactional callers (unit tests) work but do not + * get snapshot isolation. + * @returns Row-count summary of the verified archive. + * @throws {Phase6VerificationError} on the first invariant that does not + * hold; subsequent checks are skipped. + */ +export async function verifyPhase6Migration( + trx: Knex, +): Promise { + try { + await checkContentItemsCount(trx); + await checkPageMetaCount(trx); + await checkAnchorEdgesCount(trx); + await checkAnchorEdgesSum(trx); + await checkImageItemsCount(trx); + await checkResourceItemsCount(trx); + await checkContentTypePreservation(trx); + await checkUrlRoundTrip(trx); + } catch (error) { + if (error instanceof Phase6VerificationError) { + throw error; + } + throw new Phase6VerificationError({ + check: 'runtime', + context: { + underlying_error: error instanceof Error ? error.message : String(error), + }, + }); + } + return collectSummary(trx); +} + +/** + * Runs the six row counts that populate {@link Phase6VerificationSummary}. + * Separate from the invariant checks so those stay side-effect-free asserts, + * and this runs once at the end when all invariants have already held. + * @param trx - Same Knex handle passed to {@link verifyPhase6Migration}. + */ +async function collectSummary(trx: Knex): Promise { + const [contentItems, pageMeta, anchorEdges, anchorEdgesSum, imageItems, resourceItems] = + await Promise.all([ + countOf(trx, 'content_items'), + countOf(trx, 'page_meta'), + countOf(trx, 'anchor_edges'), + sumOf(trx, 'anchor_edges', 'count'), + countOf(trx, 'image_items'), + countOf(trx, 'resource_items'), + ]); + return { + contentItems, + pageMeta, + anchorEdges, + anchorEdgesSum, + imageItems, + resourceItems, + }; +} + +/** + * `SELECT count(*) FROM ` returning a plain number. + * @param trx - Knex handle. + * @param table - Table to count. + */ +async function countOf(trx: Knex, table: string): Promise { + const rows = await trx(table).count<{ n: number }[]>({ n: '*' }); + return Number(rows[0]!.n); +} + +/** + * `SELECT SUM() FROM
` returning a plain number (`null` sum + * on an empty table collapses to zero). + * @param trx - Knex handle. + * @param table - Table to sum over. + * @param column - Column to sum. + */ +async function sumOf(trx: Knex, table: string, column: string): Promise { + const rows = await trx(table).sum<{ n: number | null }[]>({ n: column }); + return Number(rows[0]!.n ?? 0); +} diff --git a/scripts/migrate-to-phase6.mjs b/scripts/migrate-to-phase6.mjs index bc5a9ff..084d4ba 100644 --- a/scripts/migrate-to-phase6.mjs +++ b/scripts/migrate-to-phase6.mjs @@ -27,12 +27,14 @@ * resource_ref_edges, image_items). All six run inside a single * knex transaction so a failure aborts the whole step; SQLite's WAL * rollback returns the DB to the pre-migration state. - * 4. **Acceptance verification** — asserts the four row-count invariants - * from issue #193's acceptance criteria: - * - `count(content_items) == count(pages)` - * - `count(page_meta) == count(pages WHERE scraped=1)` - * - `sum(anchor_edges.count) == count(anchors)` - * - `count(image_items) == count(images)` + * 4. **Acceptance verification (Phase 6-E)** — runs all eight invariant + * checks from issue #194 against the post-6-D archive. The check + * functions live in `packages/@nitpicker/crawler/src/archive/phase6e/` + * and the orchestrator `verifyPhase6Migration` chains them in the + * order documented there. Verification runs **inside** the same + * `knex.transaction()` block that ran the Phase 6-D populate step so + * a `Phase6VerificationError` rolls back every 6-D INSERT; ref tables + * populated in 6-B stay committed but are additive. * 5. **Re-tar** the work dir to the output path. * * DOM-PATH DERIVATION @@ -77,6 +79,7 @@ import { migratePhase6CEntityTables } from '../packages/@nitpicker/crawler/lib/a import { populatePhase6BRefs } from '../packages/@nitpicker/crawler/lib/archive/phase6b/populate-phase6b-refs.js'; import { matchImagesToDomPaths } from '../packages/@nitpicker/crawler/lib/archive/phase6d/match-images-to-dom-paths.js'; import { populatePhase6DEntities } from '../packages/@nitpicker/crawler/lib/archive/phase6d/populate-phase6d-entities.js'; +import { verifyPhase6Migration } from '../packages/@nitpicker/crawler/lib/archive/phase6e/verify-phase6-migration.js'; const SQLITE_DB_FILE_NAME = 'db.sqlite'; @@ -213,12 +216,19 @@ async function applyPhase6Migrations(dbPath) { console.log(' [6-D] populate entity tables'); const domPathResolver = createJsdomDomPathResolver(); const getPageHtml = createHtmlGetter(db); + /** @type {import('../packages/@nitpicker/crawler/lib/archive/phase6e/types.js').Phase6VerificationSummary} */ + let summary; await db.transaction(async (trx) => { await populatePhase6DEntities(trx, domPathResolver, getPageHtml); + console.log(' [6-E] verify 8 acceptance invariants'); + summary = await verifyPhase6Migration(trx); }); - - console.log(' [verify] acceptance counts'); - await verifyAcceptanceCounts(db); + console.log( + ` [6-E] verification passed — content_items=${summary.contentItems}, ` + + `page_meta=${summary.pageMeta}, anchor_edges=${summary.anchorEdges} ` + + `(sum count=${summary.anchorEdgesSum}), image_items=${summary.imageItems}, ` + + `resource_items=${summary.resourceItems}`, + ); await db.raw('PRAGMA wal_checkpoint(TRUNCATE)'); } finally { @@ -335,59 +345,6 @@ function decodeStoredBlob(body, codec) { throw new Error(`Unknown page_html_blobs.codec: ${codec}`); } -/** - * Asserts the four row-count invariants from issue #193's acceptance - * criteria. Throws on any mismatch so the caller can surface the failure - * to the operator. - * @param {import('knex').Knex} db - */ -async function verifyAcceptanceCounts(db) { - const contentItemsCount = await countRows(db, 'content_items'); - const pagesCount = await countRows(db, 'pages'); - if (contentItemsCount !== pagesCount) { - throw new Error( - `Acceptance failed: count(content_items)=${contentItemsCount} != count(pages)=${pagesCount}`, - ); - } - const pageMetaCount = await countRows(db, 'page_meta'); - const scrapedRows = await db('pages').where('scraped', true).count({ n: 'id' }); - const scrapedPagesCount = Number(scrapedRows[0].n); - if (pageMetaCount !== scrapedPagesCount) { - throw new Error( - `Acceptance failed: count(page_meta)=${pageMetaCount} != count(pages WHERE scraped=1)=${scrapedPagesCount}`, - ); - } - const anchorEdgesSumRows = await db('anchor_edges').sum({ n: 'count' }); - const anchorEdgesSum = Number(anchorEdgesSumRows[0].n ?? 0); - const anchorsCount = await countRows(db, 'anchors'); - if (anchorEdgesSum !== anchorsCount) { - throw new Error( - `Acceptance failed: SUM(anchor_edges.count)=${anchorEdgesSum} != count(anchors)=${anchorsCount}`, - ); - } - const imageItemsCount = await countRows(db, 'image_items'); - const imagesCount = await countRows(db, 'images'); - if (imageItemsCount !== imagesCount) { - throw new Error( - `Acceptance failed: count(image_items)=${imageItemsCount} != count(images)=${imagesCount}`, - ); - } - console.log( - ` [verify] OK — content_items=${contentItemsCount}, page_meta=${pageMetaCount}, anchor_edges(sum count)=${anchorEdgesSum}, image_items=${imageItemsCount}`, - ); -} - -/** - * Runs `SELECT count(*) FROM
` via knex and returns the count as - * a plain JS number. - * @param {import('knex').Knex} db - * @param {string} table - */ -async function countRows(db, table) { - const rows = await db(table).count({ n: '*' }); - return Number(rows[0].n); -} - try { await main(); } catch (error) {