d-zero-dev · YusukeHirao · Jul 14, 2026 · Jul 14, 2026
@@ -0,0 +1,236 @@
+import { execFileSync } from 'node:child_process';
+import { createHash } from 'node:crypto';
+import { existsSync, mkdirSync, readFileSync, rmSync } from 'node:fs';
+import path from 'node:path';
+
+import { tryParseUrl as parseUrl } from '@d-zero/shared/parse-url';
+import knex from 'knex';
+import * as tar from 'tar';
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+
+import Archive from './archive.js';
+import { peekTarTopDir } from './filesystem/peek-tar-top-dir.js';
+import { LibsqlDialect } from './libsql-dialect.js';
+
+const __filename = new URL(import.meta.url).pathname;
+const __dirname = path.dirname(__filename);
+const workingDir = path.resolve(__dirname, '__test_fixtures_migrate_phase6_script__');
+const repoRoot = path.resolve(__dirname, '..', '..', '..', '..', '..');
+const migrateScript = path.resolve(repoRoot, 'scripts', 'migrate-to-phase6.mjs');
+
+/**
+ * Builds a small but realistic 0.10 archive fixture via the production
+ * `Archive.create` + `setPage` path so downstream migration steps see a
+ * schema that matches what a real crawl would produce. Two pages, three
+ * anchors (two pointing to the same href for dedup coverage), and one
+ * shared external resource are inserted so the anchor-edges / resource
+ * populate paths have data to work on.
+ * @param filePath - Where to write the resulting `.nitpicker`.
+ */
+async function buildFixtureArchive(filePath: string): Promise<void> {
+	const archive = await Archive.create({ filePath, cwd: workingDir });
+	try {
+		await archive.setPage({
+			url: parseUrl('http://localhost/a')!,
+			redirectPaths: [],
+			isExternal: false,
+			status: 200,
+			statusText: 'OK',
+			contentLength: 100,
+			contentType: 'text/html',
+			responseHeaders: {},
+			meta: { title: 'Page A' },
+			anchorList: [
+				{
+					href: parseUrl('http://localhost/b')!,
+					textContent: 'link1',
+					isExternal: false,
+				},
+				{
+					href: parseUrl('http://localhost/b')!,
+					textContent: 'link2',
+					isExternal: false,
+				},
+			],
+			imageList: [],
+			html: '<html><body>a</body></html>',
+			isSkipped: false,
+			isTarget: true,
+		});
+		await archive.setPage({
+			url: parseUrl('http://localhost/b')!,
+			redirectPaths: [],
+			isExternal: false,
+			status: 200,
+			statusText: 'OK',
+			contentLength: 100,
+			contentType: 'text/html',
+			responseHeaders: {},
+			meta: { title: 'Page B' },
+			anchorList: [
+				{ href: parseUrl('http://localhost/a')!, textContent: 'back', isExternal: false },
+			],
+			imageList: [],
+			html: '<html><body>b</body></html>',
+			isSkipped: false,
+			isTarget: true,
+		});
+	} finally {
+		await archive.close();
+	}
+}
+
+/**
+ * Computes a SHA-256 hex digest of a file. Used to prove the migration
+ * script never mutates its input `.nitpicker` (the "restore .bak" clause
+ * of issue #194 is satisfied by out-of-place output; the input is the
+ * effective backup).
+ * @param filePath - Absolute path to the file.
+ */
+function sha256File(filePath: string): string {
+	const hash = createHash('sha256');
+	hash.update(readFileSync(filePath));
+	return hash.digest('hex');
+}
+
+/**
+ * Repacks a fixture archive after opening its inner `db.sqlite` for a
+ * caller-supplied mutation. Used to pre-populate Phase 6 tables with a
+ * phantom row so the migration's Phase 6-E verification fails.
+ *
+ * The mutation runs under `PRAGMA foreign_keys = OFF` so the caller can
+ * insert rows referencing ids that will be created (or not) later by
+ * `populate-url-refs.ts` — the phantom row is deliberately outside the
+ * ref-populate contract.
+ * @param filePath - Absolute path to the fixture `.nitpicker`.
+ * @param mutate - Callback that receives a Knex handle on the extracted DB.
+ */
+async function mutateFixture(
+	filePath: string,
+	mutate: (db: ReturnType<typeof knex>) => Promise<void>,
+): Promise<void> {
+	const innerDirName = await peekTarTopDir(filePath);
+	const stageDir = path.resolve(workingDir, `mutate-${process.pid}`);
+	mkdirSync(stageDir, { recursive: true });
+	try {
+		await tar.x({ file: filePath, cwd: stageDir });
+		const extracted = path.resolve(stageDir, innerDirName);
+		const dbPath = path.resolve(extracted, 'db.sqlite');
+		const db = knex({
+			client: LibsqlDialect,
+			connection: { filename: dbPath },
+			useNullAsDefault: true,
+		});
+		try {
+			await db.raw('PRAGMA foreign_keys = OFF');
+			await mutate(db);
+		} finally {
+			await db.destroy();
+		}
+		rmSync(filePath, { force: true });
+		await tar.c({ file: filePath, cwd: stageDir, portable: true }, [innerDirName]);
+	} finally {
+		rmSync(stageDir, { recursive: true, force: true });
+	}
+}
+
+describe('scripts/migrate-to-phase6.mjs (integration)', () => {
+	beforeEach(() => {
+		mkdirSync(workingDir, { recursive: true });
+	});
+	afterEach(() => {
+		rmSync(workingDir, { recursive: true, force: true });
+	});
+
+	it(
+		'happy path: migrates a valid 0.10 archive and passes all 8 checks',
+		{ timeout: 60_000 },
+		async () => {
+			const inputPath = path.resolve(workingDir, 'input.nitpicker');
+			const outputPath = path.resolve(workingDir, 'input.phase6.nitpicker');
+			await buildFixtureArchive(inputPath);
+
+			const stdout = execFileSync('node', [migrateScript, inputPath, outputPath], {
+				cwd: repoRoot,
+			});
+			expect(stdout.toString()).toContain('verification passed');
+
+			expect(existsSync(outputPath)).toBe(true);
+
+			const inspectDir = path.resolve(workingDir, 'inspect');
+			mkdirSync(inspectDir, { recursive: true });
+			await tar.x({ file: outputPath, cwd: inspectDir });
+			const innerDirName = await peekTarTopDir(outputPath);
+			const innerDbPath = path.resolve(inspectDir, innerDirName, 'db.sqlite');
+			const db = knex({
+				client: LibsqlDialect,
+				connection: { filename: innerDbPath },
+				useNullAsDefault: true,
+			});
+			try {
+				const contentItemsCount = await db('content_items').count<{ n: number }[]>({
+					n: '*',
+				});
+				const pagesCount = await db('pages').count<{ n: number }[]>({ n: '*' });
+				expect(Number(contentItemsCount[0]!.n)).toBe(Number(pagesCount[0]!.n));
+				const edgesCount = await db('anchor_edges').count<{ n: number }[]>({ n: '*' });
+				expect(Number(edgesCount[0]!.n)).toBeGreaterThan(0);
+			} finally {
+				await db.destroy();
+			}
+		},
+	);
+
+	it(
+		'failure path: injects a row-count mismatch, aborts with a Phase6VerificationError, leaves the input intact and no output',
+		{ timeout: 60_000 },
+		async () => {
+			const inputPath = path.resolve(workingDir, 'input.nitpicker');
+			const outputPath = path.resolve(workingDir, 'input.phase6.nitpicker');
+			await buildFixtureArchive(inputPath);
+			await mutateFixture(inputPath, async (db) => {
+				// Fresh archives from `Archive.create` already include the
+				// Phase 6-A/6-C tables (initSchema calls their creators).
+				// Insert a phantom `url_refs` + `content_items` pair so the
+				// migration's 6-D `INSERT OR IGNORE` leaves it untouched;
+				// `count(content_items) > count(pages)` after 6-D → check #1
+				// fires.
+				await db('url_refs').insert({
+					id: 999,
+					url: 'http://localhost/phantom',
+				});
+				await db('content_items').insert({
+					id: 999,
+					url_id: 999,
+					is_external: 0,
+					scraped: 1,
+					is_target: 1,
+					source: 'crawled',
+				});
+			});
+			const inputHashBefore = sha256File(inputPath);
+
+			let thrown: Error | null = null;
+			try {
+				execFileSync('node', [migrateScript, inputPath, outputPath], {
+					cwd: repoRoot,
+					stdio: 'pipe',
+				});
+			} catch (error) {
+				thrown = error as Error;
+			}
+			expect(thrown).not.toBeNull();
+			const stderr = (
+				(thrown as unknown as { stderr?: Buffer }).stderr ?? Buffer.from('')
+			).toString();
+			expect(stderr).toContain('Phase 6 verification failed');
+			expect(stderr).toContain('#1');
+
+			// Output tar was either never created, or created and then cleaned up.
+			expect(existsSync(outputPath)).toBe(false);
+
+			// Input tar bytes unchanged — the effective ".bak restore" clause.
+			expect(sha256File(inputPath)).toBe(inputHashBefore);
+		},
+	);
+});
@@ -0,0 +1,47 @@
+import type knex from 'knex';
+
+/**
+ * Inserts one row into each of `pages`, `url_refs`, and `content_items` per
+ * requested id so downstream inserts against tables that FK to
+ * `content_items(id)` (`anchor_edges`, `image_items`, `page_meta`, …) satisfy
+ * their foreign key without every spec re-authoring the same 20 lines of
+ * seed logic.
+ *
+ * Shared by Phase 6-E check specs (`check-anchor-edges-count.spec.ts`,
+ * `check-anchor-edges-sum.spec.ts`, `check-image-items-count.spec.ts`,
+ * `verify-phase6-migration.spec.ts`) so a schema change to
+ * pages/url_refs/content_items only requires editing one file. Kept in
+ * `phase6d/test-utils/` alongside {@link setup-phase6d-db.ts} because that
+ * module already provisions the tables this helper writes to.
+ *
+ * Rows are inserted with defaults suitable for count / structural
+ * invariants: `scraped=1`, `isTarget=1`, `isExternal=0`, `source='crawled'`,
+ * URL `https://example.com/<id>` and matching `url_refs` row on the same id
+ * so `url_id` binds 1:1 with `pages.id` (helpful for URL round-trip specs).
+ * @param db - Knex handle from {@link setupPhase6DDb}.
+ * @param ids - Content-item ids to create. Ordering does not matter but the
+ *   helper does not deduplicate the input list — callers should pass
+ *   distinct ids.
+ */
+export async function seedContentItems(
+	db: ReturnType<typeof knex>,
+	ids: readonly number[],
+): Promise<void> {
+	for (const id of ids) {
+		await db('pages').insert({
+			id,
+			url: `https://example.com/${id}`,
+			scraped: 1,
+			isTarget: 1,
+		});
+		await db('url_refs').insert({ id, url: `https://example.com/${id}` });
+		await db('content_items').insert({
+			id,
+			url_id: id,
+			is_external: 0,
+			scraped: 1,
+			is_target: 1,
+			source: 'crawled',
+		});
+	}
+}
@@ -0,0 +1,79 @@
+import type knex from 'knex';
+
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+
+import { seedContentItems } from '../phase6d/test-utils/seed-content-items.js';
+import { setupPhase6DDb } from '../phase6d/test-utils/setup-phase6d-db.js';
+
+import { checkAnchorEdgesCount } from './check-anchor-edges-count.js';
+import { Phase6VerificationError } from './types.js';
+
+describe('checkAnchorEdgesCount', () => {
+	let db: ReturnType<typeof knex>;
+
+	beforeEach(async () => {
+		db = await setupPhase6DDb();
+	});
+	afterEach(async () => {
+		await db.destroy();
+	});
+
+	it('passes when anchor_edges > 0 and < anchors', async () => {
+		await seedContentItems(db, [1, 2]);
+		await db('anchors').insert([
+			{ pageId: 1, hrefId: 2, hash: 'a', textContent: 'link' },
+			{ pageId: 1, hrefId: 2, hash: 'b', textContent: 'link' },
+			{ pageId: 1, hrefId: 2, hash: 'c', textContent: 'link' },
+		]);
+		await db('anchor_edges').insert({ page_id: 1, href_page_id: 2, count: 3 });
+		await expect(checkAnchorEdgesCount(db)).resolves.toBeUndefined();
+	});
+
+	it('passes when both anchors and anchor_edges are empty', async () => {
+		await expect(checkAnchorEdgesCount(db)).resolves.toBeUndefined();
+	});
+
+	it('throws when anchor_edges is empty but anchors is non-empty', async () => {
+		await seedContentItems(db, [1, 2]);
+		await db('anchors').insert({ pageId: 1, hrefId: 2, hash: 'a', textContent: 'link' });
+		await expect(checkAnchorEdgesCount(db)).rejects.toBeInstanceOf(
+			Phase6VerificationError,
+		);
+	});
+
+	it('passes when anchor_edges equals anchors count (all pairs unique — small crawls)', async () => {
+		await seedContentItems(db, [1, 2, 3]);
+		// Every (pageId, hrefId) pair is unique, so dedup preserves the row
+		// count. Check #4 (sum == count) still holds because each edge has
+		// count = 1.
+		await db('anchors').insert([
+			{ pageId: 1, hrefId: 2, hash: 'a', textContent: 'link1' },
+			{ pageId: 1, hrefId: 3, hash: 'b', textContent: 'link2' },
+		]);
+		await db('anchor_edges').insert([
+			{ page_id: 1, href_page_id: 2, count: 1 },
+			{ page_id: 1, href_page_id: 3, count: 1 },
+		]);
+		await expect(checkAnchorEdgesCount(db)).resolves.toBeUndefined();
+	});
+
+	it('throws when anchor_edges exceeds anchors (phantom rows)', async () => {
+		await seedContentItems(db, [1, 2, 3]);
+		await db('anchors').insert({ pageId: 1, hrefId: 2, hash: 'a', textContent: 'link' });
+		await db('anchor_edges').insert([
+			{ page_id: 1, href_page_id: 2, count: 1 },
+			{ page_id: 1, href_page_id: 3, count: 1 },
+		]);
+		await expect(checkAnchorEdgesCount(db)).rejects.toBeInstanceOf(
+			Phase6VerificationError,
+		);
+	});
+
+	it('throws when anchor_edges is non-empty but anchors is empty', async () => {
+		await seedContentItems(db, [1, 2]);
+		await db('anchor_edges').insert({ page_id: 1, href_page_id: 2, count: 1 });
+		await expect(checkAnchorEdgesCount(db)).rejects.toBeInstanceOf(
+			Phase6VerificationError,
+		);
+	});
+});