chore(perf): cperf regression harness + scale-validate rig + capacity docs (6/6)

.gitignore

@@ -13,3 +13,9 @@ postgres/pgsql-test/output/
 .env.local
 graphql/server/logs/
 graphql/server/*.heapsnapshot
+scripts/scale-validate/out/
+scripts/scale-validate/fleet*.json
+scripts/scale-validate/drifted.json
+scripts/scale-spike/
+packages/perf-harness/perf-out/
+/metrics.jsonl

packages/perf-harness/.gitignore

@@ -0,0 +1,3 @@
+dist/
+.smoke-out/
+perf-out/

packages/perf-harness/jest.config.js

@@ -0,0 +1,18 @@
+/** @type {import('ts-jest').JestConfigWithTsJest} */
+module.exports = {
+  preset: 'ts-jest',
+  testEnvironment: 'node',
+  transform: {
+    '^.+\\.tsx?$': [
+      'ts-jest',
+      {
+        babelConfig: false,
+        tsconfig: 'tsconfig.json',
+      },
+    ],
+  },
+  transformIgnorePatterns: [`/node_modules/*`],
+  testRegex: '(/__tests__/.*|(\\.|/)(test|spec))\\.(jsx?|tsx?)$',
+  moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node'],
+  modulePathIgnorePatterns: ['dist/*']
+};

packages/perf-harness/package.json

@@ -0,0 +1,58 @@
+{
+  "name": "@constructive-io/perf-harness",
+  "version": "0.1.0",
+  "author": "Constructive <developers@constructive.io>",
+  "description": "Reusable performance / regression harness for the Constructive GraphQL server",
+  "main": "index.js",
+  "module": "esm/index.js",
+  "types": "index.d.ts",
+  "homepage": "https://github.com/constructive-io/constructive",
+  "license": "MIT",
+  "publishConfig": {
+    "access": "public",
+    "directory": "dist"
+  },
+  "bin": {
+    "perf-harness": "index.js",
+    "cperf": "index.js"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/constructive-io/constructive"
+  },
+  "bugs": {
+    "url": "https://github.com/constructive-io/constructive/issues"
+  },
+  "scripts": {
+    "clean": "makage clean",
+    "prepack": "npm run build",
+    "build": "makage build",
+    "build:dev": "makage build --dev",
+    "dev": "ts-node ./src/index.ts",
+    "lint": "eslint . --fix",
+    "test": "jest --passWithNoTests",
+    "test:watch": "jest --watch"
+  },
+  "dependencies": {
+    "@inquirerer/utils": "^3.3.9",
+    "graphile-cache": "workspace:^",
+    "inquirerer": "^4.9.1",
+    "pg": "^8.21.0"
+  },
+  "devDependencies": {
+    "@inquirerer/test": "^1.4.3",
+    "@types/node": "^22.19.11",
+    "@types/pg": "^8.20.0",
+    "makage": "^0.3.0",
+    "ts-node": "^10.9.2"
+  },
+  "keywords": [
+    "performance",
+    "regression",
+    "harness",
+    "soak",
+    "constructive",
+    "graphile",
+    "postgres"
+  ]
+}

packages/perf-harness/src/commands/fleet.ts

@@ -0,0 +1,120 @@
+/**
+ * `fleet` command group — provision / discover / drift / canary / subset /
+ * teardown the perf tenant fleet.
+ *
+ * Each subcommand delegates to a `src/fleet/*` module, except `subset`, whose
+ * pure algorithm lives in `core/fleetfile.buildSubset`; the CLI glue (read
+ * --fleet + --drifted, write --out) is implemented inline here.
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+
+import { Argv, asBool, asInt, usageExit } from '../core/args';
+import { buildSubset, loadFleet, SubsetOpts } from '../core/fleetfile';
+import * as canary from '../fleet/canary';
+import * as discover from '../fleet/discover';
+import * as drift from '../fleet/drift';
+import * as provision from '../fleet/provision';
+import * as teardown from '../fleet/teardown';
+
+export const domain = 'fleet';
+export const summary = 'discover / provision / drift / canary / subset / teardown the perf tenant fleet';
+
+export interface Subcommand {
+  usage: string;
+  run(argv: Argv): Promise<number>;
+}
+
+const SUBSET_USAGE = `perf-harness fleet subset — derive a ramp-specific fleet manifest from a fleet + drift assignment
+
+Options:
+  --fleet <file>         fleet manifest from \`fleet discover\` (required)
+  --drifted <file>       drift assignment from \`fleet drift\` (optional)
+  --out <file>           write the subset manifest here (required)
+  --blueprints <K>       diversity subset: first K blueprint groups
+  --per-blueprint <N>    tenants per blueprint group (default: 5)
+  --tenants <N>          tenant-count subset: N group-0 tenants (single blueprint)
+  --same-bp-regex <re>   group-0 membership regex (default: ^(factory[0-9]+|marketplace_db_tenant1)$)
+  --help
+`;
+
+async function runSubset(argv: Argv): Promise<number> {
+  if (asBool(argv.help)) return usageExit(SUBSET_USAGE, 0);
+
+  const fleetFile = typeof argv.fleet === 'string' ? argv.fleet : null;
+  const out = typeof argv.out === 'string' ? argv.out : null;
+  if (!fleetFile) {
+    console.error('--fleet is required');
+    return 1;
+  }
+  if (!out) {
+    console.error('--out is required');
+    return 1;
+  }
+
+  try {
+    const fleet = loadFleet(fleetFile).tenants;
+    const drifted =
+      typeof argv.drifted === 'string'
+        ? JSON.parse(fs.readFileSync(argv.drifted, 'utf8'))
+        : { groups: {}, columnDrift: [] };
+    const o: SubsetOpts = {
+      blueprints: asInt(argv.blueprints, 0),
+      perBlueprint: asInt(argv['per-blueprint'], 5),
+      tenants: asInt(argv.tenants, 0),
+      sameBpRegex: typeof argv['same-bp-regex'] === 'string' ? argv['same-bp-regex'] : undefined
+    };
+
+    const { subset, tenants, warnings } = buildSubset(fleet, drifted, o);
+    for (const w of warnings) console.error(w);
+
+    // Preserve the original subset-fleet.mjs manifest shape (adds `derivedFrom`,
+    // which core/fleetfile.buildSubset omits).
+    const manifest = {
+      generatedAt: subset.generatedAt,
+      derivedFrom: fleetFile,
+      subset: subset.subset,
+      count: subset.count,
+      tenants: subset.tenants
+    };
+
+    fs.mkdirSync(path.dirname(path.resolve(out)), { recursive: true });
+    fs.writeFileSync(out, JSON.stringify(manifest) + '\n');
+    console.error(`[subset] ${subset.subset}: ${tenants.length} tenants -> ${out}`);
+    return 0;
+  } catch (err: any) {
+    console.error(String((err && err.message) || err));
+    return 1;
+  }
+}
+
+// Concise one-line synopses for the composed domain-level help listing; each
+// subcommand's own `--help` prints its fuller USAGE (via its `run`).
+export const subcommands: Record<string, Subcommand> = {
+  discover: {
+    usage: "fleet discover [--like '%'] [--out fleet.json] [--pretty] [--pg-* ...]",
+    run: discover.run
+  },
+  provision: {
+    usage:
+      'fleet provision [--count 1] [--prefix factory] [--blueprint marketplace] [--concurrency 1] [--dry-run] [--validate <db>] [--pg-* ...]',
+    run: provision.run
+  },
+  drift: {
+    usage: 'fleet drift [--groups K] [--per-group 4] [--column-drift N] [--out <f>] [--pg-* ...]',
+    run: drift.run
+  },
+  canary: {
+    usage: 'fleet canary --fleet <file> [--table categories] [--cleanup] [--pg-* ...]',
+    run: canary.run
+  },
+  subset: {
+    usage:
+      'fleet subset --fleet <file> --out <file> [--drifted <file>] [--blueprints K --per-blueprint N | --tenants N] [--same-bp-regex <re>]',
+    run: runSubset
+  },
+  teardown: {
+    usage: 'fleet teardown [--prefix factory] [--keep 40] [--only <db>] [--dry-run] [--pg-* ...]',
+    run: teardown.run
+  }
+};

packages/perf-harness/src/commands/load.ts

@@ -0,0 +1,31 @@
+/**
+ * `load` domain — workload drivers.
+ *
+ *   perf-harness load harness  paced GraphQL load + bleed sentinel (harness.mjs)
+ *   perf-harness load churn    schema-cache NOTIFY churn        (churn-driver.mjs)
+ *
+ * Each subcommand handles its own `--help` and returns a numeric exit code
+ * (harness: 0 ok / 1 fatal / 2 bleed-sentinel violation; churn: 0 / 1).
+ */
+import { Argv } from '../core/args';
+import { runChurn } from '../load/churn';
+import { runHarness } from '../load/harness';
+
+export const domain = 'load';
+export const summary = 'workload drivers — paced GraphQL load (bleed sentinel) and schema-cache churn';
+
+export interface Subcommand {
+  usage: string;
+  run(argv: Argv): Promise<number>;
+}
+
+export const subcommands: Record<string, Subcommand> = {
+  harness: {
+    usage: 'load harness --fleet <file> [--port 3333 --duration-sec 60 --rps 20 --mix read:.7,write:.25,meta:.05 --shape zipf|uniform|burst --auth 0|1 --out results.json]',
+    run: runHarness
+  },
+  churn: {
+    usage: 'load churn --fleet <file> [--interval-notify-sec 20 --provision-every-sec 0 --duration-sec 3600 --channel schema:update --pg-* ...]',
+    run: runChurn
+  }
+};

packages/perf-harness/src/commands/measure.ts

@@ -0,0 +1,34 @@
+/**
+ * `measure` domain — memory/heap measurement primitives.
+ *
+ *   measure collector      RSS sampler + server-metrics tailer (collector.mjs)
+ *   measure pg             PG-side docker + pg_stat_activity sampler (pg-sampler.mjs)
+ *   measure instance-heap  marginal per-class instance heap cost (measure-instance-heap.mjs)
+ */
+import { Argv } from '../core/args';
+import { runCollector } from '../measure/collector';
+import { runInstanceHeap } from '../measure/instanceHeap';
+import { runPgSampler } from '../measure/pgSampler';
+
+export const domain = 'measure';
+export const summary = 'memory/heap measurement: RSS+metrics collector, PG sampler, per-class instance-heap';
+
+export interface Subcommand {
+  usage: string;
+  run(argv: Argv): Promise<number>;
+}
+
+export const subcommands: Record<string, Subcommand> = {
+  collector: {
+    usage: 'measure collector --pid <n> [--metrics-file <f>] [--interval-sec 15] [--duration-sec 0] [--out collector.json]',
+    run: runCollector
+  },
+  pg: {
+    usage: 'measure pg [--out out/pg-soak.jsonl] [--interval-sec 30] [--duration-sec 0] [--container constructive-scale-pg] [--pg-* ...]',
+    run: runPgSampler
+  },
+  'instance-heap': {
+    usage: 'measure instance-heap --host <vhost> [--label <name>] [--port 3344] [--heap-mb 2048] [--cache-max 1] [--settle-sec 70] [--out-dir ./perf-out]',
+    run: runInstanceHeap
+  }
+};

packages/perf-harness/src/commands/regression.ts

@@ -0,0 +1,44 @@
+/**
+ * `perf-harness regression <sub>` — the one-command regression suite + catalog.
+ *
+ * Registration contract (consumed by src/index.ts): a domain name, a one-line
+ * summary, a `Subcommand` shape, and a `subcommands` map.
+ */
+import { Argv } from '../core/args';
+import { DEFAULT_BASELINE, listBaselines } from '../regression/baselines';
+import { runRegression } from '../regression/suite';
+
+export const domain = 'regression';
+export const summary = 'run the regression suite against a baseline, or list known baselines';
+
+export interface Subcommand {
+  usage: string;
+  run(argv: Argv): Promise<number>;
+}
+
+async function runBaselines(_argv: Argv): Promise<number> {
+  const rows = listBaselines();
+  console.log(`known baselines (default: ${DEFAULT_BASELINE}):`);
+  for (const b of rows) {
+    console.log(
+      `  ${b.name}  (captured ${b.capturedAt})  catalog=${b.catalogPgClassRows} rows  ` +
+        `I=${b.instanceHeapKBPerRow}KB/row  minViable=${b.minViableHeapMB}MB  ` +
+        `thresholds: errRate<=${b.thresholds.maxHealthyErrRate} p99<=${b.thresholds.maxHealthyP99Ms}ms ` +
+        `zmDelta<=${b.thresholds.maxZeroMarginalDeltaMB}MB heapKB/row<=${b.thresholds.maxInstanceHeapKBPerRow} ` +
+        `slope<=${b.thresholds.maxHeapSlopeMBPerHour}MB/h`
+    );
+  }
+  return 0;
+}
+
+export const subcommands: Record<string, Subcommand> = {
+  run: {
+    usage:
+      'regression run --fleet <manifest> [--suite quick|standard|deep] [--baseline <name>] [--port 3345] [--heap-mb 3584] [--deep-heap-mb 7168] [--out-dir ./perf-out] [--drifted <f>] [--same-bp-regex <re>] [--only <scenario,...>] [--skip <scenario,...>] [--server-cmd "..."] [--allow-hub]',
+    run: runRegression
+  },
+  baselines: {
+    usage: 'regression baselines',
+    run: runBaselines
+  }
+};

packages/perf-harness/src/commands/report.ts

@@ -0,0 +1,34 @@
+/**
+ * `perf-harness report <sub>` — post-run analysis commands.
+ *
+ * Registration contract (consumed by src/index.ts): a domain name, a one-line
+ * summary, a `Subcommand` shape, and a `subcommands` map.
+ */
+import { Argv } from '../core/args';
+import { runMerge } from '../report/merge';
+import { runPredict } from '../report/predict';
+import { runSummarize } from '../report/summarize';
+
+export const domain = 'report';
+export const summary = 'analyze run artifacts: summarize step results, merge series, predict capacity';
+
+export interface Subcommand {
+  usage: string;
+  run(argv: Argv): Promise<number>;
+}
+
+export const subcommands: Record<string, Subcommand> = {
+  summarize: {
+    usage: 'report summarize <results.jsonl> [more.jsonl ...]',
+    run: runSummarize
+  },
+  merge: {
+    usage:
+      'report merge --metrics <m.jsonl> --harness-log <h.jsonl> [--churn-log <c.jsonl>] [--ops-log <o.log>] [--pg <pg.jsonl>] [--out-dir ./perf-out]',
+    run: runMerge
+  },
+  predict: {
+    usage: 'report predict --instance-heap-bytes <n> [--heap-mb 1536,2048,3584] [--base-reserve-bytes <n>] [--build-reserve-bytes <n>]',
+    run: runPredict
+  }
+};

packages/perf-harness/src/commands/run.ts

@@ -0,0 +1,28 @@
+/**
+ * `run` domain — server lifecycle + ramp / soak orchestration.
+ *
+ * Subcommands:
+ *   run ramp                     V2 limits-discovery executor (fresh server per step)
+ *   run soak <start|status|stop> long-running soak orchestrator
+ *   run soak-ops                 provision/drop cycles during a soak
+ *
+ * `run soak` routes on the next positional (argv._[2]) inside runSoak.
+ */
+import { Argv } from '../core/args';
+import { RAMP_USAGE, runRamp } from '../run/ramp';
+import { runSoak, SOAK_USAGE } from '../run/soak';
+import { runSoakOps, SOAK_OPS_USAGE } from '../run/soakOps';
+
+export const domain = 'run';
+export const summary = 'server lifecycle + ramp / soak orchestration';
+
+export interface Subcommand {
+  usage: string;
+  run(argv: Argv): Promise<number>;
+}
+
+export const subcommands: Record<string, Subcommand> = {
+  ramp: { usage: RAMP_USAGE, run: (argv: Argv) => runRamp(argv) },
+  soak: { usage: SOAK_USAGE, run: (argv: Argv) => runSoak(argv) },
+  'soak-ops': { usage: SOAK_OPS_USAGE, run: (argv: Argv) => runSoakOps(argv) }
+};