diff --git a/graphql/server/package.json b/graphql/server/package.json index 5373750eb..3f6926dfa 100644 --- a/graphql/server/package.json +++ b/graphql/server/package.json @@ -88,6 +88,7 @@ "graphile-test": "workspace:*", "makage": "^0.3.0", "nodemon": "^3.1.14", + "pg-introspection": "1.0.1", "ts-node": "^10.9.2" } } diff --git a/graphql/server/src/diagnostics/__tests__/metrics-sampler.test.ts b/graphql/server/src/diagnostics/__tests__/metrics-sampler.test.ts index 57bce236c..ee21ede77 100644 --- a/graphql/server/src/diagnostics/__tests__/metrics-sampler.test.ts +++ b/graphql/server/src/diagnostics/__tests__/metrics-sampler.test.ts @@ -47,6 +47,8 @@ describe('collectMetricsSample', () => { expect(typeof sample.counters.evictions.lru).toBe('number'); expect(typeof sample.counters.builds).toBe('number'); expect(typeof sample.counters.buildQueueDepth).toBe('number'); + expect(typeof sample.counters.introspectionFilter.swaps).toBe('number'); + expect(typeof sample.counters.introspectionFilter.discoveries).toBe('number'); expect(typeof sample.gc).toBe('object'); // Round-trips through JSON without loss. diff --git a/graphql/server/src/diagnostics/metrics-sampler.ts b/graphql/server/src/diagnostics/metrics-sampler.ts index f16b7634f..792dfb588 100644 --- a/graphql/server/src/diagnostics/metrics-sampler.ts +++ b/graphql/server/src/diagnostics/metrics-sampler.ts @@ -5,6 +5,7 @@ import { Logger } from '@pgpmjs/logger'; import { getCacheCounters, getCacheStats } from 'graphile-cache'; import { getBuildQueueDepth, getGraphileCounters } from '../middleware/graphile'; +import { getIntrospectionFilterCounters } from '../middleware/introspection-filter'; import { getConnectionErrorGuardCounters } from './connection-error-guard'; const log = new Logger('metrics-sampler'); @@ -126,6 +127,7 @@ export interface MetricsSample { ReturnType & { buildQueueDepth: number; connGuard: ReturnType; + introspectionFilter: ReturnType; }; gc: GcStats; } @@ -153,7 +155,8 @@ export const collectMetricsSample = (): MetricsSample => { ...getCacheCounters(), ...getGraphileCounters(), buildQueueDepth: getBuildQueueDepth(), - connGuard: getConnectionErrorGuardCounters() + connGuard: getConnectionErrorGuardCounters(), + introspectionFilter: getIntrospectionFilterCounters() }, gc: snapshotGcStats() }; diff --git a/graphql/server/src/middleware/__tests__/introspection-filter.test.ts b/graphql/server/src/middleware/__tests__/introspection-filter.test.ts new file mode 100644 index 000000000..bcaef4eef --- /dev/null +++ b/graphql/server/src/middleware/__tests__/introspection-filter.test.ts @@ -0,0 +1,273 @@ +import { makeIntrospectionQuery } from 'pg-introspection'; + +import { + buildFilteredIntrospectionQuery, + createIntrospectionFilterCounters, + createIntrospectionFilterPool, + createIntrospectionInterceptor, + getIntrospectionFilterCounters, + isIntrospectionFilterEnabled, + isIntrospectionQuery +} from '../introspection-filter'; + +// The exact runtime gate substring (single backslash before the underscore). +// Derived here so tests fail loudly if the installed package ever drifts. +const GATE_SUBSTRING = + "in (select namespaces._id from namespaces where nspname <> 'information_schema' and nspname not like 'pg\\_%')"; + +const STOCK = makeIntrospectionQuery(); + +const countOccurrences = (haystack: string, needle: string): number => + haystack.split(needle).length - 1; + +const originalEnv = { ...process.env }; +afterEach(() => { + process.env = { ...originalEnv }; +}); + +// ============================================================================= +// Env flag +// ============================================================================= + +describe('isIntrospectionFilterEnabled', () => { + it("is true only for '1' or 'true'", () => { + process.env.GRAPHILE_INTROSPECTION_FILTER = '1'; + expect(isIntrospectionFilterEnabled()).toBe(true); + process.env.GRAPHILE_INTROSPECTION_FILTER = 'true'; + expect(isIntrospectionFilterEnabled()).toBe(true); + process.env.GRAPHILE_INTROSPECTION_FILTER = '0'; + expect(isIntrospectionFilterEnabled()).toBe(false); + delete process.env.GRAPHILE_INTROSPECTION_FILTER; + expect(isIntrospectionFilterEnabled()).toBe(false); + }); +}); + +// ============================================================================= +// 1. THE PINNING TEST +// ============================================================================= + +describe('buildFilteredIntrospectionQuery — pinned against installed pg-introspection', () => { + it('recognizes the stock query', () => { + expect(isIntrospectionQuery(STOCK)).toBe(true); + // The stock text carries the gate exactly four times. + expect(countOccurrences(STOCK, GATE_SUBSTRING)).toBe(4); + }); + + it('rewrites all four gates and preserves the pg_catalog type branch', () => { + const filtered = buildFilteredIntrospectionQuery(STOCK, ['t1-aaaa-app_public', 'public']); + expect(filtered).not.toBeNull(); + // The injected `= any (array[...])` gate appears exactly four times... + expect(countOccurrences(filtered as string, 'nspname = any (array[')).toBe(4); + // ...and zero stock gates remain. + expect(countOccurrences(filtered as string, GATE_SUBSTRING)).toBe(0); + // pg_catalog types survive via the intact OR branch. + expect((filtered as string).includes("'pg_catalog'::regnamespace")).toBe(true); + // Sorted, escaped literals present. + expect((filtered as string).includes("array['public', 't1-aaaa-app_public']::text[]")).toBe(true); + }); +}); + +// ============================================================================= +// 2. Gate mismatch +// ============================================================================= + +describe('buildFilteredIntrospectionQuery — gate mismatch', () => { + it('returns null when the gate does not appear exactly four times', () => { + // Remove one of the four gate occurrences -> three remain. + const doctored = STOCK.replace(GATE_SUBSTRING, 'in (select namespaces._id from namespaces where true)'); + expect(countOccurrences(doctored, GATE_SUBSTRING)).toBe(3); + expect(buildFilteredIntrospectionQuery(doctored, ['public'])).toBeNull(); + }); +}); + +// ============================================================================= +// 3. Literal escaping / defensive drops +// ============================================================================= + +describe('buildFilteredIntrospectionQuery — literal escaping', () => { + it("doubles single quotes and drops pg_ / information_schema names", () => { + const filtered = buildFilteredIntrospectionQuery(STOCK, [ + "o'brien", + 'pg_temp_1', + 'information_schema', + 'public' + ]); + expect(filtered).not.toBeNull(); + const text = filtered as string; + expect(text.includes("'o''brien'")).toBe(true); + expect(text.includes("'public'")).toBe(true); + expect(text.includes("'pg_temp_1'")).toBe(false); + // 'information_schema' appears in the stock text elsewhere, but never as an + // injected array literal. + expect(text.includes("array['information_schema'")).toBe(false); + expect(text.includes(", 'information_schema'")).toBe(false); + }); +}); + +// ============================================================================= +// 4. isIntrospectionQuery negatives +// ============================================================================= + +describe('isIntrospectionQuery — negatives', () => { + it('rejects the adaptor settings query, DEALLOCATE, and a plain SELECT', () => { + const settings = + 'select set_config(el->>0, el->>1, true) from json_array_elements($1::json) el'; + expect(isIntrospectionQuery(settings)).toBe(false); + expect(isIntrospectionQuery('deallocate all')).toBe(false); + expect(isIntrospectionQuery('select introspection_version from t')).toBe(false); + }); + + it('rejects a long query that merely mentions introspection_version', () => { + const long = 'select ' + 'a,'.repeat(3000) + " 'introspection_version' from t"; + expect(long.length).toBeGreaterThan(5000); + expect(isIntrospectionQuery(long)).toBe(false); + }); +}); + +// ============================================================================= +// 5. Closure loop + interceptor +// ============================================================================= + +const scriptedClient = (rounds: any[][]) => { + let i = 0; + return { + query: jest.fn((_sql: string, _values?: any[]) => { + const rows = rounds[i] ?? []; + i += 1; + return Promise.resolve({ rows }); + }) + }; +}; + +describe('createIntrospectionInterceptor — closure loop', () => { + beforeEach(() => { + process.env.GRAPHILE_INTROSPECTION_FILTER = '1'; + }); + + it('accumulates served ∪ public ∪ discovered, then filters', async () => { + const counters = createIntrospectionFilterCounters(); + const interceptor = createIntrospectionInterceptor({ servedSchemas: ['t1'], counters }); + // Round 1 discovers services_public; round 2 discovers nothing. + const client = scriptedClient([[{ nspname: 'services_public' }], []]); + + const swapped = await (interceptor(STOCK, client) as Promise); + + // keep set = public, t1, services_public + expect(swapped.includes("'public'")).toBe(true); + expect(swapped.includes("'t1'")).toBe(true); + expect(swapped.includes("'services_public'")).toBe(true); + expect(countOccurrences(swapped, GATE_SUBSTRING)).toBe(0); + + expect(counters.discoveries).toBe(1); + expect(counters.swaps).toBe(1); + expect(counters.closureTruncations).toBe(0); + expect(counters.keepNamespaceCount).toBe(3); + // Two closure rounds ran (the second returned nothing, ending the loop). + expect(client.query).toHaveBeenCalledTimes(2); + }); + + it('memoizes: a second call reuses the first discovery', async () => { + const counters = createIntrospectionFilterCounters(); + const interceptor = createIntrospectionInterceptor({ servedSchemas: ['t1'], counters }); + const client = scriptedClient([[], []]); + + const a = await (interceptor(STOCK, client) as Promise); + const b = await (interceptor(STOCK, client) as Promise); + expect(a).toBe(b); + expect(counters.discoveries).toBe(1); + expect(client.query).toHaveBeenCalledTimes(1); // one round (empty) only + }); + + it('rejects when the discovery query throws', async () => { + const counters = createIntrospectionFilterCounters(); + const interceptor = createIntrospectionInterceptor({ servedSchemas: ['t1'], counters }); + const client = { + query: jest.fn(() => Promise.reject(new Error('boom'))) + }; + await expect(interceptor(STOCK, client) as Promise).rejects.toThrow('boom'); + expect(counters.discoveries).toBe(1); + expect(counters.swaps).toBe(0); + }); + + it('returns null when disabled, served-empty, or non-introspection', () => { + const client = scriptedClient([[]]); + // disabled + process.env.GRAPHILE_INTROSPECTION_FILTER = '0'; + expect(createIntrospectionInterceptor({ servedSchemas: ['t1'] })(STOCK, client)).toBeNull(); + // enabled again + process.env.GRAPHILE_INTROSPECTION_FILTER = '1'; + // no served schemas + expect(createIntrospectionInterceptor({ servedSchemas: [] })(STOCK, client)).toBeNull(); + // non-introspection text + expect(createIntrospectionInterceptor({ servedSchemas: ['t1'] })('select 1', client)).toBeNull(); + }); +}); + +// ============================================================================= +// 6. createIntrospectionFilterPool (dedicated) +// ============================================================================= + +const introspectingClient = (rounds: any[][]) => { + let round = 0; + const query = jest.fn((arg: any, _values?: any[]) => { + if (typeof arg === 'string' && arg.includes('kept as')) { + const rows = rounds[round] ?? []; + round += 1; + return Promise.resolve({ rows }); + } + return Promise.resolve({ rows: [] }); + }); + return { query, release: jest.fn(), on: jest.fn() }; +}; + +const sentConfigTexts = (client: { query: jest.Mock }): string[] => + client.query.mock.calls.map((c: any[]) => (typeof c[0] === 'string' ? c[0] : c[0]?.text)); + +describe('createIntrospectionFilterPool — dedicated wrapper', () => { + beforeEach(() => { + process.env.GRAPHILE_INTROSPECTION_FILTER = '1'; + }); + + it('swaps the introspection query and leaves other queries untouched', async () => { + const counters = createIntrospectionFilterCounters(); + const client = introspectingClient([[], []]); + const pool = { connect: jest.fn().mockResolvedValue(client) }; + const wrapped = createIntrospectionFilterPool(pool, { servedSchemas: ['t1'], counters }); + + const c = await wrapped.connect(); + + // A normal query passes straight through. + await c.query('select 1 from t'); + expect(client.query).toHaveBeenCalledWith('select 1 from t'); + + // Introspection is discovered + swapped. + await c.query({ text: STOCK }); + const introspectionCall = client.query.mock.calls.find( + (call: any[]) => call[0] && typeof call[0] === 'object' && typeof call[0].text === 'string' && + call[0].text.includes('as introspection') + ); + expect(introspectionCall).toBeDefined(); + const swapped = introspectionCall[0].text as string; + expect(countOccurrences(swapped, 'nspname = any (array[')).toBe(4); + expect(swapped.includes("'t1'")).toBe(true); + expect(counters.swaps).toBe(1); + }); + + it('wraps clients handed back via callback-style connect', (done) => { + const client = introspectingClient([[]]); + const pool = { + connect: (cb: any) => cb(null, client, jest.fn()) + }; + const wrapped = createIntrospectionFilterPool(pool, { servedSchemas: ['t1'] }); + + wrapped.connect((err: any, c: any) => { + expect(err).toBeNull(); + c.query('select 1') + .then(() => { + expect(client.query).toHaveBeenCalledWith('select 1'); + done(); + }) + .catch(done); + }); + }); +}); diff --git a/graphql/server/src/middleware/graphile.ts b/graphql/server/src/middleware/graphile.ts index 0fdae99ca..288b0ea22 100644 --- a/graphql/server/src/middleware/graphile.ts +++ b/graphql/server/src/middleware/graphile.ts @@ -24,6 +24,10 @@ import { createConstructivePreset, makePgService } from 'graphile-settings'; import { getPgPool } from 'pg-cache'; import { getPgEnvOptions } from 'pg-env'; import './types'; // for Request type +import { + createIntrospectionFilterPool, + isIntrospectionFilterEnabled +} from './introspection-filter'; import { isGraphqlObservabilityEnabled } from '../diagnostics/observability'; import { HandlerCreationError } from '../errors/api-errors'; import { observeGraphileBuild } from './observability/graphile-build-stats'; @@ -312,12 +316,20 @@ const buildPreset = ( roleName: string, databaseSettings?: DatabaseSettings, ): GraphileConfig.Preset => { + // Introspection filter (opt-in via GRAPHILE_INTROSPECTION_FILTER): scope the + // instance's catalog introspection to the schemas it serves. Only active when + // the flag is on AND we have a concrete served-schema list; otherwise the pool + // selection below is byte-identical to today. + const introspectionFilterActive = isIntrospectionFilterEnabled() && schemas.length > 0; + const servicePool = introspectionFilterActive + ? createIntrospectionFilterPool(pool, { servedSchemas: schemas }) + : pool; const preset: GraphileConfig.Preset = { extends: [createConstructivePreset(databaseSettings)], plugins: [AuthCookiePlugin], pgServices: [ makePgService({ - pool, + pool: servicePool, schemas, }), ], diff --git a/graphql/server/src/middleware/introspection-filter.ts b/graphql/server/src/middleware/introspection-filter.ts new file mode 100644 index 000000000..9f68fea73 --- /dev/null +++ b/graphql/server/src/middleware/introspection-filter.ts @@ -0,0 +1,456 @@ +import { Logger } from '@pgpmjs/logger'; + +// ============================================================================= +// Introspection filter: scope PostGraphile v5 catalog introspection to the +// namespaces an instance actually serves. +// +// PostGraphile v5's introspection SQL (pg-introspection makeIntrospectionQuery) +// is UNSCOPED: it ingests the entire pg_class catalog (tens of thousands of rows +// on a shared multi-tenant database) into every built instance, and the parsed +// result's internal cross-references pin ~1.4GB of heap per instance regardless +// of how few schemas that instance serves. This module intercepts the single, +// static, parameter-less introspection statement as it flows through the pg pool +// wrapper we already own and rewrites its four namespace gates so only the +// served schemas (plus their transitive cross-schema references, plus public and +// pg_catalog) are ingested. +// +// It is intentionally FAIL-OPEN: any shape it does not recognize (gate count != +// 4, a non-introspection query, an empty served-schema set, the feature flag +// off) falls straight back to today's stock behavior. The one non-open path is a +// throwing discovery query — that fails the instance build, which the caller +// already retries on later requests, and is preferable to serving an instance +// built on an under-scoped (data-losing) catalog. +// +// The rewrite touches SQL STRING LITERALS only (single-quoted schema names in an +// `= any(array[...])`); it never emits double-quoted identifiers, so the sibling +// rewrite-pool's identifier lexer neither sees nor rewrites anything here. +// ============================================================================= + +const log = new Logger('graphile:introspection-filter'); + +// The exact namespace-gate substring emitted by pg-introspection@1.0.1's +// makeIntrospectionQuery(). It appears EXACTLY 4 times (classes.relnamespace, +// constraints.connamespace, procs.pronamespace, types.typnamespace). The types +// gate is followed by `or (typnamespace = 'pg_catalog'::regnamespace)` which we +// leave intact so pg_catalog types survive. NOTE the single backslash before the +// underscore: that is the runtime string (the .js source shows a doubled +// backslash only because it sits in a template literal). Verified by test +// against the installed package rather than hand-transcription. +const GATE_SUBSTRING = + "in (select namespaces._id from namespaces where nspname <> 'information_schema' and nspname not like 'pg\\_%')"; + +// Cheap structural signature of the introspection statement. Ordinary +// GraphQL-generated SQL can never satisfy all three of these simultaneously. +const INTROSPECTION_PREFIX = 'with\n database as ('; +const INTROSPECTION_MARKER = ')::text as introspection'; +const INTROSPECTION_VERSION_TOKEN = 'introspection_version'; + +// ----------------------------------------------------------------------------- +// Env flag +// ----------------------------------------------------------------------------- + +/** + * Whether the introspection filter is enabled via GRAPHILE_INTROSPECTION_FILTER. + * Accepts '1' or 'true'; anything else (incl. unset) means disabled. Mirrors the + * GRAPHILE_BLUEPRINT_POOLING read in blueprint.ts. + */ +export const isIntrospectionFilterEnabled = (): boolean => { + const value = process.env.GRAPHILE_INTROSPECTION_FILTER; + return value === '1' || value === 'true'; +}; + +// ----------------------------------------------------------------------------- +// Counters +// ----------------------------------------------------------------------------- + +export interface IntrospectionFilterCounters { + /** Closure-discovery passes actually run (one per intercepted introspection). */ + discoveries: number; + /** Introspection queries whose text was swapped for a filtered one. */ + swaps: number; + /** Callback-form or otherwise unfilterable introspection queries passed through. */ + failures: number; + /** Times buildFilteredIntrospectionQuery returned null (gate count != 4). */ + gateMismatches: number; + /** Closure loops that hit the 5-round cap before converging. */ + closureTruncations: number; + /** Total namespaces retained by the most recent successful filter build. */ + keepNamespaceCount: number; +} + +/** Build a zeroed counters object (companion to the interceptor opts). */ +export const createIntrospectionFilterCounters = (): IntrospectionFilterCounters => ({ + discoveries: 0, + swaps: 0, + failures: 0, + gateMismatches: 0, + closureTruncations: 0, + keepNamespaceCount: 0 +}); + +// Process-wide sink: the default counters when an interceptor/pool is created +// without its own, so getIntrospectionFilterCounters() (read by the metrics +// sampler) reflects real activity aggregated across every filtered instance. +const moduleCounters = createIntrospectionFilterCounters(); + +/** Snapshot the process-wide introspection-filter counters (metrics sampler entry point). Returns a copy so callers cannot mutate live state. */ +export const getIntrospectionFilterCounters = (): IntrospectionFilterCounters => ({ + ...moduleCounters +}); + +/** + * Record a callback-form introspection query that had to be passed through + * unfiltered (callback semantics are never rewritten). Bumps the module sink so + * the rewrite-pool integration can report it without owning its own counters. + */ +export const noteIntrospectionCallbackPassthrough = (): void => { + moduleCounters.failures += 1; +}; + +// ----------------------------------------------------------------------------- +// Query detection +// ----------------------------------------------------------------------------- + +/** + * Cheap signature check for the pg-introspection statement. Robust against + * user/GraphQL-generated SQL: requires the length floor, the real CTE prefix, + * the trailing `)::text as introspection` marker AND the introspection_version + * token together. + */ +export const isIntrospectionQuery = (text: string): boolean => { + if (typeof text !== 'string' || text.length <= 5000) return false; + return ( + text.startsWith(INTROSPECTION_PREFIX) && + text.includes(INTROSPECTION_MARKER) && + text.includes(INTROSPECTION_VERSION_TOKEN) + ); +}; + +// ----------------------------------------------------------------------------- +// Filtered-query builder +// ----------------------------------------------------------------------------- + +/** Escape a value as a SQL string literal body (single quotes doubled). */ +const sqlLiteral = (value: string): string => "'" + value.replace(/'/g, "''") + "'"; + +const isRejectedNamespace = (name: string): boolean => + name.length === 0 || + name.includes('\u0000') || + name === 'information_schema' || + name.startsWith('pg_'); + +/** + * Rewrite the four namespace gates of the stock introspection text so only + * `keepNamespaces` (sorted, deduped, escaped) are ingested. Returns null when + * the gate substring does not appear exactly 4 times (the caller then falls back + * to the stock text). Names starting 'pg_' / equal to 'information_schema' / + * containing NUL are defensively dropped from the injected array; the intact + * `or (typnamespace = 'pg_catalog'::regnamespace)` branch keeps pg_catalog + * types. + */ +export const buildFilteredIntrospectionQuery = ( + stockText: string, + keepNamespaces: string[] +): string | null => { + const occurrences = stockText.split(GATE_SUBSTRING).length - 1; + if (occurrences !== 4) return null; + + const cleaned = Array.from(new Set(keepNamespaces.filter((n) => !isRejectedNamespace(n)))).sort(); + const arrayLiteral = + cleaned.length === 0 + ? "in (select namespaces._id from namespaces where false)" + : 'in (select namespaces._id from namespaces where nspname = any (array[' + + cleaned.map(sqlLiteral).join(', ') + + ']::text[]))'; + + return stockText.split(GATE_SUBSTRING).join(arrayLiteral); +}; + +// ----------------------------------------------------------------------------- +// Namespace-closure discovery +// ----------------------------------------------------------------------------- + +// One round of closure discovery. Given the already-kept namespaces ($1), return +// NEW namespaces referenced by objects living in kept namespaces, via three arms: +// 1. FK targets: pg_constraint(contype='f') in kept classes -> confrelid ns +// 2. attribute types: live attributes of kept classes -> atttypid ns, plus that +// type's typbasetype and typelem namespaces when nonzero +// 3. proc types: pg_proc in kept namespaces -> prorettype ns + arg-type ns +// Every arm schema-qualifies pg_catalog.* (no search_path) and excludes +// information_schema, pg_* namespaces, and names already in the kept set. +const CLOSURE_ROUND_SQL = ` +with kept as ( + select n.oid + from pg_catalog.pg_namespace n + where n.nspname = any ($1::text[]) +), +kept_classes as ( + select c.oid, c.relnamespace + from pg_catalog.pg_class c + where c.relnamespace in (select oid from kept) +), +referenced as ( + -- arm 1: foreign-key target namespaces + select tc.relnamespace as nsoid + from pg_catalog.pg_constraint con + join kept_classes kc on kc.oid = con.conrelid + join pg_catalog.pg_class tc on tc.oid = con.confrelid + where con.contype = 'f' + + union + + -- arm 2: attribute type namespaces (+ base/element type namespaces) + select t.typnamespace as nsoid + from pg_catalog.pg_attribute a + join kept_classes kc on kc.oid = a.attrelid + join pg_catalog.pg_type t on t.oid = a.atttypid + where a.attnum > 0 and not a.attisdropped + + union + select bt.typnamespace as nsoid + from pg_catalog.pg_attribute a + join kept_classes kc on kc.oid = a.attrelid + join pg_catalog.pg_type t on t.oid = a.atttypid + join pg_catalog.pg_type bt on bt.oid = t.typbasetype + where a.attnum > 0 and not a.attisdropped and t.typbasetype <> 0 + + union + select et.typnamespace as nsoid + from pg_catalog.pg_attribute a + join kept_classes kc on kc.oid = a.attrelid + join pg_catalog.pg_type t on t.oid = a.atttypid + join pg_catalog.pg_type et on et.oid = t.typelem + where a.attnum > 0 and not a.attisdropped and t.typelem <> 0 + + union + + -- arm 3: proc return-type and argument-type namespaces + select rt.typnamespace as nsoid + from pg_catalog.pg_proc p + join pg_catalog.pg_type rt on rt.oid = p.prorettype + where p.pronamespace in (select oid from kept) + + union + select at.typnamespace as nsoid + from pg_catalog.pg_proc p + cross join lateral unnest(p.proargtypes) as pat(argoid) + join pg_catalog.pg_type at on at.oid = pat.argoid + where p.pronamespace in (select oid from kept) +) +select distinct n.nspname +from referenced r +join pg_catalog.pg_namespace n on n.oid = r.nsoid +where n.nspname <> 'information_schema' + and n.nspname not like 'pg\\_%' + and n.nspname <> all ($1::text[]) +`; + +const MAX_CLOSURE_ROUNDS = 5; + +/** Minimal shape of the checked-out client the discovery needs. */ +interface RawClientLike { + query(sql: string, values?: any[]): Promise<{ rows: any[] }>; +} + +/** + * Iterate CLOSURE_ROUND_SQL in JS until a round adds nothing or the round cap is + * hit. Starting set = servedSchemas ∪ {'public'}. Runs sequential queries on the + * provided (exclusively checked-out) client. Returns the accumulated keep set and + * whether the cap truncated the walk. + */ +const discoverKeepNamespaces = async ( + client: RawClientLike, + servedSchemas: string[] +): Promise<{ keep: string[]; truncated: boolean }> => { + const keep = new Set(['public']); + for (const s of servedSchemas) { + if (typeof s === 'string' && s.length > 0) keep.add(s); + } + + let truncated = true; + for (let round = 0; round < MAX_CLOSURE_ROUNDS; round++) { + const result = await client.query(CLOSURE_ROUND_SQL, [Array.from(keep)]); + const added: string[] = []; + for (const row of result.rows) { + const name = row.nspname; + if (typeof name === 'string' && !keep.has(name)) { + keep.add(name); + added.push(name); + } + } + if (added.length === 0) { + truncated = false; + break; + } + } + + return { keep: Array.from(keep), truncated }; +}; + +// ----------------------------------------------------------------------------- +// Interceptor +// ----------------------------------------------------------------------------- + +export interface IntrospectionInterceptorOptions { + /** Physical schema names this instance serves (the discovery seed, sans public). */ + servedSchemas: string[]; + /** Optional shared counters; the module sink is used when omitted. */ + counters?: IntrospectionFilterCounters; +} + +/** + * The interceptor callback: given the stock introspection text and the raw + * checked-out client, resolves to the text to actually execute. Returns null + * (synchronously) when the filter must not apply — filter disabled, no served + * schemas, or the text is not the introspection query — so the caller runs the + * original query unchanged. + * + * On the filtering path it memoizes: the closure discovery + filtered text are + * computed once per interceptor (i.e. once per instance build) and reused for any + * subsequent introspection on the same interceptor. A gate mismatch records the + * reason, warns once, and returns the STOCK text (graceful fallback). A throwing + * discovery query rejects (the instance build fails and is retried later). + */ +export type IntrospectionInterceptor = ( + text: string, + rawClient: RawClientLike +) => Promise | null; + +export const createIntrospectionInterceptor = ( + opts: IntrospectionInterceptorOptions +): IntrospectionInterceptor => { + const counters = opts.counters ?? moduleCounters; + const servedSchemas = Array.isArray(opts.servedSchemas) ? opts.servedSchemas : []; + + let memo: Promise | null = null; + let warnedGateMismatch = false; + + return (text: string, rawClient: RawClientLike): Promise | null => { + if (!isIntrospectionFilterEnabled()) return null; + if (servedSchemas.length === 0) return null; + if (!isIntrospectionQuery(text)) return null; + + if (memo) return memo; + + memo = (async (): Promise => { + counters.discoveries += 1; + const { keep, truncated } = await discoverKeepNamespaces(rawClient, servedSchemas); + if (truncated) counters.closureTruncations += 1; + + const filtered = buildFilteredIntrospectionQuery(text, keep); + if (filtered === null) { + counters.gateMismatches += 1; + if (!warnedGateMismatch) { + warnedGateMismatch = true; + log.warn( + 'introspection gate substring not found exactly 4 times; falling back to unfiltered introspection' + ); + } + return text; + } + + counters.swaps += 1; + counters.keepNamespaceCount = keep.length; + return filtered; + })(); + + // A rejected discovery must not poison later attempts (the build is retried). + memo.catch(() => { + memo = null; + }); + + return memo; + }; +}; + +// ----------------------------------------------------------------------------- +// Dedicated-instance pool wrapper +// ----------------------------------------------------------------------------- + +export interface IntrospectionFilterPoolOptions { + /** Physical schema names this dedicated instance serves. */ + servedSchemas: string[]; + /** Optional shared counters; the module sink is used when omitted. */ + counters?: IntrospectionFilterCounters; +} + +/** + * Thin Proxy over a raw pg Pool for DEDICATED (non-pooled) instances: intercepts + * a checked-out client's introspection query and swaps it for the filtered text. + * Everything else passes through untouched. No settings parsing, identifier + * rewriting, or prepared-name logic (unlike rewrite-pool). The per-checkout + * interceptor closure is a per-connection memo; release restores nothing. + */ +export const createIntrospectionFilterPool = (pool: any, opts: IntrospectionFilterPoolOptions): any => { + const counters = opts.counters ?? moduleCounters; + const servedSchemas = opts.servedSchemas; + + const wrapClient = (client: any): any => { + const interceptor = createIntrospectionInterceptor({ servedSchemas, counters }); + + const interceptQuery = (rawQuery: any, thisArg: any, args: any[]): any => { + const first = args[0]; + + // Promise-form config object with a text field is the only introspection + // shape (the @dataplan/pg adaptor runs `client.query({ text })`). A callback + // last-arg means callback semantics: never filter, count and pass through. + let text: string | undefined; + if (typeof first === 'string') text = first; + else if (first && typeof first === 'object' && typeof first.text === 'string') text = first.text; + + const hasCallback = typeof args[args.length - 1] === 'function'; + + if (text !== undefined && isIntrospectionQuery(text)) { + if (hasCallback) { + counters.failures += 1; + return rawQuery.apply(thisArg, args); + } + const swapPromise = interceptor(text, { query: (sql, values) => thisArg.query(sql, values) }); + if (swapPromise === null) { + return rawQuery.apply(thisArg, args); + } + return Promise.resolve(swapPromise).then((swapped) => { + if (swapped === text) return rawQuery.apply(thisArg, args); + if (typeof first === 'string') { + return rawQuery.apply(thisArg, [swapped, ...args.slice(1)]); + } + return rawQuery.apply(thisArg, [{ ...first, text: swapped }, ...args.slice(1)]); + }); + } + + return rawQuery.apply(thisArg, args); + }; + + return new Proxy(client, { + get(target, prop, receiver) { + if (prop === 'query') { + return (...args: any[]) => interceptQuery(target.query, target, args); + } + const value = Reflect.get(target, prop, receiver); + return typeof value === 'function' ? value.bind(target) : value; + } + }); + }; + + return new Proxy(pool, { + get(target, prop, receiver) { + if (prop === 'connect') { + return (...args: any[]) => { + if (typeof args[0] === 'function') { + const cb = args[0]; + return target.connect((err: any, client: any, done: any) => { + if (err) { + cb(err, client, done); + return; + } + cb(err, wrapClient(client), done); + }); + } + return Promise.resolve(target.connect(...args)).then((client: any) => wrapClient(client)); + }; + } + const value = Reflect.get(target, prop, receiver); + return typeof value === 'function' ? value.bind(target) : value; + } + }); +}; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 77fa6aae5..ca7b74485 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1792,6 +1792,9 @@ importers: nodemon: specifier: ^3.1.14 version: 3.1.14 + pg-introspection: + specifier: 1.0.1 + version: 1.0.1 ts-node: specifier: ^10.9.2 version: 10.9.2(@types/node@25.9.1)(typescript@5.9.3)